login
RDMA QP事件eBPF调试指南
2025/09/02·StepFun can make mistakes, check important info.
我在编写一个捕捉rdma qp事件的ebpf程序,代码大概是 #include "rdma.h" // #include "rdma_verbs.h" #include "vmlinux.h" #include <bpf/bpf_core_read.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> char LICENSE[] SEC("license") = "Dual BSD/GPL"; struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); __uint(key_size, sizeof(int)); __uint(value_size, sizeof(u32)); __uint(max_entries, 1024); } events SEC(".maps"); SEC("kprobe/mlx5_ib_modify_qp") int BPF_KPROBE(capture_modify_qp, struct ib_qp *qp, struct ib_qp_attr *attr, int attr_mask, void *udata) { // 使用栈上的临时变量来避免结构体对齐问题 int qp_state = -1; int qp_type = -1; unsigned int rqpn = 0; int qp_access_flags = 0; unsigned char port_num = 0; unsigned char alt_port_num = 0; unsigned short ib_dlid = 0; unsigned int grh_flow_label = 0; unsigned int alt_flow_label = 0; unsigned char grh_tc = 0; unsigned char sgid_port = 0; unsigned char qp_port = 0; unsigned int lqpn = 0; unsigned char sgid_raw[GID_RAW_SIZE] = { 0 }; unsigned char dgid_raw[GID_RAW_SIZE] = { 0 }; unsigned long long qp_ptr = 0; if (!attr || !qp) { return 0; } bpf_probe_read_kernel(&qp_ptr, sizeof(qp_ptr), &qp); // qp state,只有 RTR 会有 rqpn,这也是我们最关心的 bpf_probe_read_kernel(&qp_state, sizeof(qp_state), &attr->qp_state); bpf_probe_read_kernel(&qp_type, sizeof(qp_type), &qp->qp_type); // 对端的QPN,需要和节点信息一起确定一张NIC bpf_probe_read_kernel(&rqpn, sizeof(rqpn), &attr->dest_qp_num); if (rqpn > 0) { // 其实access flags没啥用,懒得删了,还要重新调整内存对齐 bpf_probe_read_kernel(&qp_access_flags, sizeof(qp_access_flags), &attr->qp_access_flags); // 对端dlid,其实没啥用 bpf_probe_read_kernel(&ib_dlid, sizeof(ib_dlid), &attr->ah_attr.ib.dlid); // flow label,对RC类型的QP而言,client和server的flow label是相同的 bpf_probe_read_kernel(&grh_flow_label, sizeof(grh_flow_label), &attr->ah_attr.grh.flow_label); bpf_probe_read_kernel(&alt_flow_label, sizeof(alt_flow_label), &attr->alt_ah_attr.grh.flow_label); // tc值,监控信息可能有用 bpf_probe_read_kernel(&grh_tc, sizeof(grh_tc), &attr->ah_attr.grh.traffic_class); // 先读取 sgid_attr 指针值,检查指针是否有效,避免非法访问 unsigned long long sgid_attr_ptr = 0; bpf_probe_read_kernel(&sgid_attr_ptr, sizeof(sgid_attr_ptr), &attr->ah_attr.grh.sgid_attr); if (sgid_attr_ptr != 0) { struct ib_gid_attr *gid_attr = (struct ib_gid_attr *)sgid_attr_ptr; bpf_probe_read_kernel(&sgid_port, sizeof(sgid_port), &gid_attr->port_num); bpf_probe_read_kernel(sgid_raw, GID_RAW_SIZE * sizeof(sgid_raw[0]), gid_attr->gid.raw); bpf_probe_read_kernel(dgid_raw, GID_RAW_SIZE * sizeof(dgid_raw[0]), attr->ah_attr.grh.dgid.raw); } // 本地QPN,因为我们只抓RC类型的QP,通过QPN就足够映射到gid,再映射到NIC bpf_probe_read_kernel(&lqpn, sizeof(lqpn), &qp->qp_num); } // 构造事件结构体 qp_ev e = { .pid = bpf_get_current_pid_tgid() >> 32, .qp_state = qp_state, .rqpn = rqpn, .qp_access_flags = qp_access_flags, .port_num = port_num, .alt_port_num = alt_port_num, .ib_dlid = ib_dlid, .grh_flow_label = grh_flow_label, .grh_tc = grh_tc, .sgid_port = sgid_port, .qp_port = qp_port, .useless = 0, .lqpn = lqpn, .sgid_raw = { 0 }, .dgid_raw = { 0 }, .alt_flow_label = alt_flow_label, .qp_type = qp_type, .qp_ptr = qp_ptr, .timestamp_ns = bpf_ktime_get_ns(), .event_type = 0, }; for (int i = 0; i < GID_RAW_SIZE; i++) { e.sgid_raw[i] = sgid_raw[i]; e.dgid_raw[i] = dgid_raw[i]; } bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &e, sizeof(e)); return 0; } SEC("kprobe/mlx5_ib_destroy_qp") int BPF_KPROBE(capture_destroy_qp, struct ib_qp *qp, void *udata) { if (!qp) { return 0; } // 使用栈上的临时变量来避免结构体对齐问题 int qp_type = -1; uint32_t lqpn = 0; bpf_probe_read_kernel(&qp_type, sizeof(qp_type), &qp->qp_type); bpf_probe_read_kernel(&lqpn, sizeof(lqpn), &qp->qp_num); qp_ev e = { .pid = bpf_get_current_pid_tgid() >> 32, .qp_state = 0, .rqpn = 0, .qp_access_flags = 0, .port_num = 0, .alt_port_num = 0, .ib_dlid = 0, .grh_flow_label = 0, .grh_tc = 0, .sgid_port = 0, .qp_port = 0, .useless = 0, .lqpn = lqpn, .sgid_raw = { 0 }, .dgid_raw = { 0 }, .alt_flow_label = 0, .qp_type = qp_type, .qp_ptr = 0, .timestamp_ns = bpf_ktime_get_ns(), .event_type = 1, }; bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &e, sizeof(e)); return 0; } 测试发现,对于IB设备的qp事件,打印出的事件中的sgid和dgid始终是空的,即都是0,在RoCE v2上就是正常的,帮我研究一下为什么