我在编写一个捕捉rdma qp事件的ebpf程序,代码大概是
#include "rdma.h"
// #include "rdma_verbs.h"
#include "vmlinux.h"
#include <bpf/bpf_core_read.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
char LICENSE[] SEC("license") = "Dual BSD/GPL";
struct {
__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
__uint(key_size, sizeof(int));
__uint(value_size, sizeof(u32));
__uint(max_entries, 1024);
} events SEC(".maps");
SEC("kprobe/mlx5_ib_modify_qp")
int BPF_KPROBE(capture_modify_qp, struct ib_qp *qp, struct ib_qp_attr *attr, int attr_mask,
void *udata)
{
// 使用栈上的临时变量来避免结构体对齐问题
int qp_state = -1;
int qp_type = -1;
unsigned int rqpn = 0;
int qp_access_flags = 0;
unsigned char port_num = 0;
unsigned char alt_port_num = 0;
unsigned short ib_dlid = 0;
unsigned int grh_flow_label = 0;
unsigned int alt_flow_label = 0;
unsigned char grh_tc = 0;
unsigned char sgid_port = 0;
unsigned char qp_port = 0;
unsigned int lqpn = 0;
unsigned char sgid_raw[GID_RAW_SIZE] = { 0 };
unsigned char dgid_raw[GID_RAW_SIZE] = { 0 };
unsigned long long qp_ptr = 0;
if (!attr || !qp) {
return 0;
}
bpf_probe_read_kernel(&qp_ptr, sizeof(qp_ptr), &qp);
// qp state,只有 RTR 会有 rqpn,这也是我们最关心的
bpf_probe_read_kernel(&qp_state, sizeof(qp_state), &attr->qp_state);
bpf_probe_read_kernel(&qp_type, sizeof(qp_type), &qp->qp_type);
// 对端的QPN,需要和节点信息一起确定一张NIC
bpf_probe_read_kernel(&rqpn, sizeof(rqpn), &attr->dest_qp_num);
if (rqpn > 0) {
// 其实access flags没啥用,懒得删了,还要重新调整内存对齐
bpf_probe_read_kernel(&qp_access_flags, sizeof(qp_access_flags),
&attr->qp_access_flags);
// 对端dlid,其实没啥用
bpf_probe_read_kernel(&ib_dlid, sizeof(ib_dlid), &attr->ah_attr.ib.dlid);
// flow label,对RC类型的QP而言,client和server的flow label是相同的
bpf_probe_read_kernel(&grh_flow_label, sizeof(grh_flow_label),
&attr->ah_attr.grh.flow_label);
bpf_probe_read_kernel(&alt_flow_label, sizeof(alt_flow_label),
&attr->alt_ah_attr.grh.flow_label);
// tc值,监控信息可能有用
bpf_probe_read_kernel(&grh_tc, sizeof(grh_tc), &attr->ah_attr.grh.traffic_class);
// 先读取 sgid_attr 指针值,检查指针是否有效,避免非法访问
unsigned long long sgid_attr_ptr = 0;
bpf_probe_read_kernel(&sgid_attr_ptr, sizeof(sgid_attr_ptr),
&attr->ah_attr.grh.sgid_attr);
if (sgid_attr_ptr != 0) {
struct ib_gid_attr *gid_attr = (struct ib_gid_attr *)sgid_attr_ptr;
bpf_probe_read_kernel(&sgid_port, sizeof(sgid_port), &gid_attr->port_num);
bpf_probe_read_kernel(sgid_raw, GID_RAW_SIZE * sizeof(sgid_raw[0]),
gid_attr->gid.raw);
bpf_probe_read_kernel(dgid_raw, GID_RAW_SIZE * sizeof(dgid_raw[0]),
attr->ah_attr.grh.dgid.raw);
}
// 本地QPN,因为我们只抓RC类型的QP,通过QPN就足够映射到gid,再映射到NIC
bpf_probe_read_kernel(&lqpn, sizeof(lqpn), &qp->qp_num);
}
// 构造事件结构体
qp_ev e = {
.pid = bpf_get_current_pid_tgid() >> 32,
.qp_state = qp_state,
.rqpn = rqpn,
.qp_access_flags = qp_access_flags,
.port_num = port_num,
.alt_port_num = alt_port_num,
.ib_dlid = ib_dlid,
.grh_flow_label = grh_flow_label,
.grh_tc = grh_tc,
.sgid_port = sgid_port,
.qp_port = qp_port,
.useless = 0,
.lqpn = lqpn,
.sgid_raw = { 0 },
.dgid_raw = { 0 },
.alt_flow_label = alt_flow_label,
.qp_type = qp_type,
.qp_ptr = qp_ptr,
.timestamp_ns = bpf_ktime_get_ns(),
.event_type = 0,
};
for (int i = 0; i < GID_RAW_SIZE; i++) {
e.sgid_raw[i] = sgid_raw[i];
e.dgid_raw[i] = dgid_raw[i];
}
bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &e, sizeof(e));
return 0;
}
SEC("kprobe/mlx5_ib_destroy_qp")
int BPF_KPROBE(capture_destroy_qp, struct ib_qp *qp, void *udata)
{
if (!qp) {
return 0;
}
// 使用栈上的临时变量来避免结构体对齐问题
int qp_type = -1;
uint32_t lqpn = 0;
bpf_probe_read_kernel(&qp_type, sizeof(qp_type), &qp->qp_type);
bpf_probe_read_kernel(&lqpn, sizeof(lqpn), &qp->qp_num);
qp_ev e = {
.pid = bpf_get_current_pid_tgid() >> 32,
.qp_state = 0,
.rqpn = 0,
.qp_access_flags = 0,
.port_num = 0,
.alt_port_num = 0,
.ib_dlid = 0,
.grh_flow_label = 0,
.grh_tc = 0,
.sgid_port = 0,
.qp_port = 0,
.useless = 0,
.lqpn = lqpn,
.sgid_raw = { 0 },
.dgid_raw = { 0 },
.alt_flow_label = 0,
.qp_type = qp_type,
.qp_ptr = 0,
.timestamp_ns = bpf_ktime_get_ns(),
.event_type = 1,
};
bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &e, sizeof(e));
return 0;
}
测试发现,对于IB设备的qp事件,打印出的事件中的sgid和dgid始终是空的,即都是0,在RoCE v2上就是正常的,帮我研究一下为什么