providers: add support for USDT tracing using libbpf/usdt

natoscott · natoscott · commit 7790db43bce5 · 2025-11-18T09:57:15.000+11:00
Provide lightweight USDT tracing as an alternative to LTTng. This piggybacks on the existing tracing code added for LTTng for a minimal set of changes. > $ sudo bpftrace -l usdt:build/lib/lib*.so:* > usdt:build/lib/libefa-rdmav59.so:rdma_core_efa:post_recv > usdt:build/lib/libefa-rdmav59.so:rdma_core_efa:post_send > usdt:build/lib/libefa-rdmav59.so:rdma_core_efa:process_completion > usdt:build/lib/libefa.so:rdma_core_efa:post_recv > usdt:build/lib/libefa.so:rdma_core_efa:post_send > usdt:build/lib/libefa.so:rdma_core_efa:process_completion > usdt:build/lib/libhns-rdmav59.so:rdma_core_hns:poll_cq > usdt:build/lib/libhns-rdmav59.so:rdma_core_hns:post_recv > usdt:build/lib/libhns-rdmav59.so:rdma_core_hns:post_send > usdt:build/lib/libhns.so:rdma_core_hns:poll_cq > usdt:build/lib/libhns.so:rdma_core_hns:post_recv > usdt:build/lib/libhns.so:rdma_core_hns:post_send > usdt:build/lib/libmlx5-rdmav59.so:rdma_core_mlx5:post_send > usdt:build/lib/libmlx5.so:rdma_core_mlx5:post_send > usdt:build/lib/librxe-rdmav59.so:rdma_core_rxe:post_send The USDT header used here is from the libbpf/usdt project at https://github.com/libbpf/usdt.git Further background discussion for this commit is included in linux-rdma#1621 Signed-off-by: Nathan Scott <nathans@redhat.com>
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -24,7 +24,7 @@
 #      Use the historical search path for providers, in the standard system library.
 #  -DNO_COMPAT_SYMS=1 (default disabled)
 #      Do not generate backwards compatibility symbols in the shared
-#      libraries. This may is necessary if using a dynmic linker that does
+#      libraries. This may be necessary if using a dynamic linker that does
 #      not support symbol versions, such as uclibc.
 #  -DIOCTL_MODE=write (default both)
 #      Disable new kABI ioctl() support and support only the legacy write
@@ -52,8 +52,10 @@
 #      Disable man pages. Allows rdma-core to be built and installed
 #      (without man pages) when neither pandoc/rst2man nor the pandoc-prebuilt
 #      directory are available.
-#   -DENABLE_LTTNG (default, no tracing support)
+#   -DENABLE_LTTNG (default, no LTTng tracing support)
 #      Enable LTTng tracing.
+#   -DENABLE_USDT (default, no USDT tracing support)
+#      Enable USDT tracing.
 
 if (${CMAKE_VERSION} VERSION_LESS "3.18.1")
 	# Centos 7 support
@@ -573,6 +575,11 @@ if (DEFINED ENABLE_LTTNG)
   add_definitions(-DLTTNG_ENABLED)
 endif()
 
+# User Static-Defined Tracing (USDT) support
+if (DEFINED ENABLE_USDT)
+  add_definitions(-DUSDT_ENABLED)
+endif()
+
 #-------------------------
 # Apply fixups
 
diff --git a/Documentation/contributing.md b/Documentation/contributing.md
@@ -66,7 +66,7 @@ that are designed to weed out bugs.
 Serious errors will result in a red X in the PR and will need to be corrected.
 Less serious errors, including checkpatch related, will show up with a green
 check but it is necessary to check the details to see that everything is
-appropriate. checkpatch is an informative too, not all of its feedback is
+appropriate. checkpatch is an informative tool, not all of its feedback is
 appropriate to fix.
 
 A build similar to AZP can be run locally using docker and the
diff --git a/providers/efa/efa_trace.h b/providers/efa/efa_trace.h
@@ -109,13 +109,19 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined (USDT_ENABLED)
 
 #ifndef __EFA_TRACE_H__
 #define __EFA_TRACE_H__
 
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#else /* __EFA_TRACE_H__*/
+
 #define rdma_tracepoint(arg...)
 
 #endif /* __EFA_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */
diff --git a/providers/efa/verbs.c b/providers/efa/verbs.c
@@ -2170,7 +2170,7 @@ static void efa_set_common_ctrl_flags(struct efa_io_tx_meta_desc *desc,
 	EFA_SET(&desc->ctrl2, EFA_IO_TX_META_DESC_COMP_REQ, 1);
 }
 
-#ifdef LTTNG_ENABLED
+#if defined(LTTNG_ENABLED) || defined(USDT_ENABLED)
 static uint32_t efa_get_wqe_length(struct efa_io_tx_wqe *tx_wqe)
 {
 	enum efa_io_send_op_type op_type;
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
@@ -678,7 +678,7 @@ static int parse_cqe_for_cq(struct hns_roce_context *ctx, struct hns_roce_cq *cq
 	return 0;
 }
 
-#ifdef LTTNG_ENABLED
+#if defined(LTTNG_ENABLED) || defined(USDT_ENABLED)
 static uint8_t read_wc_sl(struct hns_roce_qp *hr_qp,
 				   struct hns_roce_v2_cqe *cqe,
 				   struct ibv_wc *wc)
@@ -687,6 +687,7 @@ static uint8_t read_wc_sl(struct hns_roce_qp *hr_qp,
 		hr_reg_read(cqe, CQE_S_R) == CQE_FOR_RQ ?
 		wc->sl : UINT8_MAX;
 }
+#endif
 
 static uint32_t read_wc_rqpn(struct hns_roce_qp *hr_qp,
 				      struct hns_roce_v2_cqe *cqe,
@@ -750,7 +751,6 @@ static uint8_t get_send_wr_tclass(struct ibv_send_wr *wr,
 	return qp_type == IBV_QPT_UD ?
 		to_hr_ah(wr->wr.ud.ah)->av.tclass : UINT8_MAX;
 }
-#endif
 
 static int hns_roce_poll_one(struct hns_roce_context *ctx,
 			     struct hns_roce_qp **cur_qp, struct hns_roce_cq *cq,
diff --git a/providers/hns/hns_roce_u_trace.h b/providers/hns/hns_roce_u_trace.h
@@ -121,13 +121,19 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined(USDT_ENABLED)
 
 #ifndef __HNS_TRACE_H__
 #define __HNS_TRACE_H__
 
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#else
+
 #define rdma_tracepoint(arg...)
 
 #endif /* __HNS_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */
diff --git a/providers/mlx5/mlx5_trace.h b/providers/mlx5/mlx5_trace.h
@@ -47,13 +47,19 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined(USDT_ENABLED)
 
 #ifndef __MLX5_TRACE_H__
 #define __MLX5_TRACE_H__
 
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#else
+
 #define rdma_tracepoint(arg...)
 
 #endif /* __MLX5_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */
diff --git a/providers/rxe/rxe_trace.h b/providers/rxe/rxe_trace.h
@@ -47,13 +47,19 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined(USDT_ENABLED)
 
 #ifndef __RXE_TRACE_H__
 #define __RXE_TRACE_H__
 
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#else
+
 #define rdma_tracepoint(arg...)
 
 #endif /* __RXE_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */
diff --git a/util/CMakeLists.txt b/util/CMakeLists.txt
@@ -6,6 +6,7 @@ publish_internal_headers(util
   node_name_map.h
   rdma_nl.h
   symver.h
+  usdt.h
   util.h
   )
 
diff --git a/util/usdt.h b/util/usdt.h

Original file line number	Diff line number	Diff line change
`@@ -2170,7 +2170,7 @@ static void efa_set_common_ctrl_flags(struct efa_io_tx_meta_desc *desc,`
`2170`	`2170`	`EFA_SET(&desc->ctrl2, EFA_IO_TX_META_DESC_COMP_REQ, 1);`
`2171`	`2171`	`}`
`2172`	`2172`
`2173`		`-#ifdef LTTNG_ENABLED`
	`2173`	`+#if defined(LTTNG_ENABLED) \|\| defined(USDT_ENABLED)`
`2174`	`2174`	`static uint32_t efa_get_wqe_length(struct efa_io_tx_wqe *tx_wqe)`
`2175`	`2175`	`{`
`2176`	`2176`	`enum efa_io_send_op_type op_type;`
Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,7 @@ publish_internal_headers(util`
`6`	`6`	`node_name_map.h`
`7`	`7`	`rdma_nl.h`
`8`	`8`	`symver.h`
	`9`	`+ usdt.h`
`9`	`10`	`util.h`
`10`	`11`	`)`
`11`	`12`