providers: add support for USDT tracing using libbpf/usdt

Provide lightweight USDT tracing as an alternative to LTTng.
This piggybacks on the existing tracing code added for LTTng
for a minimal set of changes.

> $ sudo bpftrace -l usdt:build/lib/lib*.so:*
> usdt:build/lib/libefa-rdmav59.so:rdma_core_efa:post_recv
> usdt:build/lib/libefa-rdmav59.so:rdma_core_efa:post_send
> usdt:build/lib/libefa-rdmav59.so:rdma_core_efa:process_completion
> usdt:build/lib/libefa.so:rdma_core_efa:post_recv
> usdt:build/lib/libefa.so:rdma_core_efa:post_send
> usdt:build/lib/libefa.so:rdma_core_efa:process_completion
> usdt:build/lib/libhns-rdmav59.so:rdma_core_hns:poll_cq
> usdt:build/lib/libhns-rdmav59.so:rdma_core_hns:post_recv
> usdt:build/lib/libhns-rdmav59.so:rdma_core_hns:post_send
> usdt:build/lib/libhns.so:rdma_core_hns:poll_cq
> usdt:build/lib/libhns.so:rdma_core_hns:post_recv
> usdt:build/lib/libhns.so:rdma_core_hns:post_send
> usdt:build/lib/libmlx5-rdmav59.so:rdma_core_mlx5:post_send
> usdt:build/lib/libmlx5.so:rdma_core_mlx5:post_send
> usdt:build/lib/librxe-rdmav59.so:rdma_core_rxe:post_send

The USDT header used here is from the libbpf/usdt project at
https://github.com/libbpf/usdt.git

Further background discussion for this commit is included in
linux-rdma#1621

Signed-off-by: Nathan Scott <[email protected]>

Author: natoscott

CMakeLists.txt

@@ -24,7 +24,7 @@
 #      Use the historical search path for providers, in the standard system library.
 #  -DNO_COMPAT_SYMS=1 (default disabled)
 #      Do not generate backwards compatibility symbols in the shared
-#      libraries. This may is necessary if using a dynmic linker that does
+#      libraries. This may be necessary if using a dynamic linker that does
 #      not support symbol versions, such as uclibc.
 #  -DIOCTL_MODE=write (default both)
 #      Disable new kABI ioctl() support and support only the legacy write
@@ -52,8 +52,10 @@
 #      Disable man pages. Allows rdma-core to be built and installed
 #      (without man pages) when neither pandoc/rst2man nor the pandoc-prebuilt
 #      directory are available.
-#   -DENABLE_LTTNG (default, no tracing support)
+#   -DENABLE_LTTNG (default, no LTTng tracing support)
 #      Enable LTTng tracing.
+#   -DENABLE_USDT (default, no USDT tracing support)
+#      Enable USDT tracing.
 
 if (${CMAKE_VERSION} VERSION_LESS "3.18.1")
 	# Centos 7 support
@@ -573,6 +575,14 @@ if (DEFINED ENABLE_LTTNG)
   add_definitions(-DLTTNG_ENABLED)
 endif()
 
+# User Static-Defined Tracing (USDT) support
+if (NOT DEFINED ENABLE_USDT)
+  set(ENABLE_USDT "OFF" CACHE BOOL "Enable USDT tracing support")
+endif()
+if (ENABLE_USDT)
+  add_definitions(-DUSDT_ENABLED)
+endif()
+
 #-------------------------
 # Apply fixups
 

Documentation/contributing.md

@@ -66,7 +66,7 @@ that are designed to weed out bugs.
 Serious errors will result in a red X in the PR and will need to be corrected.
 Less serious errors, including checkpatch related, will show up with a green
 check but it is necessary to check the details to see that everything is
-appropriate. checkpatch is an informative too, not all of its feedback is
+appropriate. checkpatch is an informative tool, not all of its feedback is
 appropriate to fix.
 
 A build similar to AZP can be run locally using docker and the

providers/efa/efa_trace.h

@@ -3,7 +3,11 @@
  * Copyright 2023-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
-#if defined(LTTNG_ENABLED)
+#if defined(LTTNG_ENABLED) && defined(USDT_ENABLED)
+
+#error "Only one of LTTNG_ENABLED and USDT_ENABLED allowed"
+
+#elif defined(LTTNG_ENABLED)
 
 #undef LTTNG_UST_TRACEPOINT_PROVIDER
 #define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_efa
@@ -109,7 +113,18 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined (USDT_ENABLED)
+
+#ifndef __EFA_TRACE_H__
+#define __EFA_TRACE_H__
+
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#endif /* __EFA_TRACE_H__*/
+
+#else /* !defined(LTTNG_ENABLED) && !defined(USDT_ENABLED) */
 
 #ifndef __EFA_TRACE_H__
 #define __EFA_TRACE_H__
@@ -118,4 +133,4 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #endif /* __EFA_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */

providers/efa/verbs.c

@@ -2170,7 +2170,7 @@ static void efa_set_common_ctrl_flags(struct efa_io_tx_meta_desc *desc,
 	EFA_SET(&desc->ctrl2, EFA_IO_TX_META_DESC_COMP_REQ, 1);
 }
 
-#ifdef LTTNG_ENABLED
+#if defined(LTTNG_ENABLED) || defined(USDT_ENABLED)
 static uint32_t efa_get_wqe_length(struct efa_io_tx_wqe *tx_wqe)
 {
 	enum efa_io_send_op_type op_type;

providers/hns/hns_roce_u_hw_v2.c

@@ -678,7 +678,7 @@ static int parse_cqe_for_cq(struct hns_roce_context *ctx, struct hns_roce_cq *cq
 	return 0;
 }
 
-#ifdef LTTNG_ENABLED
+#if defined(LTTNG_ENABLED) || defined(USDT_ENABLED)
 static uint8_t read_wc_sl(struct hns_roce_qp *hr_qp,
 				   struct hns_roce_v2_cqe *cqe,
 				   struct ibv_wc *wc)
@@ -687,6 +687,7 @@ static uint8_t read_wc_sl(struct hns_roce_qp *hr_qp,
 		hr_reg_read(cqe, CQE_S_R) == CQE_FOR_RQ ?
 		wc->sl : UINT8_MAX;
 }
+#endif
 
 static uint32_t read_wc_rqpn(struct hns_roce_qp *hr_qp,
 				      struct hns_roce_v2_cqe *cqe,
@@ -750,7 +751,6 @@ static uint8_t get_send_wr_tclass(struct ibv_send_wr *wr,
 	return qp_type == IBV_QPT_UD ?
 		to_hr_ah(wr->wr.ud.ah)->av.tclass : UINT8_MAX;
 }
-#endif
 
 static int hns_roce_poll_one(struct hns_roce_context *ctx,
 			     struct hns_roce_qp **cur_qp, struct hns_roce_cq *cq,

providers/hns/hns_roce_u_trace.h

@@ -3,7 +3,11 @@
  * Copyright (c) 2025 Hisilicon Limited.
  */
 
-#if defined(LTTNG_ENABLED)
+#if defined(LTTNG_ENABLED) && defined(USDT_ENABLED)
+
+#error "Only one of LTTNG_ENABLED and USDT_ENABLED allowed"
+
+#elif defined(LTTNG_ENABLED)
 
 #undef LTTNG_UST_TRACEPOINT_PROVIDER
 #define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_hns
@@ -121,7 +125,18 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined(USDT_ENABLED)
+
+#ifndef __HNS_TRACE_H__
+#define __HNS_TRACE_H__
+
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#endif /* __HNS_TRACE_H__*/
+
+#else /* !defined(LTTNG_ENABLED) && !defined(USDT_ENABLED) */
 
 #ifndef __HNS_TRACE_H__
 #define __HNS_TRACE_H__
@@ -130,4 +145,4 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #endif /* __HNS_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */

providers/mlx5/mlx5_trace.h

@@ -3,7 +3,11 @@
  * Copyright 2023 Bytedance.com, Inc. or its affiliates. All rights reserved.
  */
 
-#if defined(LTTNG_ENABLED)
+#if defined(LTTNG_ENABLED) && defined(USDT_ENABLED)
+
+#error "Only one of LTTNG_ENABLED and USDT_ENABLED allowed"
+
+#elif defined(LTTNG_ENABLED)
 
 #undef LTTNG_UST_TRACEPOINT_PROVIDER
 #define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_mlx5
@@ -47,7 +51,18 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined(USDT_ENABLED)
+
+#ifndef __MLX5_TRACE_H__
+#define __MLX5_TRACE_H__
+
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#endif /* __MLX5_TRACE_H__*/
+
+#else /* !defined(LTTNG_ENABLED) && !defined(USDT_ENABLED) */
 
 #ifndef __MLX5_TRACE_H__
 #define __MLX5_TRACE_H__
@@ -56,4 +71,4 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #endif /* __MLX5_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */

providers/rxe/rxe_trace.h

@@ -3,7 +3,11 @@
  * Copyright 2023 Bytedance.com, Inc. or its affiliates. All rights reserved.
  */
 
-#if defined(LTTNG_ENABLED)
+#if defined(LTTNG_ENABLED) && defined(USDT_ENABLED)
+
+#error "Only one of LTTNG_ENABLED and USDT_ENABLED allowed"
+
+#elif defined(LTTNG_ENABLED)
 
 #undef LTTNG_UST_TRACEPOINT_PROVIDER
 #define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_rxe
@@ -47,7 +51,18 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined(USDT_ENABLED)
+
+#ifndef __RXE_TRACE_H__
+#define __RXE_TRACE_H__
+
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#endif /* __RXE_TRACE_H__*/
+
+#else /* !defined(LTTNG_ENABLED) && !defined(USDT_ENABLED) */
 
 #ifndef __RXE_TRACE_H__
 #define __RXE_TRACE_H__
@@ -56,4 +71,4 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #endif /* __RXE_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */

util/CMakeLists.txt

@@ -6,6 +6,7 @@ publish_internal_headers(util
   node_name_map.h
   rdma_nl.h
   symver.h
+  usdt.h
   util.h
   )