Merge pull request #4858 from DataDog/ivoanjo/libdatadog-ddsketch

[NO-TICKET] RFC: Bindings for libdatadog ddsketch API

Author: ivoanjo

Matrixfile

@@ -11,18 +11,12 @@
   'custom_cop' => {
     '' => '❌ 2.5 / ❌ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ 3.5 / ❌ jruby'
   },
-  'crashtracking' => {
+  'core_with_libdatadog_api' => {
     '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ 3.5 / ❌ jruby',
   },
   'error_tracking' => {
     '' => '❌ 2.5 / ❌ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ 3.5 / ❌ jruby',
   },
-  'process_discovery' => {
-    '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ 3.5 / ❌ jruby'
-  },
-  'stable_config' => {
-    '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ 3.5 / ❌ jruby'
-  },
   'appsec:main' => {
     '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ 3.5 / ✅ jruby'
   },

Rakefile

@@ -22,6 +22,13 @@ Dir.glob('tasks/*.rake').each { |r| import r }
 
 TEST_METADATA = eval(File.read('Matrixfile')).freeze # rubocop:disable Security/Eval
 
+CORE_WITH_LIBDATADOG_API = [
+  'spec/datadog/core/crashtracking/**/*_spec.rb',
+  'spec/datadog/core/process_discovery_spec.rb',
+  'spec/datadog/core/configuration/stable_config_spec.rb',
+  'spec/datadog/core/ddsketch_spec.rb',
+].freeze
+
 # rubocop:disable Metrics/BlockLength
 namespace :test do
   desc 'Run all tests'
@@ -70,13 +77,14 @@ namespace :spec do
     :graphql, :graphql_unified_trace_patcher, :graphql_trace_patcher, :graphql_tracing_patcher,
     :rails, :railsredis, :railsredis_activesupport, :railsactivejob,
     :elasticsearch, :http, :redis, :sidekiq, :sinatra, :hanami, :hanami_autoinstrument,
-    :profiling, :crashtracking, :error_tracking, :process_discovery, :stable_config]
+    :profiling, :core_with_libdatadog_api, :error_tracking]
 
   desc '' # "Explicitly hiding from `rake -T`"
   RSpec::Core::RakeTask.new(:main) do |t, args|
     t.pattern = 'spec/**/*_spec.rb'
     t.exclude_pattern = 'spec/**/{appsec/integration,contrib,benchmark,redis,auto_instrument,opentelemetry,profiling,crashtracking,error_tracking,rubocop}/**/*_spec.rb,' \
-                        ' spec/**/{auto_instrument,opentelemetry,process_discovery,stable_config}_spec.rb, spec/datadog/gem_packaging_spec.rb'
+                        ' spec/**/{auto_instrument,opentelemetry,process_discovery,stable_config,ddsketch}_spec.rb,' \
+                        ' spec/datadog/gem_packaging_spec.rb'
     t.rspec_opts = args.to_a.join(' ')
   end
 
@@ -202,27 +210,8 @@ namespace :spec do
   end
 
   # rubocop:disable Style/MultilineBlockChain
-  RSpec::Core::RakeTask.new(:crashtracking) do |t, args|
-    t.pattern = 'spec/datadog/core/crashtracking/**/*_spec.rb'
-    t.rspec_opts = args.to_a.join(' ')
-  end.tap do |t|
-    Rake::Task[t.name].enhance(["compile:libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"])
-  end
-  # rubocop:enable Style/MultilineBlockChain
-
-  # rubocop:disable Style/MultilineBlockChain
-  RSpec::Core::RakeTask.new(:process_discovery) do |t, args|
-    t.pattern = 'spec/datadog/core/process_discovery_spec.rb'
-    t.rspec_opts = args.to_a.join(' ')
-  end.tap do |t|
-    Rake::Task[t.name].enhance(["compile:libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"])
-  end
-  # rubocop:enable Style/MultilineBlockChain
-
-  # rubocop:disable Style/MultilineBlockChain
-  desc '' # "Explicitly hiding from `rake -T`"
-  RSpec::Core::RakeTask.new(:stable_config) do |t, args|
-    t.pattern = 'spec/datadog/core/configuration/stable_config_spec.rb'
+  RSpec::Core::RakeTask.new(:core_with_libdatadog_api) do |t, args|
+    t.pattern = CORE_WITH_LIBDATADOG_API.join(', ')
     t.rspec_opts = args.to_a.join(' ')
   end.tap do |t|
     Rake::Task[t.name].enhance(["compile:libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"])

ext/LIBDATADOG_DEVELOPMENT.md

@@ -0,0 +1,60 @@
+# Libdatadog development
+
+These instructions can quickly get outdated, so feel free to open an issue if they're not working (and/or ping @ivoanjo).
+
+## Using libdatadog builds from CI or GitHub
+
+If you're developing inside docker/natively on Linux, you can use libdatadog builds from CI and GitHub.
+
+Here's what to do:
+
+1. Create a folder for extracting libdatadog into based on your ruby platform (for instance inside the dd-trace-rb repo):
+
+```bash
+export DD_RUBY_PLATFORM=`ruby -e 'puts Gem::Platform.local.to_s'`
+echo "Current ruby platform: $DD_RUBY_PLATFORM"
+mkdir -p my-libdatadog-build/$DD_RUBY_PLATFORM
+```
+
+2. Find a libdatadog build from CI or [GitHub releases](https://github.com/DataDog/libdatadog/releases). This should match the Ruby platform seen above.
+3. Extract the libdatadog build into the folder:
+
+```bash
+# In this example the build is in my downloads; notice the use of strip-components to get the correct folder structure
+tar zxvf ~/Downloads/libdatadog-x86_64-unknown-linux-gnu.tar.gz -C my-libdatadog-build/$DD_RUBY_PLATFORM/ --strip-components=1
+# Here's how it should look after
+ls my-libdatadog-build/$DD_RUBY_PLATFORM
+bin  cmake  include  lib  LICENSE  LICENSE-3rdparty.yml  NOTICE
+```
+
+6. Tell Ruby where to find libdatadog: ```export LIBDATADOG_VENDOR_OVERRIDE=`pwd`/my-libdatadog-build/``` (Notice no platform + use of pwd for full path here)
+7. From dd-trace-rb, run `bundle exec rake clean compile`
+8. For incremental builds, usually `bundle exec rake compile` is faster and `clean` is not needed
+
+If you additionally want to run the profiler test suite, also remember to `export DD_PROFILING_MACOS_TESTING=true` and re-run `rake clean compile`.
+
+## Native development on macOS
+
+As of this writing (August 2025), the libdatadog builds on rubygems.org only support Linux.
+
+We don't officially support using libdatadog for Ruby on other platforms yet, but it is possible to use it for local development on macOS.
+(**Note that you don't need these instructions if you develop inside docker.**)
+
+Here's how you can do so:
+
+1. [Install rust](https://www.rust-lang.org/tools/install)
+2. Install `cbindgen`: `cargo install cbindgen`
+3. Clone [libdatadog](https://github.com/datadog/libdatadog)
+4. Create a folder for building into based on your ruby platform:
+
+```bash
+export DD_RUBY_PLATFORM=`ruby -e 'puts Gem::Platform.local.to_s'`
+mkdir -p my-libdatadog-build/$DD_RUBY_PLATFORM
+```
+
+5. From inside of the libdatadog repo, build libdatadog into this folder: `./build-profiling-ffi.sh my-libdatadog-build/$DD_RUBY_PLATFORM`
+6. Tell Ruby where to find libdatadog: `export LIBDATADOG_VENDOR_OVERRIDE=/adjust/this/to/be/the/full/path/to/my-libdatadog-build/` (Notice no platform here)
+7. From dd-trace-rb, run `bundle exec rake clean compile`
+8. For incremental builds, usually `bundle exec rake compile` is faster and `clean` is not needed
+
+If you additionally want to run the profiler test suite, also remember to `export DD_PROFILING_MACOS_TESTING=true` and re-run `rake clean compile`.

ext/libdatadog_api/ddsketch.c

@@ -0,0 +1,106 @@
+#include <ruby.h>
+#include <datadog/ddsketch.h>
+
+#include "datadog_ruby_common.h"
+
+static VALUE _native_new(VALUE klass);
+static void ddsketch_free(void *ptr);
+static VALUE native_add(VALUE self, VALUE point);
+static VALUE native_add_with_count(VALUE self, VALUE point, VALUE count);
+static VALUE native_count(VALUE self);
+static VALUE native_encode(VALUE self);
+NORETURN(static void raise_ddsketch_error(const char *message, ddog_VoidResult result));
+
+void ddsketch_init(VALUE core_module) {
+  VALUE ddsketch_class = rb_define_class_under(core_module, "DDSketch", rb_cObject);
+
+  rb_define_alloc_func(ddsketch_class, _native_new);
+  rb_define_method(ddsketch_class, "add", native_add, 1);
+  rb_define_method(ddsketch_class, "add_with_count", native_add_with_count, 2);
+  rb_define_method(ddsketch_class, "count", native_count, 0);
+  rb_define_method(ddsketch_class, "encode", native_encode, 0);
+}
+
+// This structure is used to define a Ruby object that stores a pointer to a ddsketch_Handle_DDSketch
+// See also https://github.com/ruby/ruby/blob/master/doc/extension.rdoc for how this works
+static const rb_data_type_t ddsketch_typed_data = {
+  .wrap_struct_name = "Datadog::DDSketch",
+  .function = {
+    .dmark = NULL, // We don't store references to Ruby objects so we don't need to mark any of them
+    .dfree = ddsketch_free,
+    .dsize = NULL, // We don't track memory usage (although it'd be cool if we did!)
+    //.dcompact = NULL, // Not needed -- we don't store references to Ruby objects
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE _native_new(VALUE klass) {
+  ddsketch_Handle_DDSketch *state = ruby_xcalloc(1, sizeof(ddsketch_Handle_DDSketch));
+
+  *state = ddog_ddsketch_new();
+
+  return TypedData_Wrap_Struct(klass, &ddsketch_typed_data, state);
+}
+
+static void ddsketch_free(void *ptr) {
+  ddsketch_Handle_DDSketch *state = (ddsketch_Handle_DDSketch *) ptr;
+  ddog_ddsketch_drop(state);
+  ruby_xfree(ptr);
+}
+
+static void raise_ddsketch_error(const char *message, ddog_VoidResult result) {
+  rb_raise(rb_eRuntimeError, "%s: %"PRIsVALUE, message, get_error_details_and_drop(&result.err));
+}
+
+static VALUE native_add(VALUE self, VALUE point) {
+  ddsketch_Handle_DDSketch *state;
+  TypedData_Get_Struct(self, ddsketch_Handle_DDSketch, &ddsketch_typed_data, state);
+
+  ddog_VoidResult result = ddog_ddsketch_add(state, NUM2DBL(point));
+
+  if (result.tag == DDOG_VOID_RESULT_ERR) raise_ddsketch_error("DDSketch add failed", result);
+
+  return self;
+}
+
+static VALUE native_add_with_count(VALUE self, VALUE point, VALUE count) {
+  ddsketch_Handle_DDSketch *state;
+  TypedData_Get_Struct(self, ddsketch_Handle_DDSketch, &ddsketch_typed_data, state);
+
+  ddog_VoidResult result = ddog_ddsketch_add_with_count(state, NUM2DBL(point), NUM2DBL(count));
+
+  if (result.tag == DDOG_VOID_RESULT_ERR) raise_ddsketch_error("DDSketch add_with_count failed", result);
+
+  return self;
+}
+
+static VALUE native_count(VALUE self) {
+  ddsketch_Handle_DDSketch *state;
+  TypedData_Get_Struct(self, ddsketch_Handle_DDSketch, &ddsketch_typed_data, state);
+
+  double count_out;
+  ddog_VoidResult result = ddog_ddsketch_count(state, &count_out);
+
+  if (result.tag == DDOG_VOID_RESULT_ERR) raise_ddsketch_error("DDSketch count failed", result);
+
+  return DBL2NUM(count_out);
+}
+
+static VALUE native_encode(VALUE self) {
+  ddsketch_Handle_DDSketch *state;
+  TypedData_Get_Struct(self, ddsketch_Handle_DDSketch, &ddsketch_typed_data, state);
+
+  ddog_Vec_U8 encoded = ddog_ddsketch_encode(state);
+
+  // Copy into a Ruby string
+  VALUE bytes = rb_str_new((const char *) encoded.ptr, encoded.len);
+
+  ddog_Vec_U8_drop(encoded);
+
+  // The sketch is consumed by encode; to make this a bit more user-friendly for
+  // a Ruby API (since we can't "kill" the Ruby object), let's re-initialize it so
+  // it can be used again.
+  *state = ddog_ddsketch_new();
+
+  return bytes;
+}

ext/libdatadog_api/init.c

@@ -5,11 +5,14 @@
 #include "process_discovery.h"
 #include "library_config.h"
 
+void ddsketch_init(VALUE core_module);
+
 void DDTRACE_EXPORT Init_libdatadog_api(void) {
   VALUE datadog_module = rb_define_module("Datadog");
   VALUE core_module = rb_define_module_under(datadog_module, "Core");
 
   crashtracker_init(core_module);
   process_discovery_init(core_module);
   library_config_init(core_module);
+  ddsketch_init(core_module);
 }

ext/libdatadog_api/macos_development.md

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'datadog/core'
+
+module Datadog
+  module Core
+    # Used to access ddsketch APIs.
+    # APIs in this class are implemented as native code.
+    class DDSketch
+      def self.supported?
+        Datadog::Core::LIBDATADOG_API_FAILURE.nil?
+      end
+
+      def initialize
+        unless self.class.supported?
+          raise(ArgumentError, "DDSketch is not supported: #{Datadog::Core::LIBDATADOG_API_FAILURE}")
+        end
+      end
+    end
+  end
+end

lib/datadog/core/ddsketch.rb

@@ -0,0 +1,26 @@
+module Datadog
+  module Core
+    class DDSketch
+      def self.supported?: () -> bool
+
+      # Adds a single point to the sketch
+      # @param point [::Numeric] The value to add to the sketch
+      # @return [true] Always returns true on success, raises RuntimeError on failure
+      def add: (::Numeric point) -> true
+
+      # Adds a point with a count to the sketch
+      # @param point [::Numeric] The value to add to the sketch
+      # @param count [::Numeric] The count/weight for this point
+      # @return [true] Always returns true on success, raises RuntimeError on failure
+      def add_with_count: (::Numeric point, ::Numeric count) -> true
+
+      # Returns the total count of points in the sketch
+      # @return [::Float] The total count of points
+      def count: () -> ::Float
+
+      # Encodes the sketch to bytes and resets it for reuse
+      # @return [::String] The encoded sketch as a binary string
+      def encode: () -> ::String
+    end
+  end
+end