[ConfigRegistry] 3/5 Generate ConfigurationKeys with source generator

tracer/src/Datadog.Trace.SourceGenerators/Constants.cs

@@ -1,4 +1,4 @@
-// <copyright file="Constants.cs" company="Datadog">
+// <copyright file="Constants.cs" company="Datadog">
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
@@ -20,5 +20,13 @@ internal static class Constants
         #nullable enable
 
 
+        """;
+
+    public const string ConfigurationGeneratorComment =
+        """
+        // This file is auto-generated from supported-configurations.json and supported-configurations-docs.yaml
+        // Do not edit this file directly. The source generator will regenerate it on build.
+        // NOTE: If you remove keys/products from the JSON, run 'dotnet clean' and remove old generated files.
+
         """;
 }

tracer/src/Datadog.Trace.SourceGenerators/Datadog.Trace.SourceGenerators.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
     <Nullable>enable</Nullable>
@@ -12,6 +12,7 @@
   <ItemGroup>
     <PackageReference Include="System.Reflection.Emit" Version="4.7.0" />
     <PackageReference Include="System.Reflection.Emit.Lightweight" Version="4.7.0" />
+    <PackageReference Include="System.Text.Json" Version="10.0.0-rc.2.25502.107" PrivateAssets="all" />
   </ItemGroup>
   <ItemGroup>    
     <Compile Include="..\Datadog.Trace\ClrProfiler\InstrumentationCategory.cs" Link="InstrumentationDefinitions\InstrumentationCategory.cs" />    

tracer/src/Datadog.Trace.SourceGenerators/Helpers/TrackingNames.cs

@@ -31,4 +31,19 @@ internal class TrackingNames
     public const string AssemblyCallTargetDefinitionSource = nameof(AssemblyCallTargetDefinitionSource);
     public const string AdoNetCallTargetDefinitionSource = nameof(AdoNetCallTargetDefinitionSource);
     public const string AdoNetSignatures = nameof(AdoNetSignatures);
+
+    // Configuration key matcher
+    public const string ConfigurationKeysParseConfiguration = nameof(ConfigurationKeysParseConfiguration);
+    public const string ConfigurationKeyMatcherDiagnostics = nameof(ConfigurationKeyMatcherDiagnostics);
+    public const string ConfigurationKeyMatcherValidData = nameof(ConfigurationKeyMatcherValidData);
+    public const string ConfigurationKeysAdditionalText = nameof(ConfigurationKeysAdditionalText);
+
+    // Configuration key generator
+    public const string ConfigurationKeysGenJsonFile = nameof(ConfigurationKeysGenJsonFile);
+    public const string ConfigurationKeysGenYamlFile = nameof(ConfigurationKeysGenYamlFile);
+    public const string ConfigurationKeysGenMappingFile = nameof(ConfigurationKeysGenMappingFile);
+    public const string ConfigurationKeysGenMergeData = nameof(ConfigurationKeysGenMergeData);
+    public const string ConfigurationKeysGenParseConfiguration = nameof(ConfigurationKeysGenParseConfiguration);
+    public const string ConfigurationKeysGenParseYaml = nameof(ConfigurationKeysGenParseYaml);
+    public const string ConfigurationKeysGenParseMapping = nameof(ConfigurationKeysGenParseMapping);
 }

tracer/src/Datadog.Trace.SourceGenerators/Helpers/YamlReader.cs

@@ -0,0 +1,212 @@
+// <copyright file="YamlReader.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Datadog.Trace.SourceGenerators.Helpers
+{
+    /// <summary>
+    /// Simple YAML parser for reading documentation strings from YAML files.
+    /// Supports basic YAML features needed for configuration documentation.
+    /// </summary>
+    internal static class YamlReader
+    {
+        /// <summary>
+        /// Parses a YAML file containing configuration key documentation.
+        /// Expects format: KEY: | followed by multi-line documentation
+        /// </summary>
+        public static Dictionary<string, string> ParseDocumentation(string yamlContent)
+        {
+            var result = new Dictionary<string, string>();
+
+            string? currentKey = null;
+            var currentDoc = new StringBuilder();
+            var inMultiLine = false;
+            var baseIndent = 0;
+
+            foreach (var line in new LineEnumerator(yamlContent.AsSpan()))
+            {
+                // Skip empty lines and comments when not in multi-line
+                if (!inMultiLine && (IsWhiteSpace(line) || line.TrimStart().StartsWith("#".AsSpan(), StringComparison.Ordinal)))
+                {
+                    continue;
+                }
+
+                // Check for new key (starts at column 0, contains colon)
+                if (!inMultiLine && line.Length > 0 && line[0] != ' ' && line.IndexOf(':') >= 0)
+                {
+                    // Save previous key if exists
+                    if (currentKey != null)
+                    {
+                        result[currentKey] = currentDoc.ToString().TrimEnd();
+                        currentDoc.Clear();
+                    }
+
+                    var colonIndex = line.IndexOf(':');
+                    currentKey = line.Slice(0, colonIndex).Trim().ToString();
+
+                    // Check if it's a multi-line string (|)
+                    var afterColon = line.Slice(colonIndex + 1).Trim();
+                    if (afterColon.Length == 1 && (afterColon[0] == '|' || afterColon[0] == '>'))
+                    {
+                        inMultiLine = true;
+                        baseIndent = -1; // Will be set on first content line
+                    }
+                    else if (afterColon.Length > 0)
+                    {
+                        // Single line value
+                        currentDoc.Append(afterColon.ToString());
+                        result[currentKey] = currentDoc.ToString();
+                        currentDoc.Clear();
+                        currentKey = null;
+                    }
+
+                    continue;
+                }
+
+                // Handle multi-line content
+                if (inMultiLine && currentKey != null)
+                {
+                    // Check if we've reached the next key (no indentation)
+                    if (line.Length > 0 && line[0] != ' ' && line.IndexOf(':') >= 0)
+                    {
+                        // Save current key and process this line as new key
+                        result[currentKey] = currentDoc.ToString().TrimEnd();
+                        currentDoc.Clear();
+                        inMultiLine = false;
+
+                        var colonIndex = line.IndexOf(':');
+                        currentKey = line.Slice(0, colonIndex).Trim().ToString();
+                        var afterColon = line.Slice(colonIndex + 1).Trim();
+                        if (afterColon.Length == 1 && (afterColon[0] == '|' || afterColon[0] == '>'))
+                        {
+                            inMultiLine = true;
+                            baseIndent = -1;
+                        }
+
+                        continue;
+                    }
+
+                    // Determine base indentation from first content line
+                    if (baseIndent == -1 && line.Length > 0 && line[0] == ' ')
+                    {
+                        baseIndent = 0;
+                        while (baseIndent < line.Length && line[baseIndent] == ' ')
+                        {
+                            baseIndent++;
+                        }
+                    }
+
+                    // Add content line (remove base indentation)
+                    if (line.Length > 0)
+                    {
+                        var contentStart = 0;
+                        while (contentStart < line.Length && contentStart < baseIndent && line[contentStart] == ' ')
+                        {
+                            contentStart++;
+                        }
+
+                        if (currentDoc.Length > 0)
+                        {
+                            currentDoc.AppendLine();
+                        }
+
+                        currentDoc.Append(line.Slice(contentStart).ToString());
+                    }
+                    else
+                    {
+                        // Empty line in multi-line content
+                        if (currentDoc.Length > 0)
+                        {
+                            currentDoc.AppendLine();
+                        }
+                    }
+                }
+            }
+
+            // Save last key
+            if (currentKey != null)
+            {
+                result[currentKey] = currentDoc.ToString().TrimEnd();
+            }
+
+            return result;
+        }
+
+        /// <summary>
+        /// Checks if a span contains only whitespace characters.
+        /// </summary>
+        private static bool IsWhiteSpace(ReadOnlySpan<char> span)
+        {
+            for (int i = 0; i < span.Length; i++)
+            {
+                if (!char.IsWhiteSpace(span[i]))
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Enumerator for iterating through lines in a ReadOnlySpan without allocations.
+        /// </summary>
+        private ref struct LineEnumerator
+        {
+            private ReadOnlySpan<char> _remaining;
+            private ReadOnlySpan<char> _current;
+            private bool _isEnumeratorActive;
+
+            public LineEnumerator(ReadOnlySpan<char> text)
+            {
+                _remaining = text;
+                _current = default;
+                _isEnumeratorActive = true;
+            }
+
+            public ReadOnlySpan<char> Current => _current;
+
+            public LineEnumerator GetEnumerator() => this;
+
+            public bool MoveNext()
+            {
+                if (!_isEnumeratorActive)
+                {
+                    return false;
+                }
+
+                if (_remaining.Length == 0)
+                {
+                    _isEnumeratorActive = false;
+                    return false;
+                }
+
+                var idx = _remaining.IndexOfAny('\r', '\n');
+                if (idx < 0)
+                {
+                    // Last line without line ending
+                    _current = _remaining;
+                    _remaining = default;
+                    return true;
+                }
+
+                _current = _remaining.Slice(0, idx);
+
+                // Skip past the line ending (\r\n or \n or \r)
+                var advance = idx + 1;
+                if (idx < _remaining.Length - 1 && _remaining[idx] == '\r' && _remaining[idx + 1] == '\n')
+                {
+                    advance = idx + 2;
+                }
+
+                _remaining = _remaining.Slice(advance);
+                return true;
+            }
+        }
+    }
+}

tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs

@@ -14,6 +14,14 @@ namespace Datadog.Trace.Configuration
     /// </summary>
     internal static partial class ConfigurationKeys
     {
+        // temporary so that ConfigurationKeys2 can resolve reference, all this is removed later
+        public const string TraceLogPath = "DD_TRACE_LOG_PATH";
+
+        /// <summary>
+        /// Configuration key to enable or disable the ActivityListener.
+        /// </summary>
+        public const string ActivityListenerEnabled = "DD_TRACE_ACTIVITY_LISTENER_ENABLED";
+
         /// <summary>
         /// Configuration key to enable experimental features.
         /// </summary>
@@ -994,7 +1002,7 @@ internal static class DataStreamsMonitoring
             /// Configuration key for enabling legacy binary headers in Data Streams Monitoring.
             /// false by default if DSM is in default state, true otherwise
             /// </summary>
-             /// <see cref="TracerSettings.IsDataStreamsLegacyHeadersEnabled"/>
+            /// <see cref="TracerSettings.IsDataStreamsLegacyHeadersEnabled"/>
             public const string LegacyHeadersEnabled = "DD_DATA_STREAMS_LEGACY_HEADERS";
         }
     }