Add Origin Detection testing (#5310)

Co-authored-by: robertomonteromiguel <[email protected]>
Co-authored-by: quinna-h <[email protected]>
Co-authored-by: wantsui <[email protected]>

Author: 4 people

tests/auto_inject/test_auto_inject_install.py

@@ -54,11 +54,50 @@ def test_profiling(self):
         logger.info(f"Done test_install for : [{context.vm_name}]")
 
 
+@features.origin_detection
 @features.container_auto_installation_script
 @scenarios.container_auto_injection_install_script
 class TestContainerAutoInjectInstallScript(base.AutoInjectBaseTest):
+    ruby_bug_platforms = [
+        "AlamaLinux_9_amd64",
+        "AlamaLinux_9_arm64",
+        "Amazon_Linux_2022_amd64",
+        "Amazon_Linux_2022_arm64",
+        "Amazon_Linux_2023_amd64",
+        "Amazon_Linux_2023_arm64",
+        "Debian_11_amd64",
+        "Debian_11_arm64",
+        "Debian_12_amd64",
+        "Debian_12_arm64",
+        "OracleLinux_9_2_amd64",
+        "OracleLinux_9_2_arm64",
+        "OracleLinux_9_3_amd64",
+        "OracleLinux_9_3_arm64",
+        "RedHat_9_0_amd64",
+        "RedHat_9_0_arm64",
+        "RockyLinux_9_amd64",
+        "RockyLinux_9_arm64",
+        "Ubuntu_22_amd64",
+        "Ubuntu_22_arm64",
+        "Ubuntu_23_04_arm64",
+        "Ubuntu_23_10_arm64",
+        "Ubuntu_24_10_amd64",
+        "Ubuntu_24_10_arm64",
+        "Ubuntu_24_amd64",
+        "Ubuntu_24_arm64",
+        "Ubuntu_25_04_amd64",
+        "Ubuntu_25_04_arm64",
+    ]
+
+    @missing_feature(
+        context.vm_name in ruby_bug_platforms,
+        library="ruby",
+        reason="""Missing TCP cgroup2 support.
+        See: https://datadoghq.atlassian.net/wiki/spaces/TS/pages/3189670032/Traces+missing+container+tags+with+cgroup+V2#Tracer-Side
+        """,
+    )
     def test_install(self):
-        self._test_install(context.virtual_machine)
+        self._test_install(context.virtual_machine, origin_detection=True)
 
 
 @features.container_auto_installation_script_profiling
@@ -200,6 +239,29 @@ def test_no_world_writeable(self):
         logger.info(f"Done test_no_world_writeable for : [{virtual_machine.name}]")
 
 
+@features.origin_detection
+@scenarios.simple_installer_auto_injection
+@scenarios.multi_installer_auto_injection
+class TestSimpleInstallerAutoInjectManualOriginDetection(base.AutoInjectBaseTest):
+    @irrelevant(
+        condition="container" not in context.weblog_variant and "alpine" not in context.weblog_variant,
+        reason="Origin detection is not supported on host environments",
+    )
+    @irrelevant(
+        context.library > "[email protected]" and context.installed_language_runtime < "3.8.0",
+        reason="python 3.7 is not supported on ddtrace >= 3.x",
+    )
+    def test_origin_detection(self):
+        virtual_machine = context.virtual_machine
+        logger.info(
+            f"Launching test_origin_detection for : [{virtual_machine.name}] [{virtual_machine.get_deployed_weblog().runtime_version}]..."
+        )
+        self._test_install(virtual_machine, origin_detection=True)
+        logger.info(
+            f"Done test_origin_detection for : [{virtual_machine.name}][{virtual_machine.get_deployed_weblog().runtime_version}]"
+        )
+
+
 @features.auto_instrumentation_appsec
 @scenarios.simple_auto_injection_appsec
 class TestSimpleInstallerAutoInjectManualAppsec(base.AutoInjectBaseTest):

tests/auto_inject/utils.py

@@ -7,21 +7,35 @@
 
 
 class AutoInjectBaseTest:
-    def _test_install(self, virtual_machine, *, profile: bool = False, appsec: bool = False):
+    def _test_install(
+        self, virtual_machine, *, profile: bool = False, appsec: bool = False, origin_detection: bool = False
+    ):
         """If there is a multicontainer app, we need to make a request to each app"""
 
         if virtual_machine.get_deployed_weblog().app_type == "multicontainer":
             for app in virtual_machine.get_deployed_weblog().multicontainer_apps:
                 vm_context_url = (
                     f"http://{virtual_machine.get_ip()}:{virtual_machine.deffault_open_port}{app.app_context_url}"
                 )
-                self._check_install(virtual_machine, vm_context_url, profile=profile, appsec=appsec)
+                self._check_install(
+                    virtual_machine, vm_context_url, profile=profile, appsec=appsec, origin_detection=origin_detection
+                )
 
         else:
             vm_context_url = f"http://{virtual_machine.get_ip()}:{virtual_machine.deffault_open_port}{virtual_machine.get_deployed_weblog().app_context_url}"
-            self._check_install(virtual_machine, vm_context_url, profile=profile, appsec=appsec)
+            self._check_install(
+                virtual_machine, vm_context_url, profile=profile, appsec=appsec, origin_detection=origin_detection
+            )
 
-    def _check_install(self, virtual_machine, vm_context_url, *, profile: bool = False, appsec: bool = False):
+    def _check_install(
+        self,
+        virtual_machine,
+        vm_context_url,
+        *,
+        profile: bool = False,
+        appsec: bool = False,
+        origin_detection: bool = False,
+    ):
         """We can easily install agent and lib injection software from agent installation script. Given a  sample application we can enable tracing using local environment variables.
         After starting application we can see application HTTP requests traces in the backend.
         Using the agent installation script we can install different versions of the software (release or beta) in different OS.
@@ -49,6 +63,8 @@ def _check_install(self, virtual_machine, vm_context_url, *, profile: bool = Fal
         validator = None
         if appsec:
             validator = self._appsec_validator
+        if origin_detection:
+            validator = self._container_tags_validator
 
         try:
             wait_backend_trace_id(request_uuid, profile=profile, validator=validator)
@@ -77,6 +93,21 @@ def _appsec_validator(self, _, trace_data):
 
         return True
 
+    def _container_tags_validator(self, _, trace_data):
+        root_id = trace_data["trace"]["root_id"]
+        root_span = trace_data["trace"]["spans"][root_id]
+
+        # Check if container tags exist in the trace metadata
+        meta = root_span.get("meta", {})
+        container_tags = meta.get("_dd.tags.container")
+
+        if container_tags:
+            logger.info(f"Found container tags: {container_tags}")
+            return True
+        else:
+            logger.error(f"No container tags found in trace. Available meta keys: {list(meta.keys())}")
+            return False
+
     def _log_trace_debug_message(self, exc: Exception, request_uuid: str) -> None:
         logger.error(
             f"❌ Exception during trace in backend verification: {exc}\n"

utils/_features.py

@@ -1921,6 +1921,16 @@ def host_auto_installation_script_profiling(test_object):
         """
         return _mark_test_object(test_object, feature_id=306, owner=_Owner.auto_instrumentation)
 
+    @staticmethod
+    def origin_detection(test_object):
+        """Origin Detection is a feature that allows the Agent to correctly detect and tag incoming custom
+        metrics or traces with their Origin (AKA container tags).
+
+        https://feature-parity.us1.prod.dog/#/?feature=362
+
+        """
+        return _mark_test_object(test_object, feature_id=362, owner=_Owner.language_platform)
+
     @staticmethod
     def container_auto_installation_script_profiling(test_object):
         """Profiling works when enabled through the agent installer script in Container environments

utils/build/virtual_machine/provisions/auto-inject/auto-inject_installer_manual.yml

@@ -9,6 +9,39 @@
       local_path: binaries/
 
   remote-command: |
+    # Check if Docker is installed and ensure it's running
+    if command -v docker >/dev/null 2>&1; then
+        echo "Docker is installed, ensuring service is enabled and running..."
+        sudo systemctl enable docker.service docker.socket
+        
+        if ! sudo systemctl is-active --quiet docker.service; then
+            echo "Docker service is not running, starting it..."
+            sudo systemctl start docker.service docker.socket
+            
+            # Wait for Docker to be fully ready
+            echo "Waiting for Docker service to be ready..."
+            MAX_WAIT=20
+            COUNTER=0
+            while [ $COUNTER -lt $MAX_WAIT ]; do
+                if sudo docker info >/dev/null 2>&1; then
+                    echo "Docker service is ready"
+                    break
+                fi
+                echo "Docker not ready yet, waiting... ($((COUNTER + 1))/$MAX_WAIT)"
+                sleep 1
+                ((COUNTER++))
+            done
+            
+            if [ $COUNTER -eq $MAX_WAIT ]; then
+                echo "WARNING: Docker service did not become ready after $MAX_WAIT seconds"
+            fi
+        else
+            echo "Docker service is already running"
+        fi
+    else
+        echo "Docker is not installed, skipping Docker service setup"
+    fi
+
     if [ "${DD_env}" == "dev" ]; then
       # To force the installer to pull from dev repositories -- agent config is set manually to datadoghq.com
       export DD_SITE="datad0g.com"
@@ -120,6 +153,12 @@
     sudo cp debug_config.yaml /etc/datadog-agent/inject/debug_config.yaml
 
     for i in 1 2 3 4 5; do  sudo datadog-installer apm instrument docker && break || sleep 1; done
+    
+    # Add dd-agent user to docker group for docker socket access
+    sudo usermod -aG docker dd-agent || echo "Warning: Could not add dd-agent to docker group"
+    
+    # Restart datadog-agent so the new group membership takes effect
+    sudo systemctl restart datadog-agent || echo "Warning: Could not restart datadog-agent"
 - os_type: windows
   copy_files:
     - name: copy-binaries

utils/build/virtual_machine/provisions/host-auto-inject-install-script/auto-inject_host_script.yml

@@ -17,7 +17,7 @@
 
     DD_APM_INSTRUMENTATION_ENABLED=host bash execute_install_script.sh
     sudo cp debug_config.yaml /etc/datadog-agent/inject/debug_config.yaml
-
+    
 - os_type: windows
   remote-command: |
     echo "TODO. Windows is not supported for installation script yet. TODO."

utils/onboarding/backend_interface.py

@@ -11,8 +11,14 @@
 
 def wait_backend_trace_id(trace_id, profile: bool = False, validator=None):
     logger.info(f"Waiting for backend trace with trace_id: {trace_id}")
-    runtime_id = _query_for_trace_id(trace_id, validator=validator)
+    results = _query_for_trace_id(trace_id, validator=validator)
+    runtime_id = results["runtime_id"]
+    if validator:
+        validator_results = results["validator"]
+        assert validator_results, f"{validator.__name__} failed to validate trace_id: {trace_id}"
+
     assert runtime_id, f"Could not find runtime-id for trace_id: {trace_id}"
+
     if profile:
         _query_for_profile(runtime_id)
 
@@ -54,23 +60,25 @@ def _headers():
 def _query_for_trace_id(trace_id, validator=None):
     url = f"{API_HOST}/api/ui/trace/{trace_id}"
 
+    results = {}
+
     trace_data = _make_request(url, headers=_headers())
     if validator:
         logger.info("Validating backend trace...")
-        if not validator(trace_id, trace_data):
-            logger.info("Backend trace is not valid")
-            return None
-        logger.info("Backend trace is valid")
+        results["validator"] = validator(trace_id, trace_data)
+        if not results["validator"]:
+            logger.info("Validator failed")
 
     root_id = trace_data["trace"]["root_id"]
     root_span = trace_data["trace"]["spans"][root_id]
     start_time = root_span["start"]
     start_date = datetime.fromtimestamp(start_time)
     if (datetime.now() - start_date).days > 1:
         logger.info("Backend trace is too old")
-        return None
-
-    return root_span["meta"]["runtime-id"]
+        results["runtime_id"] = None
+    else:
+        results["runtime_id"] = root_span["meta"]["runtime-id"]
+    return results
 
 
 def _make_request(