[translator] refine evals for llm calls (#66)

* feat: [traceloop] fix trace and evals for llm calls

* tsp: fix typo

fetch 'content' field from the traceloop entity input/output

* tsp: add check to output message

This avoids building invocation when 'parts' is empty

* tsp: explicitly disable deepeval telemetry with some versions

* splunk-deepeval: Set min-max deepeval version

This is done to make it compatible with traceloop-sdk

* deployment: revert to 0.47.5

* revert to working tsp

* remove filtering span exporter

* address review comments

* add old requirements

use pypi for splunk deepeval wrapper

Author: adityamehra

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/.env

@@ -11,7 +11,7 @@ OPENAI_API_KEY=sk-YOUR_API_KEY
 # OTEL_EXPORTER_OTLP_PROTOCOL=grpc
 
 # Traces will use this service.name
-OTEL_SERVICE_NAME=travel-plan-tl
+OTEL_SERVICE_NAME=travel-planner-tl
 
 OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
 OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental
@@ -20,4 +20,6 @@ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT_MODE=SPAN_AND_EVENT
 OTEL_INSTRUMENTATION_GENAI_EMITTERS=span_metric_event,splunk
 OTEL_INSTRUMENTATION_GENAI_EMITTERS_EVALUATION=replace-category:SplunkEvaluationResults
 OTEL_INSTRUMENTATION_GENAI_EVALS_RESULTS_AGGREGATION=true
-OTEL_INSTRUMENTATION_GENAI_DEBUG=true
+OTEL_INSTRUMENTATION_GENAI_DEBUG=false
+
+DEEPEVAL_TELEMETRY_OPT_OUT="YES"

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/Dockerfile.traceloop renamed to instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/Dockerfile.lc

@@ -18,17 +18,18 @@ COPY util/opentelemetry-util-genai-evals /app/opentelemetry-util-genai-evals
 COPY util/opentelemetry-util-genai-evals-deepeval /app/opentelemetry-util-genai-evals-deepeval
 COPY util/opentelemetry-util-genai-emitters-splunk /app/opentelemetry-util-genai-emitters-splunk
 
-# Set working directory to the example
-WORKDIR /app/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner
+# Set working directory to the traceloop example
+WORKDIR /app/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop
 
-# Install Python dependencies from requirements.traceloop.txt (excluding local -e packages)
-# First, create a temporary requirements file without the local editable packages
-RUN grep -v "^-e \.\." requirements.traceloop.txt > /tmp/requirements_external.txt && \
+# Install Python dependencies from requirements.traceloop.txt (excluding local -e packages and git+ssh)
+# First, create a temporary requirements file without the local editable packages and SSH git dependencies
+RUN grep -v "^-e" requirements.traceloop.txt | grep -v "git+ssh" > /tmp/requirements_external.txt && \
     pip install --no-cache-dir -r /tmp/requirements_external.txt && \
     rm /tmp/requirements_external.txt
 
 # Install the local packages in editable mode
 # The Traceloop translator will enable zero-code instrumentation via .pth file
+# splunk-otel-instrumentation-langchain IS installed
 RUN cd /app/opentelemetry-util-genai && \
     pip install --no-cache-dir --no-deps -e . && \
     cd /app/opentelemetry-util-genai-evals && \

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/Dockerfile.tl

@@ -0,0 +1,40 @@
+FROM python:3.12-slim
+
+# Define Traceloop SDK version as build arg (also used for opentelemetry-instrumentation-* packages)
+ARG TRACELOOP_VERSION=0.47.5
+
+WORKDIR /app
+
+# Install git for pip dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy required packages (build context is repo root)
+COPY util/opentelemetry-util-genai-traceloop-translator /app/opentelemetry-util-genai-traceloop-translator
+COPY instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner /app/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner
+
+WORKDIR /app/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop
+
+# Install Python dependencies (excluding local -e packages and git+ssh dependencies)
+RUN pip install --no-cache-dir -r requirements.traceloop.txt
+
+# Verify packages are installed correctly
+RUN python3 -c "from opentelemetry.util.genai.handler import get_telemetry_handler; print('✓ GenAI handler available')" && \
+    python3 -c "from opentelemetry.util.genai.evals import create_evaluation_manager; print('✓ Evaluation manager available')" && \
+    python3 -c "import opentelemetry.util.genai.emitters.splunk; print('✓ Splunk emitters available')" && \
+    python3 -c "import opentelemetry.util.evaluator.deepeval; print('✓ Deepeval evaluator module available')" && \
+    python3 -c "import deepeval; print('✓ Deepeval SDK installed')" && \
+    python3 -c "from opentelemetry.util.genai.traceloop import enable_traceloop_translator; print('✓ Traceloop translator available')"
+
+# Make the script executable
+RUN chmod +x main_traceloop.py
+
+# Set default environment variables
+ENV OTEL_PYTHON_LOG_CORRELATION=true \
+    OTEL_PYTHON_LOG_LEVEL=info \
+    OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
+    PYTHONUNBUFFERED=1
+
+# Run the Traceloop version
+CMD ["python3", "main_traceloop.py"]

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/trip-planner-cronjob.yaml renamed to instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/cronjob-tl-lc.yaml

@@ -7,13 +7,12 @@ metadata:
     app: trip-planner-tl
     component: telemetry
   annotations:
-    description: "Multi-agent trip planner with Traceloop translator and GenAI evaluations (Deepeval telemetry disabled)"
-    git-commit: "20251107-164153"
+    description: "Multi-agent trip planner with Traceloop translator and GenAI evaluations. splunk-otel-instrumentation-langchain is installed"
+    git-commit: "1b4045e"
 spec:
-  # Run every 10 minutes from Nov 7 (Fri) through Nov 10 (Mon) 10 AM PST
-  # Covers: Fri Nov 7 (now) through Mon Nov 10 morning
-  # NOTE: Manually suspend on Monday Nov 10 at 10 AM PST
-  schedule: "*/10 * 7-10 11 *"
+  # Run every 30 minutes from 8 AM to 5 PM PST on weekdays (Monday-Friday)
+  # Offset from travel-planner-tl by 45 minutes (runs at :15 and :45)
+  schedule: "15,45 8-17 * * 1-5"
   timeZone: "America/Los_Angeles"
   suspend: false
 
@@ -38,7 +37,7 @@ spec:
           containers:
             - name: trip-planner-traceloop
               # Multi-platform image (amd64, arm64) with git commit hash tag
-              image: admehra621/trip-planner-tl:20251107-164153
+              image: admehra621/trip-planner-tl-lc:1b4045e
               imagePullPolicy: Always
 
               env:
@@ -48,7 +47,7 @@ spec:
 
                 # === OpenTelemetry Resource Attributes ===
                 - name: OTEL_RESOURCE_ATTRIBUTES
-                  value: "deployment.environment=o11y-inframon-ai,git.commit.id=20251107-164153"
+                  value: "deployment.environment=o11y-inframon-ai,git.commit.id=1b4045e"
 
                 # === Service name for telemetry ===
                 - name: OTEL_SERVICE_NAME

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/cronjob-tl.yaml

@@ -7,8 +7,8 @@ metadata:
     app: travel-planner-tl
     component: telemetry
   annotations:
-    description: "Multi-agent travel planner with Traceloop translator and GenAI evaluations (Deepeval telemetry disabled)"
-    git-commit: "2b4a41a"
+    description: "Multi-agent travel planner with Traceloop translator and GenAI evaluations. splunk-otel-instrumentation-langchain is NOT installed"
+    git-commit: "adc36a2"
 spec:
   # Run every 30 minutes from 8 AM to 5 PM PST on weekdays (Monday-Friday)
   schedule: "*/30 8-17 * * 1-5"
@@ -36,7 +36,7 @@ spec:
           containers:
             - name: travel-planner-traceloop
               # Multi-platform image (amd64, arm64) with git commit hash tag
-              image: admehra621/travel-planner-tl:2b4a41a
+              image: admehra621/travel-planner-tl:adc36a2
               imagePullPolicy: Always
 
               env:
@@ -46,7 +46,7 @@ spec:
 
                 # === OpenTelemetry Resource Attributes ===
                 - name: OTEL_RESOURCE_ATTRIBUTES
-                  value: "deployment.environment=o11y-inframon-ai,git.commit.id=2b4a41a"
+                  value: "deployment.environment=o11y-inframon-ai,git.commit.id=adc36a2"
 
                 # === Service name for telemetry ===
                 - name: OTEL_SERVICE_NAME
@@ -64,7 +64,7 @@ spec:
 
                 # === Deepeval Telemetry Opt-Out ===
                 - name: DEEPEVAL_TELEMETRY_OPT_OUT
-                  value: "1"
+                  value: "YES"
 
                 # === GenAI Content Capture ===
                 - name: OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/requirements.old.txt

@@ -0,0 +1,84 @@
+opentelemetry-util-http==0.59b0
+orjson==3.11.4
+ormsgpack==1.11.0
+packaging==25.0
+platformdirs==4.5.0
+pluggy==1.6.0
+portalocker==3.2.0
+posthog==3.25.0
+pre-commit==3.7.0
+propcache==0.4.1
+protobuf==6.33.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pydantic==2.12.3
+pydantic-settings==2.11.0
+pydantic_core==2.41.4
+pyfiglet==1.0.4
+Pygments==2.19.2
+pylint==3.0.2
+pyright==1.1.404
+pytest==8.4.2
+pytest-asyncio==1.2.0
+pytest-cov==4.1.0
+pytest-repeat==0.9.4
+pytest-rerunfailures==12.0
+pytest-xdist==3.8.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.2.1
+PyYAML==6.0.3
+readme-renderer==42.0
+regex==2025.10.23
+requests==2.32.5
+requests-toolbelt==1.0.0
+rich==14.2.0
+rsa==4.9.1
+ruamel.yaml==0.17.21
+ruff==0.6.9
+sentry-sdk==2.43.0
+setuptools==80.9.0
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+snowballstemmer==3.0.1
+Sphinx==7.1.2
+sphinx-autodoc-typehints==1.25.2
+sphinx-rtd-theme==2.0.0rc4
+sphinxcontrib-applehelp==2.0.0
+sphinxcontrib-devhelp==2.0.0
+sphinxcontrib-htmlhelp==2.1.0
+sphinxcontrib-jquery==4.1
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==2.0.0
+sphinxcontrib-serializinghtml==2.0.0
+# Editable install with no version control (splunk-otel-genai-emitters-splunk==0.1b0.dev0)
+-e /app/opentelemetry-util-genai-emitters-splunk
+# Editable install with no version control (splunk-otel-genai-evals-deepeval==0.1b0.dev0)
+-e /app/opentelemetry-util-genai-evals-deepeval
+# Editable install with no version control (splunk-otel-util-genai==0.1b0.dev0)
+-e /app/opentelemetry-util-genai
+# Editable install with no version control (splunk-otel-util-genai-evals==0.1b0.dev0)
+-e /app/opentelemetry-util-genai-evals
+# Editable install with no version control (splunk-otel-util-genai-translator-traceloop==0.0.0.dev0)
+-e /app/opentelemetry-util-genai-traceloop-translator
+tabulate==0.9.0
+tenacity==9.1.2
+tiktoken==0.12.0
+tokenizers==0.22.1
+tomlkit==0.13.3
+tqdm==4.67.1
+traceloop-sdk==0.47.5
+typer==0.20.0
+typer-slim==0.20.0
+typing-inspection==0.4.2
+typing_extensions==4.15.0
+urllib3==2.5.0
+virtualenv==20.35.4
+webencodings==0.5.1
+websockets==15.0.1
+wheel==0.45.1
+wrapt==1.17.3
+xxhash==3.6.0
+yarl==1.22.0
+zipp==3.23.0
+zstandard==0.25.0

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/requirements.traceloop.txt

@@ -78,6 +78,7 @@ opentelemetry-exporter-otlp-proto-common @ git+https://github.com/open-telemetry
 opentelemetry-exporter-otlp-proto-grpc @ git+https://github.com/open-telemetry/opentelemetry-python.git@1f68134481c46e476a127b61a36dc69889275d15#subdirectory=exporter/opentelemetry-exporter-otlp-proto-grpc
 opentelemetry-exporter-otlp-proto-http==1.38.0
 opentelemetry-instrumentation @ git+https://github.com/open-telemetry/opentelemetry-python-contrib.git@071f68697432e7e57b31238a5998dc4c1325855c#subdirectory=opentelemetry-instrumentation
+# OpenTelemetry instrumentation packages - version must match traceloop-sdk version (0.47.5)
 opentelemetry-instrumentation-alephalpha==0.47.5
 opentelemetry-instrumentation-anthropic==0.47.5
 opentelemetry-instrumentation-bedrock==0.47.5
@@ -170,11 +171,12 @@ sphinxcontrib-jquery==4.1
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==2.0.0
 sphinxcontrib-serializinghtml==2.0.0
--e git+ssh://[email protected]/signalfx/splunk-otel-python-contrib.git@2b4a41abac7725b53c82f480833c874aca0af072#egg=splunk_otel_genai_evals_deepeval&subdirectory=util/opentelemetry-util-genai-evals-deepeval
--e git+ssh://[email protected]/signalfx/splunk-otel-python-contrib.git@2b4a41abac7725b53c82f480833c874aca0af072#egg=splunk_otel_instrumentation_langchain&subdirectory=instrumentation-genai/opentelemetry-instrumentation-langchain
--e git+ssh://[email protected]/signalfx/splunk-otel-python-contrib.git@2b4a41abac7725b53c82f480833c874aca0af072#egg=splunk_otel_util_genai&subdirectory=util/opentelemetry-util-genai
--e git+ssh://[email protected]/signalfx/splunk-otel-python-contrib.git@2b4a41abac7725b53c82f480833c874aca0af072#egg=splunk_otel_util_genai_evals&subdirectory=util/opentelemetry-util-genai-evals
--e git+ssh://[email protected]/signalfx/splunk-otel-python-contrib.git@2b4a41abac7725b53c82f480833c874aca0af072#egg=splunk_otel_util_genai_translator_traceloop&subdirectory=util/opentelemetry-util-genai-traceloop-translator
+splunk-otel-genai-evals-deepeval>=0.1.6
+splunk-otel-util-genai
+splunk-otel-util-genai-evals
+splunk-otel-genai-emitters-splunk
+# Editable install with no version control (splunk-otel-util-genai-translator-traceloop==0.0.0.dev0)
+-e /app/opentelemetry-util-genai-traceloop-translator
 tabulate==0.9.0
 tenacity==9.1.2
 tiktoken==0.12.0

util/opentelemetry-util-genai-traceloop-translator/src/opentelemetry/util/genai/processor/__init__.py

@@ -14,7 +14,6 @@
 
 """Traceloop span processor and transformation utilities."""
 
-from .filtering_span_processor import FilteringSpanProcessor
 from .traceloop_span_processor import TraceloopSpanProcessor
 
-__all__ = ["TraceloopSpanProcessor", "FilteringSpanProcessor"]
+__all__ = ["TraceloopSpanProcessor"]