address merge conflicts

Author: mtoffl01

.github/actions/pull_images/action.yml

@@ -39,7 +39,7 @@ runs:
 
     - name: clean-up
       shell: bash
-      if: inputs.library == 'php'  # PHP use heavy images
+      if: inputs.library == 'php' || inputs.library == 'ruby' || inputs.library == 'golang' # PHP/ruby/golang use heavy images
       run: sudo rm -rf /usr/local/lib/android
 
 #retry

.github/workflows/ci.yml

@@ -66,7 +66,6 @@ jobs:
     strategy:
       matrix:
         include: ${{ fromJson(needs.compute_libraries_and_scenarios.outputs.library_matrix) }}
-
       fail-fast: false
     uses: ./.github/workflows/system-tests.yml
     permissions:
@@ -87,6 +86,7 @@ jobs:
       library: ${{ matrix.library }}
       scenarios: ${{ needs.compute_libraries_and_scenarios.outputs.scenarios }}
       scenarios_groups: ${{ needs.compute_libraries_and_scenarios.outputs.scenarios_groups }}
+      excluded_scenarios: OTEL_COLLECTOR_E2E  # rely on real backend
       parametric_job_count: ${{ matrix.version == 'dev' && 2 || 1 }}  # test both use cases
       skip_empty_scenarios: true
       display_summary: true

.github/workflows/debug-harness.yml

@@ -23,13 +23,18 @@ on:
         description: How many jobs
         default: "10"
         type: string
+      dev_or_prod:
+        description: Run scenario against last release (prod) or last commit on main (dev)?
+        default: "prod"
+        type: string
 
 env:
   LIBRARY:      ${{ (github.event_name == 'workflow_dispatch' && inputs.library)     || 'php' }}
   WEBLOG:       ${{ (github.event_name == 'workflow_dispatch' && inputs.weblog)      || 'apache-mod-8.1' }}
   SCENARIO:     ${{ (github.event_name == 'workflow_dispatch' && inputs.scenario)    || 'DEBUGGER_EXPRESSION_LANGUAGE' }}
   JOB_COUNT:    ${{ (github.event_name == 'workflow_dispatch' && inputs.job_count)   || '10' }}
   AGENT_IMAGE:  ${{ (github.event_name == 'workflow_dispatch' && inputs.agent_image) || 'datadog/agent:latest' }}
+  DEV_OR_PROD:  ${{ (github.event_name == 'workflow_dispatch' && inputs.dev_or_prod) || 'prod' }}
 
 jobs:
   generate-matrix:
@@ -51,6 +56,9 @@ jobs:
         run: echo "${{ secrets.GITHUB_TOKEN }}" > "$RUNNER_TEMP/github_token.txt"
       - name: Select agent image
         run: echo ${{ env.AGENT_IMAGE }} > binaries/agent-image
+      - name: Get dev artifact
+        if: env.DEV_OR_PROD == 'dev'
+        run: ./utils/scripts/load-binary.sh ${{ env.LIBRARY }}
       - name: Build weblog
         id: build
         run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ env.LIBRARY }} -i weblog -w ${{ env.WEBLOG }} -s --github-token-file "$RUNNER_TEMP/github_token.txt"

.github/workflows/lint.yml

@@ -13,7 +13,11 @@ jobs:
         uses: ./.github/actions/install_runner
       - run: source venv/bin/activate
         shell: bash
-      - name: Black, pylint, tailing whitespaces, and yaml checks
+      - name: Install node
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: 20
+      - name: Black, pylint, tailing whitespaces, yaml, shellcheck, and language-specific checks
         shell: bash
         run: ./format.sh --check
       - if: ${{ failure() }}
@@ -24,25 +28,3 @@ jobs:
           exit 1
 
           '
-      - name: 'Install shellcheck'
-        shell: bash
-        run: sudo apt-get install -y shellcheck
-      - name: 'Run shellcheck'
-        shell: bash
-        run: ./utils/scripts/shellcheck.sh
-      - name: Install node
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
-        with:
-          node-version: 20
-      - name: 'Run nodejs express lint'
-        shell: bash
-        working-directory: ./utils/build/docker/nodejs/express
-        run: |
-          npm install
-          npm run lint
-      - name: 'Run nodejs fastify lint'
-        shell: bash
-        working-directory: ./utils/build/docker/nodejs/fastify
-        run: |
-          npm install
-          npm run lint

.github/workflows/run-end-to-end.yml

@@ -472,6 +472,11 @@ jobs:
     - name: Run OTEL_COLLECTOR scenario
       if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"OTEL_COLLECTOR"')
       run: ./run.sh OTEL_COLLECTOR
+    - name: Run OTEL_COLLECTOR_E2E scenario
+      if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"OTEL_COLLECTOR_E2E"')
+      run: ./run.sh OTEL_COLLECTOR_E2E
+      env:
+        DD_API_KEY: ${{ secrets.DD_API_KEY }}
 
     - name: Run all scenarios in replay mode
       if: success() && steps.build.outcome == 'success' && inputs._enable_replay_scenarios

.github/workflows/system-tests.yml

@@ -187,7 +187,7 @@ jobs:
     name: Build end-to-end (${{ matrix.weblog.name }})
     needs:
       - compute_parameters
-    if: ${{ needs.compute_parameters.outputs.endtoend_defs_parallel_enable == 'true' }}
+    if: ${{ needs.compute_parameters.outputs.endtoend_defs_parallel_enable == 'true' && needs.compute_parameters.outputs.endtoend_defs_parallel_weblogs != '[]'}}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -249,7 +249,7 @@ jobs:
     needs:
       - compute_parameters
       - build_end_to_end
-    if: ${{ needs.compute_parameters.outputs.endtoend_defs_parallel_enable == 'true' }}
+    if: ${{ always() && needs.build_end_to_end.result != 'failure' && needs.compute_parameters.result != 'failure' && needs.compute_parameters.outputs.endtoend_defs_parallel_enable == 'true' }}
     strategy:
       matrix:
         job: ${{ fromJson(needs.compute_parameters.outputs.endtoend_defs_parallel_jobs) }}

.promptfoo/local_cursor_provider.py

@@ -57,7 +57,7 @@ class ResponseParsingError(Exception):
 def _raise_invalid_format_error() -> None:
     """Helper function to raise invalid format error."""
     raise InvalidYAMLFormatError(
-        "Invalid YAML format - expected a list of prompt-output pairs " "or a 'responses' key containing the list"
+        "Invalid YAML format - expected a list of prompt-output pairs or a 'responses' key containing the list"
     )
 
 

.shellcheck

@@ -33,4 +33,5 @@ TODO=(
   utils/interfaces/schemas/serve.sh
   build.sh
   format.sh
+  *node_modules*
 )

docs/edit/scenarios.md

@@ -13,19 +13,10 @@ class CustomScenario(Scenario):
         """
             If needed, configure the context => mainly, only get infos from config
             At this point, logger.stdout is unavailable, so this function should not fail, unless
-            there is some config error from the user
-        """
+            there is some config error from the user.
 
-    def get_warmups(self):
-        """
-            Use this function to start anything needed to your scenario (build, run targets)
-            This function returns a list of callable that will be called sequentially
+            If any warmup must be executed at startup, they must be added here using self.warmups.add()
         """
-        warmups = super().get_warmups()
-
-        warmups.append(self.start_target)
-
-        return warmups
 
     def post_setup(self, session):
         """ called after setup functions, and before test functions """

docs/execute/binaries.md

@@ -138,15 +138,9 @@ Then run the tests from the repo root folder:
 
 ## Python library
 
-1. Add a file `binaries/python-load-from-pip`, the content will be installed by pip. Content example:
-  * `ddtrace @ git+https://github.com/DataDog/dd-trace-py.git`
-2. Add a `.tar.gz` or a `.whl` file in `binaries`, pip will install it
-3. Clone the dd-trace-py repo inside `binaries`
+1. Add a `.tar.gz` or a `.whl` file in `binaries`, pip will install it
+2. Clone the dd-trace-py repo inside `binaries`
 
-You can also run:
-```bash
-echo “ddtrace @ git+https://github.com/DataDog/dd-trace-py.git@<name-of-your-branch>” > binaries/python-load-from-pip
-```
 
 ## Ruby library