Cleanup workflow - part 1 (huggingface#42023)

* part 1

* part 2

* part 3

* part 4

* part 5

* fix 1

* check 1

* part 6

* part 7

* part 8

* part 9

* part 10: rename file

* OK: new_model_pr_merged_notification.yml

* part 11

* fix 2

* revert check

* fix

* fix

* fix

---------

Co-authored-by: ydshieh <[email protected]>

Author: ydshieh

.github/workflows/check_failed_tests.yml

@@ -64,13 +64,15 @@ jobs:
       - name: Check file
         id: check_file
         working-directory: /transformers
+        env:
+          job: ${{ inputs.job }}
         run: |
-          if [ -f ci_results_${{ inputs.job }}/new_failures.json ]; then
-            echo "`ci_results_${{ inputs.job }}/new_failures.json` exists, continue ..."
+          if [ -f "ci_results_${job}/new_failures.json" ]; then
+            echo "\`ci_results_${job}/new_failures.json\` exists, continue ..."
             echo "process=true" >> $GITHUB_ENV
             echo "process=true" >> $GITHUB_OUTPUT
           else
-            echo "`ci_results_${{ inputs.job }}/new_failures.json` doesn't exist, abort."
+            echo "\`ci_results_${job}/new_failures.json\` doesn't exist, abort."
             echo "process=false" >> $GITHUB_ENV
             echo "process=false" >> $GITHUB_OUTPUT
           fi
@@ -94,15 +96,18 @@ jobs:
       - name: Update clone
         working-directory: /transformers
         if: ${{ env.process == 'true' }}
+        env:
+          commit_sha: ${{ inputs.commit_sha || github.sha }}
         run: |
-          git fetch origin ${{ inputs.commit_sha || github.sha }}
-          git fetch && git checkout ${{ inputs.commit_sha || github.sha }}
+          git fetch origin "$commit_sha" && git checkout "$commit_sha"
 
       - name: Get `START_SHA`
         working-directory: /transformers/utils
         if: ${{ env.process == 'true' }}
+        env:
+          commit_sha: ${{ inputs.commit_sha || github.sha }}
         run: |
-          echo "START_SHA=${{ inputs.commit_sha || github.sha }}" >> $GITHUB_ENV
+          echo "START_SHA=$commit_sha" >> $GITHUB_ENV
 
       # This is used if the CI is triggered from a pull request `self-comment-ci.yml` (after security check is verified)
       - name: Extract the base commit on `main` (of the merge commit created by Github) if it is a PR
@@ -130,16 +135,20 @@ jobs:
       - name: Get `END_SHA` from previous CI runs of the same workflow
         working-directory: /transformers/utils
         if: ${{ env.process == 'true' && inputs.pr_number == '' }}
+        env:
+          ACCESS_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
         run: |
-          echo "END_SHA=$(TOKEN=${{ secrets.ACCESS_REPO_INFO_TOKEN }} python3 -c 'import os; from get_previous_daily_ci import get_last_daily_ci_run_commit; commit=get_last_daily_ci_run_commit(token=os.environ["TOKEN"], workflow_run_id=os.environ["PREV_WORKFLOW_RUN_ID"]); print(commit)')" >> $GITHUB_ENV
+          echo "END_SHA=$(TOKEN="$ACCESS_TOKEN" python3 -c 'import os; from get_previous_daily_ci import get_last_daily_ci_run_commit; commit=get_last_daily_ci_run_commit(token=os.environ["TOKEN"], workflow_run_id=os.environ["PREV_WORKFLOW_RUN_ID"]); print(commit)')" >> $GITHUB_ENV
 
       # However, for workflow runs triggered by `issue_comment` (for pull requests), we want to check against the
       # parent commit (on `main`) of the `merge_commit` (dynamically created by GitHub). In this case, the goal is to
       # see if a reported failing test is actually ONLY failing on the `merge_commit`.
       - name: Set `END_SHA`
         if: ${{ env.process == 'true' && inputs.pr_number != '' }}
+        env:
+          merge_commit_base_sha: ${{ steps.pr_info.outputs.merge_commit_base_sha }}
         run: |
-          echo "END_SHA=${{ steps.pr_info.outputs.merge_commit_base_sha }}" >> $GITHUB_ENV
+          echo "END_SHA=$merge_commit_base_sha" >> $GITHUB_ENV
 
       - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
         working-directory: /transformers
@@ -169,14 +178,20 @@ jobs:
       - name: Check failed tests
         working-directory: /transformers
         if: ${{ env.process == 'true' }}
-        run: python3 utils/check_bad_commit.py --start_commit ${{ env.START_SHA }} --end_commit ${{ env.END_SHA }} --file ci_results_${{ inputs.job }}/new_failures.json --output_file new_failures_with_bad_commit_${{ inputs.job }}_${{ matrix.run_idx }}.json
+        env:
+          job: ${{ inputs.job }}
+          run_idx: ${{ matrix.run_idx }}
+        run: python3 utils/check_bad_commit.py --start_commit "$START_SHA" --end_commit "$END_SHA" --file "ci_results_${job}/new_failures.json" --output_file "new_failures_with_bad_commit_${job}_${run_idx}.json"
 
       - name: Show results
         working-directory: /transformers
         if: ${{ env.process == 'true' }}
+        env:
+          job: ${{ inputs.job }}
+          run_idx: ${{ matrix.run_idx }}
         run: |
-          ls -l new_failures_with_bad_commit_${{ inputs.job }}_${{ matrix.run_idx }}.json
-          cat new_failures_with_bad_commit_${{ inputs.job }}_${{ matrix.run_idx }}.json
+          ls -l "new_failures_with_bad_commit_${job}_${run_idx}.json"
+          cat "new_failures_with_bad_commit_${job}_${run_idx}.json"
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
@@ -209,23 +224,28 @@ jobs:
 
       - name: Check files
         working-directory: /transformers
+        env:
+          job: ${{ inputs.job }}
         run: |
           ls -la /transformers
-          ls -la /transformers/new_failures_with_bad_commit_${{ inputs.job }}
+          ls -la "/transformers/new_failures_with_bad_commit_${job}"
 
       # Currently, we only run with a single runner by using `run_idx: [1]`. We might try to run with multiple runners
       # to further reduce the false positive caused by flaky tests, which requires further processing to merge reports.
       - name: Merge files
         shell: bash
         working-directory: /transformers
+        env:
+          job: ${{ inputs.job }}
         run: |
-          cp /transformers/new_failures_with_bad_commit_${{ inputs.job }}/new_failures_with_bad_commit_${{ inputs.job }}_1.json new_failures_with_bad_commit.json
+          cp "/transformers/new_failures_with_bad_commit_${job}/new_failures_with_bad_commit_${job}_1.json" new_failures_with_bad_commit.json
 
       - name: Update clone
         working-directory: /transformers
+        env:
+          commit_sha: ${{ inputs.commit_sha || github.sha }}
         run: |
-          git fetch origin ${{ inputs.commit_sha || github.sha }}
-          git fetch && git checkout ${{ inputs.commit_sha || github.sha }}
+          git fetch origin "$commit_sha" && git checkout "$commit_sha"
 
       - name: Process report
         shell: bash
@@ -267,9 +287,12 @@ jobs:
 
       - name: Prepare Slack report title
         working-directory: /transformers
+        env:
+          ci_event: ${{ inputs.ci_event }}
+          job: ${{ inputs.job }}
         run: |
           pip install slack_sdk
-          echo "title=$(python3 -c 'import sys; sys.path.append("utils"); from utils.notification_service import job_to_test_map; ci_event = "${{ inputs.ci_event }}"; job = "${{ inputs.job }}"; test_name = job_to_test_map[job]; title = f"New failed tests of {ci_event}" + ":" + f" {test_name}"; print(title)')" >> $GITHUB_ENV
+          echo "title=$(python3 -c 'import sys; import os; sys.path.append("utils"); from utils.notification_service import job_to_test_map; ci_event = os.environ["ci_event"]; job = os.environ["job"]; test_name = job_to_test_map[job]; title = f"New failed tests of {ci_event}" + ":" + f" {test_name}"; print(title)')" >> $GITHUB_ENV
 
       - name: Send processed report
         if: ${{ !endsWith(env.REPORT_TEXT, '{}') }}

.github/workflows/get-pr-info.yml

@@ -153,14 +153,15 @@ jobs:
 
       - name: Convert dates to timestamps
         id: get_timestamps
+        env:
+          head_commit_date: ${{ steps.pr_info.outputs.head_commit_date }}
+          merge_commit_date: ${{ steps.pr_info.outputs.merge_commit_date }}
         run: |
-          head_commit_date=${{ steps.pr_info.outputs.head_commit_date }}
-          merge_commit_date=${{ steps.pr_info.outputs.merge_commit_date }}
-          echo $head_commit_date
-          echo $merge_commit_date
+          echo "$head_commit_date"
+          echo "$merge_commit_date"
           head_commit_timestamp=$(date -d "$head_commit_date" +%s)
           merge_commit_timestamp=$(date -d "$merge_commit_date" +%s)
-          echo $head_commit_timestamp
-          echo $merge_commit_timestamp
+          echo "$head_commit_timestamp"
+          echo "$merge_commit_timestamp"
           echo "head_commit_timestamp=$head_commit_timestamp" >> $GITHUB_OUTPUT
-          echo "merge_commit_timestamp=$merge_commit_timestamp" >> $GITHUB_OUTPUT
+          echo "merge_commit_timestamp=$merge_commit_timestamp" >> $GITHUB_OUTPUT

.github/workflows/get-pr-number.yml

@@ -15,22 +15,28 @@ jobs:
     steps:
       - name: Get PR number
         shell: bash
+        env:
+          issue_number: ${{ github.event.issue.number }}
+          is_pull_request_issue: ${{ github.event.issue.pull_request != null }}
+          pr_number: ${{ github.event.pull_request.number }}
+          is_pull_request: ${{ github.event.pull_request != null }}
+          event_number: ${{ github.event.number }}
         run: |
-          if [[ "${{ github.event.issue.number }}" != "" && "${{ github.event.issue.pull_request }}" != "" ]]; then
-            echo "PR_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
-          elif [[ "${{ github.event.pull_request.number }}" != "" ]]; then
-            echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-          elif [[ "${{ github.event.pull_request }}" != "" ]]; then
-            echo "PR_NUMBER=${{ github.event.number }}" >> $GITHUB_ENV
+          if [[ "$issue_number" != "" && "$is_pull_request_issue" == "true" ]]; then
+            echo "PR_NUMBER=$issue_number" >> $GITHUB_ENV
+          elif [[ "$pr_number" != "" ]]; then
+            echo "PR_NUMBER=$pr_number" >> $GITHUB_ENV
+          elif [[ "$is_pull_request" == "true" ]]; then
+            echo "PR_NUMBER=$event_number" >> $GITHUB_ENV
           else
             echo "PR_NUMBER=" >> $GITHUB_ENV
           fi
 
       - name: Check PR number
         shell: bash
         run: |
-          echo "${{ env.PR_NUMBER }}"
+          echo "$PR_NUMBER"
 
       - name: Set PR number
         id: set_pr_number
-        run: echo "PR_NUMBER=${{ env.PR_NUMBER }}" >> "$GITHUB_OUTPUT"
+        run: echo "PR_NUMBER=$PR_NUMBER" >> "$GITHUB_OUTPUT"

.github/workflows/model_jobs.yml

@@ -62,27 +62,33 @@ jobs:
     steps:
       - name: Echo input and matrix info
         shell: bash
+        env:
+          folder_slices: ${{ inputs.folder_slices }}
+          matrix_folders: ${{ matrix.folders }}
+          slice_data: ${{ toJson(fromJson(inputs.folder_slices)[inputs.slice_id]) }}
         run: |
-          echo "${{ inputs.folder_slices }}"
-          echo "${{ matrix.folders }}"
-          echo "${{ toJson(fromJson(inputs.folder_slices)[inputs.slice_id]) }}"
+          echo "$folder_slices"
+          echo "$matrix_folders"
+          echo "$slice_data"
 
       - name: Echo folder ${{ matrix.folders }}
         shell: bash
         # For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
         # set the artifact folder names (because the character `/` is not allowed).
+        env:
+          matrix_folders_raw: ${{ matrix.folders }}
         run: |
-          echo "${{ matrix.folders }}"
-          matrix_folders=${{ matrix.folders }}
-          matrix_folders=${matrix_folders/'models/'/'models_'}
+          echo "$matrix_folders_raw"
+          matrix_folders="${matrix_folders_raw/'models/'/'models_'}"
           echo "$matrix_folders"
           echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV
 
       - name: Update clone
         working-directory: /transformers
+        env:
+          commit_sha: ${{ inputs.commit_sha || github.sha }}
         run: |
-          git fetch origin ${{ inputs.commit_sha || github.sha }}
-          git fetch && git checkout ${{ inputs.commit_sha || github.sha }}
+          git fetch origin "$commit_sha" && git checkout "$commit_sha"
 
       - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
         working-directory: /transformers
@@ -117,15 +123,17 @@ jobs:
         id: set_machine_type
         working-directory: /transformers
         shell: bash
+        env:
+          input_machine_type: ${{ inputs.machine_type }}
         run: |
-          echo "${{ inputs.machine_type }}"
+          echo "$input_machine_type"
 
-          if [ "${{ inputs.machine_type }}" = "aws-g5-4xlarge-cache" ]; then
+          if [ "$input_machine_type" = "aws-g5-4xlarge-cache" ]; then
             machine_type=single-gpu
-          elif [ "${{ inputs.machine_type }}" = "aws-g5-12xlarge-cache" ]; then
+          elif [ "$input_machine_type" = "aws-g5-12xlarge-cache" ]; then
             machine_type=multi-gpu
           else
-            machine_type=${{ inputs.machine_type }}
+            machine_type="$input_machine_type"
           fi
 
           echo "$machine_type"
@@ -134,15 +142,21 @@ jobs:
 
       - name: Create report directory if it doesn't exist
         shell: bash
+        env:
+          report_name_prefix: ${{ inputs.report_name_prefix }}
         run: |
-          mkdir -p /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
-          echo "dummy" > /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports/dummy.txt
-          ls -la /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
+          mkdir -p "/transformers/reports/${machine_type}_${report_name_prefix}_${matrix_folders}_test_reports"
+          echo "dummy" > "/transformers/reports/${machine_type}_${report_name_prefix}_${matrix_folders}_test_reports/dummy.txt"
+          ls -la "/transformers/reports/${machine_type}_${report_name_prefix}_${matrix_folders}_test_reports"
 
       - name: Run all tests on GPU
         working-directory: /transformers
+        env:
+          report_name_prefix: ${{ inputs.report_name_prefix }}
+          pytest_marker: ${{ inputs.pytest_marker }}
+          model: ${{ matrix.folders }}
         run: |
-          script -q -c "PATCH_TESTING_METHODS_TO_COLLECT_OUTPUTS=yes _PATCHED_TESTING_METHODS_OUTPUT_DIR=/transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports python3 -m pytest -rsfE -v -m '${{ inputs.pytest_marker }}' --make-reports=${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports tests/${{ matrix.folders }}" test_outputs.txt
+          script -q -c "PATCH_TESTING_METHODS_TO_COLLECT_OUTPUTS=yes _PATCHED_TESTING_METHODS_OUTPUT_DIR=/transformers/reports/${machine_type}_${report_name_prefix}_${matrix_folders}_test_reports python3 -m pytest -rsfE -v -m '${pytest_marker}' --make-reports=${machine_type}_${report_name_prefix}_${matrix_folders}_test_reports tests/${model}" test_outputs.txt
           ls -la
           # Extract the exit code from the output file
           EXIT_CODE=$(tail -1 test_outputs.txt | grep -o 'COMMAND_EXIT_CODE="[0-9]*"' | cut -d'"' -f2)
@@ -153,19 +167,25 @@ jobs:
         # This step is only to show information on Github Actions log.
         # Always mark this step as successful, even if the report directory or the file `failures_short.txt` in it doesn't exist
         continue-on-error: true
-        run: cat /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports/failures_short.txt
+        env:
+          report_name_prefix: ${{ inputs.report_name_prefix }}
+        run: cat "/transformers/reports/${machine_type}_${report_name_prefix}_${matrix_folders}_test_reports/failures_short.txt"
 
       - name: Captured information
         if: ${{ failure() }}
         continue-on-error: true
+        env:
+          report_name_prefix: ${{ inputs.report_name_prefix }}
         run: |
-          cat /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports/captured_info.txt
+          cat "/transformers/reports/${machine_type}_${report_name_prefix}_${matrix_folders}_test_reports/captured_info.txt"
 
       - name: Copy test_outputs.txt
         if: ${{ always() }}
         continue-on-error: true
+        env:
+          report_name_prefix: ${{ inputs.report_name_prefix }}
         run: |
-          cp /transformers/test_outputs.txt /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
+          cp /transformers/test_outputs.txt "/transformers/reports/${machine_type}_${report_name_prefix}_${matrix_folders}_test_reports"
 
       - name: "Test suite reports artifacts: ${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports"
         if: ${{ always() }}

.github/workflows/pr_run_slow_ci.yml renamed to .github/workflows/pr_slow_ci_suggestion.yml

@@ -1,4 +1,4 @@
-name: PR slow CI
+name: PR slow CI - Suggestion
 on:
   pull_request_target:
     types: [opened, synchronize, reopened]
@@ -59,27 +59,35 @@ jobs:
           fetch-depth: "0"
 
       - name: Write pr_files file
+        env:
+          PR_FILES: ${{ needs.get-pr-info.outputs.PR_FILES }}
         run: |
-          cat > pr_files.txt << 'EOF'
-          ${{ needs.get-pr-info.outputs.PR_FILES }}
+          cat > pr_files.txt << EOF
+          $PR_FILES
           EOF
 
       - name: Write tests_dir file
+        env:
+          tests_dir: ${{ steps.repo_content.outputs.tests_dir }}
         run: |
-          cat > tests_dir.txt << 'EOF'
-          ${{ steps.repo_content.outputs.tests_dir }}
+          cat > tests_dir.txt << EOF
+          $tests_dir
           EOF
 
       - name: Write tests_models_dir file
+        env:
+          tests_models_dir: ${{ steps.repo_content.outputs.tests_models_dir }}
         run: |
-          cat > tests_models_dir.txt << 'EOF'
-          ${{ steps.repo_content.outputs.tests_models_dir }}
+          cat > tests_models_dir.txt << EOF
+          $tests_models_dir
           EOF
 
       - name: Write tests_quantization_dir file
+        env:
+          tests_quantization_dir: ${{ steps.repo_content.outputs.tests_quantization_dir }}
         run: |
-          cat > tests_quantization_dir.txt << 'EOF'
-          ${{ steps.repo_content.outputs.tests_quantization_dir }}
+          cat > tests_quantization_dir.txt << EOF
+          $tests_quantization_dir
           EOF
 
       - name: Run script to get jobs to run