[NO-TICKET] Add seed to CI unit tests (#4953)

Author: vpellan

.github/workflows/_unit_test.yml

@@ -1,4 +1,4 @@
-name: Unit Test Template
+name: Unit Tests (Ruby version specific)
 
 on: # yamllint disable-line rule:truthy
   workflow_call:
@@ -13,6 +13,9 @@ on: # yamllint disable-line rule:truthy
       alias:
         required: true
         type: string
+      seed:
+        required: false
+        type: number
     outputs:
       lockfile:
         description: "The lockfile artifact"
@@ -21,11 +24,33 @@ on: # yamllint disable-line rule:truthy
         description: "The cache key for bundle"
         value: ${{ jobs.batch.outputs.cache-key }}
 
+  workflow_dispatch:
+    inputs:
+      engine:
+        description: "`ruby` or `jruby`"
+        required: true
+        type: string
+      version:
+        description: "The version of the engine (2.x or 3.x for Ruby, 9.x or 10.x for JRuby)"
+        required: true
+        type: string
+      alias:
+        description: "The alias of the engine (e.g. `ruby-34` for Ruby 3.4)"
+        required: true
+        type: string
+      seed:
+        description: "The seed to reproduce a CI run"
+        required: false
+        type: number
+
+permissions: {}
+
 jobs:
   batch:
     runs-on: ubuntu-24.04
     name: batch
     outputs:
+      seed: "${{ steps.set-batches.outputs.seed }}"
       batches: "${{ steps.set-batches.outputs.batches }}"
       misc: "${{ steps.set-batches.outputs.misc }}"
       cache-key: "${{ steps.bundle-cache.outputs.cache-key }}"
@@ -43,20 +68,23 @@ jobs:
 
     - id: set-batches
       name: Distribute tasks into batches
+      env:
+        CI_TEST_SEED: "${{ inputs.seed || '' }}"
       run: |
         data=$(bundle exec rake github:generate_batches)
         echo "$data" | ruby -rjson -e 'puts JSON.pretty_generate(JSON.parse(STDIN.read))'
 
         # Extract each key and set it as a separate output
+        seed_data=$(echo "$data" | ruby -rjson -e 'puts JSON.parse(STDIN.read)["seed"]')
         batches_data=$(echo "$data" | ruby -rjson -e 'puts JSON.parse(STDIN.read)["batches"].to_json')
         misc_data=$(echo "$data" | ruby -rjson -e 'puts JSON.parse(STDIN.read)["misc"].to_json')
 
-        echo "batches=$batches_data" >> "$GITHUB_OUTPUT"
-        echo "misc=$misc_data" >> "$GITHUB_OUTPUT"
+        { echo "seed=$seed_data"; echo "batches=$batches_data"; echo "misc=$misc_data"; } >> "$GITHUB_OUTPUT"
     - name: Generate batch summary
-      run: bundle exec rake github:generate_batch_summary
       env:
         batches_json: "${{ steps.set-batches.outputs.batches }}"
+        CI_TEST_SEED: "${{ steps.set-batches.outputs.seed }}"
+      run: bundle exec rake github:generate_batch_summary
 
   # `Initialize containers` step becomes quite heavily when many services are used.
   #
@@ -88,6 +116,7 @@ jobs:
     timeout-minutes: 30
     env:
       BATCHED_TASKS: "${{ toJSON(matrix.tasks) }}"
+      CI_TEST_SEED: "${{ needs.batch.outputs.seed }}"
     strategy:
       fail-fast: false
       matrix:
@@ -152,6 +181,7 @@ jobs:
     timeout-minutes: 30
     env:
       BATCHED_TASKS: "${{ toJSON(matrix.tasks) }}"
+      CI_TEST_SEED: "${{ needs.batch.outputs.seed }}"
     strategy:
       fail-fast: false
       matrix:

tasks/github.rake

@@ -46,8 +46,9 @@ namespace :github do
       end
     end
 
-    # Random!
-    matching_tasks.shuffle!
+    # Seed
+    rng = (ENV['CI_TEST_SEED'] && ENV['CI_TEST_SEED'] != '') ? Random.new(ENV['CI_TEST_SEED'].to_i) : Random.new
+    matching_tasks.shuffle!(random: rng)
 
     batch_count = 7
     batch_count *= 2 if RUBY_PLATFORM == 'java'
@@ -61,6 +62,7 @@ namespace :github do
     end
 
     data = {
+      seed: rng.seed,
       batches: batched_matrix,
       misc: {'include' => [{'batch' => "0", 'tasks' => misc_tasks}]}
     }
@@ -77,6 +79,7 @@ namespace :github do
     summary = ENV['GITHUB_STEP_SUMMARY']
 
     File.open(summary, 'a') do |f|
+      f.puts "*__Seed__: #{ENV["CI_TEST_SEED"]}*"
       data['include'].each do |batch|
         rows = batch['tasks'].map do |t|
           "* #{t["task"]} (#{t["group"]})"
@@ -118,9 +121,11 @@ namespace :github do
   task :run_batch_tests do
     tasks = JSON.parse(ENV['BATCHED_TASKS'] || {})
 
+    rng = Random.new(ENV['CI_TEST_SEED'].to_i)
+
     tasks.each do |task|
       env = {'BUNDLE_GEMFILE' => task['gemfile']}
-      cmd = "bundle exec rake spec:#{task["task"]}"
+      cmd = "bundle exec rake spec:#{task["task"]}['--seed #{rng.rand(0xFFFF)}']"
 
       Bundler.with_unbundled_env { sh(env, cmd) }
     end