From e13fb38251f02bc31a369a120cf7c1abdc70f98a Mon Sep 17 00:00:00 2001 From: Victor Pellan Date: Tue, 7 Oct 2025 13:19:51 +0200 Subject: [PATCH 1/3] Add seed to CI unit tests --- .github/workflows/_unit_test.yml | 13 ++++++++++++- tasks/github.rake | 11 ++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_unit_test.yml b/.github/workflows/_unit_test.yml index 1ca5c0f2558..59f27df6410 100644 --- a/.github/workflows/_unit_test.yml +++ b/.github/workflows/_unit_test.yml @@ -13,6 +13,9 @@ on: # yamllint disable-line rule:truthy alias: required: true type: string + seed: + required: false + type: number outputs: lockfile: description: "The lockfile artifact" @@ -26,6 +29,7 @@ jobs: runs-on: ubuntu-24.04 name: batch outputs: + seed: "${{ steps.set-batches.outputs.seed }}" batches: "${{ steps.set-batches.outputs.batches }}" misc: "${{ steps.set-batches.outputs.misc }}" cache-key: "${{ steps.bundle-cache.outputs.cache-key }}" @@ -43,20 +47,25 @@ jobs: - id: set-batches name: Distribute tasks into batches + env: + CI_TEST_SEED: "${{ inputs.seed || '' }}" run: | data=$(bundle exec rake github:generate_batches) echo "$data" | ruby -rjson -e 'puts JSON.pretty_generate(JSON.parse(STDIN.read))' # Extract each key and set it as a separate output + seed_data=$(echo "$data" | ruby -rjson -e 'puts JSON.parse(STDIN.read)["seed"]') batches_data=$(echo "$data" | ruby -rjson -e 'puts JSON.parse(STDIN.read)["batches"].to_json') misc_data=$(echo "$data" | ruby -rjson -e 'puts JSON.parse(STDIN.read)["misc"].to_json') + echo "seed=$seed_data" >> "$GITHUB_OUTPUT" echo "batches=$batches_data" >> "$GITHUB_OUTPUT" echo "misc=$misc_data" >> "$GITHUB_OUTPUT" - name: Generate batch summary - run: bundle exec rake github:generate_batch_summary env: batches_json: "${{ steps.set-batches.outputs.batches }}" + CI_TEST_SEED: "${{ steps.set-batches.outputs.seed }}" + run: bundle exec rake github:generate_batch_summary # `Initialize containers` step becomes quite heavily when many services are used. # @@ -88,6 +97,7 @@ jobs: timeout-minutes: 30 env: BATCHED_TASKS: "${{ toJSON(matrix.tasks) }}" + CI_TEST_SEED: "${{ needs.batch.outputs.seed }}" strategy: fail-fast: false matrix: @@ -152,6 +162,7 @@ jobs: timeout-minutes: 30 env: BATCHED_TASKS: "${{ toJSON(matrix.tasks) }}" + CI_TEST_SEED: "${{ needs.batch.outputs.seed }}" strategy: fail-fast: false matrix: diff --git a/tasks/github.rake b/tasks/github.rake index 461ea49633d..d173d45e854 100644 --- a/tasks/github.rake +++ b/tasks/github.rake @@ -46,8 +46,9 @@ namespace :github do end end - # Random! - matching_tasks.shuffle! + # Seed + rng = ENV['CI_TEST_SEED'] && ENV['CI_TEST_SEED'] != '' ? Random.new(ENV['CI_TEST_SEED'].to_i) : Random.new + matching_tasks.shuffle!(random: rng) batch_count = 7 batch_count *= 2 if RUBY_PLATFORM == 'java' @@ -61,6 +62,7 @@ namespace :github do end data = { + seed: rng.seed, batches: batched_matrix, misc: {'include' => [{'batch' => "0", 'tasks' => misc_tasks}]} } @@ -83,6 +85,7 @@ namespace :github do end f.puts <<~SUMMARY + *__Seed__: #{ENV['CI_TEST_SEED']}*
Batch #{batch["batch"]} (#{batch["tasks"].length} tasks) @@ -118,9 +121,11 @@ namespace :github do task :run_batch_tests do tasks = JSON.parse(ENV['BATCHED_TASKS'] || {}) + rng = ENV['CI_TEST_SEED'] && ENV['CI_TEST_SEED'] != '' ? Random.new(ENV['CI_TEST_SEED'].to_i) : Random.new + tasks.each do |task| env = {'BUNDLE_GEMFILE' => task['gemfile']} - cmd = "bundle exec rake spec:#{task["task"]}" + cmd = "bundle exec rake spec:#{task["task"]}['--seed #{rng.rand(0xFFFF)}']" Bundler.with_unbundled_env { sh(env, cmd) } end From a74462d0a0b60f6780a260b9ccea83a8405acb9c Mon Sep 17 00:00:00 2001 From: Victor Pellan Date: Tue, 7 Oct 2025 13:48:26 +0200 Subject: [PATCH 2/3] Fix Summary and add workflow dispatch to Unit test --- .github/workflows/_unit_test.yml | 22 +++++++++++++++++++++- tasks/github.rake | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_unit_test.yml b/.github/workflows/_unit_test.yml index 59f27df6410..870505c0dd2 100644 --- a/.github/workflows/_unit_test.yml +++ b/.github/workflows/_unit_test.yml @@ -1,4 +1,4 @@ -name: Unit Test Template +name: Unit Tests (Ruby version specific) on: # yamllint disable-line rule:truthy workflow_call: @@ -24,6 +24,26 @@ on: # yamllint disable-line rule:truthy description: "The cache key for bundle" value: ${{ jobs.batch.outputs.cache-key }} + workflow_dispatch: + inputs: + # TODO: Provides concurrency control for each ruby version + engine: + description: "`ruby` or `jruby`" + required: true + type: string + version: + description: "The version of the engine (2.x or 3.x for Ruby, 9.x or 10.x for JRuby)" + required: true + type: string + alias: + description: "The alias of the engine (e.g. `ruby-34` for Ruby 3.4)" + required: true + type: string + seed: + description: "The seed to reproduce a CI run" + required: false + type: number + jobs: batch: runs-on: ubuntu-24.04 diff --git a/tasks/github.rake b/tasks/github.rake index d173d45e854..72e49a27332 100644 --- a/tasks/github.rake +++ b/tasks/github.rake @@ -79,13 +79,13 @@ namespace :github do summary = ENV['GITHUB_STEP_SUMMARY'] File.open(summary, 'a') do |f| + f.puts "*__Seed__: #{ENV['CI_TEST_SEED']}*" data['include'].each do |batch| rows = batch['tasks'].map do |t| "* #{t["task"]} (#{t["group"]})" end f.puts <<~SUMMARY - *__Seed__: #{ENV['CI_TEST_SEED']}*
Batch #{batch["batch"]} (#{batch["tasks"].length} tasks) From 89b7193330c6565804ad5acd1b9acf9f846d0fe0 Mon Sep 17 00:00:00 2001 From: Victor Pellan Date: Tue, 7 Oct 2025 14:11:40 +0200 Subject: [PATCH 3/3] fix lint --- .github/workflows/_unit_test.yml | 7 +++---- tasks/github.rake | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/_unit_test.yml b/.github/workflows/_unit_test.yml index 870505c0dd2..284775d61a3 100644 --- a/.github/workflows/_unit_test.yml +++ b/.github/workflows/_unit_test.yml @@ -26,7 +26,6 @@ on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: - # TODO: Provides concurrency control for each ruby version engine: description: "`ruby` or `jruby`" required: true @@ -44,6 +43,8 @@ on: # yamllint disable-line rule:truthy required: false type: number +permissions: {} + jobs: batch: runs-on: ubuntu-24.04 @@ -78,9 +79,7 @@ jobs: batches_data=$(echo "$data" | ruby -rjson -e 'puts JSON.parse(STDIN.read)["batches"].to_json') misc_data=$(echo "$data" | ruby -rjson -e 'puts JSON.parse(STDIN.read)["misc"].to_json') - echo "seed=$seed_data" >> "$GITHUB_OUTPUT" - echo "batches=$batches_data" >> "$GITHUB_OUTPUT" - echo "misc=$misc_data" >> "$GITHUB_OUTPUT" + { echo "seed=$seed_data"; echo "batches=$batches_data"; echo "misc=$misc_data"; } >> "$GITHUB_OUTPUT" - name: Generate batch summary env: batches_json: "${{ steps.set-batches.outputs.batches }}" diff --git a/tasks/github.rake b/tasks/github.rake index 72e49a27332..9690cf31572 100644 --- a/tasks/github.rake +++ b/tasks/github.rake @@ -47,7 +47,7 @@ namespace :github do end # Seed - rng = ENV['CI_TEST_SEED'] && ENV['CI_TEST_SEED'] != '' ? Random.new(ENV['CI_TEST_SEED'].to_i) : Random.new + rng = (ENV['CI_TEST_SEED'] && ENV['CI_TEST_SEED'] != '') ? Random.new(ENV['CI_TEST_SEED'].to_i) : Random.new matching_tasks.shuffle!(random: rng) batch_count = 7 @@ -79,7 +79,7 @@ namespace :github do summary = ENV['GITHUB_STEP_SUMMARY'] File.open(summary, 'a') do |f| - f.puts "*__Seed__: #{ENV['CI_TEST_SEED']}*" + f.puts "*__Seed__: #{ENV["CI_TEST_SEED"]}*" data['include'].each do |batch| rows = batch['tasks'].map do |t| "* #{t["task"]} (#{t["group"]})" @@ -121,7 +121,7 @@ namespace :github do task :run_batch_tests do tasks = JSON.parse(ENV['BATCHED_TASKS'] || {}) - rng = ENV['CI_TEST_SEED'] && ENV['CI_TEST_SEED'] != '' ? Random.new(ENV['CI_TEST_SEED'].to_i) : Random.new + rng = Random.new(ENV['CI_TEST_SEED'].to_i) tasks.each do |task| env = {'BUNDLE_GEMFILE' => task['gemfile']}