Test all in batches

Author: vjovanov

.github/workflows/test-all-metadata.yml

@@ -27,13 +27,13 @@ jobs:
       - name: "🕸️ Populate matrix"
         id: set-matrix
         run: |
-          ./gradlew generateMatrixMatchingCoordinates -Pcoordinates=all
+          ./gradlew generateMatrixBatchedCoordinates -Pbatches=16
 
   test-all-metadata:
     if: github.repository == 'oracle/graalvm-reachability-metadata'
     name: "🧪 ${{ matrix.coordinates }} (GraalVM for JDK ${{ matrix.version }} @ ${{ matrix.os }})"
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 20
+    timeout-minutes: 120
     needs: get-all-metadata
     strategy:
       fail-fast: false
@@ -56,7 +56,7 @@ jobs:
           native-image-job-reports: 'true'
       - name: "Pull allowed docker images"
         run: |
-          ./gradlew pullAllowedDockerImages --coordinates=${{ matrix.coordinates }}
+          ./gradlew pullAllowedDockerImagesMatching -Pcoordinates=${{ matrix.coordinates }}
       - name: "Disable docker networking"
         run: bash ./.github/workflows/disable-docker.sh
       - name: "🧪 Run '${{ matrix.coordinates }}' tests"

ci.json

@@ -1,6 +1,6 @@
 {
-  "generateMatrixMatchingCoordinates": {
-    "java": ["17", "latest-ea"],
+  "generateMatrixBatchedCoordinates": {
+    "java": ["17", "21", "25", "latest-ea"],
     "os": ["ubuntu-latest"]
   },
   "generateChangedCoordinatesMatrix": {
@@ -10,5 +10,9 @@
   "generateInfrastructureChangedCoordinatesMatrix": {
     "java": ["17", "21", "25", "latest-ea"],
     "os": ["ubuntu-latest"]
+  },
+  "generateMatrixMatchingCoordinates": {
+    "java": ["17", "latest-ea"],
+    "os": ["ubuntu-latest"]
   }
 }

tests/tck-build-logic/src/main/groovy/org.graalvm.internal.tck-harness.gradle

@@ -10,9 +10,12 @@ plugins {
 }
 
 import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
 import org.graalvm.internal.tck.ContributionTask
 import org.graalvm.internal.tck.DockerTask
 import org.graalvm.internal.tck.ConfigFilesChecker
+import org.graalvm.internal.tck.DockerUtils
+import org.graalvm.internal.tck.PullImagesFromFileTask
 import org.graalvm.internal.tck.ScaffoldTask
 import org.graalvm.internal.tck.GrypeTask
 import org.graalvm.internal.tck.TestedVersionUpdaterTask
@@ -42,7 +45,47 @@ def writeGithubOutput(String key, String value) {
 }
 
 String coordinateFilter = Objects.requireNonNullElse(project.findProperty("coordinates"), "")
-List<String> matchingCoordinates = tck.getMatchingCoordinates(coordinateFilter)
+
+// Support fractional batching coordinates in the form "k/n" (e.g., "1/16")
+boolean isFractionalBatch(String s) {
+    return s != null && (s ==~ /\d+\/\d+/)
+}
+
+List<Integer> parseFraction(String s) {
+    def m = s =~ /(\d+)\/(\d+)/
+    if (!m.matches()) return null
+    int k = (m[0][1] as int)
+    int n = (m[0][2] as int)
+    return [k, n]
+}
+
+List<String> computeBatchedCoordinates(List<String> allCoords, int k, int n) {
+    if (n <= 0) throw new GradleException("Invalid batches denominator: ${n}")
+    if (k < 1 || k > n) throw new GradleException("Invalid batch index: ${k}/${n}")
+    def sorted = new ArrayList<String>(allCoords)
+    java.util.Collections.sort(sorted)
+    int target = (k - 1)
+    def result = []
+    int i = 0
+    for (String c : sorted) {
+        if ((i % n) == target) {
+            result.add(c)
+        }
+        i++
+    }
+    return result
+}
+
+List<String> matchingCoordinates
+if (isFractionalBatch(coordinateFilter)) {
+    def frac = parseFraction(coordinateFilter)
+    int k = frac[0]
+    int n = frac[1]
+    List<String> all = tck.getMatchingCoordinates("all")
+    matchingCoordinates = computeBatchedCoordinates(all, k, n)
+} else {
+    matchingCoordinates = tck.getMatchingCoordinates(coordinateFilter)
+}
 
 // gradle test -Pcoordinates=<maven-coordinates>
 Provider<Task> test = tasks.register("test", DefaultTask) { task ->
@@ -78,6 +121,79 @@ tasks.named("check").configure {
     dependsOn(checkstyle)
 }
 
+final String METADATA_GROUP = "Metadata"
+
+Provider<Task> pullAllowedDockerImagesMatching = tasks.register("pullAllowedDockerImagesMatching", DefaultTask) { task ->
+    task.setDescription("Pull allowed docker images for all matching coordinates")
+    task.setGroup(METADATA_GROUP)
+    task.doFirst {
+        if (matchingCoordinates == null || matchingCoordinates.isEmpty()) {
+            throw new GradleException("No matching coordinates found for property 'coordinates'. Provide -Pcoordinates=<filter> or a fractional batch 'k/n'.")
+        }
+        println "Pulling allowed docker images for ${matchingCoordinates.size()} coordinate(s):"
+        matchingCoordinates.each { c -> println(" - ${c}") }
+    }
+}
+
+// Collect unique required images across matching coordinates and filter by allowed list
+tasks.register("collectRequiredAllowedDockerImagesMatching", DefaultTask) { task ->
+    task.setDescription("Collect unique allowed docker images required by matching coordinates into a file")
+    task.setGroup(METADATA_GROUP)
+    File out = new File(buildDir, "required-docker-images-union.txt")
+    // Declare the output so Gradle can track it
+    task.outputs.file(out)
+    task.doFirst {
+        if (matchingCoordinates == null || matchingCoordinates.isEmpty()) {
+            throw new GradleException("No matching coordinates found for property 'coordinates'. Provide -Pcoordinates=<filter> or a fractional batch 'k/n'.")
+        }
+        Set<String> unionRequired = new LinkedHashSet<>()
+        matchingCoordinates.each { c ->
+            def parts = c.split(":")
+            if (parts.length < 3) {
+                logger.warn("Skipping invalid coordinates: ${c}")
+                return
+            }
+            def group = parts[0]
+            def artifact = parts[1]
+            def version = parts[2]
+            File f = project.file("tests/src/${group}/${artifact}/${version}/required-docker-images.txt")
+            if (f.exists()) {
+                f.readLines()
+                        .collect { it?.trim() }
+                        .findAll { it && !it.startsWith("#") }
+                        .each { unionRequired.add(it) }
+            }
+        }
+
+        Set<String> allowed = DockerUtils.getAllAllowedImages()
+        def notAllowed = unionRequired.findAll { !allowed.contains(it) }
+        if (!notAllowed.isEmpty()) {
+            throw new GradleException("The following images are not in the allowed list: ${notAllowed}. " +
+                    "If you need them, add Dockerfiles under tests/tck-build-logic/src/main/resources/allowed-docker-images " +
+                    "per CONTRIBUTING.md, or adjust required-docker-images.txt files.")
+        }
+
+        out.parentFile.mkdirs()
+        def finalList = unionRequired.findAll { allowed.contains(it) }.toList()
+        out.text = finalList.join(System.lineSeparator())
+        println "Collected ${finalList.size()} required allowed image(s):"
+        finalList.each { println(" - ${it}") }
+    }
+}
+
+// Pull the collected unique images (once)
+tasks.register("pullRequiredAllowedDockerImagesMatching", PullImagesFromFileTask.class) { task ->
+    task.setDescription("Pull unique allowed docker images required by matching coordinates")
+    task.setGroup(METADATA_GROUP)
+    task.dependsOn("collectRequiredAllowedDockerImagesMatching")
+    task.imagesFile.set(new File(buildDir, "required-docker-images-union.txt"))
+}
+
+// Rewire the aggregate to depend on the unique pull task
+pullAllowedDockerImagesMatching.configure {
+    dependsOn("pullRequiredAllowedDockerImagesMatching")
+}
+
 // Here we want to configure all test and checkstyle tasks for all filtered subprojects
 for (String coordinates in matchingCoordinates) {
     String testTaskName = generateTaskName("test", coordinates)
@@ -127,6 +243,13 @@ for (String coordinates in matchingCoordinates) {
     javaTest.configure {
         dependsOn(javaTestTaskName)
     }
+
+    String pullDockerTaskName = generateTaskName("pullAllowedDockerImages", coordinates)
+    if ((!tasks.getNames().contains(pullDockerTaskName))) {
+        tasks.register(pullDockerTaskName, DockerTask.class) { t ->
+            t.setCoordinates(coordinates)
+        }
+    }
 }
 
 // gradle diff -PbaseCommit=<base-commit> -PnewCommit=<new-commit>
@@ -157,15 +280,23 @@ if (project.hasProperty("baseCommit")) {
     }
 }
 
-def matrixDefault = [
-        "version": [
-            "17",
-            "latest-ea"
-        ],
-        "os"     : ["ubuntu-latest"]
-]
+Map<String, Object> loadCi() {
+    File f = project.file("ci.json")
+    if (!f.exists()) {
+        throw new GradleException("Missing ci.json in repo root")
+    }
+    new JsonSlurper().parse(f) as Map<String, Object>
+}
 
-final String METADATA_GROUP = "Metadata"
+Map<String, List<String>> matrixDefaultsFor(String section) {
+    def ci = loadCi()
+    def sec = ci[section]
+    if (sec == null) throw new GradleException("Missing '${section}' in ci.json")
+    [
+        "version": (sec["java"] as List<String>),
+        "os": (sec["os"] as List<String>)
+    ]
+}
 
 // gradle generateMatrixMatchingCoordinates -Pcoordinates=<maven-coordinates>
 Provider<Task> generateMatrixMatchingCoordinates = tasks.register("generateMatrixMatchingCoordinates", DefaultTask) { task ->
@@ -175,7 +306,25 @@ Provider<Task> generateMatrixMatchingCoordinates = tasks.register("generateMatri
         def matrix = [
                 "coordinates": matchingCoordinates
         ]
-        matrix.putAll(matrixDefault)
+        matrix.putAll(matrixDefaultsFor("generateMatrixMatchingCoordinates"))
+        writeGithubOutput("matrix", JsonOutput.toJson(matrix))
+    }
+}
+
+ // gradle generateMatrixBatchedCoordinates [-Pbatches=<n>]
+Provider<Task> generateMatrixBatchedCoordinates = tasks.register("generateMatrixBatchedCoordinates", DefaultTask) { task ->
+    task.setDescription("Returns matrix definition populated with fractional batch coordinates (k/n)")
+    task.setGroup(METADATA_GROUP)
+    task.doFirst {
+        int batches = project.hasProperty("batches") ? Integer.parseInt(project.findProperty("batches").toString()) : 16
+        if (batches <= 0) {
+            throw new GradleException("Invalid 'batches' value: ${batches}")
+        }
+        List<String> coords = (1..batches).collect { i -> "${i}/${batches}" }
+        def matrix = [
+                "coordinates": coords
+        ]
+        matrix.putAll(matrixDefaultsFor("generateMatrixBatchedCoordinates"))
         writeGithubOutput("matrix", JsonOutput.toJson(matrix))
     }
 }
@@ -193,13 +342,11 @@ Provider<Task> generateChangedCoordinatesMatrix = tasks.register("generateChange
         def matrix = [
                 "coordinates": noneFound ? [] : diffCoordinates
         ]
-        matrix.putAll(matrixDefault)
+        matrix.putAll(matrixDefaultsFor("generateChangedCoordinatesMatrix"))
         if (noneFound) {
             println "No changed coordinates were found!"
         }
 
-        matrix.version.add("21")
-        matrix.version.add("25")
 
         writeGithubOutput("matrix", JsonOutput.toJson(matrix))
         writeGithubOutput("none-found", noneFound.toString())
@@ -215,7 +362,7 @@ Provider<Task> generateInfrastructureChangedCoordinatesMatrix = tasks.register("
         boolean infraChanged = true
         // Pre-selected modules; keep stable for infra verification
         List<String> preselectedModules = [
-                'org.bouncycastle:bcpkix-jdk15on',
+                'ch.qos.logback:logback-classic',
                 'org.bouncycastle:bcpkix-jdk18on',
                 'io.netty:netty-common',
                 'io.grpc:grpc-core',
@@ -243,10 +390,8 @@ Provider<Task> generateInfrastructureChangedCoordinatesMatrix = tasks.register("
         def matrix = [
             "coordinates": selected
         ]
-        matrix.putAll(matrixDefault)
+        matrix.putAll(matrixDefaultsFor("generateInfrastructureChangedCoordinatesMatrix"))
 
-        matrix.version.add("21")
-        matrix.version.add("25")
 
         writeGithubOutput("matrix", JsonOutput.toJson(matrix))
         writeGithubOutput("none-found", "false")

tests/tck-build-logic/src/main/java/org/graalvm/internal/tck/PullImagesFromFileTask.java

@@ -0,0 +1,53 @@
+package org.graalvm.internal.tck;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.tasks.InputFile;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.process.ExecOperations;
+import org.gradle.api.GradleException;
+
+import javax.inject.Inject;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+/**
+ * Pulls docker images listed in a text file (one image per line).
+ * Fails the task immediately if any image pull fails to ensure batch execution fails on single error.
+ */
+public abstract class PullImagesFromFileTask extends DefaultTask {
+
+    @InputFile
+    public abstract RegularFileProperty getImagesFile();
+
+    @Inject
+    protected abstract ExecOperations getExecOperations();
+
+    @TaskAction
+    public void run() throws IOException {
+        File inFile = getImagesFile().get().getAsFile();
+        if (!inFile.exists()) {
+            getLogger().lifecycle("No required docker images collected: {}", inFile.getAbsolutePath());
+            return;
+        }
+
+        List<String> images = Files.readAllLines(inFile.toPath()).stream()
+                .map(String::trim)
+                .filter(s -> !s.isEmpty() && !s.startsWith("#"))
+                .toList();
+
+        for (String image : images) {
+            getLogger().lifecycle("Pulling image {}...", image);
+            try {
+                getExecOperations().exec(spec -> {
+                    spec.setExecutable("docker");
+                    spec.args("pull", image);
+                });
+            } catch (Exception e) {
+                throw new GradleException("Failed to pull image " + image + ": " + e.getMessage(), e);
+            }
+        }
+    }
+}