add multi-arch image build for linux/amd64 and linux/arm64

.build/.versions.yml

@@ -63,5 +63,5 @@ compatibility-matrix:
 build-matrix:
   python_version: ['3.10', '3.11', '3.12']
   spark_version: [3.3.4, 3.4.4, 3.5.6, 4.0.1]
-  java_version: [11, 17]
+  java_version: [17]
   scala_version: [2.13]

.build/python/src/okdp/extension/tagging/apply_tags.py

@@ -19,18 +19,19 @@
 
 class Tagging:
 
-    def __init__(self, short_image_name: str, registry: str, owner: str,):
-        self.short_image_name, self.tag = short_image_name.split(":")
+    def __init__(self, image_name: str, registry: str, owner: str, platform: str,):
+        self.image_name, self.tag = image_name.split(":")
         self.registry = registry
         self.owner = owner
+        self.platform = platform
 
     def apply_tags(self) -> None:
         """
-        Tags <registry>/<owner>/<short_image_name>:tag with the tags reported by all taggers for this image
+        Tags <registry>/<owner>/<image_name>:tag with the tags reported by all taggers for this image
         """
-        LOGGER.info(f"Tagging image: {self.short_image_name}")
+        LOGGER.info(f"Tagging image: {self.image_name}")
 
-        image = f"{self.registry}/{self.owner}/{self.short_image_name}:{self.tag}"
+        image = f"{self.registry}/{self.owner}/{self.image_name}:{self.tag}-{self.platform}"
 
         tags = self.generate_tags()
 
@@ -40,13 +41,13 @@ def apply_tags(self) -> None:
 
     def generate_tags(self) -> list[str]:
         """
-        Generate tags for the image <registry>/<owner>/<short_image_name>:latest
+        Generate tags for the image <registry>/<owner>/<image_name>:latest
         """
-        LOGGER.info(f"Tagging image: {self.short_image_name}")
-        taggers, _ = get_taggers_and_manifests(self.short_image_name)
+        LOGGER.info(f"Tagging image: {self.image_name}")
+        taggers, _ = get_taggers_and_manifests(self.image_name)
 
-        image = f"{self.registry}/{self.owner}/{self.short_image_name}:{ self.tag }"
-        tags = [f"{self.registry}/{self.owner}/{self.short_image_name}:{ self.tag }"]
+        image = f"{self.registry}/{self.owner}/{self.image_name}:{self.tag}-{self.platform}"
+        tags = [f"{self.registry}/{self.owner}/{self.image_name}:{self.tag}-{self.platform}"]
         with DockerRunner(image) as container:
             for tagger in taggers:
                 tagger_name = tagger.__class__.__name__
@@ -55,7 +56,7 @@ def generate_tags(self) -> list[str]:
                     f"Calculated tag, tagger_name: {tagger_name} tag_value: {tag_value}"
                 )
                 tags.append(
-                    f"{self.registry}/{self.owner}/{self.short_image_name}:{tag_value}"
+                    f"{self.registry}/{self.owner}/{self.image_name}:{tag_value}-{self.platform}"
                 )
 
         return tags
@@ -66,9 +67,9 @@ def generate_tags(self) -> list[str]:
 
     arg_parser = argparse.ArgumentParser()
     arg_parser.add_argument(
-        "--short-image-name",
+        "--image-name",
         required=True,
-        help="Short image name",
+        help="Image name:tag",
     )
     arg_parser.add_argument(
         "--registry",
@@ -82,8 +83,15 @@ def generate_tags(self) -> list[str]:
         required=True,
         help="Owner of the image",
     )
+    arg_parser.add_argument(
+        "--platform",
+        required=True,
+        type=str,
+        choices=["amd64", "arm64"],
+        help="Platform",
+    )
     args = arg_parser.parse_args()
 
-    tagging = Tagging(args.short_image_name, args.registry, args.owner)
+    tagging = Tagging(args.image_name, args.registry, args.owner, args.platform)
 
     tagging.apply_tags()

.build/python/tests/okdp/extension/tagging/test_tagging.py

@@ -45,9 +45,9 @@ def test_long_tagger(mock_container, mock_exec_cmd):
 def test_generate_tags(mock_get_taggers_and_manifests, mock_container, mock_exec_cmd):
     """Test generate_tags output."""
     expected_tags = [
-        "ghcr.io/owner/pyspark-notebook:latest",
-        "ghcr.io/owner/pyspark-notebook:spark-3.4.1-python-3.12-java-17-scala-2.13",
-        "ghcr.io/owner/pyspark-notebook:spark-3.4.1-python-3.12.3-java-17.0.2-scala-2.13.12-hub-4.0.1-lab-4.0.5",
+        "ghcr.io/owner/pyspark-notebook:2025-09-22-amd64",
+        "ghcr.io/owner/pyspark-notebook:spark-3.4.1-python-3.12-java-17-scala-2.13-amd64",
+        "ghcr.io/owner/pyspark-notebook:spark-3.4.1-python-3.12.3-java-17.0.2-scala-2.13.12-hub-4.0.1-lab-4.0.5-amd64",
     ]
 
     mock_get_taggers_and_manifests.return_value = (
@@ -58,10 +58,10 @@ def test_generate_tags(mock_get_taggers_and_manifests, mock_container, mock_exec
         [spark_info_manifest],
     )
 
-    t = Tagging("pyspark-notebook:latest", "ghcr.io", "owner")
+    t = Tagging("pyspark-notebook:2025-09-22", "ghcr.io", "owner", "amd64")
     tags = t.generate_tags()
 
-    assert tags[0] == "ghcr.io/owner/pyspark-notebook:latest"
+    assert tags[0] == "ghcr.io/owner/pyspark-notebook:2025-09-22-amd64"
     for tag in expected_tags:
       assert tag in tags
     assert len(tags) == len(expected_tags)
@@ -75,7 +75,7 @@ def tag_value(self, container): return "tag1"
     mock_apply_tags_docker_runner.return_value.__enter__.return_value = mock_container
     mock_apply_tags_docker_runner.return_value.__exit__.return_value = None
 
-    t = Tagging("pyspark-notebook:latest", "ghcr.io", "owner")
+    t = Tagging("pyspark-notebook:2025-09-22", "ghcr.io", "owner", "arm64")
     t.apply_tags()
 
     # Check docker["tag", ...] was called with the right args

.github/actions/docker-tags-merge/action.yml

@@ -0,0 +1,63 @@
+name: 'Multi-arch Image Tagger'
+description: 'Applies tags and prepares amd64 and arm64 images for creating multi-arch manifests'
+inputs:
+  image:
+    description: Image name with tag (including registry/repo, e.g. docker.io/myuser/myimage:mytag)
+    required: true
+  registry:
+    description: The registry used to push images into
+    required: true
+  platform_amd64:
+    description: Target image platform for AMD64 architecture
+    required: true
+  platform_arm64:
+    description: Target image platform for AMD64 architecture
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Prepare image push 📦
+      run: |
+        # The short image name (without tag) is necessary to push to the registry
+        echo "SHORT_IMAGE_NAME=${{ inputs.image }}" | awk -F: '{print $1}' >> $GITHUB_ENV
+      shell: bash
+
+    # Pull the images locally for the two platforms: amd64 and arm64
+    - name: Pull ${{ inputs.image }} image for ${{ inputs.platform_amd64 }} platform ⬇️
+      run: |
+        docker pull ${{ inputs.registry }}/$OWNER/${{ inputs.image }}-${{ inputs.platform_amd64 }}
+      shell: bash
+
+    - name: Pull ${{ inputs.image }} image for ${{ inputs.platform_arm64 }} platform ⬇️
+      run: |
+        docker pull ${{ inputs.registry }}/$OWNER/${{ inputs.image }}-${{ inputs.platform_arm64 }}
+      shell: bash
+
+    - name: Apply tags to ${{ inputs.image }} image and platform ${{ inputs.platform_amd64 }} 🏷
+      run: |
+        python3 -m okdp.extension.tagging.apply_tags --image-name ${{ inputs.image }} \
+                --registry ${{ inputs.registry }} \
+                --owner $OWNER \
+                --platform ${{ inputs.platform_amd64 }}
+      shell: bash
+
+    - name: Fetch all tags for ${{ inputs.image }} image and platform ${{ inputs.platform_amd64 }} 🏷
+      run: |
+        tags=$(docker images ${{ inputs.registry }}/$OWNER/$SHORT_IMAGE_NAME \
+          --format "{{.Repository}}:{{.Tag}}" \
+          | grep -v -- "-${{ inputs.platform_arm64 }}$" \
+          | sed "s/-${{ inputs.platform_amd64 }}$//" \
+          | tr '\n' ' ')
+        echo "IMAGE_TAGS=$tags" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Apply tags to ${{ inputs.image }} image and platform ${{ inputs.platform_arm64 }} 🏷
+      run: |
+        for tag in $IMAGE_TAGS
+        do
+          docker tag ${{ inputs.registry }}/$OWNER/${{ inputs.image }}-${{ inputs.platform_arm64 }} ${tag}-${{ inputs.platform_arm64 }}
+        done
+      shell: bash
+

.github/workflows/build-base-images-template.yml

@@ -25,15 +25,11 @@ on:
         required: false
         type: string
         default: ""
-      runs-on:
-        description: GitHub Actions Runner image
-        required: true
-        type: string
 
 jobs:
 
   docker-stacks-foundation:
-    uses: ./.github/workflows/build-image-template.yml
+    uses: ./.github/workflows/build-multi-arch-image-template.yml
     with:
       parent-image: ""
       image: docker-stacks-foundation:${{ inputs.python_dev_tag }}
@@ -42,42 +38,38 @@ jobs:
       registry: ${{ inputs.registry }}
       publish_to_registry: ${{ inputs.publish_to_registry }}
       git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
-      runs-on: ubuntu-24.04
     secrets: inherit
 
   base-notebook:
-    uses: ./.github/workflows/build-image-template.yml
+    uses: ./.github/workflows/build-multi-arch-image-template.yml
     with:
       parent-image: docker-stacks-foundation:${{ inputs.python_dev_tag }}
       image: base-notebook:${{ inputs.python_dev_tag }}
       registry: ${{ inputs.registry }}
       publish_to_registry: ${{ inputs.publish_to_registry }}
       git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
-      runs-on: ubuntu-24.04
     secrets: inherit
     needs: [docker-stacks-foundation]
 
   minimal-notebook:
-    uses: ./.github/workflows/build-image-template.yml
+    uses: ./.github/workflows/build-multi-arch-image-template.yml
     with:
       parent-image: base-notebook:${{ inputs.python_dev_tag }}
       image: minimal-notebook:${{ inputs.python_dev_tag }}
       registry: ${{ inputs.registry }}
       publish_to_registry: ${{ inputs.publish_to_registry }}
       git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
-      runs-on: ubuntu-24.04
     secrets: inherit
     needs: [base-notebook]
 
   scipy-notebook:
-    uses: ./.github/workflows/build-image-template.yml
+    uses: ./.github/workflows/build-multi-arch-image-template.yml
     with:
       parent-image: minimal-notebook:${{ inputs.python_dev_tag }}
       image: scipy-notebook:${{ inputs.python_dev_tag }}
       registry: ${{ inputs.registry }}
       publish_to_registry: ${{ inputs.publish_to_registry }}
       git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
-      runs-on: ubuntu-24.04
     secrets: inherit
     needs: [minimal-notebook]
 

.github/workflows/build-datascience-images-template.yml

@@ -21,65 +21,56 @@ on:
         required: false
         type: string
         default: ""
-      runs-on:
-        description: GitHub Actions Runner image
-        required: true
-        type: string
 
 jobs:
   r:
-    uses: ./.github/workflows/build-image-template.yml
+    uses: ./.github/workflows/build-multi-arch-image-template.yml
     with:
       parent-image: minimal-notebook:${{ inputs.python_dev_tag }}
       image: r-notebook:${{ inputs.python_dev_tag }}
       registry: ${{ inputs.registry }}
       publish_to_registry: ${{ inputs.publish_to_registry }}
       git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
-      runs-on: ${{ inputs.runs-on }}
     secrets: inherit
 
   datascience:
-    uses: ./.github/workflows/build-image-template.yml
+    uses: ./.github/workflows/build-multi-arch-image-template.yml
     with:
       parent-image: scipy-notebook:${{ inputs.python_dev_tag }}
       image: datascience-notebook:${{ inputs.python_dev_tag }}
       registry: ${{ inputs.registry }}
       publish_to_registry: ${{ inputs.publish_to_registry }}
       git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
-      runs-on: ${{ inputs.runs-on }}
     secrets: inherit
 
   # julia:
-  #   uses: ./.github/workflows/build-image-template.yml
+  #   uses: ./.github/workflows/build-multi-arch-image-template.yml
   #   with:
   #     parent-image: minimal-notebook:${{ inputs.python_dev_tag }}
   #     image: julia-notebook
   #     registry: ${{ inputs.registry }}
   #     publish_to_registry: ${{ inputs.publish_to_registry }}
   #     git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
-  #     runs-on: ${{ inputs.runs-on }}
   #   secrets: inherit
 
   # tensorflow:
-  #   uses: ./.github/workflows/build-image-template.yml
+  #   uses: ./.github/workflows/build-multi-arch-image-template.yml
   #   with:
   #     parent-image: scipy-notebook:${{ inputs.python_dev_tag }}
   #     image: tensorflow-notebook
   #     registry: ${{ inputs.registry }}
   #     publish_to_registry: ${{ inputs.publish_to_registry }}
   #     git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
-  #     runs-on: ${{ inputs.runs-on }}
   #   secrets: inherit
 
   # pytorch:
-  #   uses: ./.github/workflows/build-image-template.yml
+  #   uses: ./.github/workflows/build-multi-arch-image-template.yml
   #   with:
   #     parent-image: scipy-notebook:${{ inputs.python_dev_tag }}
   #     image: pytorch-notebook
   #     registry: ${{ inputs.registry }}
   #     publish_to_registry: ${{ inputs.publish_to_registry }}
   #     git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
-  #     runs-on: ${{ inputs.runs-on }}
   #   secrets: inherit