SOFTWARE-6168: Feature/restore gha changes

.github/workflows/build-containers.yml

@@ -28,77 +28,29 @@ jobs:
         with:
           fetch-depth: 0
 
-      - id: image-list
-        run: |
-          ORG_DIR=opensciencegrid
-          # Get the list of files changed based on the type of event
-          # kicking off the GHA:
-          # 1. For the main branch, diff the previous state of main vs
-          #    the current commit
-          # 2. For other branches (i.e., on someone's fork), diff main
-          #    vs the current commit
-          # 3. For PRs, diff the base ref vs the current commit
-          # 4. For everything else (e.g., dispatches), build all images
-          if [[ $GITHUB_EVENT_NAME == 'pull_request' ]] ||
-             [[ $GITHUB_EVENT_NAME == 'push' ]]; then
-               if [[ $GITHUB_EVENT_NAME == 'pull_request' ]]; then
-                   BASE=$(git merge-base origin/$GITHUB_BASE_REF HEAD)
-               elif [[ $GITHUB_REF == 'refs/heads/main' ]]; then
-                   BASE=${{github.event.before}}
-               else
-                   BASE=origin/main
-               fi
-               # List image root dirs where files have changed and the
-               # root dir exists. Example value:
-               # "opensciencegrid/vo-frontend opensciencegrid/ospool-cm"
-               images=$(git diff --name-only \
-                                 "$BASE" \
-                                 "$GITHUB_SHA" |
-                        egrep "^$ORG_DIR/" |
-                        cut -d/ -f -2 |
-                        sort |
-                        uniq |
-                        xargs -I {} find . -type d \
-                                           -wholename ./{} \
-                                           -printf "%P\n")
-          else
-               # List all image root dirs. Example value:
-               # "opensciencegrid/vo-frontend opensciencegrid/ospool-cm"
-               images=$(find $ORG_DIR -mindepth 1 \
-                                      -maxdepth 1 \
-                                      -type d \
-                                      -printf "$ORG_DIR/%P\n")
-          fi
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+          cache: 'pip' # caching pip dependencies
+      - run: pip install -r requirements.txt
 
-            image_json=$(echo -n "${images:-dummy}" | jq -Rcs '.|split("\n") | map(select(. != ""))')
-            echo "$image_json" > image_list.json
-            echo "images=$(echo $images | tr '\n' ' ')" >> $GITHUB_OUTPUT
-            echo "image_list=$image_json" >> $GITHUB_OUTPUT
+      - id: image-list
+        run: python3 scripts/detect-changed-images.py --before ${{github.event.before}}
 
       - name: Display image list
         run: cat image_list.json
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.x'
-
       - name: Set matrix output
         id: set-matrix
-        run: |
-          # Run Python script and capture JSON output
-          matrix_json=$(python scripts/build-job-matrix.py ${{ steps.image-list.outputs.images }} | tail -n +2)
-          # Use jq to extract the 'include' part of the JSON
-          matrix=$(echo "$matrix_json" | jq -c '.include')
-          echo "::set-output name=matrix::$matrix"
-
+        run: python3 scripts/build-job-matrix.py ${{ steps.image-list.outputs.images }}
 
       - name: Verify matrix content
         run: |
           echo "Content of matrix:"
           echo "${{ steps.set-matrix.outputs.matrix }}"
 
-  build:
+  build-push-image:
     runs-on: ubuntu-latest
     needs: build-image-list
     if: ${{ needs.build-image-list.outputs.image_list != '["dummy"]' }}
@@ -111,105 +63,87 @@ jobs:
         include: ${{ fromJson(needs.build-image-list.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v3
-        # Example of a matrix configuration string:
-        # {el9-23-development-True-False}
       - name: Print raw matrix configuration
         run: |
-          echo "Raw matrix configuration: ${{ matrix.config }}"
-
-      - name: Set environment variables
-        id: set-env-vars
-        run: |
-          CONFIG="${{ matrix.config }}"
-          BASE_OS=$(echo $CONFIG | awk -F'-' '{print $1}')
-          OSG_SERIES=$(echo $CONFIG | awk -F'-' '{print $2}')
-          BASE_REPO=$(echo $CONFIG | awk -F'-' '{print $3}')
-          CONTEXT="opensciencegrid/${{ matrix.name }}"
-          echo "BASE_OS=${BASE_OS}" >> $GITHUB_ENV
-          echo "OSG_SERIES=${OSG_SERIES}" >> $GITHUB_ENV
-          echo "BASE_REPO=${BASE_REPO}" >> $GITHUB_ENV
-          echo "CONTEXT=${CONTEXT}" >> $GITHUB_ENV
+          echo "Raw matrix configuration: ${{ matrix }}"
 
-      - name: Validate Environment Variables
+      - name: Validate Build Matrix
         run: |
           echo "Validating environment variables:"
-          if [ -z "$BASE_OS" ] || [ -z "$OSG_SERIES" ] || [ -z "$BASE_REPO" ] || [ -z "$CONTEXT" ]; then
-            echo "Error: One or more environment variables are not set."
+          if [ -z "${{ matrix.base_os }}" ] || [ -z "${{ matrix.osg_series }}" ] || [ -z "${{ matrix.base_repo }}" ] || [ -z "${{ matrix.context }}" ]; then
+            echo "Error: One or more build matrix variables are not set."
             exit 1
           else
-            echo "All required environment variables are set."
+            echo "All required build matrix variables are set."
           fi
 
-      - name: Build Image
-        continue-on-error: ${{ matrix.yum_repo == 'development' }}
-        uses: opensciencegrid/[email protected]
-        with:
-          osg_series: ${{ env.OSG_SERIES }}
-          osg_repo: ${{ env.BASE_REPO }}
-          context: ${{ env.CONTEXT }}
-          base_os: ${{ env.BASE_OS }}
-
-  push:
-    runs-on: ubuntu-latest
-    if: >-
-      github.ref == 'refs/heads/main' &&
-      github.event_name != 'pull_request' &&
-      github.repository_owner == 'opensciencegrid' &&
-      needs.build-image-list.outputs.image_list != '["dummy"]'
-    needs: [make-date-tag, build-image-list, build]
-    strategy:
-      fail-fast: false
-      matrix:
-        include: ${{ fromJson(needs.build-image-list.outputs.matrix) }}
-    steps:
-      - uses: actions/checkout@v3
-        # Example of a matrix configuration string:
-        # {el9-23-development-True-False}
-      - name: Print raw matrix configuration
-        run: |
-          echo "Raw matrix configuration: ${{ matrix.config }}"
-
-      - name: Set environment variables
-        id: set-env-vars
+      - name: Set Image Name
         run: |
-          CONFIG="${{ matrix.config }}"
-          BASE_OS=$(echo $CONFIG | awk -F'-' '{print $1}')
-          OSG_SERIES=$(echo $CONFIG | awk -F'-' '{print $2}')
-          BASE_REPO=$(echo $CONFIG | awk -F'-' '{print $3}')
-          CONTEXT="opensciencegrid/${{ matrix.name }}"
-          echo "BASE_OS=${BASE_OS}" >> $GITHUB_ENV
-          echo "OSG_SERIES=${OSG_SERIES}" >> $GITHUB_ENV
-          echo "BASE_REPO=${BASE_REPO}" >> $GITHUB_ENV
-          echo "CONTEXT=${CONTEXT}" >> $GITHUB_ENV
+          BASE_OS=${{ matrix.base_os }}
+          OSG_SERIES=${{ matrix.osg_series }}
+          BASE_REPO=${{ matrix.base_repo }}
+          ORGANIZATION=$(echo ${{ matrix.context }} | cut -d'/' -f1)
 
-      - name: Validate Environment Variables
-        run: |
-          echo "Validating environment variables:"
-          if [ -z "$BASE_OS" ] || [ -z "$OSG_SERIES" ] || [ -z "$BASE_REPO" ] || [ -z "$CONTEXT" ]; then
-            echo "Error: One or more environment variables are not set."
-            exit 1
+          if [ -n "${{ matrix.tag_override }}" ] ; then
+            echo "IMAGE_NAME=${ORGANIZATION}/${{ matrix.name }}:${{ matrix.tag_override }}" >> $GITHUB_ENV
           else
-            echo "All required environment variables are set."
+            echo "IMAGE_NAME=${ORGANIZATION}/${{ matrix.name }}:${OSG_SERIES}-${BASE_REPO}" >> $GITHUB_ENV
           fi
+
+      - name: Pull External Repo
+        if: ${{ matrix.upstream }}
+        run: |
+          mkdir -p ${{ matrix.context }}
+          git clone ${{ matrix.upstream }} ${{ matrix.context }}
+          (cd ${{matrix.context}} && git fetch && git reset --hard ${{ matrix.upstream_ref }})
+
+
+      - name: Build Image
+        continue-on-error: ${{ matrix.yum_repo == 'development' }}
+        uses: opensciencegrid/[email protected]
+        with:
+          clean_before_build: false
+          osg_series: ${{ matrix.osg_series }}
+          osg_repo: ${{ matrix.base_repo }}
+          context: ${{ matrix.context }}
+          base_os: ${{ matrix.base_os }}
+          output_image: ${{ env.IMAGE_NAME }}
 
       - name: Push to Harbor (${OSG_SERIES}-${BASE_REPO})
-        uses: opensciencegrid/push-container-action@main
+        if: >-
+          github.ref == 'refs/heads/main' &&
+          github.event_name != 'pull_request' &&
+          needs.build-image-list.outputs.image_list != '["dummy"]'
+        #github.repository_owner == 'opensciencegrid' &&
+        uses: opensciencegrid/[email protected]
         with:
-          repo: ${{ env.BASE_REPO }}
-          osg_series: ${{ env.OSG_SERIES }}
-          context: ${{ env.CONTEXT }}
-          base_os: ${{ env.BASE_OS }}
+          clean_before_build: false
+          push_image: true
+          osg_series: ${{ matrix.osg_series }}
+          osg_repo: ${{ matrix.base_repo }}
+          context: ${{ matrix.context }}
+          base_os: ${{ matrix.base_os }}
+          output_image: ${{ env.IMAGE_NAME }}
           registry_url: hub.opensciencegrid.org
           registry_user: ${{ secrets.OSG_HARBOR_ROBOT_USER }}
           registry_pass: ${{ secrets.OSG_HARBOR_ROBOT_PASSWORD }}
 
       - name: Push to Docker Hub (${OSG_SERIES}-${BASE_REPO})
-        uses: opensciencegrid/push-container-action@main
+        if: >-
+          startsWith(env.IMAGE_NAME, 'opensciencegrid') &&
+          github.ref == 'refs/heads/main' &&
+          github.event_name != 'pull_request' &&
+          needs.build-image-list.outputs.image_list != '["dummy"]' 
+        #github.repository_owner == 'opensciencegrid' &&
+        uses: opensciencegrid/[email protected]
         with:
-          repo: ${{ env.BASE_REPO }}
-          osg_series: ${{ env.OSG_SERIES }}
-          context: ${{ env.CONTEXT }}
-          base_os: ${{ env.BASE_OS }}
+          clean_before_build: false
+          push_image: true
+          osg_series: ${{ matrix.osg_series }}
+          osg_repo: ${{ matrix.base_repo }}
+          context: ${{ matrix.context }}
+          base_os: ${{ matrix.base_os }}
+          output_image: ${{ env.IMAGE_NAME }}
           registry_url: docker.io
           registry_user: ${{ secrets.DOCKER_USERNAME }}
           registry_pass: ${{ secrets.DOCKER_PASSWORD }}

iris-hep/kuantifier/build-config.json

@@ -0,0 +1,12 @@
+{
+  "standard_build": true,
+  "repo_build": false,
+  "base_os": ["el9"],
+  "osg_series": ["24"],
+  "base_repo": ["release"],
+  "context": "kuantifier",
+
+  "tag": "1.0.2",
+  "upstream": "https://github.com/rptaylor/kuantifier.git",
+  "upstream_ref": "v1.3.2"
+}

requirements.txt

@@ -0,0 +1 @@
+GitPython

scripts/build-job-matrix.py

@@ -4,6 +4,7 @@
 from itertools import product
 
 DEFAULT_CONFIG_PATH = 'opensciencegrid/default-build-config.json'
+GITHUB_OUTPUT = os.environ['GITHUB_OUTPUT']
 
 
 def load_config(config_path, default_config=None):
@@ -41,6 +42,11 @@ def main(image_dirs):
         osg_series_list = config['osg_series']
         base_repo_list = config['base_repo']
 
+        build_context = config.get('context', '')
+        tag_override = config.get('tag', '')
+        upstream_repo = config.get('upstream', '')
+        upstream_ref = config.get('upstream_ref', '')
+
         combinations = product(
             base_os_list,
             osg_series_list,
@@ -55,17 +61,34 @@ def main(image_dirs):
         # structures in the matrix construction and it offers:
         # 1. Simplicity: Using a single string to represent configurations is straightforward and easy to understand.
         # 2. Integration: A single string is easily passed to external tools and systems that manage builds.
-            configuration_string = f"{base_os}-{osg_series}-{base_repo}-{config['standard_build']}-{config['repo_build']}"
-            include_list.append({"name": image_name, "config": configuration_string})
+            include_list.append({
+                "name": image_name, 
+                "base_os": base_os,
+                "osg_series": osg_series,
+                "base_repo": base_repo,
+                "tag_override": tag_override,
+                "standard_build": config['standard_build'],
+                "repo_build": config['repo_build'],
+                "context": os.path.join(image_dir, build_context),
+                "upstream": upstream_repo,
+                "upstream_ref": upstream_ref
+            })
 
     sys.stdout.flush()
-    json_output = json.dumps({"include": include_list}, indent=4)
-    print(json_output)
 
+    if not include_list:
+        # Add a dummy value to the matrix so later GHA steps don't auto-error
+        include_list.append('dummy')
 
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        sys.exit(f"Usage: {sys.argv[0]} <image_dirs>")
+    # Write the include list to GITHUB_OUTPUT
+    with open(GITHUB_OUTPUT, 'a') as github_output:
+        github_output.writelines([
+            f"matrix={json.dumps(include_list)}\n"
+        ])
+    print(include_list)
 
+
+if __name__ == "__main__":
+    # If run with zero arguments, output an empty list
     image_dirs = sys.argv[1:]
     main(image_dirs)