Merge branch 'dev' into ruigao/cilium-update

Author: hippogr

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+. @microsoft/ltpadmin
+.github @microsoft/ltpadmin

.github/workflows/build-all.yaml

@@ -0,0 +1,111 @@
+name: Build All Services
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches: [main, 'release/*']
+  pull_request:
+    branches: [main, dev, 'release/*']
+  release:
+    types: [published]
+  workflow_dispatch:
+
+env:
+  TAG: ${{ github.run_number }}
+
+jobs:
+  build:
+    name: Build All
+    runs-on: [self-hosted, paicicd]
+    timeout-minutes: 120
+    environment: auto-test
+    container:
+      image: ubuntu:latest
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
+    steps:
+      - name: Install git
+        run: |
+          DEBIAN_FRONTEND=noninteractive apt update
+          DEBIAN_FRONTEND=noninteractive apt install -y git
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: false
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.ref_name }}
+
+      - name: Get All Services
+        id: all
+        run: |
+          services=$(ls -1d src/* | awk -F'/' '{print $2}' | tr '\n' ' ')
+          skipped_services="base-image cleaning-image dev-box marketplace-db marketplace-restserver marketplace-webportal utilization-reporter"
+          for skip in $skipped_services; do
+            services=$(echo $services | sed "s/\b$skip\b//g")
+          done
+          echo "All services: $services"
+          echo "services=$services" >> $GITHUB_OUTPUT
+
+      - name: Install Package
+        if: steps.all.outputs.services != ''
+        run: |
+          DEBIAN_FRONTEND=noninteractive apt install -y python3 python-is-python3 pip git unzip docker-cli ca-certificates curl apt-transport-https lsb-release gnupg parallel
+          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+
+      - name: Install python libs
+        if: steps.all.outputs.services != ''
+        run: python -m pip install --break-system-packages pyyaml jinja2 paramiko etcd3 protobuf==3.20.3 kubernetes gitpython
+
+      - name: Decode and unzip config file
+        if: steps.all.outputs.services != ''
+        run: |
+          echo "${{ secrets.CONFIG_FILE_B64 }}" | base64 -d > config.zip
+          mkdir -p $GITHUB_WORKSPACE/config
+          unzip -o config.zip -d $GITHUB_WORKSPACE/config
+          ls -l $GITHUB_WORKSPACE/config
+
+      - name: Arrange Config Files
+        if: steps.all.outputs.services != ''
+        run: |
+          rm -rf /tmp/auth-configuration
+          mv $GITHUB_WORKSPACE/config/auth-configuration /tmp/
+          ls -l /tmp/auth-configuration
+
+      - name: Build Images of Services
+        if: steps.all.outputs.services != ''
+        run: |
+          all_services="${{ steps.all.outputs.services }}"
+          echo "Building: $all_services"
+          echo "--------------------------------"
+          failed_services=""
+          for service in $all_services; do
+            if [[ "$service" =~ alert-manager ]]; then
+              echo "alert-manager is in the changed services"
+              # Build specific images in alert-manager
+              echo "Building specific alert-manager images"
+              $GITHUB_WORKSPACE/build/pai_build.py build \
+                -c $GITHUB_WORKSPACE/config/cluster-configuration \
+                -s alert-manager \
+                -i abnormal-detector,alert-handler,alert-parser,cert-expiration-checker,cluster-utilization,job-data-recorder,job-status-change-notification,node-failure-detection,node-issue-classifier,nvidia-gpu-low-perf-fixer,redis-monitoring
+            fi
+            echo "Building service: $service"
+            if python3 $GITHUB_WORKSPACE/build/pai_build.py build \
+              -c $GITHUB_WORKSPACE/config/cluster-configuration \
+              -s $service; then
+              echo "✓ Successfully built: $service"
+            else
+              echo "✗ Failed to build: $service"
+              failed_services="$failed_services $service"
+            fi
+          done
+            
+          if [ -n "$failed_services" ]; then
+            echo "::error::Failed to build services:$failed_services"
+            echo "FAILED_SERVICES=$failed_services" 
+            exit 1
+          else
+            echo "All services built successfully"
+          fi

.github/workflows/build-deploy-changes.yaml

@@ -6,9 +6,9 @@ permissions:
 
 on:
   push:
-    branches: [main, dev, 'release/*']
+    branches: [main, dev, "release/*"]
   pull_request:
-    branches: [main, dev, 'release/*']
+    branches: [main, dev, "release/*"]
 
 env:
   TAG: ${{ github.run_number }}
@@ -55,9 +55,8 @@ jobs:
           changed_files=$(git diff --name-only "$base_sha" "$head_sha")
           echo "Changed files: $changed_files"
 
-          # extract service folders under src/, skip alert-manager
+          # extract service folders under src/
           folders=$(echo "$changed_files" | grep '^src/' \
-            | grep -v 'alert-manager' \
             |  awk -F'/' '{print $2}' \
             | sort -u | tr '\n' ' ')
           echo "Changed folders: $folders"
@@ -104,6 +103,18 @@ jobs:
         run: |
           changed_services="${{ steps.changes.outputs.folders }}"
           echo "Building: $changed_services"
+          if [[ "$changed_services" == *"alert-manager"* ]]; then
+            echo "alert-manager is in the changed services"
+            changed_services=$(echo $changed_services | sed 's/alert-manager//g')
+            # build specific images in alert-manager
+            echo "Building specific alert-manager images"
+            $GITHUB_WORKSPACE/build/pai_build.py build \
+              -c $GITHUB_WORKSPACE/config/cluster-configuration \
+              -s alert-manager
+              -i abnormal-detector,alert-handler,alert-parser,cert-expiration-checker,cluster-utilization,job-data-recorder,job-status-change-notification,node-failure-detection,node-issue-classifier,nvidia-gpu-low-perf-fixer,redis-monitoring
+          fi
+
+          echo "Changed services after removing alert-manager: $changed_services"
           $GITHUB_WORKSPACE/build/pai_build.py build \
             -c $GITHUB_WORKSPACE/config/cluster-configuration \
             -s $changed_services
@@ -122,6 +133,24 @@ jobs:
         run: |
           changed_services="${{ steps.changes.outputs.folders }}"
           echo "Pushing: $changed_services"
+          # check whether alert-manager is in the changed services
+          echo "Changed services before removing alert-manager: $changed_services"
+          if [[ "$changed_services" == *"alert-manager"* ]]; then
+            echo "alert-manager is in the changed services"
+            changed_services=$(echo $changed_services | sed 's/alert-manager//g')
+            # push specific images in alert-manager to GHCR
+            echo "Pushing specific alert-manager images to GHCR"
+            $GITHUB_WORKSPACE/build/pai_build.py push \
+              -c $GITHUB_WORKSPACE/config/cluster-configuration \
+              -s alert-manager \
+              -i abnormal-detector,alert-handler,alert-parser,cert-expiration-checker,cluster-utilization,job-data-recorder,job-status-change-notification,node-failure-detection,node-issue-classifier,nvidia-gpu-low-perf-fixer,redis-monitoring \
+              --docker-registry ghcr.io \
+              --docker-namespace ${GITHUB_REPOSITORY_OWNER} \
+              --docker-username ${{ github.actor }} \
+              --docker-password ${{ secrets.GITHUB_TOKEN }}
+          fi
+
+          echo "Changed services after removing alert-manager: $changed_services"
           $GITHUB_WORKSPACE/build/pai_build.py push \
             -c $GITHUB_WORKSPACE/config/cluster-configuration \
             -s $changed_services \
@@ -133,24 +162,24 @@ jobs:
       - name: Azure CLI get credentials and deploy
         if: steps.check.outputs.has_changed == 'true'
         run: |
-            az version
-            az login --identity --client-id ${{ secrets.AZURE_MANAGED_IDENTITY_CLIENT_ID }}
-            az aks install-cli
-            az aks get-credentials \
-              --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \
-              --name ${{ secrets.KUBERNETES_CLUSTER }} \
-              --overwrite-existing
-            kubelogin convert-kubeconfig -l azurecli
-            kubectl config use-context ${{ secrets.KUBERNETES_CLUSTER }}
-            echo "${{ secrets.PAI_CLUSTER_NAME }}" > cluster_id
-            echo "Stopping changed pai services \"${{ steps.changes.outputs.folders }}\" on ${{ secrets.PAI_CLUSTER_NAME }} ..."
-            $GITHUB_WORKSPACE/paictl.py service stop -n ${{ steps.changes.outputs.folders }} < cluster_id 
-            echo "Pushing config to cluster \"${{ secrets.PAI_CLUSTER_NAME }}\" ..."
-            $GITHUB_WORKSPACE/paictl.py config push -m service -p $GITHUB_WORKSPACE/config/cluster-configuration < cluster_id 
-            echo "Starting to update \"${{ steps.changes.outputs.folders }}\" on ${{ secrets.PAI_CLUSTER_NAME }} ..."
-            $GITHUB_WORKSPACE/paictl.py service start -n ${{ steps.changes.outputs.folders }} < cluster_id 
-            kubectl get pod
-            kubectl get service
+          az version
+          az login --identity --client-id ${{ secrets.AZURE_MANAGED_IDENTITY_CLIENT_ID }}
+          az aks install-cli
+          az aks get-credentials \
+            --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \
+            --name ${{ secrets.KUBERNETES_CLUSTER }} \
+            --overwrite-existing
+          kubelogin convert-kubeconfig -l azurecli
+          kubectl config use-context ${{ secrets.KUBERNETES_CLUSTER }}
+          echo "${{ secrets.PAI_CLUSTER_NAME }}" > cluster_id
+          echo "Stopping changed pai services \"${{ steps.changes.outputs.folders }}\" on ${{ secrets.PAI_CLUSTER_NAME }} ..."
+          $GITHUB_WORKSPACE/paictl.py service stop -n ${{ steps.changes.outputs.folders }} < cluster_id 
+          echo "Pushing config to cluster \"${{ secrets.PAI_CLUSTER_NAME }}\" ..."
+          $GITHUB_WORKSPACE/paictl.py config push -m service -p $GITHUB_WORKSPACE/config/cluster-configuration < cluster_id 
+          echo "Starting to update \"${{ steps.changes.outputs.folders }}\" on ${{ secrets.PAI_CLUSTER_NAME }} ..."
+          $GITHUB_WORKSPACE/paictl.py service start -n ${{ steps.changes.outputs.folders }} < cluster_id 
+          kubectl get pod
+          kubectl get service
 
   test:
     name: Test rest-server
@@ -169,4 +198,3 @@ jobs:
             exit 1
           fi
           echo "Virtual cluster info: $vc_info"
-

build/core/build_center.py

@@ -28,14 +28,16 @@
 
 class BuildCenter:
 
-    def __init__(self, build_config, process_list, type):
+    def __init__(self, build_config, process_list, type, arg_config=None):
 
         self.logger = logging.getLogger(__name__)
         build_utility.setup_logger_config(self.logger)
 
         self.build_config = build_config
         self.task_type = type
 
+        self.arg_config = arg_config
+
         self.process_list = [service.lower() for service in process_list] if process_list is not None else None
 
         # Initialize docker_cli instance
@@ -138,7 +140,7 @@ def build_center(self):
                     for inedge in self.graph.services[item].inedges:
                         build_worker.copy_dependency_folder(os.path.join(self.codeDir,inedge),
                         os.path.join(self.graph.services[item].path,self.dependencyDir+inedge))
-                    build_worker.build_single_component(self.graph.services[item])
+                    build_worker.build_single_component(self.graph.services[item], self.arg_config.imagelist)
             self.logger.info("Build all components succeed")
 
         except Exception as e:

build/core/build_handler.py

@@ -40,7 +40,7 @@ def __init__(self, docker_cli):
         self.dependencyDir = 'dependency'
 
 
-    def build_single_component(self, service):
+    def build_single_component(self, service, imagelist=None):
 
         self.logger.info("Starts to build {0}".format(service.service_name))
 
@@ -53,7 +53,8 @@ def build_single_component(self, service):
         for dockerfile_prefix in service.docker_files:
             image_name = os.path.splitext(dockerfile_prefix)[0]
             dockerfile = os.path.join(service.path, 'build/' + dockerfile_prefix + '.dockerfile')
-            self.docker_cli.docker_image_build(image_name, dockerfile, service.path)
+            if imagelist is None or image_name in imagelist:
+                self.docker_cli.docker_image_build(image_name, dockerfile, service.path)
 
         post_build = os.path.join(service.path, self.build_post)
         if os.path.exists(post_build):

build/pai_build.py

@@ -39,7 +39,7 @@ def load_build_config(config_dir):
 
 
 def build_service(args, config_model):
-    pai_build = build_center.BuildCenter(config_model, args.service, 'k8s')
+    pai_build = build_center.BuildCenter(config_model, args.service, 'k8s', args)
     pai_build.set_build_cache_type(args.nocache)
     pai_build.build_center()
 
@@ -98,6 +98,13 @@ def main():
         action='store_true',
         help="Build the service using cache or not"
     )
+    build_parser.add_argument(
+        '-i', '--imagelist',
+        type=str,
+        nargs='+',
+        default=None,
+        help="The image list you want to build"
+    )
     build_parser.set_defaults(func=build_service)
 
     # Push commands

contrib/aks/scripts/config-ipoib.sh

@@ -5,9 +5,21 @@
 
 set -x
 
-DEBIAN_FRONTEND=noninteractive apt-get update -y
-DEBIAN_FRONTEND=noninteractive apt-get install -y network-manager net-tools rsync || echo "Failed in apt install"
-
+wait_for_dpkg_lock() {
+    if ! timeout 300 bash -c \
+        'while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 \
+                || pgrep -x "apt|apt-get|dpkg|unattended-upgrades" >/dev/null; do
+            sleep 3
+        done'
+    then
+        echo "Timed out waiting for dpkg lock."
+        exit 124
+    fi
+    bash -c 'exec "$@"' -- "$@"
+}
+
+wait_for_dpkg_lock bash -c 'DEBIAN_FRONTEND=noninteractive apt-get update -y'
+wait_for_dpkg_lock bash -c 'DEBIAN_FRONTEND=noninteractive apt-get install -y network-manager net-tools rsync || echo "Failed in apt install"'
 
 # rename
 INIT=0