Merge branch 'dev' into ruigao/cilium-update

Author: hippogr

.github/workflows/build-all.yaml

@@ -5,12 +5,16 @@ permissions:
 
 on:
   push:
-    branches: [main, 'release/*']
-  pull_request:
-    branches: [main, dev, 'release/*']
+    branches: ['release/*']
   release:
     types: [published]
   workflow_dispatch:
+    inputs:
+      branch:
+        description: 'The branch name or tag to run the workflow on'
+        required: true
+        default: 'main'
+        type: string
 
 env:
   TAG: ${{ github.run_number }}
@@ -36,7 +40,7 @@ jobs:
         with:
           fetch-depth: 0
           submodules: false
-          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.ref_name }}
+          ref: ${{ github.event.inputs.branch || github.ref }}
 
       - name: Get All Services
         id: all

.github/workflows/build-deploy-changes.yaml

@@ -171,13 +171,27 @@ jobs:
             --overwrite-existing
           kubelogin convert-kubeconfig -l azurecli
           kubectl config use-context ${{ secrets.KUBERNETES_CLUSTER }}
+            # Replace "webportal" with "webportal-dind" if "webportal" is changed
+            services_to_deploy="${{ steps.changes.outputs.folders }}"
+            if echo " $services_to_deploy " | grep -q " webportal "; then
+              tmp=""
+              for s in $services_to_deploy; do
+                [ "$s" = "webportal" ] && continue
+                [ "$s" = "webportal-dind" ] && continue
+                tmp="$tmp $s"
+              done
+              services_to_deploy="$tmp webportal-dind"
+              services_to_deploy=$(echo "$services_to_deploy" | xargs)
+            fi
+            echo "Final services to deploy: $services_to_deploy"
+
           echo "${{ secrets.PAI_CLUSTER_NAME }}" > cluster_id
-          echo "Stopping changed pai services \"${{ steps.changes.outputs.folders }}\" on ${{ secrets.PAI_CLUSTER_NAME }} ..."
-          $GITHUB_WORKSPACE/paictl.py service stop -n ${{ steps.changes.outputs.folders }} < cluster_id 
+          echo "Stopping changed pai services $services_to_deploy on ${{ secrets.PAI_CLUSTER_NAME }} ..."
+          $GITHUB_WORKSPACE/paictl.py service stop -n $services_to_deploy < cluster_id
           echo "Pushing config to cluster \"${{ secrets.PAI_CLUSTER_NAME }}\" ..."
-          $GITHUB_WORKSPACE/paictl.py config push -m service -p $GITHUB_WORKSPACE/config/cluster-configuration < cluster_id 
-          echo "Starting to update \"${{ steps.changes.outputs.folders }}\" on ${{ secrets.PAI_CLUSTER_NAME }} ..."
-          $GITHUB_WORKSPACE/paictl.py service start -n ${{ steps.changes.outputs.folders }} < cluster_id 
+          $GITHUB_WORKSPACE/paictl.py config push -m service -p $GITHUB_WORKSPACE/config/cluster-configuration < cluster_id
+          echo "Starting to update $services_to_deploy on ${{ secrets.PAI_CLUSTER_NAME }} ..."
+          $GITHUB_WORKSPACE/paictl.py service start -n $services_to_deploy < cluster_id
           kubectl get pod
           kubectl get service
 

src/rest-server/deploy/rest-server.yaml.template

@@ -174,6 +174,8 @@ spec:
           value: {{ cluster_cfg['rest-server']['hived-computing-device-envs'] }}
         - name: ALERT_MANAGER_URL
           value: "{{ cluster_cfg['alert-manager']['url'] }}"
+        - name: IMAGE_REGEX
+          value: "{{ cluster_cfg['rest-server']['image-regex'] | default("") }}"
         ports:
         - name: rest-server
           containerPort: 8080

src/rest-server/src/config/launcher.js

@@ -84,6 +84,7 @@ const k8sLauncherConfigSchema = Joi.object()
     jobRestrictionGitScriptName: Joi.string().required(),
     clstoreHostPath: Joi.string().empty(''),
     clstoreJobPath: Joi.string().empty(''),
+    imageRegex: Joi.string().empty(''),
   })
   .required();
 
@@ -156,6 +157,7 @@ if (launcherType === 'k8s') {
     jobRestrictionGitScriptName: process.env.JOB_RESTRICTION_GIT_SCRIPT_NAME || 'unset',
     clstoreHostPath: process.env.CLUSTER_LOCAL_STORAGE_HOST_PATH,
     clstoreJobPath: process.env.CLUSTER_LOCAL_STORAGE_JOB_PATH,
+    imageRegex: process.env.IMAGE_REGEX || '',
   };
 
   const { error, value } = k8sLauncherConfigSchema.validate(launcherConfig);

src/rest-server/src/config/v2/protocol.js

@@ -173,6 +173,11 @@ const protocolSchema = {
       },
       minItems: 1,
     },
+    jobType: {
+      type: 'string',
+      enum: ['inference', 'training', 'others'],
+      default: 'others',
+    },
     parameters: {
       type: 'object',
       additionalProperties: true,

src/rest-server/src/controllers/v2/job.js

@@ -86,6 +86,24 @@ const list = asyncHandler(async (req, res) => {
     if ('tagsNotContain' in req.query) {
       tagsNotContainFilter.name = req.query.tagsNotContain.split(',');
     }
+    if ('jobType' in req.query) {
+      // validate jobType values
+      const validJobTypes = ['inference', 'training', 'others'];
+      const requestedTypes = req.query.jobType.split(',');
+      const invalidTypes = requestedTypes.filter(type => !validJobTypes.includes(type));
+      if (invalidTypes.length > 0) {
+        throw createError(
+          'Bad Request',
+          'InvalidParametersError',
+          `Invalid job type(s): ${invalidTypes.join(', ')}`
+        );
+      }
+      if (Array.isArray(tagsContainFilter.name)) {
+        tagsContainFilter.name.push(...requestedTypes);
+      } else {
+        tagsContainFilter.name = requestedTypes;
+      }
+    }
     if ('keyword' in req.query) {
       // match text in username, jobname, or vc
       filters[Op.or] = [
@@ -199,6 +217,7 @@ const update = asyncHandler(async (req, res) => {
   const jobName = res.locals.protocol.name;
   const userName = req.user.username;
   const frameworkName = `${userName}~${jobName}`;
+  const jobType = res.locals.protocol.jobType || 'others';
 
   // check duplicate job
   try {
@@ -216,6 +235,7 @@ const update = asyncHandler(async (req, res) => {
     }
   }
   await job.put(frameworkName, res.locals.protocol, req.body);
+  await job.addTag(frameworkName, jobType);
   res.status(status('Accepted')).json({
     status: status('Accepted'),
     message: `Update job ${jobName} for user ${userName} successfully.`,
@@ -368,10 +388,10 @@ const getLogs = asyncHandler(async (req, res) => {
     throw error.code === 'NoTaskLogError'
       ? error
       : createError(
-          'Internal Server Error',
-          'UnknownError',
-          'Failed to get log list',
-        );
+        'Internal Server Error',
+        'UnknownError',
+        'Failed to get log list',
+      );
   }
 });
 

src/rest-server/src/middlewares/v2/protocol.js

@@ -24,6 +24,8 @@ const hived = require('@pai/middlewares/v2/hived');
 const { enabledHived } = require('@pai/config/launcher');
 const protocolSchema = require('@pai/config/v2/protocol');
 const asyncHandler = require('@pai/middlewares/v2/asyncHandler');
+const logger = require('@pai/config/logger');
+const launcherConfig = require('@pai/config/launcher'); 
 
 const mustacheWriter = new mustache.Writer();
 
@@ -53,6 +55,19 @@ const render = (template, dict, tags = ['<%', '%>']) => {
   return result.trim();
 };
 
+const getImageName = (prerequisites) => { 
+  if (
+    typeof prerequisites !== 'object' ||
+    typeof prerequisites.dockerimage !== 'object'
+  ) {
+    return null;
+  }
+
+  const dockerImages = Object.values(prerequisites.dockerimage);
+  const img = dockerImages.find(p => p.type === 'dockerimage' && p.uri);
+  return img?.uri ?? null;
+};
+
 const protocolValidate = (protocolYAML) => {
   const protocolObj = yaml.load(protocolYAML);
   if (!protocolSchema.validate(protocolObj)) {
@@ -87,6 +102,42 @@ const protocolValidate = (protocolYAML) => {
         prerequisiteSet.add(item.name);
       }
     }
+    const imageRegexPattern = launcherConfig.imageRegex;
+    let imageRegex;
+
+    try {
+      if (imageRegexPattern && imageRegexPattern.length > 0) {
+        imageRegex = new RegExp(imageRegexPattern);
+      }
+    } catch (error) {
+      logger.info(`Invalid imageRegex pattern: ${imageRegexPattern}. Error: ${error.message}`);
+      throw createError(
+          'Internal Server Error',
+          'InvalidImageRegexError',
+          `The provided imageRegex pattern "${imageRegexPattern}" is invalid.`
+      );
+    }
+
+    if (imageRegex) {
+      const imageName = getImageName(protocolObj.prerequisites);
+      // Check if the imageName matches the imageRegex
+      if (!imageName) {
+        throw createError(
+            'Bad Request',
+            'NoDockerImageError',
+            'No valid docker image found in prerequisites.'
+        );
+      }
+      // Check if the imageName matches the imageRegex
+      const match = imageRegex.test(imageName);
+      if (!match) {
+        throw createError(
+            'Bad Request',
+            'InvalidImageError',
+            `The image ${imageName} is not allowed.`
+        );
+      }
+    }
   }
   protocolObj.prerequisites = prerequisites;
   // convert deployments list to dict
@@ -120,6 +171,34 @@ const protocolValidate = (protocolYAML) => {
       }
     }
   }
+
+  // check jobType
+  if ('jobType' in protocolObj) {
+    if (protocolObj.jobType === 'inference') {
+      // check parameters for inference job
+      if (!('parameters' in protocolObj)) {
+        throw createError(
+          'Bad Request',
+          'InvalidProtocolError',
+          `The following parameters must be specified for inference job:
+INTERNAL_SERVER_IP=$PAI_HOST_IP_taskrole_0
+INTERNAL_SERVER_PORT=$PAI_PORT_LIST_taskrole_0_http
+API_KEY=[any string]`,
+        );
+      }
+      const requiredParams = ['INTERNAL_SERVER_IP', 'INTERNAL_SERVER_PORT', 'API_KEY'];
+      for (const param of requiredParams) {
+        if (!(param in protocolObj.parameters)) {
+          throw createError(
+            'Bad Request',
+            'InvalidProtocolError',
+            `Parameter ${param} must be specified for inference job.`,
+          );
+        }
+      }
+    }
+  }
+
   for (const taskRole of Object.keys(protocolObj.taskRoles)) {
     for (const field of prerequisiteFields) {
       if (
@@ -194,7 +273,7 @@ const protocolRender = (protocolObj) => {
         ],
       $output:
         protocolObj.prerequisites.output[
-          protocolObj.taskRoles[taskRole].output
+        protocolObj.taskRoles[taskRole].output
         ],
       $data:
         protocolObj.prerequisites.data[protocolObj.taskRoles[taskRole].data],

src/rest-server/src/middlewares/v2/quota.js

@@ -75,19 +75,6 @@ const getReqeustedGpuCount = (protocol) => {
   return gpuCount;
 };
 
-function getImageName(prerequisites) {
-  if (
-    typeof prerequisites !== 'object' ||
-    typeof prerequisites.dockerimage !== 'object'
-  ) {
-    return null;
-  }
-
-  const dockerImages = Object.values(prerequisites.dockerimage);
-  const img = dockerImages.find(p => p.type === 'dockerimage' && p.uri);
-  return img?.uri ?? null;
-}
-
 const getJobPriority = (protocol) => {
   return protocol.extras?.hivedScheduler?.jobPriorityClass || null;
 };
@@ -104,21 +91,6 @@ const check = async (req, res, next) => {
     const userPrioritySet = userInfo.extension?.jobPriority ?? null;
     const userPriorityExpiration = userInfo.extension?.jobExpiration ?? null;
 
-    const imageName = getImageName(jobProtocol.prerequisites);
-    const imageRepo = imageName.split('/')[0].trim().toLowerCase();
-    const isAzureCR = /\.azurecr\.io$/i.test(imageRepo);
-    // Reject if it’s not ACR
-    if (!isAzureCR) {
-      return next(
-        createError(
-          'Forbidden',
-          'InvalidImageError',
-          `The image ${imageRepo || imageName} is not allowed. ` +
-          `Please use an image from Azure Container Registry (ACR).`
-        )
-      );
-    }
-
     if (jobPriority === 'prod') {
       if (userPrioritySet !== 1) {
         logger.debug(

src/rest-server/src/models/v2/job/k8s.js

@@ -1000,33 +1000,43 @@ const list = async (
     Object.keys(tagsContainFilter).length !== 0 ||
     Object.keys(tagsNotContainFilter).length !== 0
   ) {
-    filters.name = {};
-    // tagsContain
+    // Build name filters by querying Tag table directly to avoid using QueryGenerator
+    const nameFilter = {};
+
+    // tagsContain -> include frameworks whose name appears in Tag rows matched by tagsContainFilter
     if (Object.keys(tagsContainFilter).length !== 0) {
-      const queryContainFrameworkName = databaseModel.sequelize.dialect.QueryGenerator.selectQuery(
-        'tags',
-        {
-          attributes: ['frameworkName'],
-          where: tagsContainFilter,
-        },
-      );
-      filters.name[Sequelize.Op.in] = Sequelize.literal(`
-          (${queryContainFrameworkName.slice(0, -1)})
-      `);
+      const containRows = await databaseModel.Tag.findAll({
+        attributes: ['frameworkName'],
+        where: tagsContainFilter,
+        raw: true,
+      });
+      const containNames = [...new Set(containRows.map((r) => r.frameworkName))];
+      // if no tags match, result is empty
+      if (containNames.length === 0) {
+        if (withTotalCount) {
+          return { totalCount: 0, data: [] };
+        } else {
+          return [];
+        }
+      }
+      nameFilter[Sequelize.Op.in] = containNames;
     }
-    // tagsNotContain
+
+    // tagsNotContain -> exclude frameworks whose name appears in Tag rows matched by tagsNotContainFilter
     if (Object.keys(tagsNotContainFilter).length !== 0) {
-      const queryNotContainFrameworkName = databaseModel.sequelize.dialect.QueryGenerator.selectQuery(
-        'tags',
-        {
-          attributes: ['frameworkName'],
-          where: tagsNotContainFilter,
-        },
-      );
-      filters.name[Sequelize.Op.notIn] = Sequelize.literal(`
-          (${queryNotContainFrameworkName.slice(0, -1)})
-      `);
+      const notContainRows = await databaseModel.Tag.findAll({
+        attributes: ['frameworkName'],
+        where: tagsNotContainFilter,
+        raw: true,
+      });
+      const notContainNames = [...new Set(notContainRows.map((r) => r.frameworkName))];
+      if (notContainNames.length > 0) {
+        nameFilter[Sequelize.Op.notIn] = notContainNames;
+      }
     }
+
+    // merge with any existing name filter
+    filters.name = Object.assign({}, filters.name || {}, nameFilter);
   }
 
   frameworks = await databaseModel.Framework.findAll({

src/rest-server/src/utils/error.d.ts

@@ -33,8 +33,11 @@ declare type Code =
     'ForbiddenUserError' |
     'ForbiddenKeyError' |
     'IncorrectPasswordError' |
+    'InvalidImageError' |
+    'InvalidImageRegexError' |
     'InvalidParametersError' |
     'NoApiError' |
+    'NoDockerImageError' |
     'NoJobError' |
     'NoJobConfigError' |
     'NoJobSshInfoError' |