Deploy-Test-Cleanup Pipeline #113
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy-Test-Cleanup Pipeline | |
| on: | |
| push: | |
| branches: | |
| - main # Adjust this to the branch you want to trigger the deployment on | |
| - dev | |
| - demo | |
| schedule: | |
| - cron: "0 10,22 * * *" # Runs at 10:00 AM and 10:00 PM GMT | |
| env: | |
| GPT_CAPACITY: 250 | |
| TEXT_EMBEDDING_CAPACITY: 200 | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| RESOURCE_GROUP_NAME: ${{ steps.get_webapp_url.outputs.RESOURCE_GROUP_NAME }} | |
| KUBERNETES_RESOURCE_GROUP_NAME: ${{ steps.get_webapp_url.outputs.KUBERNETES_RESOURCE_GROUP_NAME }} | |
| WEBAPP_URL: ${{ steps.get_webapp_url.outputs.WEBAPP_URL }} | |
| OPENAI_RESOURCE_NAME: ${{ steps.get_webapp_url.outputs.OPENAI_RESOURCE_NAME }} | |
| DOCUMENT_INTELLIGENCE_RESOURCE_NAME: ${{ steps.get_webapp_url.outputs.DOCUMENT_INTELLIGENCE_RESOURCE_NAME }} | |
| VALID_REGION: ${{ steps.get_webapp_url.outputs.VALID_REGION }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 # Checks out your repository | |
| - name: Install Azure CLI | |
| shell: bash | |
| run: | | |
| curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash | |
| az --version # Verify installation | |
| - name: Install Kubernetes CLI (kubectl) | |
| shell: bash | |
| run: | | |
| az aks install-cli | |
| az extension add --name aks-preview | |
| - name: Install Helm | |
| shell: bash | |
| run: | | |
| curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null | |
| sudo apt-get install apt-transport-https --yes | |
| echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list | |
| sudo apt-get update | |
| sudo apt-get install helm | |
| - name: Set up Docker | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| driver: docker | |
| - name: Run Quota Check | |
| id: quota-check | |
| shell: pwsh | |
| run: | | |
| $ErrorActionPreference = "Stop" # Ensure that any error stops the pipeline | |
| # Path to the PowerShell script for quota check | |
| $quotaCheckScript = "Deployment/checkquota.ps1" | |
| # Check if the script exists and is executable (not needed for PowerShell like chmod) | |
| if (-not (Test-Path $quotaCheckScript)) { | |
| Write-Host "❌ Error: Quota check script not found." | |
| exit 1 | |
| } | |
| # Run the script | |
| .\Deployment\checkquota.ps1 | |
| # If the script fails, check for the failure message | |
| $quotaFailedMessage = "No region with sufficient quota found" | |
| $output = Get-Content "Deployment/checkquota.ps1" | |
| if ($output -contains $quotaFailedMessage) { | |
| echo "QUOTA_FAILED=true" >> $GITHUB_ENV | |
| } | |
| env: | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| GPT_MIN_CAPACITY: ${{ env.GPT_CAPACITY }} | |
| TEXT_EMBEDDING_MIN_CAPACITY: ${{ env.TEXT_EMBEDDING_CAPACITY }} | |
| AZURE_REGIONS: "${{ vars.AZURE_REGIONS }}" | |
| - name: Send Notification on Quota Failure | |
| if: env.QUOTA_FAILED == 'true' | |
| shell: pwsh | |
| run: | | |
| $RUN_URL = "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| # Construct the email body | |
| $EMAIL_BODY = @" | |
| { | |
| "body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> <a href='$RUN_URL'>$RUN_URL</a></p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>" | |
| } | |
| "@ | |
| # Send the notification | |
| try { | |
| $response = Invoke-RestMethod -Uri "${{ secrets.LOGIC_APP_URL }}" -Method Post -ContentType "application/json" -Body $EMAIL_BODY | |
| Write-Host "Notification sent successfully." | |
| } catch { | |
| Write-Host "❌ Failed to send notification." | |
| } | |
| - name: Fail Pipeline if Quota Check Fails | |
| if: env.QUOTA_FAILED == 'true' | |
| run: exit 1 | |
| - name: Generate Environment Name | |
| id: generate_environment_name | |
| shell: bash | |
| run: | | |
| set -e | |
| TIMESTAMP_SHORT=$(date +%s | tail -c 5) # Last 4-5 digits of epoch seconds | |
| RANDOM_SUFFIX=$(head /dev/urandom | tr -dc 'a-z0-9' | head -c 8) # 8 random alphanum chars | |
| UNIQUE_ENV_NAME="${TIMESTAMP_SHORT}${RANDOM_SUFFIX}" # Usually ~12-13 chars | |
| echo "ENVIRONMENT_NAME=${UNIQUE_ENV_NAME}" >> $GITHUB_ENV | |
| echo "Generated ENVIRONMENT_NAME: ${UNIQUE_ENV_NAME}" | |
| - name: Run Deployment Script with Input | |
| shell: pwsh | |
| run: | | |
| cd Deployment | |
| $input = @" | |
| ${{ secrets.AZURE_TENANT_ID }} | |
| ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| ${{ env.ENVIRONMENT_NAME }} | |
| CanadaCentral | |
| ${{ env.VALID_REGION }} | |
| ${{ secrets.EMAIL }} | |
| yes | |
| "@ | |
| $input | pwsh ./resourcedeployment.ps1 | |
| Write-Host "Resource Group Name is ${{ env.rg_name }}" | |
| Write-Host "Kubernetes resource group are ${{ env.krg_name }}" | |
| env: | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| - name: Extract Web App URL and Increase TPM | |
| id: get_webapp_url | |
| shell: bash | |
| run: | | |
| # Save the resource group name and Kubernetes resource group name to GITHUB_OUTPUT | |
| echo "RESOURCE_GROUP_NAME=${{ env.rg_name }}" >> $GITHUB_OUTPUT | |
| echo "KUBERNETES_RESOURCE_GROUP_NAME=${{ env.krg_name }}" >> $GITHUB_OUTPUT | |
| echo "VALID_REGION=${{ env.VALID_REGION }}" >> $GITHUB_OUTPUT | |
| if az account show &> /dev/null; then | |
| echo "Azure CLI is authenticated." | |
| else | |
| echo "Azure CLI is not authenticated. Logging in..." | |
| az login --service-principal --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} | |
| fi | |
| az account set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| # Get the Web App URL and save it to GITHUB_OUTPUT | |
| echo "Retrieving Web App URL..." | |
| public_ip_name=$(az network public-ip list --resource-group ${{ env.krg_name }} --query "[?contains(name, 'kubernetes-')].name" -o tsv) | |
| fqdn=$(az network public-ip show --resource-group ${{ env.krg_name }} --name $public_ip_name --query "dnsSettings.fqdn" -o tsv) | |
| if [ -n "$fqdn" ]; then | |
| echo "WEBAPP_URL=https://$fqdn" >> $GITHUB_OUTPUT | |
| echo "Web App URL is https://$fqdn" | |
| else | |
| echo "Failed to retrieve Web App URL." | |
| exit 1 | |
| fi | |
| # Get Azure OpenAI resource name | |
| openai_resource_name=$(az cognitiveservices account list --resource-group ${{ env.rg_name }} --query "[?kind=='OpenAI'].name | [0]" -o tsv) | |
| if [ -z "$openai_resource_name" ]; then | |
| echo "No Azure OpenAI resource found in the resource group." | |
| exit 1 | |
| fi | |
| echo "OpenAI resource name is $openai_resource_name" | |
| echo "OPENAI_RESOURCE_NAME=$openai_resource_name" >> $GITHUB_OUTPUT | |
| # Get Azure Document Intelligence resource name | |
| document_intelligence_resource_name=$(az cognitiveservices account list --resource-group ${{ env.rg_name }} --query "[?kind=='FormRecognizer'].name | [0]" -o tsv) | |
| if [ -z "$document_intelligence_resource_name" ]; then | |
| echo "No Azure Document Intelligence resource found in the resource group." | |
| else | |
| echo "Document Intelligence resource name is $document_intelligence_resource_name" | |
| echo "DOCUMENT_INTELLIGENCE_RESOURCE_NAME=$document_intelligence_resource_name" >> $GITHUB_OUTPUT | |
| fi | |
| # Increase the TPM for the Azure OpenAI models | |
| echo "Increasing TPM for Azure OpenAI models..." | |
| openai_gpt_deployment_url="/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ env.rg_name }}/providers/Microsoft.CognitiveServices/accounts/$openai_resource_name/deployments/gpt-4o-mini?api-version=2023-05-01" | |
| az rest -m put -u "$openai_gpt_deployment_url" -b "{'sku':{'name':'GlobalStandard','capacity':${{ env.GPT_CAPACITY }}},'properties': {'model': {'format': 'OpenAI','name': 'gpt-4o-mini','version': '2024-07-18'}}}" | |
| if [ $? -ne 0 ]; then | |
| echo "Failed to increase TPM for GPT deployment." | |
| exit 1 | |
| else | |
| echo "Successfully increased TPM for GPT deployment." | |
| fi | |
| openai_embedding_deployment_url="/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ env.rg_name }}/providers/Microsoft.CognitiveServices/accounts/$openai_resource_name/deployments/text-embedding-large?api-version=2023-05-01" | |
| az rest -m put -u "$openai_embedding_deployment_url" -b "{'sku':{'name':'GlobalStandard','capacity': ${{ env.TEXT_EMBEDDING_CAPACITY }}},'properties': {'model': {'format': 'OpenAI','name': 'text-embedding-3-large','version': '1'}}}" | |
| if [ $? -ne 0 ]; then | |
| echo "Failed to increase TPM for Text Embedding deployment." | |
| exit 1 | |
| else | |
| echo "Successfully increased TPM for Text Embedding deployment." | |
| fi | |
| - name: Validate Deployment | |
| shell: bash | |
| run: | | |
| webapp_url="${{ steps.get_webapp_url.outputs.WEBAPP_URL }}" | |
| echo "Validating web app at: $webapp_url" | |
| # Enhanced health check with retry logic | |
| max_attempts=7 | |
| attempt=1 | |
| success=false | |
| while [ $attempt -le $max_attempts ] && [ "$success" = false ]; do | |
| echo "Attempt $attempt/$max_attempts: Checking web app health..." | |
| # Check if web app responds | |
| http_code=$(curl -s -o /dev/null -w "%{http_code}" "$webapp_url" || echo "000") | |
| if [ "$http_code" -eq 200 ]; then | |
| echo "✅ Web app is healthy (HTTP $http_code)" | |
| success=true | |
| elif [ "$http_code" -eq 404 ]; then | |
| echo "❌ Web app not found (HTTP 404)" | |
| break | |
| elif [ "$http_code" -eq 503 ] || [ "$http_code" -eq 502 ]; then | |
| echo "⚠️ Web app temporarily unavailable (HTTP $http_code), retrying..." | |
| sleep 20 | |
| else | |
| echo "⚠️ Web app returned HTTP $http_code, retrying..." | |
| sleep 20 | |
| fi | |
| attempt=$((attempt + 1)) | |
| done | |
| if [ "$success" = false ]; then | |
| echo "❌ Web app validation failed after $max_attempts attempts" | |
| exit 1 | |
| fi | |
| - name: Run Post Deployment Script | |
| shell: pwsh | |
| run: | | |
| Write-Host "Running post deployment script to upload files..." | |
| cd Deployment | |
| try { | |
| .\uploadfiles.ps1 -EndpointUrl ${{ steps.get_webapp_url.outputs.WEBAPP_URL }} | |
| Write-Host "ExitCode: $LASTEXITCODE" | |
| if ($LASTEXITCODE -eq $null -or $LASTEXITCODE -eq 0) { | |
| Write-Host "✅ Post deployment script completed successfully." | |
| } else { | |
| Write-Host "❌ Post deployment script failed with exit code: $LASTEXITCODE" | |
| exit 1 | |
| } | |
| } | |
| catch { | |
| Write-Host "❌ Post deployment script failed with error: $($_.Exception.Message)" | |
| exit 1 | |
| } | |
| - name: Logout from Azure | |
| if: always() | |
| shell: bash | |
| run: | | |
| if az account show &> /dev/null; then | |
| echo "Logging out from Azure..." | |
| az logout | |
| echo "Logged out from Azure successfully." | |
| else | |
| echo "Azure CLI is not authenticated. Skipping logout." | |
| fi | |
| e2e-test: | |
| needs: deploy | |
| uses: ./.github/workflows/test-automation.yml | |
| with: | |
| DKM_URL: ${{ needs.deploy.outputs.WEBAPP_URL }} | |
| secrets: inherit | |
| cleanup-deployment: | |
| if: always() | |
| needs: [deploy, e2e-test] | |
| runs-on: ubuntu-latest | |
| env: | |
| RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }} | |
| KUBERNETES_RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.KUBERNETES_RESOURCE_GROUP_NAME }} | |
| OPENAI_RESOURCE_NAME: ${{ needs.deploy.outputs.OPENAI_RESOURCE_NAME }} | |
| DOCUMENT_INTELLIGENCE_RESOURCE_NAME: ${{ needs.deploy.outputs.DOCUMENT_INTELLIGENCE_RESOURCE_NAME }} | |
| VALID_REGION: ${{ needs.deploy.outputs.VALID_REGION }} | |
| steps: | |
| - name: Install Azure CLI | |
| shell: bash | |
| run: | | |
| curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash | |
| az --version # Verify installation | |
| - name: Login to Azure | |
| shell: bash | |
| run: | | |
| az login --service-principal --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} | |
| az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}" | |
| - name: Delete Resource Groups | |
| if: env.RESOURCE_GROUP_NAME != '' | |
| shell: bash | |
| run: | | |
| az group delete --name ${{ env.RESOURCE_GROUP_NAME }} --yes --no-wait | |
| az group delete --name ${{ env.KUBERNETES_RESOURCE_GROUP_NAME }} --yes --no-wait | |
| - name: Wait for Resource Deletion to Complete | |
| shell: bash | |
| run: | | |
| echo "Waiting for Azure OpenaAI and Document Intelligence resources to be deleted..." | |
| sleep 60 | |
| retries=0 | |
| max_retries=3 | |
| sleep_duration=60 | |
| while [ $retries -lt $max_retries ]; do | |
| aoai_exists=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --name ${{ env.OPENAI_RESOURCE_NAME }} --query "[0].name" -o tsv) | |
| di_exists=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --name ${{ env.DOCUMENT_INTELLIGENCE_RESOURCE_NAME }} --query "[0].name" -o tsv) | |
| if [ -z "$aoai_exists" ] && [ -z "$di_exists" ]; then | |
| echo "Resources deleted successfully." | |
| break | |
| else | |
| echo "Resources still exist, retrying in $((sleep_duration * (retries + 1))) seconds..." | |
| sleep $((sleep_duration * (retries + 1))) | |
| retries=$((retries + 1)) | |
| fi | |
| done | |
| - name: Purging the Resources | |
| if: success() | |
| shell: bash | |
| run: | | |
| echo "Purging the Azure OpenAI and Document Intelligence resources..." | |
| if [ -z "${{ env.OPENAI_RESOURCE_NAME }}" ]; then | |
| echo "No Azure OpenAI resource to purge." | |
| else | |
| echo "Purging Azure OpenAI resource..." | |
| az cognitiveservices account purge --name ${{ env.OPENAI_RESOURCE_NAME }} --resource-group ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.VALID_REGION }} | |
| fi | |
| if [ -z "${{ env.DOCUMENT_INTELLIGENCE_RESOURCE_NAME }}" ]; then | |
| echo "No Azure Document Intelligence resource to purge." | |
| else | |
| echo "Purging Azure Document Intelligence resources..." | |
| az cognitiveservices account purge --name ${{ env.DOCUMENT_INTELLIGENCE_RESOURCE_NAME }} --resource-group ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.VALID_REGION }} | |
| fi | |
| - name: Send Notification on Failure | |
| if: failure() || needs.deploy.result == 'failure' | |
| shell: pwsh | |
| run: | | |
| # Define the RUN_URL variable | |
| $RUN_URL = "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| # Construct the email body using a Here-String | |
| $EMAIL_BODY = @" | |
| { | |
| "body": "<p>Dear Team,</p><p>The Document Knowledge Mining Automation process encountered an issue.</p><p><strong>Build URL:</strong> <a href='$RUN_URL'>$RUN_URL</a></p><p>Please investigate promptly.</p><p>Best regards,<br>Your Automation Team</p>" | |
| } | |
| "@ | |
| # Send the notification with error handling | |
| try { | |
| curl -X POST "${{ secrets.LOGIC_APP_URL }}" ` | |
| -H "Content-Type: application/json" ` | |
| -d "$EMAIL_BODY" | |
| } catch { | |
| Write-Output "Failed to send notification." | |
| } | |
| - name: Logout from Azure | |
| if: always() | |
| shell: bash | |
| run: | | |
| if az account show &> /dev/null; then | |
| echo "Logging out from Azure..." | |
| az logout | |
| echo "Logged out from Azure successfully." | |
| else | |
| echo "Azure CLI is not authenticated. Skipping logout." | |
| fi |