Adding MacOS and Windows hosts values to prod-02 (#9519)

meyrevived · web-flow · commit eecc329a4d97 · 2025-12-09T10:12:22.000Z
* KFLUXINFRA-2686: Adding MacOS and Windows hosts values to prod-02 * KFLUXINFRA-2686 II: reverting changes to p02's cluster-queue.yaml * KFLUXINFRA-2686 III: Fixing host-values.yaml for p02, adding a script-generated cluser-queue and updating MPC's version on p02 * KFLUXINFRA-2686 IV: Fixing host-values.yaml for p02 properly dividing windows and macos config values between dynamicConfigs and archDefaults * KFLUXINFRA-2686 V: Formatting the user-data scripts for macos and windows, reverting MPC's version update from production-downstream/base/ to only p02's kustomization.yaml * KFLUXINFRA-2686 VI: Fix for errors from forbid-clusterpolicies.yaml check * KFLUXINFRA-2686 VI: Adding fix to MacOS cloud-init script from #9513
diff --git a/components/kueue/production/stone-prod-p02/queue-config/cluster-queue.yaml b/components/kueue/production/stone-prod-p02/queue-config/cluster-queue.yaml
@@ -149,6 +149,8 @@ spec:
     - linux-x86-64
     - local
     - localhost
+    - macos-mac2metal-arm64
+    - windows-amd64
     flavors:
     - name: platform-group-3
       resources:
@@ -174,6 +176,10 @@ spec:
         nominalQuota: '1000'
       - name: localhost
         nominalQuota: '1000'
+      - name: macos-mac2metal-arm64
+        nominalQuota: '5'
+      - name: windows-amd64
+        nominalQuota: '5'
   stopPolicy: None
 ---
 apiVersion: kueue.x-k8s.io/v1beta1
diff --git a/components/multi-platform-controller/production-downstream/stone-prod-p02/host-values.yaml b/components/multi-platform-controller/production-downstream/stone-prod-p02/host-values.yaml
@@ -11,6 +11,15 @@ archDefaults:
     key-name: "konflux-prod-int-mab01"
     security-group-id: "sg-0903aedd465be979e"
     subnet-id: "subnet-02c476f8d2a4ae05e"
+  windows-amd64:
+    ami: "ami-0cf643428c5013531"
+    key-name: "konflux-prod-int-mab01"
+    security-group-id: "sg-0903aedd465be979e"
+    subnet-id: "subnet-02c476f8d2a4ae05e"
+  macos-mac2metal-arm64:
+    ami: "ami-000ce2c23b96216d3"
+    host-resource-group-arn: "arn:aws:resource-groups:us-east-1:381491906438:group/macos-servers"
+    license-configuration-arn: "arn:aws:license-manager:us-east-1:381491906438:license-configuration:lic-becd775af7ca09c097f1fa94e495e148"
 
 
 dynamicConfigs:
@@ -189,6 +198,246 @@ dynamicConfigs:
     sudo-commands: "/usr/bin/podman, /usr/bin/rm /usr/share/containers/mounts.conf"
     disk: "200"
 
+  macos-mac2metal-arm64:
+    user-data: |
+      #!/bin/bash
+      set -eu
+      set -x
+
+      user="konflux-builder"
+
+      # Check if user already exists
+      if ! id "$user" &>/dev/null; then
+        # Generate random password
+        random_password=$(openssl rand -base64 32)
+
+        # Create user
+        sudo sysadminctl -addUser "$user" -fullName "Konflux Builder" -password "$random_password" -home /Users/$user
+
+        # Clear password from variable
+        unset random_password
+      else
+        echo "User $user already exists, skipping user creation"
+      fi
+
+      # Create home directory if it doesn't exist
+      sudo mkdir -p /Users/$user
+
+      # Create SSH directory
+      sudo mkdir -p /Users/$user/.ssh
+
+      # Remove existing SSH keys if they exist
+      sudo rm -f /Users/$user/.ssh/id_rsa /Users/$user/.ssh/id_rsa.pub
+
+      # Generate new SSH keys
+      sudo ssh-keygen -t rsa -b 4096 -f /Users/$user/.ssh/id_rsa -N "" -C ""
+
+      # Set proper permissions on .ssh directory
+      sudo chmod 700 /Users/$user/.ssh
+
+      # Create/overwrite authorized_keys
+      sudo chmod 600 /Users/$user/.ssh/authorized_keys 2>/dev/null || true
+      sudo cat /Users/$user/.ssh/id_rsa.pub | sudo tee /Users/$user/.ssh/authorized_keys > /dev/null
+      sudo cat /Users/$user/.ssh/id_rsa | sudo tee /Users/ec2-user/$user > /dev/null
+
+      # Set ownership of entire home directory to ensure user has full control
+      sudo chown -R $user:staff /Users/$user
+
+      # Set ownership of the copied private key to ec2-user
+      sudo chown ec2-user:staff /Users/ec2-user/$user
+      sudo chmod 600 /Users/ec2-user/$user
+
+  windows-amd64:
+    user-data: |
+      <powershell>
+      ## -----------------------------------------
+      ## --------- Helper Functions --------------
+      ## -----------------------------------------
+      function Wait-Folder {
+        param(
+          [Parameter(Mandatory=$true)]
+          [string]$FolderPath,
+          [Parameter(Mandatory=$false)]
+          [int]$TimeoutSeconds = 30
+        )
+
+        Write-Host "Waiting for folder '${FolderPath}' to be created"
+
+        # Start a timer
+        $stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
+
+        while (-not (Test-Path -Path ${FolderPath})) {
+          # Check if we have exceeded the timeout
+          if ($stopwatch.Elapsed.TotalSeconds -ge $TimeoutSeconds) {
+            Write-Error "Timeout reached! Folder was not created within $TimeoutSeconds seconds."
+            $stopwatch.Stop()
+            return $false
+          }
+
+          Write-Host "Waiting for folder..." -NoNewline
+          Start-Sleep -Seconds 1
+        }
+
+        $stopwatch.Stop()
+        return $true
+      }
+
+      ## -----------------------------------------
+      ## --------- Create Local User -------------
+      ## -----------------------------------------
+      $user = "konflux-builder"
+
+      if ((Get-LocalUser -Name "${user}" -ErrorAction SilentlyContinue) -eq $null) {
+        # Generate random password
+        $password = (-join([char[]](33..122) | Get-Random -Count 30))
+        $securePassword = (ConvertTo-SecureString $password -AsPlainText -Force)
+
+        # Create user
+        New-LocalUser -Name $user -Password $securePassword -Description "Konflux Builder" | Out-Null
+        Add-LocalGroupMember -Group 'Users' -Member "${user}"
+        Add-LocalGroupMember -Group 'OpenSSH Users' -Member "${user}"
+
+        # Create a Credential Object for the new user
+        $userCred = New-Object System.Management.Automation.PSCredential($user, $securePassword)
+
+        # Start a dummy Process as the new User
+        # This is required to have the user home folder initialized.
+        # TODO: can we do better?
+        Start-Process -FilePath "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" `
+          -Credential ${userCred} `
+          -ArgumentList "-Command exit" `
+          -LoadUserProfile `
+          -WindowStyle Hidden `
+          -WorkingDirectory "C:\Users\" `
+          -Wait
+
+        # Create SSH Key for user
+        Write-Host "Creating SSH Key for user '${user}'"
+        $tempKey = "${env:TEMP}\${user}"
+
+        if (Test-Path "${tempKey}") { Remove-Item -Force "${tempKey}" }
+        if (Test-Path "${tempKey}.pub") { Remove-Item -Force "${tempKey}.pub" }
+
+        ssh-keygen -t rsa -f "${tempKey}" -N `"`" | Out-Null
+
+        # Move private key to a secure location and restrict access to it
+        $privateKeyPath = "C:\Users\Administrator\${user}"
+        mv "${env:TEMP}\${user}" "${privateKeyPath}"
+
+        $ACL = Get-Acl "${privateKeyPath}"
+        $Ar = New-Object System.Security.AccessControl.FileSystemAccessRule("NT AUTHORITY\SYSTEM", "FullControl", "Allow")
+        $ACL.SetAccessRule($Ar)
+        $Ar = New-Object System.Security.AccessControl.FileSystemAccessRule("BUILTIN\Administrators", "FullControl", "Allow")
+        $ACL.SetAccessRule($Ar)
+        Set-Acl "${privateKeyPath}" ${ACL}
+
+        # Initialize user home folder
+        $userHome = "C:\Users\${user}"
+        Write-Host "Waiting for User Home '${userHome}' to be created"
+
+        # Ensure User's home folder is eventually created
+        if (-not (Wait-Folder -FolderPath ${userHome})) {
+          Write-Error "Folder '${userHome}' not found! Cleanup..." -ForegroundColor Red
+          if (Test-Path "\${tempKey}") { Remove-Item -Force "${tempKey}" }
+          if (Test-Path "\${tempKey}.pub") { Remove-Item -Force "${tempKey}.pub" }
+          exit 1
+        }
+
+        # Set up SSH Keys for User
+        Write-Host "User Home found. Configuring SSH access" -ForegroundColor Green
+        New-Item -ItemType Directory -Force -Path "${userHome}\.ssh"
+        New-Item -ItemType Directory -Force -Path "${userHome}\build"
+
+        # Copying and removing to preserve file permissions! Do not use `mv`! :)
+        cp "${tempKey}.pub" "${userHome}\.ssh\authorized_keys"
+        rm "${tempKey}.pub"
+      }
+
+      ## -------------------------------------------------
+      ## --------- Enable Windows Containers -------------
+      ## -------------------------------------------------
+      Invoke-WebRequest -UseBasicParsing "https://raw.githubusercontent.com/microsoft/Windows-Containers/Main/helpful_tools/Install-DockerCE/install-docker-ce.ps1" -o install-docker-ce.ps1
+      .\install-docker-ce.ps1 -NoRestart
+
+      if (${global:RebootRequired}) {
+        Restart-Computer
+        exit
+      }
+
+      # Create docker-users group and add konflux-builder to it
+      if ((Get-LocalGroup -Name 'docker-users') -eq $null) {
+        New-LocalGroup -Name 'docker-users' -Description 'Docker Users'
+      }
+
+      if ((Get-LocalGroupMember -Group 'docker-users' -Member 'konflux-builder') -eq $null) {
+        Add-LocalGroupMember -Group 'docker-users' -Member "${user}"
+      }
+
+      # Allow the docker-users group to use docker
+      $dockerConfigPath = "C:\ProgramData\docker\config\daemon.json"
+      $existingConfig = Get-Content $dockerConfigPath -Raw | ConvertFrom-Json
+
+      if ((${existingConfig}.group) -eq $null) {
+        $existingConfig | Add-Member -NotePropertyName "group" -NotePropertyValue "docker-users" -Force
+        $existingConfig | ConvertTo-Json -Depth 10 | Set-Content $dockerConfigPath
+        Restart-Service docker
+      }
+
+      # Exclude docker in Windows Defender
+      Add-MpPreference -ExclusionProcess "dockerd.exe"
+      Add-MpPreference -ExclusionProcess "docker.exe"
+      Add-MpPreference -ExclusionProcess "containerd.exe"
+      Add-MpPreference -ExclusionProcess "vmcompute.exe"
+
+      ## -----------------------------------------
+      ## --------- Configure OpenSSH -------------
+      ## -----------------------------------------
+      # Install OpenSSH Server
+      Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
+
+      # Start the sshd service and set it to start automatically
+      Start-Service sshd
+      Set-Service -Name sshd -StartupType 'Automatic'
+
+      # Grab the Public Key from AWS Metadata and configure authorized_keys
+      # This allows you to log in with your .pem/.ppk file instead of a password
+      $MAGIC_IP = "169.254.169.254"
+      $IMDS_TOKEN = Invoke-RestMethod -Uri "http://${MAGIC_IP}/latest/api/token" -Method 'PUT' -Headers @{'X-aws-ec2-metadata-token-ttl-seconds' = '21600'}
+      $PUBKEY = Invoke-RestMethod -Uri "http://${MAGIC_IP}/latest/meta-data/public-keys/0/openssh-key" -Headers @{'X-aws-ec2-metadata-token' = $IMDS_TOKEN}
+
+      # Ensure SSH_PATH folder was created
+      $SSH_PATH = "C:\ProgramData\ssh"
+      Write-Host "Waiting for SSH Folder"
+
+      if (-not (Wait-Folder -FolderPath ${SSH_PATH})) {
+        Write-Error "Folder '${SSH_PATH}' not found! Exiting..." -ForegroundColor Red
+        exit 1
+      }
+
+      Write-Host "Folder '${SSH_PATH}' found"
+
+      # Add key to administrators_authorized_keys
+      $PUBKEY | Out-File -FilePath "$SSH_PATH\administrators_authorized_keys" -Encoding ascii
+
+      # Fix permissions (ACLs) for the authorized_keys file
+      # OpenSSH is strict: only System and Administrators should have access
+      $ACL = Get-Acl "$SSH_PATH\administrators_authorized_keys"
+      $Ar = New-Object System.Security.AccessControl.FileSystemAccessRule("NT AUTHORITY\SYSTEM", "FullControl", "Allow")
+      $ACL.SetAccessRule($Ar)
+      $Ar = New-Object System.Security.AccessControl.FileSystemAccessRule("BUILTIN\Administrators", "FullControl", "Allow")
+      $ACL.SetAccessRule($Ar)
+      Set-Acl "$SSH_PATH\administrators_authorized_keys" $ACL
+
+      # Restart sshd to apply key changes
+      Restart-Service sshd
+
+      # Configure the Firewall to allow SSH (Port 22)
+      Remove-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' | Out-Null
+      New-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 -RemoteAddress any
+      Write-Output "Firewall Rule 'OpenSSH-Server-In-TCP' updated"
+      </powershell>
+      <persist>true</persist>
+
   linux-root-amd64:
     iops: "16000"
     throughput: "1000"
diff --git a/components/multi-platform-controller/production-downstream/stone-prod-p02/kustomization.yaml b/components/multi-platform-controller/production-downstream/stone-prod-p02/kustomization.yaml
@@ -4,8 +4,14 @@ kind: Kustomization
 namespace: multi-platform-controller
 
 resources:
-- ../base
+- ../../base/common
+- ../../base/rbac
 - external-secrets.yaml
+- https://github.com/konflux-ci/multi-platform-controller/deploy/operator?ref=6347caa2deb2dce08f4f3a100cb96f23ae7f14c6
+- https://github.com/konflux-ci/multi-platform-controller/deploy/otp?ref=6347caa2deb2dce08f4f3a100cb96f23ae7f14c6
+
+components:
+  - ../../k-components/manager-resources
 
 helmGlobals:
   chartHome: ../../base
@@ -15,3 +21,11 @@ helmCharts:
   releaseName: host-config
   namespace: multi-platform-controller
   valuesFile: host-values.yaml
+
+images:
+  - name: multi-platform-controller
+    newName: quay.io/konflux-ci/multi-platform-controller
+    newTag: 6347caa2deb2dce08f4f3a100cb96f23ae7f14c6
+  - name: multi-platform-otp-server
+    newName: quay.io/konflux-ci/multi-platform-controller-otp-service
+    newTag: 6347caa2deb2dce08f4f3a100cb96f23ae7f14c6
