Enable unattended upgrades on Ubuntu nodes for soak testing

Author: Supreeth095

.github/workflows/integration-tests.yaml

@@ -1,6 +1,9 @@
 name: Integration tests
 
 on:
+  workflow_dispatch: {}
+  schedule:
+    - cron: "0 3 * * *" # every night
   push:
     branches:
       - "master"

scripts/lib/cluster.sh

@@ -30,6 +30,9 @@ function up-test-cluster() {
     if [[ "$RUN_BOTTLEROCKET_TEST" == true ]]; then
         echo "Copying bottlerocket config to $CLUSTER_CONFIG"
         cp $CLUSTER_TEMPLATE_PATH/bottlerocket.yaml $CLUSTER_CONFIG
+    elif [[ "$RUN_UBUNTU_TEST" == true ]]; then
+        echo "Copying ubuntu config to $CLUSTER_CONFIG"
+        cp $CLUSTER_TEMPLATE_PATH/ubuntu.yaml $CLUSTER_CONFIG
     elif [[ "$RUN_PERFORMANCE_TESTS" == true ]]; then
         echo "Copying perf test cluster config to $CLUSTER_CONFIG"
         cp $CLUSTER_TEMPLATE_PATH/perf-cluster.yml $CLUSTER_CONFIG

scripts/run-integration-tests.sh

@@ -29,6 +29,7 @@ ARCH=$(go env GOARCH)
 : "${RUN_KOPS_TEST:=false}"
 : "${RUN_BOTTLEROCKET_TEST:=false}"
 : "${RUN_PERFORMANCE_TESTS:=false}"
+: "${RUN_SOAK_TEST:=false}"
 : "${RUNNING_PERFORMANCE:=false}"
 : "${KOPS_VERSION=v1.33.0-beta.1}"
 
@@ -57,6 +58,9 @@ on_error() {
     if [[ $RUN_PERFORMANCE_TESTS == true ]]; then
         emit_cloudwatch_metric "performance_test_status" "0"
     fi
+    if [[ $RUN_SOAK_TEST == true ]]; then
+        emit_cloudwatch_metric "soak_test_status" "0"
+    fi
     # Make sure we destroy any cluster that was created if we hit run into an
     # error when attempting to run tests against the 
     if [[ $RUNNING_PERFORMANCE == false ]]; then
@@ -284,6 +288,30 @@ if [[ "$RUN_PERFORMANCE_TESTS" == true ]]; then
     uninstall_cw_agent
 fi
 
+if [[ "$RUN_SOAK_TEST" == true ]]; then
+    echo "*******************************************************************************"
+    echo "Running soak tests on current image"
+    echo ""
+    START=$SECONDS
+    
+    GINKGO_TEST_BUILD="$SCRIPT_DIR/../test/build"
+    TEST_IMAGE_REGISTRY=${TEST_IMAGE_REGISTRY:-"617930562442.dkr.ecr.us-west-2.amazonaws.com"}
+    
+    (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="SOAK_TEST" -v --timeout 3h --fail-on-pending $GINKGO_TEST_BUILD/cni.test -- \
+        --cluster-kubeconfig="$KUBECONFIG" \
+        --cluster-name="$CLUSTER_NAME" \
+        --aws-region="$REGION" \
+        --aws-vpc-id="$VPC_ID" \
+        --ng-name-label-key="kubernetes.io/os" \
+        --ng-name-label-val="linux" \
+        --test-image-registry=$TEST_IMAGE_REGISTRY \
+        --publish-cw-metrics=true \
+        $ENDPOINT_OPTION)
+    
+    SOAK_DURATION=$((SECONDS - START))
+    echo "TIMELINE: Soak tests took $SOAK_DURATION seconds."
+fi
+
 if [[ $RUN_KOPS_TEST == true ]]; then
     run_kops_conformance
 fi
@@ -299,6 +327,9 @@ if [[ "$DEPROVISION" == true ]]; then
     elif [[ "$RUN_PERFORMANCE_TESTS" == true ]]; then
         eksctl delete cluster $CLUSTER_NAME
         emit_cloudwatch_metric "performance_test_status" "1"
+    elif [[ "$RUN_SOAK_TEST" == true ]]; then
+        eksctl delete cluster $CLUSTER_NAME --disable-nodegroup-eviction
+        emit_cloudwatch_metric "soak_test_status" "1"
     else
         down-test-cluster
     fi

scripts/test/config/ubuntu.yaml

@@ -0,0 +1,52 @@
+---
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+
+metadata:
+  name: CLUSTER_NAME_PLACEHOLDER
+  region: us-west-2
+
+managedNodeGroups:
+  - name: cni-test-arm64-ubuntu-mng
+    instanceType: c6g.xlarge
+    desiredCapacity: 3
+    amiFamily: UbuntuPro2404
+    minSize: 3
+    maxSize: 3
+    volumeSize: 40
+    releaseVersion: ""
+    preBootstrapCommands:
+      - |
+        # Enable unattended upgrades for soak testing
+        apt-get update
+        apt-get install -y unattended-upgrades
+        echo 'Unattended-Upgrade::Automatic-Reboot "false";' >> /etc/apt/apt.conf.d/50unattended-upgrades
+        echo 'Unattended-Upgrade::Remove-Unused-Dependencies "true";' >> /etc/apt/apt.conf.d/50unattended-upgrades
+        systemctl enable unattended-upgrades
+        systemctl start unattended-upgrades
+    tags:
+      group: amazon-vpc-cni-k8s-arm64-ubuntu
+  - name: cni-test-x86-ubuntu-mng
+    instanceType: c5.xlarge
+    desiredCapacity: 3
+    amiFamily: UbuntuPro2404
+    minSize: 3
+    maxSize: 3
+    volumeSize: 40
+    releaseVersion: ""
+    preBootstrapCommands:
+      - |
+        # Enable unattended upgrades for soak testing
+        apt-get update
+        apt-get install -y unattended-upgrades
+        echo 'Unattended-Upgrade::Automatic-Reboot "false";' >> /etc/apt/apt.conf.d/50unattended-upgrades
+        echo 'Unattended-Upgrade::Remove-Unused-Dependencies "true";' >> /etc/apt/apt.conf.d/50unattended-upgrades
+        systemctl enable unattended-upgrades
+        systemctl start unattended-upgrades
+    tags:
+      group: amazon-vpc-cni-k8s-x86-ubuntu
+availabilityZones:
+  - us-west-2a
+  - us-west-2b
+  - us-west-2c
+  - us-west-2d