Introduce multi-version support for e2e tests

Fix issue with podman

Remove exit(1)

Author: henrybear327

.github/env

@@ -1,3 +1,3 @@
 golang-version=1.23
 kind-version=v0.25.0
-kind-image=kindest/node:v1.31.2
+kind-image=kindest/node:v1.32.0

.github/workflows/e2e-tests.yaml

@@ -17,8 +17,8 @@ env:
   HTTPS_ENABLE: true
   NETBOX_RESTORATION_HASH_FIELD_NAME: netboxOperatorRestorationHash
 jobs:
-  e2e-tests:
-    name: E2E tests for netbox operator
+  e2e-tests-3-7-8:
+    name: Against netbox version 3.7.8
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
@@ -28,7 +28,6 @@ jobs:
     - name: Import environment variables from file
       run: |
         cat ".github/env" >> "$GITHUB_ENV"
-        echo "E2E_DIAGNOSTIC_DIRECTORY=$(mktemp -d)" >> "$GITHUB_ENV"
     - name: Start kind cluster
       uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
       with:
@@ -47,27 +46,70 @@ jobs:
         kubectl get pods -A
         echo "Cluster information"
         kubectl cluster-info
-    - name: Setup kind cluster with required software such as NetBox
+    - name: Run e2e tests
       run: |
-        make create-kind
-    - name: Deploy NetBox operator to the kind cluster
+        make test-e2e-3.7.8
+  e2e-tests-4-0-11:
+    name: Against netbox version 4.0.11
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      with: 
+        go-version: 1.23.4
+    - name: Import environment variables from file
       run: |
-        make deploy-kind
-    - name: Run tests
-      env:
-        E2E_DIAGNOSTIC_DIRECTORY: ${{ env.E2E_DIAGNOSTIC_DIRECTORY }}
+        cat ".github/env" >> "$GITHUB_ENV"
+    - name: Start kind cluster
+      uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+      with:
+        version: ${{ env.kind-version }}
+        node_image: ${{ env.kind-image }}
+        wait: 300s
+        config: ./tests/e2e/kind-config.yaml
+        cluster_name: e2e
+    - name: Wait for cluster to finish bootstraping
+      run: |
+        echo "Waiting for all nodes to be ready..."
+        kubectl wait --for=condition=Ready nodes --all --timeout=120s
+        kubectl get nodes
+        echo "Waiting for all pods to be ready..."
+        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s
+        kubectl get pods -A
+        echo "Cluster information"
+        kubectl cluster-info
+    - name: Run e2e tests
       run: |
-        # # Very straight forward way of implementing a test and checking the result
-        # kubectl apply -f config/samples/netbox_v1_prefixclaim.yaml
-        # kubectl get prefixclaim,prefix,ipaddressclaim,ipaddress,iprange,iprangeclaim
-        # kubectl wait --for=condition=ready --timeout=30s prefixclaim.netbox.dev/prefixclaim-sample
-
-        # Use Chainsaw
-        make test-e2e
-    - name: Upload diagnostics artifact
-      if: ${{ failure() }}
-      uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        make test-e2e-4.0.11
+  e2e-tests-4-1-8:
+    name: Against netbox version 4.1.8
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      with: 
+        go-version: 1.23.4
+    - name: Import environment variables from file
+      run: |
+        cat ".github/env" >> "$GITHUB_ENV"
+    - name: Start kind cluster
+      uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
       with:
-        name: cluster-state
-        path: ${{ env.E2E_DIAGNOSTIC_DIRECTORY }}
-        retention-days: 15
+        version: ${{ env.kind-version }}
+        node_image: ${{ env.kind-image }}
+        wait: 300s
+        config: ./tests/e2e/kind-config.yaml
+        cluster_name: e2e
+    - name: Wait for cluster to finish bootstraping
+      run: |
+        echo "Waiting for all nodes to be ready..."
+        kubectl wait --for=condition=Ready nodes --all --timeout=120s
+        kubectl get nodes
+        echo "Waiting for all pods to be ready..."
+        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s
+        kubectl get pods -A
+        echo "Cluster information"
+        kubectl cluster-info
+    - name: Run e2e tests
+      run: |
+        make test-e2e-4.1.8

.gitignore

@@ -9,7 +9,10 @@ Dockerfile.cross
 
 # Test binary, built with `go test -c`
 *.test
-database.sql
+
+# Test/Intermediate files
+kind/load-data-job/load-data.sh
+kind/load-data-job/dockerfile
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out

Makefile

@@ -237,6 +237,25 @@ generate_mocks: ## TODO: auto install go install go.uber.org/mock/mockgen@latest
 	mkdir -p ${GEN_DIR}
 	mockgen -destination ${GEN_DIR}/${NETBOX_MOCKS_OUTPUT_FILE} -source=${INTERFACE_DEFITIONS_DIR}
 
-.PHONY: test-e2e
-test-e2e: install-$(GO_PACKAGE_NAME_CHAINSAW)
-	chainsaw test --namespace e2e
+# e2e tests
+E2E_PARAM := --namespace e2e --parallel 3 --apply-timeout 3m --assert-timeout 3m --delete-timeout 3m --error-timeout 3m --exec-timeout 3m # --skip-delete (add this argument for local debugging)
+.PHONY: create-kind-3.7.8
+create-kind-3.7.8:
+	./kind/local-env.sh --version 3.7.8
+.PHONY: test-e2e-3.7.8
+test-e2e-3.7.8: create-kind-3.7.8 deploy-kind install-$(GO_PACKAGE_NAME_CHAINSAW) 
+	chainsaw test $(E2E_PARAM)
+
+.PHONY: create-kind-4.0.11
+create-kind-4.0.11:
+	./kind/local-env.sh --version 4.0.11
+.PHONY: test-e2e-4.0.11
+test-e2e-4.0.11: create-kind-4.0.11 deploy-kind install-$(GO_PACKAGE_NAME_CHAINSAW) 
+	chainsaw test $(E2E_PARAM)
+
+.PHONY: create-kind-4.1.8
+create-kind-4.1.8:
+	./kind/local-env.sh --version 4.1.8
+.PHONY: test-e2e-4.1.8
+test-e2e-4.1.8: create-kind-4.1.8 deploy-kind install-$(GO_PACKAGE_NAME_CHAINSAW) 
+	chainsaw test $(E2E_PARAM)

kind/load-data-job/README.md

@@ -0,0 +1,7 @@
+# load-data-job
+
+Due to database schema changes cross major/minor NetBox versions, we have to `patch` the SQL files and demo data link on-the-fly.
+
+The default values stems from the NetBox 4.1.x version. So the patching will only happen for 3.7.x and 4.0.x versions. 
+
+Please see `../local-env.sh`, that's where all the patching happen.

kind/load-data-job/dockerfile renamed to kind/load-data-job/dockerfile.orig

@@ -1,4 +1,4 @@
 FROM python:3.12
 ADD main.py .
-RUN pip install pynetbox
+RUN pip install -Iv pynetbox==7.4.1
 CMD ["python", "./main.py"] 

kind/load-data-job/load-data.sh renamed to kind/load-data-job/load-data.orig.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -o errexit
 
 TMP_SQL_FILE=$(mktemp /tmp/netbox-data-dump.XXXXXXX.sql) || exit 1
 curl -k https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql > "${TMP_SQL_FILE}"

kind/load-data-job/main.py

@@ -1,7 +1,6 @@
 import pynetbox
 from pprint import pprint
 from dataclasses import dataclass
-import sys
 
 print("Starting to load data onto NetBox through API")
 try:
@@ -11,7 +10,7 @@
     )
 except pynetbox.RequestError as e:
     pprint(e.error)
-    sys.exit(1)
+   
 print("Connected to NetBoxAPI")
 
 # insert Tenants
@@ -47,7 +46,7 @@ class Tenant:
         )
     except pynetbox.RequestError as e:
         pprint(e.error)
-        sys.exit(1)
+       
 print("Tenants loaded")
 
 # insert Sites
@@ -88,13 +87,14 @@ class Site:
         )
     except pynetbox.RequestError as e:
         pprint(e.error)
-        sys.exit(1)
+       
 print("Sites loaded")
 
 # create custom fields and associate custom fields with IP/IPRange/Prefix
 @dataclass
 class CustomField:
-    object_types: list[str]
+    content_types: list[str] # for v3
+    object_types: list[str] # for v4
     type: str
     name: str
     label: str
@@ -104,6 +104,7 @@ class CustomField:
 
 custom_fields = [
     CustomField(
+        content_types=["ipam.ipaddress", "ipam.iprange", "ipam.prefix"],
         object_types=["ipam.ipaddress", "ipam.iprange", "ipam.prefix"],
         type="text",
         name="netboxOperatorRestorationHash",
@@ -113,6 +114,7 @@ class CustomField:
         filter_logic="exact"
     ),
     CustomField(
+        content_types=["ipam.ipaddress", "ipam.iprange", "ipam.prefix"],
         object_types=["ipam.ipaddress", "ipam.iprange", "ipam.prefix"],
         type="text",
         name="example_field",
@@ -122,6 +124,7 @@ class CustomField:
         filter_logic="exact"
     ),
     CustomField(
+        content_types=["ipam.prefix"],
         object_types=["ipam.prefix"],
         type="text",
         name="environment",
@@ -131,6 +134,7 @@ class CustomField:
         filter_logic="exact"
     ),
     CustomField(
+        content_types=["ipam.prefix"],
         object_types=["ipam.prefix"],
         type="text",
         name="poolName",
@@ -140,6 +144,7 @@ class CustomField:
         filter_logic="exact"
     ),
     CustomField(
+        content_types=["ipam.prefix"],
         object_types=["ipam.prefix"],
         type="boolean",
         name="cfDataTypeBool",
@@ -149,6 +154,7 @@ class CustomField:
         filter_logic="exact"
     ),
     CustomField(
+        content_types=["ipam.prefix"],
         object_types=["ipam.prefix"],
         type="integer",
         name="cfDataTypeInteger",
@@ -162,6 +168,7 @@ class CustomField:
 for custom_field in custom_fields:
     try:
         nb.extras.custom_fields.create(
+            content_types=custom_field.content_types,
             object_types=custom_field.object_types,
             type=custom_field.type,
             name=custom_field.name,
@@ -173,7 +180,7 @@ class CustomField:
         )
     except pynetbox.RequestError as e:
         pprint(e.error)
-        sys.exit(1)
+       
 print("Custom fields loaded")
 
 # for debugging
@@ -461,5 +468,5 @@ class Prefix:
         )
     except pynetbox.RequestError as e:
         pprint(e.error)
-        sys.exit(1)
+       
 print("Prefixes loaded")