Add e2e tests for ConfigMap feature (#558)

* wip - ensure e2e tests for configmap versioning

Signed-off-by: Katyanna Moura <[email protected]>

* Enable ConfigMap support on e2e tests

* enable versioned configmaps during e2e tests

* pass testing variable to stackset factory to make assertions

* add e2e tests for preventing traffic switch when configmap is missing

---------

Signed-off-by: Katyanna Moura <[email protected]>
Co-authored-by: Martin Linkhorst <[email protected]>

Author: katyanna

cmd/e2e/README.md

@@ -32,7 +32,7 @@ watch -n 10 "kubectl get -n foo stackset,stack,ing,ep,deployment"
 kubectl delete namespace foo; kubectl create namespace foo
 make
 ./build/stackset-controller --apiserver=http://127.0.0.1:8001 \
---enable-routegroup-support --controller-id=foo \
+--enable-configmap-support --enable-routegroup-support --controller-id=foo \
 --cluster-domain=${CLUSTER_DOMAIN} --cluster-domain=${CLUSTER_DOMAIN_INTERNAL}
 ```
 4. rebuild e2e test and run e2e tests in `foo` namespace

cmd/e2e/basic_test.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"slices"
 	"sort"
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	rgv1 "github.com/szuecs/routegroup-client/apis/zalando.org/v1"
 	"github.com/zalando-incubator/stackset-controller/controller"
@@ -16,6 +18,7 @@ import (
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
@@ -35,6 +38,7 @@ type TestStacksetSpecFactory struct {
 	ingress                       bool
 	routegroup                    bool
 	externalIngress               bool
+	configMap                     bool
 	limit                         int32
 	scaleDownTTL                  int64
 	replicas                      int32
@@ -51,6 +55,7 @@ type TestStacksetSpecFactory struct {
 func NewTestStacksetSpecFactory(stacksetName string) *TestStacksetSpecFactory {
 	return &TestStacksetSpecFactory{
 		stacksetName:           stacksetName,
+		configMap:              false,
 		ingress:                false,
 		externalIngress:        false,
 		limit:                  4,
@@ -62,6 +67,11 @@ func NewTestStacksetSpecFactory(stacksetName string) *TestStacksetSpecFactory {
 	}
 }
 
+func (f *TestStacksetSpecFactory) ConfigMap() *TestStacksetSpecFactory {
+	f.configMap = true
+	return f
+}
+
 func (f *TestStacksetSpecFactory) Behavior(stabilizationWindowSeconds int32) *TestStacksetSpecFactory {
 	f.hpaBehavior = true
 	f.hpaStabilizationWindowSeconds = stabilizationWindowSeconds
@@ -103,7 +113,7 @@ func (f *TestStacksetSpecFactory) SubResourceAnnotations(annotations map[string]
 	return f
 }
 
-func (f *TestStacksetSpecFactory) Create(stackVersion string) zv1.StackSetSpec {
+func (f *TestStacksetSpecFactory) Create(t *testing.T, stackVersion string) zv1.StackSetSpec {
 	var result = zv1.StackSetSpec{
 		StackLifecycle: zv1.StackLifecycle{
 			Limit:               pint32(f.limit),
@@ -134,6 +144,42 @@ func (f *TestStacksetSpecFactory) Create(stackVersion string) zv1.StackSetSpec {
 		},
 	}
 
+	if f.configMap {
+		configMapName := fmt.Sprintf("%s-%s-configmap", f.stacksetName, stackVersion)
+		configMap := &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: configMapName,
+			},
+			Data: map[string]string{
+				"key": "value",
+			},
+		}
+
+		_, err := configMapInterface().Create(context.Background(), configMap, metav1.CreateOptions{})
+		require.NoError(t, err)
+
+		result.StackTemplate.Spec.ConfigurationResources = []zv1.ConfigurationResourcesSpec{
+			{
+				ConfigMapRef: corev1.LocalObjectReference{
+					Name: configMapName,
+				},
+			},
+		}
+
+		result.StackTemplate.Spec.StackSpec.PodTemplate.Spec.Volumes = []corev1.Volume{
+			{
+				Name: "config-volume",
+				VolumeSource: corev1.VolumeSource{
+					ConfigMap: &corev1.ConfigMapVolumeSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: configMapName,
+						},
+					},
+				},
+			},
+		}
+	}
+
 	if f.autoscaler {
 		result.StackTemplate.Spec.Autoscaler = &zv1.Autoscaler{
 			MaxReplicas: f.hpaMaxReplicas,
@@ -287,6 +333,23 @@ func verifyStack(t *testing.T, stacksetName, currentVersion string, stacksetSpec
 		require.EqualValues(t, expectedRef, hpa.Spec.ScaleTargetRef)
 	}
 
+	// Verify ConfigMaps
+	for _, rsc := range stacksetSpec.StackTemplate.Spec.ConfigurationResources {
+		configMap, err := waitForConfigMap(t, rsc.ConfigMapRef.Name)
+		require.NoError(t, err)
+		assert.EqualValues(t, stackResourceLabels, configMap.Labels)
+		assert.Contains(t, configMap.Name, stack.Name)
+		assert.Equal(t, map[string]string{"key": "value"}, configMap.Data)
+		assert.Equal(t, []metav1.OwnerReference{
+			{
+				APIVersion: "zalando.org/v1",
+				Kind:       "Stack",
+				Name:       stack.Name,
+				UID:        stack.UID,
+			},
+		}, configMap.OwnerReferences)
+	}
+
 	verifyStackIngressSources(
 		t,
 		stack,
@@ -576,6 +639,7 @@ func verifyStacksetRouteGroup(t *testing.T, stacksetName string, stacksetSpec zv
 func testStacksetCreate(
 	t *testing.T,
 	testName string,
+	configmap bool,
 	hpa,
 	ingress,
 	routegroup,
@@ -589,6 +653,9 @@ func testStacksetCreate(
 		stacksetName := fmt.Sprintf("stackset-create-%s-%s", ingType, testName)
 		stackVersion := "v1"
 		stacksetSpecFactory := NewTestStacksetSpecFactory(stacksetName)
+		if configmap {
+			stacksetSpecFactory.ConfigMap()
+		}
 		if hpa {
 			stacksetSpecFactory.Autoscaler(
 				1,
@@ -611,7 +678,7 @@ func testStacksetCreate(
 		if len(subResourceAnnotations) > 0 {
 			stacksetSpecFactory.SubResourceAnnotations(subResourceAnnotations)
 		}
-		stacksetSpec := stacksetSpecFactory.Create(stackVersion)
+		stacksetSpec := stacksetSpecFactory.Create(t, stackVersion)
 
 		var err error
 		switch ingType {
@@ -730,7 +797,7 @@ func testStacksetUpdate(
 				oldSubResourceAnnotations,
 			)
 		}
-		stacksetSpec := stacksetSpecFactory.Create(initialVersion)
+		stacksetSpec := stacksetSpecFactory.Create(t, initialVersion)
 
 		var err error
 		switch ingType {
@@ -840,7 +907,7 @@ func testStacksetUpdate(
 			)
 		}
 
-		updatedSpec := stacksetSpecFactory.Create(updatedVersion)
+		updatedSpec := stacksetSpecFactory.Create(t, updatedVersion)
 		err = updateStackset(stacksetName, updatedSpec)
 		require.NoError(t, err)
 
@@ -941,27 +1008,31 @@ func testStacksetUpdate(
 }
 
 func TestStacksetCreateBasic(t *testing.T) {
-	testStacksetCreate(t, "basic", false, false, false, false, false, testAnnotationsCreate)
+	testStacksetCreate(t, "basic", false, false, false, false, false, false, testAnnotationsCreate)
+}
+
+func TestStacksetCreateConfigMap(t *testing.T) {
+	testStacksetCreate(t, "configmap", true, false, false, false, false, false, testAnnotationsCreate)
 }
 
 func TestStacksetCreateHPA(t *testing.T) {
-	testStacksetCreate(t, "hpa", true, false, false, false, false, testAnnotationsCreate)
+	testStacksetCreate(t, "hpa", false, true, false, false, false, false, testAnnotationsCreate)
 }
 
 func TestStacksetCreateIngress(t *testing.T) {
-	testStacksetCreate(t, "ingress", false, true, false, false, false, testAnnotationsCreate)
+	testStacksetCreate(t, "ingress", false, false, true, false, false, false, testAnnotationsCreate)
 }
 
 func TestStacksetCreateRouteGroup(t *testing.T) {
-	testStacksetCreate(t, "routegroup", false, false, true, false, false, testAnnotationsCreate)
+	testStacksetCreate(t, "routegroup", false, false, false, true, false, false, testAnnotationsCreate)
 }
 
 func TestStacksetCreateExternalIngress(t *testing.T) {
-	testStacksetCreate(t, "externalingress", false, false, false, true, false, testAnnotationsCreate)
+	testStacksetCreate(t, "externalingress", false, false, false, false, true, false, testAnnotationsCreate)
 }
 
 func TestStacksetCreateUpdateStrategy(t *testing.T) {
-	testStacksetCreate(t, "updatestrategy", false, false, false, false, true, testAnnotationsCreate)
+	testStacksetCreate(t, "updatestrategy", false, false, false, false, false, true, testAnnotationsCreate)
 }
 
 func TestStacksetUpdateBasic(t *testing.T) {

cmd/e2e/broken_stack_test.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
@@ -18,15 +20,15 @@ func TestBrokenStacks(t *testing.T) {
 
 	firstVersion := "v1"
 	firstStack := fmt.Sprintf("%s-%s", stacksetName, firstVersion)
-	spec := factory.Create(firstVersion)
+	spec := factory.Create(t, firstVersion)
 	err := createStackSet(stacksetName, 0, spec)
 	require.NoError(t, err)
 	_, err = waitForStack(t, stacksetName, firstVersion)
 	require.NoError(t, err)
 
 	unhealthyVersion := "v2"
 	unhealthyStack := fmt.Sprintf("%s-%s", stacksetName, unhealthyVersion)
-	spec = factory.Create(unhealthyVersion)
+	spec = factory.Create(t, unhealthyVersion)
 	spec.StackTemplate.Spec.Service.Ports = []corev1.ServicePort{
 		{
 			Protocol:   corev1.ProtocolTCP,
@@ -55,7 +57,7 @@ func TestBrokenStacks(t *testing.T) {
 	// Create a healthy stack
 	healthyVersion := "v3"
 	healthyStack := fmt.Sprintf("%s-%s", stacksetName, healthyVersion)
-	spec = factory.Create(healthyVersion)
+	spec = factory.Create(t, healthyVersion)
 	err = updateStackset(stacksetName, spec)
 	require.NoError(t, err)
 	_, err = waitForStack(t, stacksetName, healthyVersion)
@@ -70,7 +72,87 @@ func TestBrokenStacks(t *testing.T) {
 	// Create another healthy stack so we can test GC
 	finalVersion := "v4"
 	finalStack := fmt.Sprintf("%s-%s", stacksetName, finalVersion)
-	spec = factory.Create(finalVersion)
+	spec = factory.Create(t, finalVersion)
+	err = updateStackset(stacksetName, spec)
+	require.NoError(t, err)
+	_, err = waitForStack(t, stacksetName, finalVersion)
+	require.NoError(t, err)
+
+	finalWeights := map[string]float64{finalStack: 100}
+	err = setDesiredTrafficWeightsStackset(stacksetName, finalWeights)
+	require.NoError(t, err)
+	err = trafficWeightsUpdatedStackset(t, stacksetName, weightKindActual, finalWeights, nil).await()
+	require.NoError(t, err)
+
+	// Check that the unhealthy stack was deleted
+	for _, stack := range []string{unhealthyStack, firstStack} {
+		err := resourceDeleted(t, "stack", stack, stackInterface()).withTimeout(time.Second * 60).await()
+		require.NoError(t, err)
+	}
+
+}
+
+func TestBrokenStackWithConfigMaps(t *testing.T) {
+	t.Parallel()
+
+	stacksetName := "stackset-broken-stacks-with-configmap"
+	factory := NewTestStacksetSpecFactory(stacksetName).Ingress().ConfigMap().StackGC(1, 30)
+
+	firstVersion := "v1"
+	firstStack := fmt.Sprintf("%s-%s", stacksetName, firstVersion)
+	spec := factory.Create(t, firstVersion)
+	err := createStackSet(stacksetName, 0, spec)
+	require.NoError(t, err)
+	_, err = waitForStack(t, stacksetName, firstVersion)
+	require.NoError(t, err)
+
+	unhealthyVersion := "v2"
+	unhealthyStack := fmt.Sprintf("%s-%s", stacksetName, unhealthyVersion)
+	spec = factory.Create(t, unhealthyVersion)
+	for _, cr := range spec.StackTemplate.Spec.ConfigurationResources {
+		if cr.ConfigMapRef.Name != "" {
+			err := configMapInterface().Delete(context.Background(), cr.ConfigMapRef.Name, metav1.DeleteOptions{})
+			require.NoError(t, err)
+		}
+	}
+	err = updateStackset(stacksetName, spec)
+	require.NoError(t, err)
+	_, err = waitForStack(t, stacksetName, unhealthyVersion)
+	require.NoError(t, err)
+
+	_, err = waitForIngress(t, stacksetName)
+	require.NoError(t, err)
+
+	initialWeights := map[string]float64{firstStack: 100}
+	err = trafficWeightsUpdatedStackset(t, stacksetName, weightKindActual, initialWeights, nil).await()
+	require.NoError(t, err)
+
+	// Switch traffic to the second stack, this should fail
+	desiredWeights := map[string]float64{unhealthyStack: 100}
+	err = setDesiredTrafficWeightsStackset(stacksetName, desiredWeights)
+	require.NoError(t, err)
+	err = trafficWeightsUpdatedStackset(t, stacksetName, weightKindActual, desiredWeights, nil).await()
+	require.Error(t, err)
+
+	// Create a healthy stack
+	healthyVersion := "v3"
+	healthyStack := fmt.Sprintf("%s-%s", stacksetName, healthyVersion)
+	spec = factory.Create(t, healthyVersion)
+	err = updateStackset(stacksetName, spec)
+	require.NoError(t, err)
+	_, err = waitForStack(t, stacksetName, healthyVersion)
+	require.NoError(t, err)
+
+	healthyWeights := map[string]float64{healthyStack: 100}
+	err = setDesiredTrafficWeightsStackset(stacksetName, healthyWeights)
+	require.NoError(t, err)
+	err = trafficWeightsUpdatedStackset(t, stacksetName, weightKindActual, healthyWeights, nil).await()
+	require.NoError(t, err)
+
+	// Create another healthy stack so we can test GC
+	finalVersion := "v4"
+	finalStack := fmt.Sprintf("%s-%s", stacksetName, finalVersion)
+	spec = factory.Create(t, finalVersion)
 	err = updateStackset(stacksetName, spec)
 	require.NoError(t, err)
 	_, err = waitForStack(t, stacksetName, finalVersion)

cmd/e2e/configure_hpa_test.go

@@ -25,7 +25,7 @@ func TestConfigureAutoscaling(t *testing.T) {
 		Autoscaler(1, 10, metrics).
 		Behavior(stabilizationWindow)
 	firstVersion := "v1"
-	spec := factory.Create(firstVersion)
+	spec := factory.Create(t, firstVersion)
 	err := createStackSet(stacksetName, 0, spec)
 	require.NoError(t, err)
 

cmd/e2e/generated_autoscaler_test.go

@@ -44,7 +44,7 @@ func TestGenerateAutoscaler(t *testing.T) {
 
 	factory := NewTestStacksetSpecFactory(stacksetName).Ingress().Autoscaler(1, 10, metrics)
 	firstVersion := "v1"
-	spec := factory.Create(firstVersion)
+	spec := factory.Create(t, firstVersion)
 	err := createStackSet(stacksetName, 0, spec)
 	require.NoError(t, err)