update unit tests and integration tests for CNINode (#278)

Author: haouc

pkg/node/manager/manager_test.go

@@ -898,3 +898,57 @@ func Test_GetEniConfigName(t *testing.T) {
 		})
 	}
 }
+
+func Test_TrunkEnabledInCNINode(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	dataStoreWithUnManagedNode := map[string]node.Node{v1Node.Name: unManagedNode}
+
+	mock := NewMock(ctrl, dataStoreWithUnManagedNode)
+
+	testCases := []struct {
+		features []rcV1alpha1.Feature
+		managed  bool
+		msg      string
+	}{
+		{
+			features: []rcV1alpha1.Feature{},
+			managed:  false,
+			msg:      "no feature is added and node is not managed",
+		},
+		{
+			features: []rcV1alpha1.Feature{
+				{
+					Name:  rcV1alpha1.SecurityGroupsForPods,
+					Value: "",
+				},
+			},
+			managed: true,
+			msg:     "no SGP feature is added and node is not managed",
+		},
+		{
+			features: []rcV1alpha1.Feature{
+				{
+					Name:  rcV1alpha1.CustomNetworking,
+					Value: "default",
+				},
+			},
+			managed: false,
+			msg:     "SGP feature is added and node is managed",
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.msg, func(t *testing.T) {
+			mock.MockK8sAPI.EXPECT().GetCNINode(types.NamespacedName{Name: v1Node.Name}).Return(&rcV1alpha1.CNINode{
+				Spec: rcV1alpha1.CNINodeSpec{
+					Features: test.features,
+				},
+			}, nil).Times(1)
+			managed, err := mock.Manager.trunkEnabledInCNINode(v1Node)
+			assert.NoError(t, err)
+			assert.Equal(t, test.managed, managed)
+		})
+	}
+}

test/framework/framework.go

@@ -15,6 +15,7 @@ package framework
 
 import (
 	eniConfig "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
+	cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
 	sgp "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1beta1"
 	ec2Manager "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/aws/ec2"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/configmap"
@@ -72,6 +73,7 @@ func New(options Options) *Framework {
 	clientgoscheme.AddToScheme(k8sSchema)
 	sgp.AddToScheme(k8sSchema)
 	eniConfig.AddToScheme(k8sSchema)
+	cninode.AddToScheme(k8sSchema)
 
 	stopChan := ctrl.SetupSignalHandler()
 	cache, err := cache.New(config, cache.Options{Scheme: k8sSchema})

test/framework/resource/k8s/node/manager.go

@@ -16,6 +16,7 @@ package node
 import (
 	"context"
 
+	cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -28,6 +29,9 @@ type Manager interface {
 	AddLabels(nodeList []v1.Node, label map[string]string) error
 	RemoveLabels(nodeList []v1.Node, label map[string]string) error
 	GetNode(node *v1.Node) (*v1.Node, error)
+	GetNodeList() (*v1.NodeList, error)
+	GetCNINode(node *v1.Node) (*cninode.CNINode, error)
+	GetCNINodeList() (*cninode.CNINodeList, error)
 }
 
 type defaultManager struct {
@@ -95,3 +99,21 @@ func (d *defaultManager) GetNode(node *v1.Node) (*v1.Node, error) {
 	err := d.k8sClient.Get(context.TODO(), utils.NamespacedName(node), observedNode)
 	return observedNode, err
 }
+
+func (d *defaultManager) GetCNINode(node *v1.Node) (*cninode.CNINode, error) {
+	cniNode := &cninode.CNINode{}
+	err := d.k8sClient.Get(context.TODO(), types.NamespacedName{Name: node.Name}, cniNode)
+	return cniNode, err
+}
+
+func (d *defaultManager) GetCNINodeList() (*cninode.CNINodeList, error) {
+	list := &cninode.CNINodeList{}
+	err := d.k8sClient.List(context.TODO(), list)
+	return list, err
+}
+
+func (d *defaultManager) GetNodeList() (*v1.NodeList, error) {
+	list := &v1.NodeList{}
+	err := d.k8sClient.List(context.TODO(), list)
+	return list, err
+}

test/integration/perpodsg/perpodsg_test.go

@@ -16,6 +16,7 @@ package perpodsg_test
 import (
 	"time"
 
+	cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1beta1"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch"
@@ -26,6 +27,7 @@ import (
 	podWrapper "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/pod"
 	sgpWrapper "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/sgp"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils"
+	"github.com/samber/lo"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -35,6 +37,29 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+var _ = Describe("CNINode Veification", func() {
+	Describe("verify CNINode mapping to nodes", func() {
+		Context("when nodes are ready", func() {
+			It("should have same number of CNINode no matter which mode", func() {
+				cniNodes, err := frameWork.NodeManager.GetCNINodeList()
+				Expect(err).NotTo(HaveOccurred())
+				nodes, err := frameWork.NodeManager.GetNodeList()
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(nodes.Items)).To(Equal(len(cniNodes.Items)))
+				nameMatched := true
+				for _, node := range nodes.Items {
+					if !lo.ContainsBy(cniNodes.Items, func(cniNode cninode.CNINode) bool {
+						return cniNode.Name == node.Name
+					}) {
+						nameMatched = false
+					}
+				}
+				Expect(nameMatched).To(BeTrue())
+			})
+		})
+	})
+})
+
 var _ = Describe("Branch ENI Pods", func() {
 	var (
 		securityGroupPolicy *v1beta1.SecurityGroupPolicy
@@ -427,44 +452,55 @@ var _ = Describe("Branch ENI Pods", func() {
 			It("pod should not run when un-managed and run when managed", func() {
 				node := targetedNodes[0]
 
-				By("verifying node has trunk ENI label present")
-				// This label is added by IPAM-D
-				_, found := node.Labels[config.HasTrunkAttachedLabel]
-				Expect(found).To(BeTrue())
-
-				// This should never happens as once the trunk is attached,
-				// this label will not be removed again. This is for testing
-				// purposes to make a managed node an un-managed node
-				By("removing the has-trunk-attached label from the node")
-				err = frameWork.NodeManager.RemoveLabels(targetedNodes,
-					map[string]string{config.HasTrunkAttachedLabel: "true"})
-
-				firstPod := podTemplate.DeepCopy()
-				By("creating a Pod on the un-managed node and verifying it fails")
-				_, err = frameWork.PodManager.CreateAndWaitTillPodIsRunning(ctx, firstPod, utils.ResourceCreationTimeout)
-				Expect(err).To(HaveOccurred())
-
-				By("deleting the pod")
-				err = frameWork.PodManager.DeleteAndWaitTillPodIsDeleted(ctx, firstPod)
-				Expect(err).ToNot(HaveOccurred())
-
-				// Currently we wait for some time before removing the trunk from cache
-				// to allow evicted Pods's event to be received and their Branch ENIs be
-				// removed. In this period if we try to make the node managed again, it will
-				// fail
-				time.Sleep(branch.NodeDeleteRequeueRequestDelay)
-
-				By("adding the has trunk ENI label")
-				err = frameWork.NodeManager.AddLabels(targetedNodes,
-					map[string]string{config.HasTrunkAttachedLabel: "true"})
+				By("verifying node has CNINode present")
+				cniNode, err := frameWork.NodeManager.GetCNINode(&node)
 				Expect(err).ToNot(HaveOccurred())
-
-				By("creating the Pod on now managed node and verify it runs")
-				secondPod := podTemplate.DeepCopy()
-				secondPod, err = frameWork.PodManager.CreateAndWaitTillPodIsRunning(ctx, secondPod, utils.ResourceCreationTimeout)
-				Expect(err).ToNot(HaveOccurred())
-
-				verify.VerifyNetworkingOfPodUsingENI(*secondPod, []string{securityGroupID1})
+				Expect(cniNode.Name).To(Equal(node.Name))
+
+				// we don't support changing SGP managed node to unmanaged node
+				// after using CNINode, no longer like node label the feature in CNINode Spec shouldn't be modified
+				// only run this test for old label based mode
+				if !lo.ContainsBy(cniNode.Spec.Features, func(addedFeature cninode.Feature) bool {
+					return addedFeature.Name == cninode.SecurityGroupsForPods
+				}) {
+					if _, found := node.Labels[config.HasTrunkAttachedLabel]; found {
+						// This should never happens as once the trunk is attached,
+						// this label will not be removed again. This is for testing
+						// purposes to make a managed node an un-managed node
+						By("removing the has-trunk-attached label from the node")
+						err = frameWork.NodeManager.RemoveLabels(targetedNodes,
+							map[string]string{config.HasTrunkAttachedLabel: "true"})
+						Expect(err).To(HaveOccurred())
+
+						firstPod := podTemplate.DeepCopy()
+						By("creating a Pod on the un-managed node and verifying it fails")
+						_, err = frameWork.PodManager.CreateAndWaitTillPodIsRunning(ctx, firstPod, utils.ResourceCreationTimeout)
+						Expect(err).To(HaveOccurred())
+
+						By("deleting the pod")
+						err = frameWork.PodManager.DeleteAndWaitTillPodIsDeleted(ctx, firstPod)
+						Expect(err).ToNot(HaveOccurred())
+
+						// Currently we wait for some time before removing the trunk from cache
+						// to allow evicted Pods's event to be received and their Branch ENIs be
+						// removed. In this period if we try to make the node managed again, it will
+						// fail
+						time.Sleep(branch.NodeDeleteRequeueRequestDelay)
+
+						By("adding the has trunk ENI label")
+						err = frameWork.NodeManager.AddLabels(targetedNodes,
+							map[string]string{config.HasTrunkAttachedLabel: "true"})
+						Expect(err).ToNot(HaveOccurred())
+
+						By("creating the Pod on now managed node and verify it runs")
+						secondPod := podTemplate.DeepCopy()
+						secondPod, err = frameWork.PodManager.CreateAndWaitTillPodIsRunning(ctx, secondPod, utils.ResourceCreationTimeout)
+						Expect(err).ToNot(HaveOccurred())
+
+						verify.VerifyNetworkingOfPodUsingENI(*secondPod, []string{securityGroupID1})
+
+					}
+				}
 			})
 		})
 
@@ -485,7 +521,7 @@ var _ = Describe("Branch ENI Pods", func() {
 				pod := podTemplate.DeepCopy()
 
 				By("creating pod which should not run since controller is down")
-				pod, err = frameWork.PodManager.CreateAndWaitTillPodIsRunning(ctx, pod, time.Second*10)
+				_, err = frameWork.PodManager.CreateAndWaitTillPodIsRunning(ctx, pod, time.Second*10)
 				Expect(err).To(HaveOccurred())
 
 				By("scaling the controller deployment to 2")