Skip to content

Commit 21961a7

Browse files
sushrksushrk
authored
add CNINode integration tests (#479)
* add CNINode integration tests * address PR comments * updating log statements * add retry in VerifyCNINode
1 parent 3e74da5 commit 21961a7

File tree

14 files changed

+338
-73
lines changed

14 files changed

+338
-73
lines changed

‎pkg/utils/helper.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ import (
2727

2828
"github.com/go-logr/logr"
2929
corev1 "k8s.io/api/core/v1"
30-
v1 "k8s.io/api/core/v1"
3130
"k8s.io/apimachinery/pkg/api/meta"
3231
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3332
"k8s.io/apimachinery/pkg/labels"
@@ -237,7 +236,7 @@ func GetSourceAcctAndArn(roleARN, region, clusterName string) (string, string, s
237236

238237
// PodHasENIRequest will return true if first container of pod spec has request for eni indicating
239238
// it needs trunk interface from vpc-rc
240-
func PodHasENIRequest(pod *v1.Pod) bool {
239+
func PodHasENIRequest(pod *corev1.Pod) bool {
241240
if pod == nil {
242241
return false
243242
}

‎pkg/utils/set.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313

1414
package utils
1515

16+
import (
17+
"github.com/aws/aws-sdk-go/service/ec2"
18+
)
19+
1620
// Difference returns a-b, elements present in a and not in b
1721
func Difference[T comparable](a, b []T) (diff []T) {
1822
m := make(map[T]struct{})
@@ -35,3 +39,11 @@ func GetKeyValSlice(m map[string]string) (key []string, val []string) {
3539
}
3640
return
3741
}
42+
43+
func GetTagKeyValueMap(tagSet []*ec2.Tag) map[string]string {
44+
m := make(map[string]string)
45+
for _, tag := range tagSet {
46+
m[*tag.Key] = *tag.Value
47+
}
48+
return m
49+
}

‎scripts/test/run-integration-tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ function run_integration_tests(){
4747
echo "skipping Windows tests"
4848
fi
4949
(cd $INTEGRATION_TEST_DIR/webhook && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=5m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail
50-
# (cd $INTEGRATION_TEST_DIR/cninode && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=10m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail
50+
(cd $INTEGRATION_TEST_DIR/cninode && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=10m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail
5151

5252
if [[ "$TEST_RESULT" == fail ]]; then
5353
exit 1

‎test/framework/framework.go‎

Lines changed: 33 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
eniConfig "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
1818
cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
1919
sgp "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1beta1"
20+
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/aws/autoscaling"
2021
ec2Manager "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/aws/ec2"
2122
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/configmap"
2223
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/controller"
@@ -42,21 +43,22 @@ import (
4243
)
4344

4445
type Framework struct {
45-
Options Options
46-
K8sClient client.Client
47-
ec2Client *ec2.EC2
48-
DeploymentManager deployment.Manager
49-
PodManager pod.Manager
50-
EC2Manager *ec2Manager.Manager
51-
SAManager serviceaccount.Manager
52-
NSManager namespace.Manager
53-
SGPManager *sgpManager.Manager
54-
SVCManager service.Manager
55-
JobManager jobs.Manager
56-
NodeManager node.Manager
57-
ControllerManager controller.Manager
58-
RBACManager rbac.Manager
59-
ConfigMapManager configmap.Manager
46+
Options Options
47+
K8sClient client.Client
48+
ec2Client *ec2.EC2
49+
DeploymentManager deployment.Manager
50+
PodManager pod.Manager
51+
EC2Manager *ec2Manager.Manager
52+
SAManager serviceaccount.Manager
53+
NSManager namespace.Manager
54+
SGPManager *sgpManager.Manager
55+
SVCManager service.Manager
56+
JobManager jobs.Manager
57+
NodeManager node.Manager
58+
ControllerManager controller.Manager
59+
RBACManager rbac.Manager
60+
ConfigMapManager configmap.Manager
61+
AutoScalingManager autoscaling.Manager
6062
}
6163

6264
func New(options Options) *Framework {
@@ -91,20 +93,21 @@ func New(options Options) *Framework {
9193
ec2 := ec2.New(sess, &aws.Config{Region: aws.String(options.AWSRegion)})
9294

9395
return &Framework{
94-
K8sClient: k8sClient,
95-
ec2Client: ec2,
96-
PodManager: pod.NewManager(k8sClient, k8sSchema, config),
97-
DeploymentManager: deployment.NewManager(k8sClient),
98-
EC2Manager: ec2Manager.NewManager(ec2, options.AWSVPCID),
99-
SAManager: serviceaccount.NewManager(k8sClient, config),
100-
NSManager: namespace.NewManager(k8sClient),
101-
SGPManager: sgpManager.NewManager(k8sClient),
102-
SVCManager: service.NewManager(k8sClient),
103-
JobManager: jobs.NewManager(k8sClient),
104-
NodeManager: node.NewManager(k8sClient),
105-
ControllerManager: controller.NewManager(k8sClient),
106-
RBACManager: rbac.NewManager(k8sClient),
107-
ConfigMapManager: configmap.NewManager(k8sClient),
108-
Options: options,
96+
K8sClient: k8sClient,
97+
ec2Client: ec2,
98+
PodManager: pod.NewManager(k8sClient, k8sSchema, config),
99+
DeploymentManager: deployment.NewManager(k8sClient),
100+
EC2Manager: ec2Manager.NewManager(ec2, options.AWSVPCID),
101+
SAManager: serviceaccount.NewManager(k8sClient, config),
102+
NSManager: namespace.NewManager(k8sClient),
103+
SGPManager: sgpManager.NewManager(k8sClient),
104+
SVCManager: service.NewManager(k8sClient),
105+
JobManager: jobs.NewManager(k8sClient),
106+
NodeManager: node.NewManager(k8sClient),
107+
ControllerManager: controller.NewManager(k8sClient),
108+
RBACManager: rbac.NewManager(k8sClient),
109+
ConfigMapManager: configmap.NewManager(k8sClient),
110+
AutoScalingManager: autoscaling.NewManager(sess),
111+
Options: options,
109112
}
110113
}

‎test/framework/resource/aws/autoscaling/manager.go‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
package autoscaling
15+
16+
import (
17+
"fmt"
18+
19+
"github.com/aws/aws-sdk-go/aws"
20+
"github.com/aws/aws-sdk-go/aws/session"
21+
"github.com/aws/aws-sdk-go/service/autoscaling"
22+
"github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface"
23+
)
24+
25+
type Manager interface {
26+
DescribeAutoScalingGroup(autoScalingGroupName string) ([]*autoscaling.Group, error)
27+
UpdateAutoScalingGroup(asgName string, desiredSize, minSize, maxSize int64) error
28+
}
29+
30+
type defaultManager struct {
31+
autoscalingiface.AutoScalingAPI
32+
}
33+
34+
func NewManager(session *session.Session) Manager {
35+
return &defaultManager{
36+
AutoScalingAPI: autoscaling.New(session),
37+
}
38+
}
39+
40+
func (d defaultManager) DescribeAutoScalingGroup(autoScalingGroupName string) ([]*autoscaling.Group, error) {
41+
describeAutoScalingGroupIp := &autoscaling.DescribeAutoScalingGroupsInput{
42+
AutoScalingGroupNames: aws.StringSlice([]string{autoScalingGroupName}),
43+
}
44+
asg, err := d.AutoScalingAPI.DescribeAutoScalingGroups(describeAutoScalingGroupIp)
45+
if err != nil {
46+
return nil, err
47+
}
48+
if len(asg.AutoScalingGroups) == 0 {
49+
return nil, fmt.Errorf("failed to find asg %s", autoScalingGroupName)
50+
}
51+
52+
return asg.AutoScalingGroups, nil
53+
}
54+
55+
func (d defaultManager) UpdateAutoScalingGroup(asgName string, desiredSize, minSize, maxSize int64) error {
56+
updateASGInput := &autoscaling.UpdateAutoScalingGroupInput{
57+
AutoScalingGroupName: aws.String(asgName),
58+
DesiredCapacity: aws.Int64(desiredSize),
59+
MaxSize: aws.Int64(maxSize),
60+
MinSize: aws.Int64(minSize),
61+
}
62+
_, err := d.AutoScalingAPI.UpdateAutoScalingGroup(updateASGInput)
63+
return err
64+
}

‎test/framework/resource/k8s/node/manager.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ package node
1515

1616
import (
1717
"context"
18+
"strings"
1819

1920
cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
2021
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils"
@@ -32,6 +33,7 @@ type Manager interface {
3233
GetNodeList() (*v1.NodeList, error)
3334
GetCNINode(node *v1.Node) (*cninode.CNINode, error)
3435
GetCNINodeList() (*cninode.CNINodeList, error)
36+
GetInstanceID(node *v1.Node) string
3537
}
3638

3739
type defaultManager struct {
@@ -117,3 +119,11 @@ func (d *defaultManager) GetNodeList() (*v1.NodeList, error) {
117119
err := d.k8sClient.List(context.TODO(), list)
118120
return list, err
119121
}
122+
123+
func (d *defaultManager) GetInstanceID(node *v1.Node) string {
124+
if node.Spec.ProviderID != "" {
125+
id := strings.Split(node.Spec.ProviderID, "/")
126+
return id[len(id)-1]
127+
}
128+
return ""
129+
}

‎test/framework/resource/k8s/node/wrapper.go‎

Lines changed: 54 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,30 +15,71 @@ package node
1515

1616
import (
1717
"context"
18+
"fmt"
1819

20+
cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
1921
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils"
2022
. "github.com/onsi/ginkgo/v2"
2123
. "github.com/onsi/gomega"
24+
"github.com/samber/lo"
2225
v1 "k8s.io/api/core/v1"
2326
"k8s.io/apimachinery/pkg/util/wait"
2427
)
2528

26-
func GetNodeAndWaitTillCapacityPresent(manager Manager, ctx context.Context, os string, expectedResource string) *v1.NodeList {
27-
29+
func GetNodeAndWaitTillCapacityPresent(manager Manager, os string, expectedResource string) *v1.NodeList {
2830
observedNodeList := &v1.NodeList{}
2931
var err error
30-
err = wait.Poll(utils.PollIntervalShort, utils.ResourceCreationTimeout, func() (bool, error) {
31-
By("checking nodes have capacity present")
32-
observedNodeList, err = manager.GetNodesWithOS(os)
33-
Expect(err).ToNot(HaveOccurred())
34-
for _, node := range observedNodeList.Items {
35-
_, found := node.Status.Allocatable[v1.ResourceName(expectedResource)]
36-
if !found {
37-
return false, nil
32+
err = wait.PollUntilContextTimeout(context.Background(), utils.PollIntervalShort, utils.ResourceCreationTimeout, true,
33+
func(ctx context.Context) (bool, error) {
34+
By("checking nodes have capacity present")
35+
observedNodeList, err = manager.GetNodesWithOS(os)
36+
Expect(err).ToNot(HaveOccurred())
37+
for _, node := range observedNodeList.Items {
38+
_, found := node.Status.Allocatable[v1.ResourceName(expectedResource)]
39+
if !found {
40+
return false, nil
41+
}
3842
}
39-
}
40-
return true, nil
41-
})
43+
return true, nil
44+
})
4245
Expect(err).ToNot(HaveOccurred())
4346
return observedNodeList
4447
}
48+
49+
// VerifyCNINode checks if the number of CNINodes is equal to number of nodes in the cluster, and verifies 1:1 mapping between CNINode and Node objects
50+
// Returns nil if count and 1:1 mapping exists, else returns error
51+
func VerifyCNINode(manager Manager) error {
52+
var cniNodeList *cninode.CNINodeList
53+
var nodeList *v1.NodeList
54+
var err error
55+
By("checking number of CNINodes match number of nodes in the cluster")
56+
err = wait.PollUntilContextTimeout(context.Background(), utils.PollIntervalShort, utils.PollTimeout, true,
57+
func(ctx context.Context) (bool, error) {
58+
if cniNodeList, err = manager.GetCNINodeList(); err != nil {
59+
return false, nil
60+
}
61+
if nodeList, err = manager.GetNodeList(); err != nil {
62+
return false, nil
63+
}
64+
if len(nodeList.Items) != len(cniNodeList.Items) {
65+
return false, nil
66+
}
67+
return true, nil
68+
})
69+
if err != nil {
70+
return fmt.Errorf("number of CNINodes does not match number of nodes in the cluster")
71+
}
72+
By("checking CNINode list matches node list")
73+
nameMatched := true
74+
for _, node := range nodeList.Items {
75+
if !lo.ContainsBy(cniNodeList.Items, func(cniNode cninode.CNINode) bool {
76+
return cniNode.Name == node.Name
77+
}) {
78+
nameMatched = false
79+
}
80+
}
81+
if !nameMatched {
82+
return fmt.Errorf("CNINode list does not match node list")
83+
}
84+
return nil
85+
}

‎test/framework/utils/poll.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ const (
1919
PollIntervalShort = 2 * time.Second
2020
PollIntervalMedium = 10 * time.Second
2121
PollIntervalLong = 20 * time.Second
22+
PollTimeout = 30 * time.Second
2223
// ResourceCreationTimeout is the number of seconds till the controller waits
2324
// for the resource creation to complete
2425
ResourceCreationTimeout = 120 * time.Second

‎test/integration/cninode/cninode_suite_test.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
package cninode_test
15+
16+
import (
17+
"testing"
18+
19+
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework"
20+
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/node"
21+
. "github.com/onsi/ginkgo/v2"
22+
. "github.com/onsi/gomega"
23+
)
24+
25+
func TestCNINode(t *testing.T) {
26+
RegisterFailHandler(Fail)
27+
RunSpecs(t, "CNINode Test Suite")
28+
}
29+
30+
var frameWork *framework.Framework
31+
var _ = BeforeSuite(func() {
32+
By("creating a framework")
33+
frameWork = framework.New(framework.GlobalOptions)
34+
35+
By("verify at least 2 nodes are available")
36+
nodeList, err := frameWork.NodeManager.GetNodeList()
37+
Expect(err).ToNot(HaveOccurred())
38+
Expect(len(nodeList.Items)).To(BeNumerically(">", 1))
39+
40+
By("verify CNINode count")
41+
err = node.VerifyCNINode(frameWork.NodeManager)
42+
Expect(err).ToNot(HaveOccurred())
43+
})
44+
45+
// Verify CNINode count before and after test remains same
46+
var _ = AfterSuite(func() {
47+
By("verify CNINode count")
48+
err := node.VerifyCNINode(frameWork.NodeManager)
49+
Expect(err).ToNot(HaveOccurred())
50+
})

0 commit comments

Comments
 (0)