monitoring stale cni node resources

Author: yash97

controllers/crds/cninode_controller.go

@@ -52,14 +52,22 @@ var (
 			Help: "The number of requests that failed when controller tried to recreate the CNINode",
 		},
 	)
+	staleCNINodeCount = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "stale_cniNode_count",
+			Help: "The number of stale CNINode resources with deletion timestamp older than 15 minutes",
+		},
+	)
 )
 
 func prometheusRegister() {
 	prometheusRegistered = true
 
 	metrics.Registry.MustRegister(
 		recreateCNINodeCallCount,
-		recreateCNINodeErrCount)
+		recreateCNINodeErrCount,
+		staleCNINodeCount,
+	)
 
 	prometheusRegistered = true
 }
@@ -171,6 +179,11 @@ func (r *CNINodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		return ctrl.Result{}, nil
 
 	} else { // CNINode is marked for deletion
+		if time.Since(cniNode.GetDeletionTimestamp().Time).Minutes() > 15 {
+			// delete stale CNINode resource
+			r.log.Info("stale CNINode resource", "cniNode", cniNode.Name)
+			staleCNINodeCount.Inc()
+		}
 		if !nodeFound {
 			//  node is also deleted, proceed with running the cleanup routine and remove the finalizer
 

controllers/crds/cninode_controller_test.go

@@ -3,6 +3,7 @@ package crds
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
 	mock_api "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api"
@@ -12,6 +13,7 @@ import (
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2/api/cleanup"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
 	"github.com/golang/mock/gomock"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -24,7 +26,8 @@ import (
 )
 
 type CNINodeMock struct {
-	Reconciler CNINodeReconciler
+	Reconciler        CNINodeReconciler
+	initialStaleCount float64
 }
 
 var (
@@ -163,6 +166,50 @@ func TestCNINodeReconcile(t *testing.T) {
 				assert.Equal(t, res, reconcile.Result{})
 			},
 		},
+		{
+			name: "verify stale CNINode metric is incremented when deletion timestamp is older than 15 minutes",
+			args: args{
+				mockNode: nil,
+				mockCNINode: &v1alpha1.CNINode{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: mockName,
+						Labels: map[string]string{
+							config.NodeLabelOS: config.OSLinux,
+						},
+						Finalizers:        []string{config.NodeTerminationFinalizer},
+						DeletionTimestamp: &metav1.Time{Time: metav1.Now().Add(-20 * time.Minute)}, // 20 minutes ago
+					},
+					Spec: v1alpha1.CNINodeSpec{
+						Tags: map[string]string{
+							config.NetworkInterfaceNodeIDKey: "i-1234567890",
+						},
+					},
+				},
+			},
+			prepare: func(f *fields) {
+				// Get initial metric value
+				initialValue := testutil.ToFloat64(staleCNINodeCount)
+
+				f.mockCNINode.Reconciler.newResourceCleaner = func(nodeID string, eC2Wrapper ec2API.EC2Wrapper, vpcID string) cleanup.ResourceCleaner {
+					return f.mockResourceCleaner
+				}
+				f.mockResourceCleaner.EXPECT().DeleteLeakedResources().Times(1).Return(nil)
+				f.mockFinalizerManager.EXPECT().
+					RemoveFinalizers(gomock.Any(), gomock.Any(), config.NodeTerminationFinalizer).
+					Return(nil)
+
+				// Store initial value for comparison in asserts
+				f.mockCNINode.initialStaleCount = initialValue
+			},
+			asserts: func(res reconcile.Result, err error, cniNode *v1alpha1.CNINode) {
+				assert.NoError(t, err)
+				assert.Equal(t, res, reconcile.Result{})
+
+				// Verify the stale metric was incremented
+				finalValue := testutil.ToFloat64(staleCNINodeCount)
+				assert.Greater(t, finalValue,float64(0))
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

go.mod

@@ -45,6 +45,7 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/gnostic-models v0.6.9 // indirect
 	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	golang.org/x/sync v0.13.0 // indirect