Skip to content

🐛 (go/v4): Fix flaky metrics e2e tests when webhooks are scaffolded #5204

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 15, 2025
+228 −97
Merged

🐛 (go/v4): Fix flaky metrics e2e tests when webhooks are scaffolded #5204

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 21 additions & 7 deletions docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,24 +197,38 @@ var _ = Describe("Manager", Ordered, func() {
Expect(err).NotTo(HaveOccurred())
Expect(token).NotTo(BeEmpty())

By("waiting for the metrics endpoint to be ready")
verifyMetricsEndpointReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
By("ensuring the controller pod is ready")
verifyControllerPodReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "pod", controllerPodName, "-n", namespace,
"-o", "jsonpath={.status.conditions[?(@.type=='Ready')].status}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
g.Expect(output).To(Equal("True"), "Controller pod not ready")
}
Eventually(verifyMetricsEndpointReady).Should(Succeed())
Eventually(verifyControllerPodReady, 3*time.Minute, time.Second).Should(Succeed())

By("verifying that the controller manager is serving the metrics server")
verifyMetricsServerStarted := func(g Gomega) {
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
g.Expect(output).To(ContainSubstring("Serving metrics server"),
"Metrics server not yet started")
}
Eventually(verifyMetricsServerStarted).Should(Succeed())
Eventually(verifyMetricsServerStarted, 3*time.Minute, time.Second).Should(Succeed())

By("waiting for the webhook service endpoints to be ready")
verifyWebhookEndpointsReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "endpointslices.discovery.k8s.io", "-n", namespace,
"-l", "kubernetes.io/service-name=project-webhook-service",
"-o", "jsonpath={range .items[*]}{range .endpoints[*]}{.addresses[*]}{end}{end}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook endpoints should exist")
g.Expect(output).ShouldNot(BeEmpty(), "Webhook endpoints not yet ready")
}
Eventually(verifyWebhookEndpointsReady, 3*time.Minute, time.Second).Should(Succeed())
Copy link
Member Author

@camilamacedo86 camilamacedo86 Nov 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mayuka-c that is the code that we must to inject.
Without it we still risking the flakes

Why?

Without waiting for the webhook service to publish endpoints, the curl pod can hit the validating webhook while it’s still initializing, producing the same “connection refused” failure. The EndpointSlice check is what gives us a deterministic signal that the webhook server is actually accepting traffic before we launch the curl metrics pod.

Copy link
Contributor

@mayuka-c mayuka-c Nov 15, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes this would be required. I mean the approach which was taken here (To have the marker to check for webhook readiness).

Sorry, if I caused some misunderstandings here :)

Copy link
Member Author

@camilamacedo86 camilamacedo86 Nov 15, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not at all — you helped a lot!
You identified the issue, and your effort was absolutely not lost.
I’ve added you as a co-author of this PR to give you proper credit for your contribution. 🙌

Screenshot 2025-11-15 at 16 37 52


// +kubebuilder:scaffold:e2e-metrics-webhooks-readiness

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
Expand Down
17 changes: 10 additions & 7 deletions docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,24 +192,27 @@ var _ = Describe("Manager", Ordered, func() {
Expect(err).NotTo(HaveOccurred())
Expect(token).NotTo(BeEmpty())

By("waiting for the metrics endpoint to be ready")
verifyMetricsEndpointReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
By("ensuring the controller pod is ready")
verifyControllerPodReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "pod", controllerPodName, "-n", namespace,
"-o", "jsonpath={.status.conditions[?(@.type=='Ready')].status}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
g.Expect(output).To(Equal("True"), "Controller pod not ready")
}
Eventually(verifyMetricsEndpointReady).Should(Succeed())
Eventually(verifyControllerPodReady, 3*time.Minute, time.Second).Should(Succeed())

By("verifying that the controller manager is serving the metrics server")
verifyMetricsServerStarted := func(g Gomega) {
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
g.Expect(output).To(ContainSubstring("Serving metrics server"),
"Metrics server not yet started")
}
Eventually(verifyMetricsServerStarted).Should(Succeed())
Eventually(verifyMetricsServerStarted, 3*time.Minute, time.Second).Should(Succeed())

// +kubebuilder:scaffold:e2e-metrics-webhooks-readiness

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
Expand Down
28 changes: 21 additions & 7 deletions docs/book/src/multiversion-tutorial/testdata/project/test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,24 +204,38 @@ var _ = Describe("Manager", Ordered, func() {
Expect(err).NotTo(HaveOccurred())
Expect(token).NotTo(BeEmpty())

By("waiting for the metrics endpoint to be ready")
verifyMetricsEndpointReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
By("ensuring the controller pod is ready")
verifyControllerPodReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "pod", controllerPodName, "-n", namespace,
"-o", "jsonpath={.status.conditions[?(@.type=='Ready')].status}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
g.Expect(output).To(Equal("True"), "Controller pod not ready")
}
Eventually(verifyMetricsEndpointReady).Should(Succeed())
Eventually(verifyControllerPodReady, 3*time.Minute, time.Second).Should(Succeed())

By("verifying that the controller manager is serving the metrics server")
verifyMetricsServerStarted := func(g Gomega) {
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
g.Expect(output).To(ContainSubstring("Serving metrics server"),
"Metrics server not yet started")
}
Eventually(verifyMetricsServerStarted).Should(Succeed())
Eventually(verifyMetricsServerStarted, 3*time.Minute, time.Second).Should(Succeed())

By("waiting for the webhook service endpoints to be ready")
verifyWebhookEndpointsReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "endpointslices.discovery.k8s.io", "-n", namespace,
"-l", "kubernetes.io/service-name=project-webhook-service",
"-o", "jsonpath={range .items[*]}{range .endpoints[*]}{.addresses[*]}{end}{end}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook endpoints should exist")
g.Expect(output).ShouldNot(BeEmpty(), "Webhook endpoints not yet ready")
}
Eventually(verifyWebhookEndpointsReady, 3*time.Minute, time.Second).Should(Succeed())

// +kubebuilder:scaffold:e2e-metrics-webhooks-readiness

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
Expand Down
26 changes: 13 additions & 13 deletions docs/book/src/reference/markers/scaffold.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,20 +95,20 @@ properly registered with the manager, so that the controller can reconcile the r

## List of `+kubebuilder:scaffold` Markers

| Marker | Usual Location | Function |
|--------------------------------------------|------------------------------|---------------------------------------------------------------------------------|
| `+kubebuilder:scaffold:imports` | `main.go` | Marks where imports for new controllers, webhooks, or APIs should be injected. |
| `+kubebuilder:scaffold:scheme` | `init()` in `main.go` | Used to add API versions to the scheme for runtime. |
| `+kubebuilder:scaffold:builder` | `main.go` | Marks where new controllers should be registered with the manager. |
| `+kubebuilder:scaffold:webhook` | `webhooks suite tests` files | Marks where webhook setup functions are added. |
| `+kubebuilder:scaffold:crdkustomizeresource`| `config/crd` | Marks where CRD custom resource patches are added. |
| `+kubebuilder:scaffold:crdkustomizewebhookpatch` | `config/crd` | Marks where CRD webhook patches are added. |
| `+kubebuilder:scaffold:crdkustomizecainjectionns` | `config/default` | Marks where CA injection patches are added for the conversion webhooks. |
| `+kubebuilder:scaffold:crdkustomizecainjectioname` | `config/default` | Marks where CA injection patches are added for the conversion webhooks. |
| Marker | Usual Location | Function |
|--------------------------------------------------------------------------------|------------------------------|---------------------------------------------------------------------------------|
| `+kubebuilder:scaffold:imports` | `main.go` | Marks where imports for new controllers, webhooks, or APIs should be injected. |
| `+kubebuilder:scaffold:scheme` | `init()` in `main.go` | Used to add API versions to the scheme for runtime. |
| `+kubebuilder:scaffold:builder` | `main.go` | Marks where new controllers should be registered with the manager. |
| `+kubebuilder:scaffold:webhook` | `webhooks suite tests` files | Marks where webhook setup functions are added. |
| `+kubebuilder:scaffold:crdkustomizeresource` | `config/crd` | Marks where CRD custom resource patches are added. |
| `+kubebuilder:scaffold:crdkustomizewebhookpatch` | `config/crd` | Marks where CRD webhook patches are added. |
| `+kubebuilder:scaffold:crdkustomizecainjectionns` | `config/default` | Marks where CA injection patches are added for the conversion webhooks. |
| `+kubebuilder:scaffold:crdkustomizecainjectioname` | `config/default` | Marks where CA injection patches are added for the conversion webhooks. |
| **(No longer supported)** `+kubebuilder:scaffold:crdkustomizecainjectionpatch` | `config/crd` | Marks where CA injection patches are added for the webhooks. Replaced by `+kubebuilder:scaffold:crdkustomizecainjectionns` and `+kubebuilder:scaffold:crdkustomizecainjectioname` |
| `+kubebuilder:scaffold:manifestskustomizesamples` | `config/samples` | Marks where Kustomize sample manifests are injected. |
| `+kubebuilder:scaffold:e2e-webhooks-checks` | `test/e2e` | Adds e2e checks for webhooks depending on the types of webhooks scaffolded. |

| `+kubebuilder:scaffold:manifestskustomizesamples` | `config/samples` | Marks where Kustomize sample manifests are injected. |
| `+kubebuilder:scaffold:e2e-webhooks-checks` | `test/e2e` | Adds e2e checks for webhooks depending on the types of webhooks scaffolded. |
| `+kubebuilder:scaffold:e2e-metrics-webhooks-readiness` | `test/e2e` | Adds readiness logic so metrics e2e tests wait for webhook service endpoints before creating pods. |
<aside class="warning">
<h3> **(No longer supported)** `+kubebuilder:scaffold:crdkustomizecainjectionpatch` </h3>

Expand Down
89 changes: 60 additions & 29 deletions pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
log "log/slog"
"os"
"path/filepath"
"strings"

"sigs.k8s.io/kubebuilder/v4/pkg/machinery"
)
Expand All @@ -31,7 +32,10 @@ var (
_ machinery.Inserter = &WebhookTestUpdater{}
)

const webhookChecksMarker = "e2e-webhooks-checks"
const (
webhookChecksMarker = "e2e-webhooks-checks"
metricsWebhookReadinessMarker = "e2e-metrics-webhooks-readiness"
)

// Test defines the basic setup for the e2e test
type Test struct {
Expand Down Expand Up @@ -75,6 +79,7 @@ func (*WebhookTestUpdater) GetIfExistsAction() machinery.IfExistsAction {
func (f *WebhookTestUpdater) GetMarkers() []machinery.Marker {
return []machinery.Marker{
machinery.NewMarkerFor(f.GetPath(), webhookChecksMarker),
machinery.NewMarkerFor(f.GetPath(), metricsWebhookReadinessMarker),
}
}

Expand All @@ -99,36 +104,46 @@ func (f *WebhookTestUpdater) GetCodeFragments() machinery.CodeFragmentsMap {
markers := f.GetMarkers()

for _, marker := range markers {
if !bytes.Contains(content, []byte(marker.String())) {
markerStr := marker.String()
if !bytes.Contains(content, []byte(markerStr)) {
log.Warn("Marker not found in file, skipping webhook test code injection",
"marker", marker.String(),
"marker", markerStr,
"file_path", filePath)
continue // skip this marker
}

var fragments []string
fragments = append(fragments, webhookChecksFragment)
switch {
case strings.Contains(markerStr, webhookChecksMarker):
var fragments []string
fragments = append(fragments, webhookChecksFragment)

if f.Resource != nil && f.Resource.HasDefaultingWebhook() {
mutatingWebhookCode := fmt.Sprintf(mutatingWebhookChecksFragment, f.ProjectName)
fragments = append(fragments, mutatingWebhookCode)
}
if f.Resource != nil && f.Resource.HasDefaultingWebhook() {
mutatingWebhookCode := fmt.Sprintf(mutatingWebhookChecksFragment, f.ProjectName)
fragments = append(fragments, mutatingWebhookCode)
}

if f.Resource != nil && f.Resource.HasValidationWebhook() {
validatingWebhookCode := fmt.Sprintf(validatingWebhookChecksFragment, f.ProjectName)
fragments = append(fragments, validatingWebhookCode)
}
if f.Resource != nil && f.Resource.HasValidationWebhook() {
validatingWebhookCode := fmt.Sprintf(validatingWebhookChecksFragment, f.ProjectName)
fragments = append(fragments, validatingWebhookCode)
}

if f.Resource != nil && f.Resource.HasConversionWebhook() {
conversionWebhookCode := fmt.Sprintf(
conversionWebhookChecksFragment,
f.Resource.Kind,
f.Resource.Plural+"."+f.Resource.Group+"."+f.Resource.Domain,
)
fragments = append(fragments, conversionWebhookCode)
}
if f.Resource != nil && f.Resource.HasConversionWebhook() {
conversionWebhookCode := fmt.Sprintf(
conversionWebhookChecksFragment,
f.Resource.Kind,
f.Resource.Plural+"."+f.Resource.Group+"."+f.Resource.Domain,
)
fragments = append(fragments, conversionWebhookCode)
}

codeFragments[marker] = fragments
if len(fragments) > 0 {
codeFragments[marker] = fragments
}
case strings.Contains(markerStr, metricsWebhookReadinessMarker):
webhookServiceName := fmt.Sprintf("%s-webhook-service", f.ProjectName)
fragments := []string{fmt.Sprintf(metricsWebhookReadinessFragment, webhookServiceName)}
codeFragments[marker] = fragments
}
}

if len(codeFragments) == 0 {
Expand Down Expand Up @@ -198,6 +213,19 @@ const conversionWebhookChecksFragment = `It("should have CA injection for %[1]s

`

const metricsWebhookReadinessFragment = `By("waiting for the webhook service endpoints to be ready")
verifyWebhookEndpointsReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "endpointslices.discovery.k8s.io", "-n", namespace,
"-l", "kubernetes.io/service-name=%s",
"-o", "jsonpath={range .items[*]}{range .endpoints[*]}{.addresses[*]}{end}{end}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook endpoints should exist")
g.Expect(output).ShouldNot(BeEmpty(), "Webhook endpoints not yet ready")
}
Eventually(verifyWebhookEndpointsReady, 3*time.Minute, time.Second).Should(Succeed())

`

var testCodeTemplate = `//go:build e2e
// +build e2e

Expand Down Expand Up @@ -375,24 +403,27 @@ var _ = Describe("Manager", Ordered, func() {
Expect(err).NotTo(HaveOccurred())
Expect(token).NotTo(BeEmpty())

By("waiting for the metrics endpoint to be ready")
verifyMetricsEndpointReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
By("ensuring the controller pod is ready")
verifyControllerPodReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "pod", controllerPodName, "-n", namespace,
"-o", "jsonpath={.status.conditions[?(@.type=='Ready')].status}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
g.Expect(output).To(Equal("True"), "Controller pod not ready")
}
Eventually(verifyMetricsEndpointReady).Should(Succeed())
Eventually(verifyControllerPodReady, 3*time.Minute, time.Second).Should(Succeed())

By("verifying that the controller manager is serving the metrics server")
verifyMetricsServerStarted := func(g Gomega) {
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
g.Expect(output).To(ContainSubstring("Serving metrics server"),
"Metrics server not yet started")
}
Eventually(verifyMetricsServerStarted).Should(Succeed())
Eventually(verifyMetricsServerStarted, 3*time.Minute, time.Second).Should(Succeed())

// +kubebuilder:scaffold:e2e-metrics-webhooks-readiness

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
Expand Down
Loading
Loading