From 95057f134d1477aec01f58824fe7b964596e140d Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Tue, 9 Jun 2026 23:11:03 -0400 Subject: [PATCH 1/5] fix(e2e): add TLS connectivity probe to webhook readiness checks The webhook endpoint IP check passes before the TLS server is ready, causing pod creation failures when Kubernetes calls the mutating webhook. Add a curl-based TLS probe after the endpoint IP check at all 7 locations where webhook readiness is verified. Fixes: kagenti/kagenti-operator#420 Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- kagenti-operator/test/e2e/e2e_test.go | 35 +++++++++++++++++++++++++++ kagenti-operator/test/utils/utils.go | 23 ++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/kagenti-operator/test/e2e/e2e_test.go b/kagenti-operator/test/e2e/e2e_test.go index 3e090b4e..e9ed7b8f 100644 --- a/kagenti-operator/test/e2e/e2e_test.go +++ b/kagenti-operator/test/e2e/e2e_test.go @@ -344,6 +344,11 @@ var _ = Describe("AuthBridge Injection E2E", Ordered, func() { g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for webhook TLS to accept connections") + Eventually(func() error { + return utils.ProbeWebhookTLS(controllerNamespace) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) + By("creating auth bridge test namespace") cmd := exec.Command("kubectl", "create", "ns", authBridgeTestNamespace) _, err := utils.Run(cmd) @@ -696,6 +701,11 @@ var _ = Describe("AgentCard E2E", Ordered, func() { g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for webhook TLS to accept connections") + Eventually(func() error { + return utils.ProbeWebhookTLS(controllerNamespace) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) + By("creating test namespace with labels") cmd := exec.Command("kubectl", "create", "ns", testNamespace) _, err := utils.Run(cmd) @@ -1039,6 +1049,11 @@ rules: g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for webhook TLS to accept connections") + Eventually(func() error { + return utils.ProbeWebhookTLS(controllerNamespace) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) + By("creating test namespace") cmd := exec.Command("kubectl", "create", "ns", agentRuntimeTestNamespace) _, err := utils.Run(cmd) @@ -1513,6 +1528,11 @@ rules: g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for webhook TLS to accept connections") + Eventually(func() error { + return utils.ProbeWebhookTLS(controllerNamespace) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) + By("setting KAGENTI_SPIRE_TRUST_DOMAIN env var") envCmd := exec.Command("kubectl", "set", "env", "deployment/"+controllerDeployment, "-n", controllerNamespace, "KAGENTI_SPIRE_TRUST_DOMAIN=example.org") @@ -1974,6 +1994,11 @@ rules: g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for webhook TLS to accept connections") + Eventually(func() error { + return utils.ProbeWebhookTLS(controllerNamespace) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) + By("creating skill discovery test namespace") cmd := exec.Command("kubectl", "create", "ns", skillDiscoveryTestNamespace) _, err := utils.Run(cmd) @@ -2134,6 +2159,11 @@ rules: g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("waiting for webhook TLS to accept connections after restart") + Eventually(func() error { + return utils.ProbeWebhookTLS(controllerNamespace) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) }) It("should populate linkedSkills from annotation", func() { @@ -2438,6 +2468,11 @@ var _ = Describe("Istio Mesh Enrollment E2E", Ordered, func() { g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for webhook TLS to accept connections") + Eventually(func() error { + return utils.ProbeWebhookTLS(controllerNamespace) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) + By("creating test namespace") cmd := exec.Command("kubectl", "create", "ns", istioMeshTestNamespace) _, err := utils.Run(cmd) diff --git a/kagenti-operator/test/utils/utils.go b/kagenti-operator/test/utils/utils.go index 2a3e58ed..5c215966 100644 --- a/kagenti-operator/test/utils/utils.go +++ b/kagenti-operator/test/utils/utils.go @@ -751,3 +751,26 @@ func UncommentCode(filename, target, prefix string) error { // nolint:gosec return os.WriteFile(filename, out.Bytes(), 0644) } + +// ProbeWebhookTLS runs a temporary curl pod that attempts a TLS connection to the +// webhook service. Returns nil if the webhook is accepting TLS connections, or an +// error if it is not yet ready. Intended for use inside an Eventually() block. +func ProbeWebhookTLS(namespace string) error { + podName := fmt.Sprintf("webhook-probe-%d", time.Now().UnixNano()%100000) + svcURL := fmt.Sprintf("https://kagenti-operator-webhook-service.%s.svc:443/", namespace) + + cmd := exec.Command("kubectl", "run", podName, + "--rm", "-i", "--restart=Never", + "--image=curlimages/curl:latest", + "-n", namespace, + "--", "curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", + "--max-time", "5", svcURL) + output, err := Run(cmd) + if err != nil { + return fmt.Errorf("webhook TLS probe failed: %w", err) + } + if strings.TrimSpace(output) == "000" { + return fmt.Errorf("webhook TLS probe got HTTP 000 (connection refused)") + } + return nil +} From 68f9a292d00d5ebba14ebd7e422a788f854ffb81 Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Tue, 9 Jun 2026 23:34:43 -0400 Subject: [PATCH 2/5] fix(e2e): add security context to webhook probe pod The curl probe pod was rejected by the restricted PodSecurity policy in CI. Add required securityContext fields (runAsNonRoot, capabilities drop ALL, seccompProfile RuntimeDefault) via --overrides. Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- kagenti-operator/test/utils/utils.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/kagenti-operator/test/utils/utils.go b/kagenti-operator/test/utils/utils.go index 5c215966..51111e90 100644 --- a/kagenti-operator/test/utils/utils.go +++ b/kagenti-operator/test/utils/utils.go @@ -759,12 +759,29 @@ func ProbeWebhookTLS(namespace string) error { podName := fmt.Sprintf("webhook-probe-%d", time.Now().UnixNano()%100000) svcURL := fmt.Sprintf("https://kagenti-operator-webhook-service.%s.svc:443/", namespace) + // Security context required for clusters with "restricted" PodSecurity enforcement. + overrides := `{ + "spec": { + "securityContext": {"runAsNonRoot": true, "seccompProfile": {"type": "RuntimeDefault"}}, + "containers": [{ + "name": "` + podName + `", + "image": "curlimages/curl:latest", + "command": ["curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", "` + svcURL + `"], + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": {"drop": ["ALL"]}, + "runAsNonRoot": true, + "seccompProfile": {"type": "RuntimeDefault"} + } + }] + } + }` + cmd := exec.Command("kubectl", "run", podName, "--rm", "-i", "--restart=Never", "--image=curlimages/curl:latest", - "-n", namespace, - "--", "curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", - "--max-time", "5", svcURL) + "--overrides", overrides, + "-n", namespace) output, err := Run(cmd) if err != nil { return fmt.Errorf("webhook TLS probe failed: %w", err) From 3efc274fd91392880de4ca59e9f8bba140ffb1d8 Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Wed, 10 Jun 2026 00:05:51 -0400 Subject: [PATCH 3/5] fix(e2e): use --connect-only for webhook TLS probe The webhook server only handles admission POST requests and does not respond to HTTP GET. Using --connect-only verifies the TLS handshake succeeds without waiting for an HTTP response, avoiding a 5s timeout on every probe attempt. Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- kagenti-operator/test/utils/utils.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/kagenti-operator/test/utils/utils.go b/kagenti-operator/test/utils/utils.go index 51111e90..b31f1b97 100644 --- a/kagenti-operator/test/utils/utils.go +++ b/kagenti-operator/test/utils/utils.go @@ -752,21 +752,21 @@ func UncommentCode(filename, target, prefix string) error { return os.WriteFile(filename, out.Bytes(), 0644) } -// ProbeWebhookTLS runs a temporary curl pod that attempts a TLS connection to the -// webhook service. Returns nil if the webhook is accepting TLS connections, or an -// error if it is not yet ready. Intended for use inside an Eventually() block. +// ProbeWebhookTLS runs a temporary curl pod that verifies the webhook service +// TLS handshake succeeds. Uses --connect-only to avoid waiting for an HTTP +// response (the webhook only handles admission POST requests). Returns nil if +// the TLS connection succeeds, or an error if not yet ready. func ProbeWebhookTLS(namespace string) error { podName := fmt.Sprintf("webhook-probe-%d", time.Now().UnixNano()%100000) - svcURL := fmt.Sprintf("https://kagenti-operator-webhook-service.%s.svc:443/", namespace) + svcURL := fmt.Sprintf("https://kagenti-operator-webhook-service.%s.svc:443", namespace) - // Security context required for clusters with "restricted" PodSecurity enforcement. overrides := `{ "spec": { "securityContext": {"runAsNonRoot": true, "seccompProfile": {"type": "RuntimeDefault"}}, "containers": [{ "name": "` + podName + `", "image": "curlimages/curl:latest", - "command": ["curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", "` + svcURL + `"], + "command": ["curl", "-sk", "--connect-only", "--connect-timeout", "5", "` + svcURL + `"], "securityContext": { "allowPrivilegeEscalation": false, "capabilities": {"drop": ["ALL"]}, @@ -782,12 +782,9 @@ func ProbeWebhookTLS(namespace string) error { "--image=curlimages/curl:latest", "--overrides", overrides, "-n", namespace) - output, err := Run(cmd) + _, err := Run(cmd) if err != nil { return fmt.Errorf("webhook TLS probe failed: %w", err) } - if strings.TrimSpace(output) == "000" { - return fmt.Errorf("webhook TLS probe got HTTP 000 (connection refused)") - } return nil } From 1564816d37540b0822448c15349fc86ef93784db Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Wed, 10 Jun 2026 00:35:08 -0400 Subject: [PATCH 4/5] fix(e2e): replace curl probe with pod readiness check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The curl probe pod approach fails in CI due to image pull delays and kubectl run timeout issues. Replace with a simple check that the controller pod is fully Ready — the manager's readiness probe on port 8081 gates on complete initialization including webhook server. Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- kagenti-operator/test/e2e/e2e_test.go | 42 ++++++++++++------------- kagenti-operator/test/utils/utils.go | 45 ++++++++------------------- 2 files changed, 34 insertions(+), 53 deletions(-) diff --git a/kagenti-operator/test/e2e/e2e_test.go b/kagenti-operator/test/e2e/e2e_test.go index e9ed7b8f..da60d992 100644 --- a/kagenti-operator/test/e2e/e2e_test.go +++ b/kagenti-operator/test/e2e/e2e_test.go @@ -344,10 +344,10 @@ var _ = Describe("AuthBridge Injection E2E", Ordered, func() { g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) - By("waiting for webhook TLS to accept connections") + By("waiting for controller pod to be fully Ready") Eventually(func() error { - return utils.ProbeWebhookTLS(controllerNamespace) - }, 2*time.Minute, 5*time.Second).Should(Succeed()) + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) By("creating auth bridge test namespace") cmd := exec.Command("kubectl", "create", "ns", authBridgeTestNamespace) @@ -701,10 +701,10 @@ var _ = Describe("AgentCard E2E", Ordered, func() { g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) - By("waiting for webhook TLS to accept connections") + By("waiting for controller pod to be fully Ready") Eventually(func() error { - return utils.ProbeWebhookTLS(controllerNamespace) - }, 2*time.Minute, 5*time.Second).Should(Succeed()) + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) By("creating test namespace with labels") cmd := exec.Command("kubectl", "create", "ns", testNamespace) @@ -1049,10 +1049,10 @@ rules: g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) - By("waiting for webhook TLS to accept connections") + By("waiting for controller pod to be fully Ready") Eventually(func() error { - return utils.ProbeWebhookTLS(controllerNamespace) - }, 2*time.Minute, 5*time.Second).Should(Succeed()) + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) By("creating test namespace") cmd := exec.Command("kubectl", "create", "ns", agentRuntimeTestNamespace) @@ -1528,10 +1528,10 @@ rules: g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) - By("waiting for webhook TLS to accept connections") + By("waiting for controller pod to be fully Ready") Eventually(func() error { - return utils.ProbeWebhookTLS(controllerNamespace) - }, 2*time.Minute, 5*time.Second).Should(Succeed()) + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) By("setting KAGENTI_SPIRE_TRUST_DOMAIN env var") envCmd := exec.Command("kubectl", "set", "env", "deployment/"+controllerDeployment, @@ -1994,10 +1994,10 @@ rules: g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) - By("waiting for webhook TLS to accept connections") + By("waiting for controller pod to be fully Ready") Eventually(func() error { - return utils.ProbeWebhookTLS(controllerNamespace) - }, 2*time.Minute, 5*time.Second).Should(Succeed()) + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) By("creating skill discovery test namespace") cmd := exec.Command("kubectl", "create", "ns", skillDiscoveryTestNamespace) @@ -2160,10 +2160,10 @@ rules: g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) - By("waiting for webhook TLS to accept connections after restart") + By("waiting for controller pod to be fully Ready after restart") Eventually(func() error { - return utils.ProbeWebhookTLS(controllerNamespace) - }, 2*time.Minute, 5*time.Second).Should(Succeed()) + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) }) It("should populate linkedSkills from annotation", func() { @@ -2468,10 +2468,10 @@ var _ = Describe("Istio Mesh Enrollment E2E", Ordered, func() { g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) - By("waiting for webhook TLS to accept connections") + By("waiting for controller pod to be fully Ready") Eventually(func() error { - return utils.ProbeWebhookTLS(controllerNamespace) - }, 2*time.Minute, 5*time.Second).Should(Succeed()) + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) By("creating test namespace") cmd := exec.Command("kubectl", "create", "ns", istioMeshTestNamespace) diff --git a/kagenti-operator/test/utils/utils.go b/kagenti-operator/test/utils/utils.go index b31f1b97..e2a9b853 100644 --- a/kagenti-operator/test/utils/utils.go +++ b/kagenti-operator/test/utils/utils.go @@ -752,39 +752,20 @@ func UncommentCode(filename, target, prefix string) error { return os.WriteFile(filename, out.Bytes(), 0644) } -// ProbeWebhookTLS runs a temporary curl pod that verifies the webhook service -// TLS handshake succeeds. Uses --connect-only to avoid waiting for an HTTP -// response (the webhook only handles admission POST requests). Returns nil if -// the TLS connection succeeds, or an error if not yet ready. -func ProbeWebhookTLS(namespace string) error { - podName := fmt.Sprintf("webhook-probe-%d", time.Now().UnixNano()%100000) - svcURL := fmt.Sprintf("https://kagenti-operator-webhook-service.%s.svc:443", namespace) - - overrides := `{ - "spec": { - "securityContext": {"runAsNonRoot": true, "seccompProfile": {"type": "RuntimeDefault"}}, - "containers": [{ - "name": "` + podName + `", - "image": "curlimages/curl:latest", - "command": ["curl", "-sk", "--connect-only", "--connect-timeout", "5", "` + svcURL + `"], - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": {"drop": ["ALL"]}, - "runAsNonRoot": true, - "seccompProfile": {"type": "RuntimeDefault"} - } - }] - } - }` - - cmd := exec.Command("kubectl", "run", podName, - "--rm", "-i", "--restart=Never", - "--image=curlimages/curl:latest", - "--overrides", overrides, - "-n", namespace) - _, err := Run(cmd) +// ProbeWebhookReady checks that the controller pod is fully Ready (all +// containers passing readiness probes). The manager's readiness probe gates +// on full initialization including the webhook server startup. +func ProbeWebhookReady(namespace string) error { + cmd := exec.Command("kubectl", "get", "pods", + "-l", "control-plane=controller-manager", + "-n", namespace, + "-o", "jsonpath={.items[0].status.conditions[?(@.type==\"Ready\")].status}") + output, err := Run(cmd) if err != nil { - return fmt.Errorf("webhook TLS probe failed: %w", err) + return fmt.Errorf("failed to check controller pod readiness: %w", err) + } + if strings.TrimSpace(output) != "True" { + return fmt.Errorf("controller pod not yet Ready (status=%q)", output) } return nil } From ce59ce4e9646f33bf9bc0fac2aec7d2298eac713 Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Wed, 10 Jun 2026 09:42:37 -0400 Subject: [PATCH 5/5] fix: gate readyz on webhook TLS server startup Add webhookServer.StartedChecker() as a readyz check so the pod Ready condition reflects actual webhook TLS availability on :9443, not just the health endpoint being up. Replace the fragile .items[0] jsonpath probe with kubectl wait which handles rolling restarts correctly. Addresses review feedback from #421. Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- kagenti-operator/cmd/main.go | 4 ++++ kagenti-operator/test/utils/utils.go | 20 +++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/kagenti-operator/cmd/main.go b/kagenti-operator/cmd/main.go index 446f3aa5..85c2c7ff 100644 --- a/kagenti-operator/cmd/main.go +++ b/kagenti-operator/cmd/main.go @@ -688,6 +688,10 @@ func main() { setupLog.Error(err, "unable to set up ready check") os.Exit(1) } + if err := mgr.AddReadyzCheck("webhook", webhookServer.StartedChecker()); err != nil { + setupLog.Error(err, "unable to set up webhook ready check") + os.Exit(1) + } setupLog.Info("starting manager") if err := mgr.Start(ctx); err != nil { diff --git a/kagenti-operator/test/utils/utils.go b/kagenti-operator/test/utils/utils.go index e2a9b853..d9d3740e 100644 --- a/kagenti-operator/test/utils/utils.go +++ b/kagenti-operator/test/utils/utils.go @@ -752,20 +752,18 @@ func UncommentCode(filename, target, prefix string) error { return os.WriteFile(filename, out.Bytes(), 0644) } -// ProbeWebhookReady checks that the controller pod is fully Ready (all -// containers passing readiness probes). The manager's readiness probe gates -// on full initialization including the webhook server startup. +// ProbeWebhookReady checks that the controller pod is fully Ready. +// The readyz endpoint gates on webhookServer.StartedChecker(), so Ready +// means the webhook TLS server is accepting connections on :9443. func ProbeWebhookReady(namespace string) error { - cmd := exec.Command("kubectl", "get", "pods", - "-l", "control-plane=controller-manager", + cmd := exec.Command("kubectl", "wait", + "--for=condition=Ready", + "pod", "-l", "control-plane=controller-manager", "-n", namespace, - "-o", "jsonpath={.items[0].status.conditions[?(@.type==\"Ready\")].status}") - output, err := Run(cmd) + "--timeout=5s") + _, err := Run(cmd) if err != nil { - return fmt.Errorf("failed to check controller pod readiness: %w", err) - } - if strings.TrimSpace(output) != "True" { - return fmt.Errorf("controller pod not yet Ready (status=%q)", output) + return fmt.Errorf("controller pod not yet Ready: %w", err) } return nil }