diff --git a/kagenti-operator/cmd/main.go b/kagenti-operator/cmd/main.go index 446f3aa5..85c2c7ff 100644 --- a/kagenti-operator/cmd/main.go +++ b/kagenti-operator/cmd/main.go @@ -688,6 +688,10 @@ func main() { setupLog.Error(err, "unable to set up ready check") os.Exit(1) } + if err := mgr.AddReadyzCheck("webhook", webhookServer.StartedChecker()); err != nil { + setupLog.Error(err, "unable to set up webhook ready check") + os.Exit(1) + } setupLog.Info("starting manager") if err := mgr.Start(ctx); err != nil { diff --git a/kagenti-operator/test/e2e/e2e_test.go b/kagenti-operator/test/e2e/e2e_test.go index 3e090b4e..da60d992 100644 --- a/kagenti-operator/test/e2e/e2e_test.go +++ b/kagenti-operator/test/e2e/e2e_test.go @@ -344,6 +344,11 @@ var _ = Describe("AuthBridge Injection E2E", Ordered, func() { g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for controller pod to be fully Ready") + Eventually(func() error { + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("creating auth bridge test namespace") cmd := exec.Command("kubectl", "create", "ns", authBridgeTestNamespace) _, err := utils.Run(cmd) @@ -696,6 +701,11 @@ var _ = Describe("AgentCard E2E", Ordered, func() { g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for controller pod to be fully Ready") + Eventually(func() error { + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("creating test namespace with labels") cmd := exec.Command("kubectl", "create", "ns", testNamespace) _, err := utils.Run(cmd) @@ -1039,6 +1049,11 @@ rules: g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for controller pod to be fully Ready") + Eventually(func() error { + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("creating test namespace") cmd := exec.Command("kubectl", "create", "ns", agentRuntimeTestNamespace) _, err := utils.Run(cmd) @@ -1513,6 +1528,11 @@ rules: g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for controller pod to be fully Ready") + Eventually(func() error { + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("setting KAGENTI_SPIRE_TRUST_DOMAIN env var") envCmd := exec.Command("kubectl", "set", "env", "deployment/"+controllerDeployment, "-n", controllerNamespace, "KAGENTI_SPIRE_TRUST_DOMAIN=example.org") @@ -1974,6 +1994,11 @@ rules: g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for controller pod to be fully Ready") + Eventually(func() error { + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("creating skill discovery test namespace") cmd := exec.Command("kubectl", "create", "ns", skillDiscoveryTestNamespace) _, err := utils.Run(cmd) @@ -2134,6 +2159,11 @@ rules: g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("waiting for controller pod to be fully Ready after restart") + Eventually(func() error { + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) }) It("should populate linkedSkills from annotation", func() { @@ -2438,6 +2468,11 @@ var _ = Describe("Istio Mesh Enrollment E2E", Ordered, func() { g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("waiting for controller pod to be fully Ready") + Eventually(func() error { + return utils.ProbeWebhookReady(controllerNamespace) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + By("creating test namespace") cmd := exec.Command("kubectl", "create", "ns", istioMeshTestNamespace) _, err := utils.Run(cmd) diff --git a/kagenti-operator/test/utils/utils.go b/kagenti-operator/test/utils/utils.go index 2a3e58ed..d9d3740e 100644 --- a/kagenti-operator/test/utils/utils.go +++ b/kagenti-operator/test/utils/utils.go @@ -751,3 +751,19 @@ func UncommentCode(filename, target, prefix string) error { // nolint:gosec return os.WriteFile(filename, out.Bytes(), 0644) } + +// ProbeWebhookReady checks that the controller pod is fully Ready. +// The readyz endpoint gates on webhookServer.StartedChecker(), so Ready +// means the webhook TLS server is accepting connections on :9443. +func ProbeWebhookReady(namespace string) error { + cmd := exec.Command("kubectl", "wait", + "--for=condition=Ready", + "pod", "-l", "control-plane=controller-manager", + "-n", namespace, + "--timeout=5s") + _, err := Run(cmd) + if err != nil { + return fmt.Errorf("controller pod not yet Ready: %w", err) + } + return nil +}