From c35437f524b2467aa57e90429c26a2e57a8a5291 Mon Sep 17 00:00:00 2001 From: Akram Date: Fri, 10 Apr 2026 12:25:30 +0200 Subject: [PATCH] Add configurable SPIFFE ID template support and e2e tests for ClientRegistration controller Fixes kagenti/kagenti-operator#268 The client registration controller was hardcoding the SPIFFE ID format as spiffe://{trustDomain}/ns/{namespace}/sa/{serviceAccount}, but SPIRE's ClusterSPIFFEID resources allow custom spiffeIDTemplate configurations. This caused confusion when the operator-generated client IDs didn't match the actual SPIFFE IDs issued to workloads. Add comprehensive end-to-end tests for the ClientRegistration controller that verify OAuth client registration in Keycloak and Secret management. Changes: - Add --spire-id-template flag (default: spiffe://{{.TrustDomain}}/ns/{{.Namespace}}/sa/{{.ServiceAccount}}) - Support SPIRE_ID_TEMPLATE environment variable for configuration - Refactor resolveKeycloakClientID to use text/template for rendering - Update tests to verify template rendering with custom formats - Document the new flag and environment variable in operator docs - Mock Keycloak server that simulates admin API for client registration - authbridge-config and keycloak-admin-secret ConfigMaps/Secrets - Agent Deployments for testing non-SPIRE and SPIRE modes The template supports variables: {{.TrustDomain}}, {{.Namespace}}, {{.ServiceAccount}} Command-line flag takes precedence over environment variable. Assisted-By: Claude (Anthropic AI) Signed-off-by: Akram --- kagenti-operator/cmd/main.go | 10 + .../operator-managed-client-registration.md | 4 +- .../clientregistration_controller.go | 30 +- .../clientregistration_controller_test.go | 15 +- kagenti-operator/test/e2e/README.md | 53 ++- .../test/e2e/clientregistration_e2e_test.go | 358 ++++++++++++++++++ kagenti-operator/test/e2e/fixtures.go | 229 +++++++++++ 7 files changed, 690 insertions(+), 9 deletions(-) create mode 100644 kagenti-operator/test/e2e/clientregistration_e2e_test.go diff --git a/kagenti-operator/cmd/main.go b/kagenti-operator/cmd/main.go index fded58d5..d51aaa44 100644 --- a/kagenti-operator/cmd/main.go +++ b/kagenti-operator/cmd/main.go @@ -89,6 +89,7 @@ func main() { var enableMLflow bool var spireTrustDomain string + var spireIDTemplate string var spireTrustBundleConfigMapName string var spireTrustBundleConfigMapNS string var spireTrustBundleConfigMapKey string @@ -131,6 +132,14 @@ func main() { flag.StringVar(&spireTrustDomain, "spire-trust-domain", "", "SPIRE trust domain for identity binding (e.g. 'example.org')") + + spireIDTemplateDefault := "spiffe://{{.TrustDomain}}/ns/{{.Namespace}}/sa/{{.ServiceAccount}}" + if envTemplate := os.Getenv("SPIRE_ID_TEMPLATE"); envTemplate != "" { + spireIDTemplateDefault = envTemplate + } + flag.StringVar(&spireIDTemplate, "spire-id-template", spireIDTemplateDefault, + "SPIFFE ID template for Keycloak client IDs (must match ClusterSPIFFEID spiffeIDTemplate, "+ + "can also be set via SPIRE_ID_TEMPLATE env var)") flag.StringVar(&spireTrustBundleConfigMapName, "spire-trust-bundle-configmap", "", "Name of the ConfigMap containing the SPIRE trust bundle (SPIFFE JSON format)") flag.StringVar(&spireTrustBundleConfigMapNS, "spire-trust-bundle-configmap-namespace", "", @@ -404,6 +413,7 @@ func main() { APIReader: mgr.GetAPIReader(), Scheme: mgr.GetScheme(), SpireTrustDomain: spireTrustDomain, + SpireIDTemplate: spireIDTemplate, KeycloakAdminTokenCache: &keycloak.CachedAdminTokenProvider{}, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ClientRegistration") diff --git a/kagenti-operator/docs/operator-managed-client-registration.md b/kagenti-operator/docs/operator-managed-client-registration.md index c81236cf..5a6cfd5a 100644 --- a/kagenti-operator/docs/operator-managed-client-registration.md +++ b/kagenti-operator/docs/operator-managed-client-registration.md @@ -77,7 +77,7 @@ Other workloads are ignored by this controller. 3. Read **`keycloak-admin-secret`** (admin username/password). 4. Compute **Keycloak client ID**: - If `SPIRE_ENABLED` is not true: `namespace/workloadName`. - - If SPIRE is enabled: `spiffe:///ns//sa/` (requires a **non-default** `serviceAccountName` and operator **`--spire-trust-domain`**). + - If SPIRE is enabled: render the `--spire-id-template` with trust domain, namespace, and service account (default format: `spiffe:///ns//sa/`). Requires a **non-default** `serviceAccountName`, operator **`--spire-trust-domain`**, and **`--spire-id-template`**. 5. **Register or fetch** the client via Keycloak admin API (`internal/keycloak`). 6. **Create or update** the credentials Secret; set **owner** to the Deployment/StatefulSet. 7. **Patch** the pod template annotation `kagenti.io/keycloak-client-credentials-secret-name` to the deterministic secret name. @@ -88,6 +88,7 @@ Other workloads are ignored by this controller. |-----------|-------------|------| | Operator (controller) | `--enable-operator-client-registration` (default **false**) | Master switch for the ClientRegistration controller. | | Operator (controller) | `--spire-trust-domain` | Required for SPIFFE-shaped client IDs when `authbridge-config` has `SPIRE_ENABLED=true`. | +| Operator (controller) | `--spire-id-template` (default `spiffe://{{.TrustDomain}}/ns/{{.Namespace}}/sa/{{.ServiceAccount}}`) | SPIFFE ID template for Keycloak client IDs. Must match the `spiffeIDTemplate` in ClusterSPIFFEID resources. Can also be set via `SPIRE_ID_TEMPLATE` environment variable. | | Operator (webhook) | `--enable-client-registration` | Cluster-wide gate for client-registration **injection** (precedence still applies). | | Operator (webhook) | Feature gates file (`/etc/kagenti/feature-gates/feature-gates.yaml`) | `clientRegistration`, `injectTools`, `globalEnabled`, etc., same as for injected sidecars. | @@ -110,6 +111,7 @@ Other workloads are ignored by this controller. ### 3.3 Operator configuration - When `authbridge-config` sets `SPIRE_ENABLED=true`, configure **`--spire-trust-domain`** to match the SPIRE server trust domain (same value as used for workload SPIFFE IDs). +- Configure **`--spire-id-template`** (or set the **`SPIRE_ID_TEMPLATE`** environment variable) to match the `spiffeIDTemplate` used in your ClusterSPIFFEID resources. The default template `spiffe://{{.TrustDomain}}/ns/{{.Namespace}}/sa/{{.ServiceAccount}}` matches the standard format. If you use a custom template in SPIRE (e.g., with pod name or labels), update this flag or environment variable accordingly. The flag takes precedence over the environment variable if both are set. - Ensure the operator can read **`authbridge-config`** and **`keycloak-admin-secret`** in agent namespaces, and create/update **`kagenti-keycloak-client-credentials-*`** Secrets there (see RBAC below). ### 3.4 RBAC: why Secret rules are cluster-wide diff --git a/kagenti-operator/internal/controller/clientregistration_controller.go b/kagenti-operator/internal/controller/clientregistration_controller.go index 66aa1818..239f8061 100644 --- a/kagenti-operator/internal/controller/clientregistration_controller.go +++ b/kagenti-operator/internal/controller/clientregistration_controller.go @@ -8,11 +8,13 @@ you may not use this file except in compliance with the License. package controller import ( + "bytes" "context" "crypto/sha256" "encoding/hex" "fmt" "strings" + "text/template" "time" appsv1 "k8s.io/api/apps/v1" @@ -61,6 +63,7 @@ type ClientRegistrationReconciler struct { Scheme *runtime.Scheme SpireTrustDomain string + SpireIDTemplate string // KeycloakAdminTokenCache caches admin password-grant tokens by Keycloak URL and credentials to // avoid a token request on every reconcile. If nil, PasswordGrantToken is used without caching. KeycloakAdminTokenCache *keycloak.CachedAdminTokenProvider @@ -180,7 +183,7 @@ func (r *ClientRegistrationReconciler) reconcileOne( spireEnabled := strings.EqualFold(strings.TrimSpace(ab.SpireEnabled), "true") clientName := ns + "/" + workloadName - clientID, err := resolveKeycloakClientID(ns, workloadName, template.Spec.ServiceAccountName, spireEnabled, r.SpireTrustDomain) + clientID, err := resolveKeycloakClientID(ns, workloadName, template.Spec.ServiceAccountName, spireEnabled, r.SpireTrustDomain, r.SpireIDTemplate) if err != nil { logger.Info("cannot resolve Keycloak client id yet", "reason", err.Error()) return ctrl.Result{RequeueAfter: 30 * time.Second}, nil @@ -375,7 +378,7 @@ func readClusterFeatureGates(ctx context.Context, c client.Reader) (globalOn, cl return globalOn, clientReg, injectTools, nil } -func resolveKeycloakClientID(namespace, workloadName, serviceAccount string, spireEnabled bool, trustDomain string) (string, error) { +func resolveKeycloakClientID(namespace, workloadName, serviceAccount string, spireEnabled bool, trustDomain, spireIDTemplate string) (string, error) { sa := strings.TrimSpace(serviceAccount) if sa == "" { sa = "default" @@ -389,7 +392,28 @@ func resolveKeycloakClientID(namespace, workloadName, serviceAccount string, spi if trustDomain == "" { return "", fmt.Errorf("SPIRE enabled: operator --spire-trust-domain is required for operator-managed client registration") } - return fmt.Sprintf("spiffe://%s/ns/%s/sa/%s", trustDomain, namespace, sa), nil + if spireIDTemplate == "" { + return "", fmt.Errorf("SPIRE enabled: operator --spire-id-template is required for operator-managed client registration") + } + + // Parse and render the SPIFFE ID template + tmpl, err := template.New("spiffe-id").Parse(spireIDTemplate) + if err != nil { + return "", fmt.Errorf("invalid SPIFFE ID template: %w", err) + } + + data := map[string]string{ + "TrustDomain": trustDomain, + "Namespace": namespace, + "ServiceAccount": sa, + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return "", fmt.Errorf("failed to render SPIFFE ID template: %w", err) + } + + return buf.String(), nil } func keycloakClientCredentialsSecretName(namespace, workload string) string { diff --git a/kagenti-operator/internal/controller/clientregistration_controller_test.go b/kagenti-operator/internal/controller/clientregistration_controller_test.go index 9acb183f..012744c7 100644 --- a/kagenti-operator/internal/controller/clientregistration_controller_test.go +++ b/kagenti-operator/internal/controller/clientregistration_controller_test.go @@ -119,18 +119,27 @@ func TestParsePlatformClientIDs(t *testing.T) { } func TestResolveKeycloakClientID(t *testing.T) { - id, err := resolveKeycloakClientID("ns1", "dep", "", false, "") + defaultTemplate := "spiffe://{{.TrustDomain}}/ns/{{.Namespace}}/sa/{{.ServiceAccount}}" + + id, err := resolveKeycloakClientID("ns1", "dep", "", false, "", "") if err != nil || id != "ns1/dep" { t.Fatalf("non-spire: %q %v", id, err) } - _, err = resolveKeycloakClientID("ns1", "dep", "", true, "example.org") + _, err = resolveKeycloakClientID("ns1", "dep", "", true, "example.org", defaultTemplate) if err == nil { t.Fatal("expected error for default SA with SPIRE") } - id, err = resolveKeycloakClientID("ns1", "dep", "mysa", true, "example.org") + id, err = resolveKeycloakClientID("ns1", "dep", "mysa", true, "example.org", defaultTemplate) if err != nil || id != "spiffe://example.org/ns/ns1/sa/mysa" { t.Fatalf("spire: %q %v", id, err) } + + // Test custom template + customTemplate := "spiffe://{{.TrustDomain}}/workload/{{.Namespace}}/{{.ServiceAccount}}" + id, err = resolveKeycloakClientID("ns1", "dep", "mysa", true, "example.org", customTemplate) + if err != nil || id != "spiffe://example.org/workload/ns1/mysa" { + t.Fatalf("custom template: %q %v", id, err) + } } func clientRegistrationTestScheme(t *testing.T) *runtime.Scheme { diff --git a/kagenti-operator/test/e2e/README.md b/kagenti-operator/test/e2e/README.md index 84a54197..93fb586c 100644 --- a/kagenti-operator/test/e2e/README.md +++ b/kagenti-operator/test/e2e/README.md @@ -1,9 +1,10 @@ # E2E Tests -End-to-end tests for the kagenti-operator. The suite runs 8 specs: +End-to-end tests for the kagenti-operator. The suite runs 11 specs: - **Manager tests** (2 specs) — controller pod readiness and Prometheus metrics - **AgentCard tests** (6 specs) — webhook validation, auto-discovery, duplicate prevention, audit mode, and SPIRE signature verification +- **ClientRegistration tests** (3 specs) — Keycloak client registration, SPIFFE ID templates, and Secret creation ## Prerequisites @@ -20,7 +21,7 @@ The test suite auto-detects Docker vs Podman. No env vars needed. # Create a fresh Kind cluster kind delete cluster 2>/dev/null; kind create cluster -# Run all 8 specs (~7 min) +# Run all 11 specs (~9 min) make test-e2e ``` @@ -46,6 +47,9 @@ go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="should reject AgentCard|should n # Signature verification tests only (~5 min) go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="SignatureInvalidAudit|should verify signed" +# ClientRegistration tests only (~3 min) +go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="ClientRegistration" + # Manager tests only go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="Manager" ``` @@ -66,6 +70,9 @@ kind delete cluster | Duplicate prevention | Without signature | Webhook rejects a second AgentCard targeting the same workload | | Audit mode | With signature | Unsigned card syncs (Synced=True) but reports SignatureVerified=False with reason SignatureInvalidAudit | | Signed agent | With signature | SPIRE-signed card gets SignatureVerified=True, correct SPIFFE ID, Synced=True, and Bound=True | +| Client registration (non-SPIRE) | ClientRegistration | Controller creates client credentials Secret with `namespace/workloadName` format and patches pod template annotation | +| Client registration (SPIRE default) | ClientRegistration | Controller creates SPIFFE ID using default template `spiffe://{domain}/ns/{namespace}/sa/{serviceAccount}` | +| Client registration (SPIRE custom) | ClientRegistration | Controller respects `SPIRE_ID_TEMPLATE` env var for custom SPIFFE ID formats | ## Architecture @@ -198,6 +205,48 @@ Test verifies: `SignatureVerified=True` (reason `SignatureValid`), `signatureSpiffeId = spiffe://example.org/ns/e2e-agentcard-test/sa/signed-agent-sa`, `Synced=True`, `Bound=True`. +#### Client registration (non-SPIRE mode) + +Controller is patched with `--enable-operator-client-registration=true --spire-trust-domain=example.org`. +Deploys a mock Keycloak server, `authbridge-config` ConfigMap with `SPIRE_ENABLED=false`, and +`keycloak-admin-secret`. Deploys `clientreg-agent` workload. + +1. Controller watches the Deployment (has `kagenti.io/type=agent` label) +2. Reads `authbridge-config` and `keycloak-admin-secret` from workload namespace +3. Computes client ID as `namespace/workloadName` (non-SPIRE format) +4. Registers client with mock Keycloak via admin API +5. Creates Secret with `client-id.txt` and `client-secret.txt` (owner: Deployment) +6. Patches Deployment pod template with `kagenti.io/keycloak-client-credentials-secret-name` annotation + +Test verifies: Secret created, contains both keys, client ID format is `e2e-clientreg-test/clientreg-agent`, +and pod template annotation references the Secret name. + +#### Client registration (SPIRE mode with default template) + +Updates `authbridge-config` to `SPIRE_ENABLED=true`. Deploys `clientreg-spire-agent` with dedicated +ServiceAccount `clientreg-spire-sa`. + +1. Controller detects SPIRE is enabled +2. Validates ServiceAccount is not `default` +3. Renders default SPIFFE ID template: `spiffe://{{.TrustDomain}}/ns/{{.Namespace}}/sa/{{.ServiceAccount}}` +4. Computes client ID as `spiffe://example.org/ns/e2e-clientreg-test/sa/clientreg-spire-sa` +5. Registers client and creates Secret with SPIFFE ID as client ID + +Test verifies: Client ID matches expected SPIFFE format with default template. + +#### Client registration (SPIRE mode with custom template) + +Patches operator Deployment to set env var `SPIRE_ID_TEMPLATE=spiffe://{{.TrustDomain}}/workload/{{.Namespace}}/{{.ServiceAccount}}`. +Waits for operator restart. Deploys `clientreg-custom-agent` with ServiceAccount `clientreg-custom-sa`. + +1. Controller reads custom template from environment variable +2. Renders custom template with workload metadata +3. Computes client ID as `spiffe://example.org/workload/e2e-clientreg-test/clientreg-custom-sa` +4. Registers client and creates Secret + +Test verifies: Client ID matches expected SPIFFE format with custom template. Restores operator +to default configuration after test. + ## Troubleshooting **Stale cluster state** — if you see errors about namespaces being terminated or cert-manager TLS failures, delete and recreate the cluster: diff --git a/kagenti-operator/test/e2e/clientregistration_e2e_test.go b/kagenti-operator/test/e2e/clientregistration_e2e_test.go new file mode 100644 index 00000000..71bda43e --- /dev/null +++ b/kagenti-operator/test/e2e/clientregistration_e2e_test.go @@ -0,0 +1,358 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +*/ + +package e2e + +import ( + "fmt" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/kagenti/operator/test/utils" +) + +const clientRegNamespace = "e2e-clientreg-test" + +var _ = Describe("ClientRegistration E2E", Ordered, func() { + var origArgs []string + + BeforeAll(func() { + By("deploying controller") + Expect(utils.DeployController(namespace, projectImage)).To(Succeed(), "Failed to deploy controller") + + By("waiting for controller-manager to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", + "-n", namespace, + "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .status.phase }}{{ end }}{{ end }}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Running")) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("patching operator to enable ClientRegistration controller") + var err error + origArgs, err = utils.PatchControllerArgs(namespace, "kagenti-operator-controller-manager", []string{ + "--enable-operator-client-registration=true", + "--spire-trust-domain=example.org", + }) + Expect(err).NotTo(HaveOccurred(), "Failed to patch operator deployment") + + By("creating clientreg test namespace") + cmd := exec.Command("kubectl", "create", "ns", clientRegNamespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create clientreg test namespace") + + By("labeling the clientreg namespace for restricted PSA and agentcard") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", clientRegNamespace, + "pod-security.kubernetes.io/enforce=restricted", + "agentcard=true") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label clientreg namespace") + + By("deploying mock Keycloak server") + cmd = exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(mockKeycloakFixture()) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy mock Keycloak") + + By("waiting for mock Keycloak to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", "mock-keycloak", + "-n", clientRegNamespace, "-o", "jsonpath={.status.readyReplicas}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + }) + + AfterAll(func() { + By("cleaning up clientreg test namespace") + cmd := exec.Command("kubectl", "delete", "ns", clientRegNamespace, "--ignore-not-found") + _, _ = utils.Run(cmd) + + By("restoring operator to default args") + if origArgs != nil { + _ = utils.RestoreControllerArgs(namespace, "kagenti-operator-controller-manager", origArgs) + } + + utils.UndeployController() + + By("removing manager namespace") + cmd = exec.Command("kubectl", "delete", "ns", namespace, "--ignore-not-found") + _, _ = utils.Run(cmd) + }) + + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + By("Fetching deployment details on failure") + cmd := exec.Command("kubectl", "get", "deployments", "-n", clientRegNamespace, "-o", "wide") + output, _ := utils.Run(cmd) + _, _ = fmt.Fprintf(GinkgoWriter, "Deployments:\n%s\n", output) + + By("Fetching secrets on failure") + cmd = exec.Command("kubectl", "get", "secrets", "-n", clientRegNamespace) + output, _ = utils.Run(cmd) + _, _ = fmt.Fprintf(GinkgoWriter, "Secrets:\n%s\n", output) + + By("Fetching configmaps on failure") + cmd = exec.Command("kubectl", "get", "configmaps", "-n", clientRegNamespace) + output, _ = utils.Run(cmd) + _, _ = fmt.Fprintf(GinkgoWriter, "ConfigMaps:\n%s\n", output) + + By("Fetching operator logs on failure") + cmd = exec.Command("kubectl", "logs", "-n", namespace, "-l", "control-plane=controller-manager", "--tail=100") + output, _ = utils.Run(cmd) + _, _ = fmt.Fprintf(GinkgoWriter, "Operator logs:\n%s\n", output) + } + }) + + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(2 * time.Second) + + Context("Non-SPIRE mode", func() { + BeforeAll(func() { + By("deploying authbridge-config (SPIRE_ENABLED=false)") + cmd := exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(authbridgeConfigFixture(false)) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("deploying keycloak-admin-secret") + cmd = exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(keycloakAdminSecretFixture()) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create client credentials Secret for agent workload", func() { + By("deploying clientreg-agent") + cmd := exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(clientRegAgentFixture()) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for Deployment to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", "clientreg-agent", + "-n", clientRegNamespace, "-o", "jsonpath={.status.readyReplicas}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + }).Should(Succeed()) + + By("verifying client credentials Secret was created") + var secretName string + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "secrets", "-n", clientRegNamespace, + "-l", "app.kubernetes.io/managed-by=kagenti-operator", + "-o", "jsonpath={.items[0].metadata.name}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty(), "Expected client credentials Secret to be created") + secretName = output + }).Should(Succeed()) + + By("verifying Secret contains client-id.txt and client-secret.txt") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "secret", secretName, "-n", clientRegNamespace, + "-o", "jsonpath={.data}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("client-id.txt")) + g.Expect(output).To(ContainSubstring("client-secret.txt")) + }).Should(Succeed()) + + By("verifying client ID format is namespace/workloadName") + cmd = exec.Command("kubectl", "get", "secret", secretName, "-n", clientRegNamespace, + "-o", "jsonpath={.data.client-id\\.txt}") + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + // Decode base64 + cmd = exec.Command("base64", "-d") + cmd.Stdin = strings.NewReader(output) + clientID, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(clientID)).To(Equal(clientRegNamespace + "/clientreg-agent")) + + By("verifying pod template has keycloak-client-credentials-secret-name annotation") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", "clientreg-agent", + "-n", clientRegNamespace, + "-o", "jsonpath={.spec.template.metadata.annotations.kagenti\\.io/keycloak-client-credentials-secret-name}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal(secretName)) + }).Should(Succeed()) + }) + }) + + Context("SPIRE mode with default template", func() { + BeforeAll(func() { + By("deploying authbridge-config (SPIRE_ENABLED=true)") + cmd := exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(authbridgeConfigFixture(true)) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create SPIFFE ID using default template", func() { + By("deploying clientreg-spire-agent with dedicated ServiceAccount") + cmd := exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(clientRegAgentSpireFixture()) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for Deployment to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", "clientreg-spire-agent", + "-n", clientRegNamespace, "-o", "jsonpath={.status.readyReplicas}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + }).Should(Succeed()) + + By("verifying client credentials Secret was created") + var secretName string + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "secrets", "-n", clientRegNamespace, + "--selector=app.kubernetes.io/managed-by=kagenti-operator", + "-o", "jsonpath={.items[?(@.metadata.ownerReferences[0].name=='clientreg-spire-agent')].metadata.name}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty(), "Expected client credentials Secret for SPIRE agent") + secretName = strings.TrimSpace(output) + }).Should(Succeed()) + + By("verifying client ID uses SPIFFE ID format with default template") + cmd = exec.Command("kubectl", "get", "secret", secretName, "-n", clientRegNamespace, + "-o", "jsonpath={.data.client-id\\.txt}") + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + // Decode base64 + cmd = exec.Command("base64", "-d") + cmd.Stdin = strings.NewReader(output) + clientID, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + // Default template: spiffe://{{.TrustDomain}}/ns/{{.Namespace}}/sa/{{.ServiceAccount}} + expectedClientID := fmt.Sprintf("spiffe://example.org/ns/%s/sa/clientreg-spire-sa", clientRegNamespace) + Expect(strings.TrimSpace(clientID)).To(Equal(expectedClientID)) + }) + }) + + Context("SPIRE mode with custom template", func() { + It("should create SPIFFE ID using custom template from env var", func() { + By("patching operator deployment with custom SPIRE_ID_TEMPLATE") + customTemplate := "spiffe://{{.TrustDomain}}/workload/{{.Namespace}}/{{.ServiceAccount}}" + cmd := exec.Command("kubectl", "set", "env", "deployment/kagenti-operator-controller-manager", + "-n", namespace, + fmt.Sprintf("SPIRE_ID_TEMPLATE=%s", customTemplate)) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for operator rollout after env var change") + err = utils.WaitForRollout("kagenti-operator-controller-manager", namespace, 2*time.Minute) + Expect(err).NotTo(HaveOccurred()) + + By("deploying a new agent to test custom template") + customAgentYAML := `apiVersion: v1 +kind: ServiceAccount +metadata: + name: clientreg-custom-sa + namespace: ` + clientRegNamespace + ` +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clientreg-custom-agent + namespace: ` + clientRegNamespace + ` + labels: + kagenti.io/type: agent + protocol.kagenti.io/a2a: "" +spec: + replicas: 1 + selector: + matchLabels: + app: clientreg-custom-agent + template: + metadata: + labels: + app: clientreg-custom-agent + kagenti.io/type: agent + protocol.kagenti.io/a2a: "" + spec: + serviceAccountName: clientreg-custom-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: pause + image: registry.k8s.io/pause:3.9 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL +` + cmd = exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(customAgentYAML) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for custom agent Deployment to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", "clientreg-custom-agent", + "-n", clientRegNamespace, "-o", "jsonpath={.status.readyReplicas}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + }).Should(Succeed()) + + By("verifying client ID uses custom SPIFFE ID template") + var secretName string + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "secrets", "-n", clientRegNamespace, + "--selector=app.kubernetes.io/managed-by=kagenti-operator", + "-o", "jsonpath={.items[?(@.metadata.ownerReferences[0].name=='clientreg-custom-agent')].metadata.name}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty()) + secretName = strings.TrimSpace(output) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + cmd = exec.Command("kubectl", "get", "secret", secretName, "-n", clientRegNamespace, + "-o", "jsonpath={.data.client-id\\.txt}") + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + // Decode base64 + cmd = exec.Command("base64", "-d") + cmd.Stdin = strings.NewReader(output) + clientID, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + // Custom template: spiffe://{{.TrustDomain}}/workload/{{.Namespace}}/{{.ServiceAccount}} + expectedClientID := fmt.Sprintf("spiffe://example.org/workload/%s/clientreg-custom-sa", clientRegNamespace) + Expect(strings.TrimSpace(clientID)).To(Equal(expectedClientID)) + + By("restoring operator to default template") + cmd = exec.Command("kubectl", "set", "env", "deployment/kagenti-operator-controller-manager", + "-n", namespace, "SPIRE_ID_TEMPLATE-") + _, _ = utils.Run(cmd) + }) + }) +}) diff --git a/kagenti-operator/test/e2e/fixtures.go b/kagenti-operator/test/e2e/fixtures.go index 4bfc099a..118dbdf8 100644 --- a/kagenti-operator/test/e2e/fixtures.go +++ b/kagenti-operator/test/e2e/fixtures.go @@ -465,3 +465,232 @@ spec: agentcard: "true" ` } + +// mockKeycloakFixture returns YAML for a mock Keycloak server (for ClientRegistration tests). +// The mock server simulates the Keycloak admin API for client registration. +func mockKeycloakFixture() string { + return `apiVersion: apps/v1 +kind: Deployment +metadata: + name: mock-keycloak + namespace: ` + testNamespace + ` +spec: + replicas: 1 + selector: + matchLabels: + app: mock-keycloak + template: + metadata: + labels: + app: mock-keycloak + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: mock-keycloak + image: docker.io/python:3.11-slim + imagePullPolicy: IfNotPresent + command: + - python3 + - -c + - | + import http.server, json, base64, uuid + clients = {} + class H(http.server.BaseHTTPRequestHandler): + def do_POST(self): + # Admin token endpoint + if self.path == '/realms/master/protocol/openid-connect/token': + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + resp = {'access_token': 'mock-admin-token', 'token_type': 'bearer'} + self.wfile.write(json.dumps(resp).encode()) + # Create client endpoint + elif self.path.startswith('/admin/realms/') and self.path.endswith('/clients'): + length = int(self.headers.get('Content-Length', 0)) + body = self.rfile.read(length).decode('utf-8') + client_data = json.loads(body) + client_id = client_data.get('clientId', '') + secret = str(uuid.uuid4()) + clients[client_id] = {'id': str(uuid.uuid4()), 'clientId': client_id, 'secret': secret} + self.send_response(201) + self.send_header('Location', '/admin/realms/test/clients/' + clients[client_id]['id']) + self.end_headers() + else: + self.send_response(404) + self.end_headers() + def do_GET(self): + # Get clients by clientId + if self.path.startswith('/admin/realms/') and 'clientId=' in self.path: + client_id = self.path.split('clientId=')[1].split('&')[0] + if client_id in clients: + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps([clients[client_id]]).encode()) + else: + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(b'[]') + # Get client secret + elif '/client-secret' in self.path: + for cid, data in clients.items(): + if data['id'] in self.path: + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({'value': data['secret']}).encode()) + return + self.send_response(404) + self.end_headers() + else: + self.send_response(404) + self.end_headers() + def log_message(self, *a): pass + http.server.HTTPServer(('', 8080), H).serve_forever() + ports: + - containerPort: 8080 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL +--- +apiVersion: v1 +kind: Service +metadata: + name: mock-keycloak + namespace: ` + testNamespace + ` +spec: + selector: + app: mock-keycloak + ports: + - port: 8080 + targetPort: 8080 +` +} + +// authbridgeConfigFixture returns YAML for authbridge-config ConfigMap (for ClientRegistration tests). +func authbridgeConfigFixture(spireEnabled bool) string { + spireEnabledStr := "false" + if spireEnabled { + spireEnabledStr = "true" + } + return `apiVersion: v1 +kind: ConfigMap +metadata: + name: authbridge-config + namespace: ` + testNamespace + ` +data: + KEYCLOAK_URL: "http://mock-keycloak.` + testNamespace + `.svc:8080" + KEYCLOAK_REALM: "test" + SPIRE_ENABLED: "` + spireEnabledStr + `" +` +} + +// keycloakAdminSecretFixture returns YAML for keycloak-admin-secret (for ClientRegistration tests). +func keycloakAdminSecretFixture() string { + return `apiVersion: v1 +kind: Secret +metadata: + name: keycloak-admin-secret + namespace: ` + testNamespace + ` +type: Opaque +stringData: + KEYCLOAK_ADMIN_USERNAME: "admin" + KEYCLOAK_ADMIN_PASSWORD: "admin" +` +} + +// clientRegAgentFixture returns YAML for an agent Deployment for client registration tests (non-SPIRE). +func clientRegAgentFixture() string { + return `apiVersion: apps/v1 +kind: Deployment +metadata: + name: clientreg-agent + namespace: ` + testNamespace + ` + labels: + kagenti.io/type: agent + protocol.kagenti.io/a2a: "" + app.kubernetes.io/name: clientreg-agent +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: clientreg-agent + kagenti.io/type: agent + template: + metadata: + labels: + app.kubernetes.io/name: clientreg-agent + kagenti.io/type: agent + protocol.kagenti.io/a2a: "" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: pause + image: registry.k8s.io/pause:3.9 + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL +` +} + +// clientRegAgentSpireFixture returns YAML for an agent Deployment for SPIRE-enabled client registration tests. +func clientRegAgentSpireFixture() string { + return `apiVersion: v1 +kind: ServiceAccount +metadata: + name: clientreg-spire-sa + namespace: ` + testNamespace + ` +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clientreg-spire-agent + namespace: ` + testNamespace + ` + labels: + kagenti.io/type: agent + protocol.kagenti.io/a2a: "" + app.kubernetes.io/name: clientreg-spire-agent +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: clientreg-spire-agent + kagenti.io/type: agent + template: + metadata: + labels: + app.kubernetes.io/name: clientreg-spire-agent + kagenti.io/type: agent + protocol.kagenti.io/a2a: "" + spec: + serviceAccountName: clientreg-spire-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: pause + image: registry.k8s.io/pause:3.9 + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL +` +}