diff --git a/.coverage-baseline b/.coverage-baseline index 9ddeb3cd..2ba01570 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -76.5 +76.6 diff --git a/.github/RELEASES.md b/.github/RELEASES.md index 939ebe54..29d4597b 100644 --- a/.github/RELEASES.md +++ b/.github/RELEASES.md @@ -86,7 +86,7 @@ Push to main 4. **When Release PR is Merged**: - GitHub Release created with tag (e.g., `v0.2.0`) — first as a **draft**, so every signed - asset (`install.yaml`, SBOM, `.sigstore.json` signatures, `.intoto.jsonl` attestations) + asset (`crds.yaml`, `install.yaml`, SBOM, `.sigstore.json` signatures, `.intoto.jsonl` attestations) can be attached before it goes public; immutable releases reject post-publish uploads, so `publish-release` flips the draft to published only after every asset is in place. - The linux/amd64 + linux/arm64 image digests already built and scanned by that diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7adb6b0..e046c378 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -446,6 +446,7 @@ jobs: with: name: release-bundle path: | + dist/crds.yaml dist/install.yaml gitops-reverser.tgz if-no-files-found: error @@ -809,7 +810,7 @@ jobs: # place gitops-reverser mirrors REMOTE clusters (GitTarget.spec.kubeConfig). # kcp workspaces are cheap logical clusters, so its own runner installs a # small kcp control plane and provisions three workspaces as source clusters. - # docs/design/config-plane-split.md. + # docs/finished/config-plane-split.md. # # `task test-e2e-source-cluster` pins its own label filter and Ginkgo procs # (SOURCE_CLUSTER_GINKGO_PROCS=1 — the specs share one kcp port-forward), so diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a27ac4db..f5edef1d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -219,8 +219,11 @@ jobs: sbom.spdx.json.intoto.jsonl body: | ## Installation - ### Quick Install (Single YAML) + ### Quick Install (plain manifests) + The CRDs ship as their own file and must be applied **first** — the bundle + contains a custom resource that cannot be created before its CRD exists: ```bash + kubectl apply -f https://github.com/ConfigButler/gitops-reverser/releases/download/${{ needs.release-please.outputs.tag_name }}/crds.yaml kubectl apply -f https://github.com/ConfigButler/gitops-reverser/releases/download/${{ needs.release-please.outputs.tag_name }}/install.yaml ``` ### Helm Chart @@ -314,12 +317,17 @@ jobs: # The chart is already signed above via its OCI digest, but OpenSSF # Scorecard's Signed-Releases check only looks at the GitHub *release - # assets*, not the OCI registry. Sign and attest install.yaml directly so - # the release carries a signature (*.sigstore.json) and SLSA provenance - # (*.intoto.jsonl) next to it. + # assets*, not the OCI registry. Sign and attest BOTH installer files + # directly so the release carries a signature (*.sigstore.json) and SLSA + # provenance (*.intoto.jsonl) next to each. The installer ships as two + # files — crds.yaml is applied first, then install.yaml (see the release + # notes); each is signed independently so either can be verified alone. - name: Sign install.yaml (cosign keyless) run: cosign sign-blob --bundle install.yaml.sigstore.json --yes dist/install.yaml + - name: Sign crds.yaml (cosign keyless) + run: cosign sign-blob --bundle crds.yaml.sigstore.json --yes dist/crds.yaml + - name: Attest SLSA provenance for install.yaml id: attest-install-yaml uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 @@ -329,7 +337,16 @@ jobs: - name: Rename provenance bundle to the Scorecard-recognized suffix run: cp "${{ steps.attest-install-yaml.outputs.bundle-path }}" install.yaml.intoto.jsonl - - name: Upload install.yaml as release asset + - name: Attest SLSA provenance for crds.yaml + id: attest-crds-yaml + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-path: dist/crds.yaml + + - name: Rename crds provenance bundle to the Scorecard-recognized suffix + run: cp "${{ steps.attest-crds-yaml.outputs.bundle-path }}" crds.yaml.intoto.jsonl + + - name: Upload installer bundle as release assets uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 with: tag_name: ${{ needs.release-please.outputs.tag_name }} @@ -338,6 +355,9 @@ jobs: # uploads). draft: true files: | + dist/crds.yaml + crds.yaml.sigstore.json + crds.yaml.intoto.jsonl dist/install.yaml install.yaml.sigstore.json install.yaml.intoto.jsonl diff --git a/.gitignore b/.gitignore index ab72cf9d..37decd61 100644 --- a/.gitignore +++ b/.gitignore @@ -41,8 +41,10 @@ __debug* charts/gitops-reverser/crds/*.yaml charts/gitops-reverser/config/*.yaml -# Generated single-file installer artifact (not tracked). -/dist/install.yaml +# Generated installer artifacts (not tracked). `task dist-install` emits the split bundle +# — crds.yaml (applied first) + install.yaml — and CI regenerates both before packaging a +# release. Ignore the whole directory so a new artifact never lands in a commit by accident. +/dist/ *.o.yaml *.ignore.* diff --git a/PROJECT b/PROJECT index 1a02e9ac..bfcca565 100644 --- a/PROJECT +++ b/PROJECT @@ -41,4 +41,12 @@ resources: kind: ClusterWatchRule path: github.com/ConfigButler/gitops-reverser/api/v1alpha3 version: v1alpha3 +- api: + crdVersion: v1 + namespaced: false + controller: true + domain: configbutler.ai + kind: ClusterProvider + path: github.com/ConfigButler/gitops-reverser/api/v1alpha3 + version: v1alpha3 version: "3" diff --git a/README.md b/README.md index 4cab3268..69c5c3eb 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,9 @@ Production use should follow an environment-specific review. Deferred directions This brings up the **demo**: a starter `GitProvider`, `GitTarget`, and `WatchRule` in a `gitops-reverser-quickstart-demo` namespace. It watches ConfigMaps in that namespace and writes them to `/live-cluster` on the `main` branch. It runs in **`configured-author` mode** (one -committer identity, no Redis) by default. +committer identity, no Redis) by default. The chart also renders the cluster-scoped `default` +`ClusterProvider`, so the starter target's omitted source reference resolves to the operator's own +cluster. ![Config basics diagram showing the relationship between GitProvider, GitTarget, and WatchRule](docs/images/config-basics.excalidraw.svg) @@ -164,9 +166,9 @@ credentials Secrets are accepted as-is (they must have **write** access). See [`docs/github-setup-guide.md`](docs/github-setup-guide.md) for the full GitHub guide and HTTPS/PAT fallback. -**4. Install GitOps Reverser with the demo** +**4. Install GitOps Reverser with the demo enabled** -A single install enables the demo and points the starter `GitProvider` at your repo: +Point the starter `GitProvider` at your repo and install: ```bash helm install gitops-reverser \ diff --git a/SECURITY.md b/SECURITY.md index 099bb9b8..8e0a7121 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,6 +8,20 @@ model, and operational risks for your environment. For what the controller can access and which pieces are sensitive, see [`docs/security-model.md`](docs/security-model.md). +## Shared audit-ingress trust model + +Audit ingress uses mutual TLS, but the current multi-cluster routes use the client certificate to +authenticate membership in the shared audit CA—not to bind a sender to one `ClusterProvider`. A +source that holds the shared client credential can therefore submit audit facts for any configured +provider, whether the provider is selected by `/audit-webhook/` or by shared-stream +annotation routing. + +This is an explicit, accepted operating assumption for now: use shared ingress only when a highly +privileged control plane manages all participating source clusters, protects the shared credential, +and treats those sources as mutually trusted. It is not a tenant-isolation boundary. Deploy separate +instances or keep attribution disabled when one source must not be able to influence another source's +Git author history. Provider-bound client identities may be added if that stronger boundary is needed. + ## Reporting a vulnerability Please do not open a public GitHub issue for security-sensitive reports. diff --git a/Taskfile-build.yml b/Taskfile-build.yml index 0a74af26..19f01bf9 100644 --- a/Taskfile-build.yml +++ b/Taskfile-build.yml @@ -399,7 +399,7 @@ tasks: IMG: '{{.DEV_IMG}}' dist-install: - desc: Generate consolidated YAML from the Helm chart + desc: Generate the split installer bundle (dist/crds.yaml + dist/install.yaml) from the Helm chart deps: - helm-sync sources: @@ -409,6 +409,7 @@ tasks: - charts/gitops-reverser/Chart.yaml - charts/gitops-reverser/values.yaml - charts/gitops-reverser/templates/** + - charts/gitops-reverser/crds/*.yaml - exclude: api/**/*_test.go - exclude: internal/**/*_test.go - exclude: cmd/**/*_test.go @@ -416,17 +417,26 @@ tasks: - exclude: internal/**/zz_generated.deepcopy.go - exclude: cmd/**/zz_generated.deepcopy.go generates: + - dist/crds.yaml - dist/install.yaml cmds: - | mkdir -p dist + # CRDs ship as their OWN file, applied FIRST. The bundle contains the reserved `default` + # ClusterProvider — a custom resource that cannot be applied in the same `kubectl apply` as + # the CRD defining it: kubectl builds its RESTMapper up front and fails with + # `no matches for kind "ClusterProvider"`. So: kubectl apply -f dist/crds.yaml, then + # kubectl apply -f dist/install.yaml. `helm install` is unaffected — it installs + # charts/gitops-reverser/crds/ in its own first phase, which is why --include-crds is gone + # here. Each chart CRD file already begins with its own `---`, so a plain cat is valid + # multi-doc YAML. + cat charts/gitops-reverser/crds/*.yaml > dist/crds.yaml {{.HELM}} template {{.INSTALL_NAME}} charts/gitops-reverser \ --namespace {{.NAMESPACE}} \ --set labels.managedBy=kubectl \ --set createNamespace=true \ --set queue.redis.addr={{.DEFAULT_AUDIT_REDIS_ADDR}} \ --set queue.redis.auth.existingSecret=valkey-auth \ - --include-crds \ > dist/install.yaml clean: diff --git a/api/v1alpha3/clusterprovider_types.go b/api/v1alpha3/clusterprovider_types.go new file mode 100644 index 00000000..f3e85750 --- /dev/null +++ b/api/v1alpha3/clusterprovider_types.go @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha3 + +import ( + meta "github.com/fluxcd/pkg/apis/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" +) + +// DefaultClusterProviderName is the conventionally opinionated ClusterProvider name that an +// omitted GitTarget.spec.clusterProviderRef points at. That defaulting is its ONLY special +// behavior: it is an ordinary user-created object that may omit kubeConfig (the operator's own +// in-cluster config) or set it to mirror a remote cluster, it is existence-gated on its +// /audit-webhook/default route like every other name, and it is never created by the operator. +const DefaultClusterProviderName = "default" + +// ClusterProviderReference references the cluster-scoped ClusterProvider a GitTarget sources +// FROM. It is the read-side peer of GitProviderReference (which names the WRITE destination): +// a GitTarget names one ClusterProvider by name and its author-attribution facts, kube client, +// and namespace authorization all follow from that single reference. Group and Kind are typed +// (with defaults) for consistency with the project's other typed references. +type ClusterProviderReference struct { + // API Group of the referent. + // +kubebuilder:default=configbutler.ai + // +kubebuilder:validation:Enum=configbutler.ai + Group string `json:"group,omitempty"` + + // Kind of the referent. + // Optional because this reference currently only supports a single kind (ClusterProvider). + // +optional + // +kubebuilder:validation:Enum=ClusterProvider + // +kubebuilder:default=ClusterProvider + Kind string `json:"kind,omitempty"` + + // Name of the referent. + // +required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` +} + +// AllowedNamespaces is the deny-by-default namespace-access policy a ClusterProvider carries. +// A cluster-scoped provider holds a credential that can read a lot of a remote cluster; any +// GitTarget that references it makes the operator mirror that cluster's state into the target's +// destination. So which namespaces may reference the provider is authorization, not routing: an +// empty policy (no names, no selector) means NO namespace may reference the provider. Names and +// selector are ORed — a namespace is allowed if it is listed OR matches the selector. +type AllowedNamespaces struct { + // Names is an explicit allow-list of namespace names that may reference this provider. + // +optional + // +listType=set + Names []string `json:"names,omitempty"` + + // Selector is a label selector matched against Namespace labels; a namespace whose labels + // match may reference this provider. ORed with Names. + // +optional + Selector *metav1.LabelSelector `json:"selector,omitempty"` +} + +// ClusterProviderSpec defines the desired state of ClusterProvider. +// +// kubeConfig is IMMUTABLE and OPTIONAL: which physical cluster a provider name means must not +// silently change under the GitTargets bound to it, and an OMITTED kubeConfig means the operator's +// own (in-cluster) cluster. That choice is free for EVERY name, "default" included — a provider +// named "default" may just as well carry a kubeConfig and mirror a remote cluster. The name is an +// identity, not a claim about which cluster it points at. +// +// +kubebuilder:validation:XValidation:rule="has(self.kubeConfig) == has(oldSelf.kubeConfig) && (!has(self.kubeConfig) || self.kubeConfig == oldSelf.kubeConfig)",message="spec.kubeConfig is immutable; delete and recreate the ClusterProvider to point a name at a different cluster" +// +// configMapRef (Flux workload-identity auth) is present in meta.KubeConfigReference's schema but +// deferred here; reject it so the v1alpha3 contract is "secretRef only". +// +kubebuilder:validation:XValidation:rule="!has(self.kubeConfig) || !has(self.kubeConfig.configMapRef)",message="spec.kubeConfig.configMapRef (workload-identity auth) is not yet supported; use secretRef" +// +// secretRef.name comes from the external meta.KubeConfigReference schema, which marks it required +// but permits the empty string; an empty name can never resolve a Secret, so reject it here. +// +kubebuilder:validation:XValidation:rule="!has(self.kubeConfig) || !has(self.kubeConfig.secretRef) || size(self.kubeConfig.secretRef.name) > 0",message="spec.kubeConfig.secretRef.name must not be empty" +type ClusterProviderSpec struct { + // KubeConfig names the SOURCE CLUSTER this provider represents and the credentials to reach + // it (Flux's meta.KubeConfigReference, embedded verbatim). OMITTED means the operator's own + // in-cluster cluster, for any provider name. IMMUTABLE. The referenced Secret is + // resolved from the operator's namespace — the credential for a cluster never has to live on + // that cluster. When secretRef.key is empty the resolver reads "value" then "value.yaml" + // (Flux's order). Only secretRef is honored (configMapRef is rejected); unsafe kubeconfigs + // (exec auth, insecure-skip-tls-verify) are rejected with a Validated=False reason unless the + // operator opts in via flags. + // +optional + KubeConfig *meta.KubeConfigReference `json:"kubeConfig,omitempty"` + + // AllowedNamespaces is the deny-by-default policy for which namespaces may reference this + // provider from a GitTarget. Empty (or omitted) means no namespace may reference it. + // +optional + AllowedNamespaces *AllowedNamespaces `json:"allowedNamespaces,omitempty"` + + // QPS overrides the operator's outgoing kube-client query-per-second throttle for this + // cluster's watches and discovery. Omitted, the operator-wide --source-cluster-qps applies. + // Ignored when kubeConfig is omitted (the in-cluster client is not per-provider). + // +optional + // +kubebuilder:validation:Minimum=1 + QPS *int32 `json:"qps,omitempty"` + + // Burst overrides the operator's outgoing kube-client burst for this cluster. Omitted, the + // operator-wide --source-cluster-burst applies. Ignored when kubeConfig is omitted. + // +optional + // +kubebuilder:validation:Minimum=1 + Burst *int32 `json:"burst,omitempty"` +} + +// ClusterProviderStatus defines the observed state of ClusterProvider. +type ClusterProviderStatus struct { + // ObservedGeneration is the latest generation observed by the controller. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Conditions report the provider's readiness: Validated (kubeconfig inputs are safe and + // resolvable, asserted without a network dial) plus the aggregated Ready and the kstatus + // Reconciling/Stalled pair. Runtime reachability/discovery health and a last-audit-event + // timestamp are deferred until authenticated remote ingest wires them from the watch engine. + // +optional + // +listType=map + // +listMapKey=type + // +patchMergeKey=type + // +patchStrategy=merge + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` +// +kubebuilder:printcolumn:name="Validated",type=string,JSONPath=`.status.conditions[?(@.type=="Validated")].status`,priority=1 +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].message`,priority=1 +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// ClusterProvider is the cluster-scoped, read-side peer of GitProvider: it names a SOURCE cluster +// a GitTarget mirrors FROM, and is the home for that cluster's connectivity credential +// (spec.kubeConfig), namespace-access authorization (spec.allowedNamespaces), and per-cluster +// status. Its NAME is the cluster's identity everywhere: the /audit-webhook/ ingress route +// and the attribution fact-index key. No name is special: "default" is merely the name +// GitTarget.spec.clusterProviderRef defaults to when omitted. Whether a provider is the operator's +// own cluster or a remote follows from spec.kubeConfig (omitted = in-cluster), not from its name, +// so "default" may just as well name a remote cluster. +// +// Security model: +// - ClusterProvider is cluster-scoped and requires platform-admin permissions to create. +// - A GitTarget may reference it only from a namespace its spec.allowedNamespaces admits +// (deny-by-default), enforced at admission AND before any watch starts. +type ClusterProvider struct { + metav1.TypeMeta `json:",inline"` + + // metadata is a standard object metadata. + // +optional + metav1.ObjectMeta `json:"metadata,omitempty,omitzero"` + + // spec defines the desired state of ClusterProvider. + // +required + Spec ClusterProviderSpec `json:"spec"` + + // status defines the observed state of ClusterProvider. + // +optional + Status ClusterProviderStatus `json:"status,omitempty,omitzero"` +} + +// +kubebuilder:object:root=true + +// ClusterProviderList contains a list of ClusterProvider. +type ClusterProviderList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []ClusterProvider `json:"items"` +} + +// IsInCluster reports whether this provider represents the operator's own (in-cluster) cluster — +// i.e. it has no kubeConfig. The provider name is irrelevant: any name, including "default", may +// either omit kubeConfig for the in-cluster client or set it for a remote cluster. +func (p *ClusterProvider) IsInCluster() bool { + return p.Spec.KubeConfig == nil +} + +// AllowsNamespace reports whether a namespace (by name and labels) may reference this provider +// from a GitTarget, per spec.allowedNamespaces. It is DENY-BY-DEFAULT: a provider with no +// allowedNamespaces policy (neither names nor selector) admits no namespace. Names and selector +// are ORed. This is the single authorization predicate shared by the admission webhook and the +// reconcile-time refusal, so the two can never diverge. A malformed selector is a configuration +// error surfaced to the caller (not a silent allow). +func (p *ClusterProvider) AllowsNamespace(nsName string, nsLabels map[string]string) (bool, error) { + policy := p.Spec.AllowedNamespaces + if policy == nil { + return false, nil + } + for _, n := range policy.Names { + if n == nsName { + return true, nil + } + } + if policy.Selector != nil { + sel, err := metav1.LabelSelectorAsSelector(policy.Selector) + if err != nil { + return false, err + } + if sel.Matches(labels.Set(nsLabels)) { + return true, nil + } + } + return false, nil +} + +func init() { + SchemeBuilder.Register(&ClusterProvider{}, &ClusterProviderList{}) +} diff --git a/api/v1alpha3/gittarget_types.go b/api/v1alpha3/gittarget_types.go index dc5cf337..cfb8951f 100644 --- a/api/v1alpha3/gittarget_types.go +++ b/api/v1alpha3/gittarget_types.go @@ -3,7 +3,6 @@ package v1alpha3 import ( - meta "github.com/fluxcd/pkg/apis/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -47,19 +46,11 @@ type GitProviderReference struct { // +kubebuilder:validation:XValidation:rule="self.branch == oldSelf.branch",message="spec.branch is immutable; delete and recreate the GitTarget to change its destination" // +kubebuilder:validation:XValidation:rule="self.path == oldSelf.path",message="spec.path is immutable; delete and recreate the GitTarget to change its destination" // -// spec.kubeConfig is immutable — the source of a folder's content is destination identity, like -// providerRef/branch/path above. Delete and recreate to change the cluster a GitTarget mirrors. -// +kubebuilder:validation:XValidation:rule="has(self.kubeConfig) == has(oldSelf.kubeConfig) && (!has(self.kubeConfig) || self.kubeConfig == oldSelf.kubeConfig)",message="spec.kubeConfig is immutable; delete and recreate the GitTarget to change the cluster it mirrors" -// -// configMapRef (provider / workload-identity auth) is present in meta.KubeConfigReference's schema -// but not yet implemented here; reject it at admission so the v1alpha3 contract is "secretRef only". -// Deleting this one rule, plus wiring the provider path in the resolver, is the whole future enablement. -// +kubebuilder:validation:XValidation:rule="!has(self.kubeConfig) || !has(self.kubeConfig.configMapRef)",message="spec.kubeConfig.configMapRef (provider auth) is not yet supported; use secretRef" -// -// secretRef.name comes from the external meta.KubeConfigReference schema, which marks it required but -// permits the empty string; an empty name is meaningless (it can never resolve a Secret), so reject it -// at admission rather than letting it surface later as a Validated=False "Secret not found". -// +kubebuilder:validation:XValidation:rule="!has(self.kubeConfig) || !has(self.kubeConfig.secretRef) || size(self.kubeConfig.secretRef.name) > 0",message="spec.kubeConfig.secretRef.name must not be empty" +// spec.clusterProviderRef names the SOURCE cluster a GitTarget mirrors FROM (see its field doc). It +// is immutable — a folder's source cluster is part of what the folder means, like +// providerRef/branch/path above — and defaults to a ClusterProvider named "default", so it is +// always populated (never nil) and always jumpable. +// +kubebuilder:validation:XValidation:rule="self.clusterProviderRef == oldSelf.clusterProviderRef",message="spec.clusterProviderRef is immutable; delete and recreate the GitTarget to change the cluster it mirrors" type GitTargetSpec struct { // ProviderRef references the GitProvider that backs this target. // Immutable: delete and recreate the GitTarget to change its destination. @@ -96,18 +87,17 @@ type GitTargetSpec struct { // +optional Placement *GitTargetPlacementSpec `json:"placement,omitempty"` - // KubeConfig names the SOURCE CLUSTER this GitTarget mirrors FROM: the kubeconfig - // determines both the cluster and the credentials to reach it. Omitted means the cluster - // the operator runs in, the single-cluster default that behaves exactly as before. Its - // Secret is read from the GitTarget's OWN namespace, on the cluster the operator runs in — - // the credential for a cluster never has to live on that cluster. When SecretRef.Key is - // empty the resolver reads "value" then "value.yaml" (Flux's order). Immutable: the source - // of a folder's content is part of what the folder means; delete and recreate to change it. - // Only kubeConfig.secretRef is honored in v1alpha3 (configMapRef is rejected at admission); - // unsafe kubeconfigs (exec auth providers, insecure-skip-tls-verify) are rejected by the - // controller with a legible Validated=False reason unless the operator opts in via flags. + // ClusterProviderRef names the SOURCE cluster this GitTarget mirrors FROM, by referencing a + // cluster-scoped ClusterProvider by name. The ClusterProvider is the home for that cluster's + // connectivity credential, namespace-access authorization, and author-attribution mode. This + // DEFAULTS to {name: "default"} — a user-created provider by that conventional name, which may + // be in-cluster or remote — so a target that omits it persists with a concrete, jumpable ref + // rather than an implicit nil. The operator never creates that provider; a GitTarget naming one + // that does not exist is held unready. Immutable: a folder's source cluster is part of what the + // folder means; delete and recreate to change it. + // +kubebuilder:default={name: "default"} // +optional - KubeConfig *meta.KubeConfigReference `json:"kubeConfig,omitempty"` + ClusterProviderRef *ClusterProviderReference `json:"clusterProviderRef,omitempty"` } // GitTargetPlacementSpec declares where NEW resources are written when no document @@ -210,6 +200,7 @@ type GitTargetStreamsStatus struct { // +kubebuilder:printcolumn:name="StreamsRunning",type=string,JSONPath=`.status.conditions[?(@.type=="StreamsRunning")].status`,priority=1 // +kubebuilder:printcolumn:name="SourceReachable",type=string,JSONPath=`.status.conditions[?(@.type=="SourceClusterReachable")].reason`,priority=1 // +kubebuilder:printcolumn:name="ProviderReady",type=string,JSONPath=`.status.conditions[?(@.type=="GitProviderReady")].status`,priority=1 +// +kubebuilder:printcolumn:name="ClusterProviderReady",type=string,JSONPath=`.status.conditions[?(@.type=="ClusterProviderReady")].status`,priority=1 // +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].message`,priority=1 // +kubebuilder:printcolumn:name="Encryption",type=string,JSONPath=`.spec.encryption.provider`,priority=1 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` @@ -231,22 +222,25 @@ type GitTarget struct { Status GitTargetStatus `json:"status,omitempty,omitzero"` } -// SourceClusterID renders the identity the watch data plane keys a GitTarget's source -// cluster on: "//", where namespace/name locate the kubeconfig Secret -// in the GitTarget's own (config-plane) namespace and key is the SecretRef key AS WRITTEN in -// spec — an empty key is its own identity, distinct from an explicit one, because the -// resolver's value→value.yaml fallback only runs when the key is omitted. A GitTarget with no -// spec.kubeConfig (or no secretRef) mirrors the cluster the operator runs in and returns "", -// the local-cluster id every source-cluster-unaware code path already lands on. -// -// Neither a namespace, a Secret name, nor a Secret data key may contain "/", so the three -// segments are unambiguous; the resolver splits them back with SplitN(id, "/", 3). -func (g *GitTarget) SourceClusterID() string { - if g.Spec.KubeConfig == nil || g.Spec.KubeConfig.SecretRef == nil { - return "" +// SourceCluster is the identity the watch data plane keys a GitTarget's source cluster on: the +// referenced ClusterProvider's NAME. It defaults to "default" when clusterProviderRef is unset — +// so a source-cluster-unaware caller still gets a concrete, non-empty name, and there is no "" +// sentinel. That name is a convention, not a claim about which physical cluster it is. The name is +// the cluster's identity everywhere: the fact-index key, the GVK→GVR registry key, and the +// /audit-webhook/ route. +func (g *GitTarget) SourceCluster() string { + if g.Spec.ClusterProviderRef == nil || g.Spec.ClusterProviderRef.Name == "" { + return DefaultClusterProviderName } - ref := g.Spec.KubeConfig.SecretRef - return g.Namespace + "/" + ref.Name + "/" + ref.Key + return g.Spec.ClusterProviderRef.Name +} + +// IsLocalSource reports whether this GitTarget references the "default" ClusterProvider, which the +// watch data plane maps to its local cluster context. It is a NAME test, not a claim about the +// physical cluster: a "default" provider may carry a kubeConfig. It only supplies the pre-discovery +// default for SourceClusterReachable, which the watch manager overwrites as soon as it is wired. +func (g *GitTarget) IsLocalSource() bool { + return g.SourceCluster() == DefaultClusterProviderName } // +kubebuilder:object:root=true diff --git a/api/v1alpha3/helpers_test.go b/api/v1alpha3/helpers_test.go new file mode 100644 index 00000000..daac6abb --- /dev/null +++ b/api/v1alpha3/helpers_test.go @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha3 + +import ( + "testing" + + meta "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// IsInCluster must follow spec.kubeConfig ALONE. Nothing may key "is this the operator's own +// cluster?" off the provider name: "default" is only the name an omitted clusterProviderRef +// resolves to, and it may just as well carry a kubeConfig for a remote cluster. If a name ever +// became a proxy for in-cluster-ness, a remote provider named "default" would be dialed with the +// operator's own credentials against the wrong cluster. +func TestIsInCluster_FollowsKubeConfigNotName(t *testing.T) { + t.Parallel() + + secretRef := &meta.KubeConfigReference{ + SecretRef: &meta.SecretKeyReference{Name: "remote-kubeconfig"}, + } + + tests := []struct { + name string + provider ClusterProvider + want bool + }{ + { + name: "omitted kubeConfig means the operator's own cluster", + provider: ClusterProvider{ObjectMeta: metav1.ObjectMeta{Name: "prod"}}, + want: true, + }, + { + name: "the name \"default\" does not by itself mean in-cluster", + provider: ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: DefaultClusterProviderName}, + Spec: ClusterProviderSpec{KubeConfig: secretRef}, + }, + want: false, + }, + { + name: "a non-default name without kubeConfig is still in-cluster", + provider: ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "some-remote-sounding-name"}, + }, + want: true, + }, + { + name: "any kubeConfig means remote", + provider: ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "prod"}, + Spec: ClusterProviderSpec{KubeConfig: secretRef}, + }, + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tc.want, tc.provider.IsInCluster()) + }) + } +} + +// AllowsNamespace is the authorization predicate behind the reconcile-time refusal, and there is +// no admission webhook backstopping it — that reconcile call site is the ENTIRE boundary. So every +// contract below is a security boundary rather than a routing convenience: a cluster-scoped +// provider holds a credential that can read a whole remote cluster, and referencing it from a +// GitTarget mirrors that cluster into the target's Git destination. +func TestAllowsNamespace_Authorization(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + policy *AllowedNamespaces + nsName string + labels map[string]string + want bool + wantErr bool + }{ + { + // Deny-by-default: a provider whose author never wrote a policy grants nothing. + // The dangerous inversion would be treating "no policy" as "no restriction", which + // would silently open a freshly created provider to every namespace in the cluster. + name: "nil policy denies", + policy: nil, + nsName: "team-a", + want: false, + }, + { + // A policy object that exists but says nothing is the same statement as no policy: + // it enumerates zero namespaces, so it admits zero namespaces. + name: "empty policy denies", + policy: &AllowedNamespaces{}, + nsName: "team-a", + want: false, + }, + { + name: "listed name is allowed", + policy: &AllowedNamespaces{Names: []string{"team-a", "team-b"}}, + nsName: "team-b", + want: true, + }, + { + // Names is an exact allow-list, never a prefix or substring match; otherwise an + // attacker could create "team-a-evil" and inherit "team-a"'s grant. + name: "unlisted name is denied", + policy: &AllowedNamespaces{Names: []string{"team-a"}}, + nsName: "team-a-evil", + want: false, + }, + { + name: "name match is case-sensitive and exact", + policy: &AllowedNamespaces{Names: []string{"team-a"}}, + nsName: "Team-A", + want: false, + }, + { + name: "selector match is allowed", + policy: &AllowedNamespaces{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "prod"}}, + }, + nsName: "anything", + labels: map[string]string{"tier": "prod"}, + want: true, + }, + { + name: "selector miss is denied", + policy: &AllowedNamespaces{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "prod"}}, + }, + nsName: "anything", + labels: map[string]string{"tier": "dev"}, + want: false, + }, + { + // Names and Selector are ORed, so a listed namespace stays allowed even when its + // labels do not match. Requiring both would be a silent tightening that breaks + // existing grants. + name: "name allows even when the selector misses", + policy: &AllowedNamespaces{ + Names: []string{"team-a"}, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "prod"}}, + }, + nsName: "team-a", + labels: map[string]string{"tier": "dev"}, + want: true, + }, + { + name: "selector allows even when the name is unlisted", + policy: &AllowedNamespaces{ + Names: []string{"team-a"}, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "prod"}}, + }, + nsName: "team-z", + labels: map[string]string{"tier": "prod"}, + want: true, + }, + { + // GOTCHA worth pinning loudly: an EMPTY selector is the Kubernetes "match everything" + // selector, not "match nothing". `selector: {}` therefore grants every namespace in + // the cluster. It is deliberate (it is how a platform admin says "any namespace"), + // but it looks identical to an accidentally blank field, so the behavior must be + // asserted rather than rediscovered in production. + name: "empty selector matches every namespace", + policy: &AllowedNamespaces{Selector: &metav1.LabelSelector{}}, + nsName: "any-namespace-at-all", + want: true, + }, + { + // ...including a namespace that carries no labels at all. + name: "empty selector matches a namespace with no labels", + policy: &AllowedNamespaces{Selector: &metav1.LabelSelector{}}, + nsName: "bare", + labels: nil, + want: true, + }, + { + // A malformed selector must FAIL CLOSED: the error is surfaced to the caller AND the + // allow answer is false. Returning true on a parse failure would turn a typo in a + // provider's policy into a cluster-wide grant. + name: "invalid selector fails closed", + policy: &AllowedNamespaces{ + Selector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "tier", Operator: "NotAnOperator", Values: []string{"prod"}}, + }, + }, + }, + nsName: "team-a", + labels: map[string]string{"tier": "prod"}, + want: false, + wantErr: true, + }, + { + // The name allow-list short-circuits before the selector is parsed, so a listed + // namespace is admitted even when the selector alongside it is malformed. Pinned + // because it is the one path where a broken policy does not surface an error. + name: "listed name short-circuits an invalid selector", + policy: &AllowedNamespaces{ + Names: []string{"team-a"}, + Selector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "tier", Operator: "NotAnOperator"}, + }, + }, + }, + nsName: "team-a", + labels: map[string]string{"tier": "prod"}, + want: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + provider := &ClusterProvider{Spec: ClusterProviderSpec{AllowedNamespaces: tc.policy}} + + allowed, err := provider.AllowsNamespace(tc.nsName, tc.labels) + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.want, allowed) + }) + } +} + +// SourceCluster must always return a concrete, non-empty name so callers never have to handle a "" +// sentinel: it is the fact-index key, the GVK→GVR registry key, and the /audit-webhook/ +// route. An empty value there would collapse distinct clusters onto one key. +func TestSourceCluster_DefaultsToDefaultProviderName(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + ref *ClusterProviderReference + want string + }{ + { + name: "omitted ref resolves to the default provider name", + ref: nil, + want: DefaultClusterProviderName, + }, + { + // An empty Name is rejected by CRD validation, but the helper must not depend on + // admission having run — an unvalidated object still has to yield a usable key. + name: "empty name resolves to the default provider name", + ref: &ClusterProviderReference{Name: ""}, + want: DefaultClusterProviderName, + }, + { + name: "explicit name is returned verbatim", + ref: &ClusterProviderReference{Name: "prod-eu"}, + want: "prod-eu", + }, + { + name: "explicitly naming default is the same as omitting the ref", + ref: &ClusterProviderReference{ + Group: "configbutler.ai", + Kind: "ClusterProvider", + Name: DefaultClusterProviderName, + }, + want: DefaultClusterProviderName, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + target := &GitTarget{Spec: GitTargetSpec{ClusterProviderRef: tc.ref}} + + assert.Equal(t, tc.want, target.SourceCluster()) + assert.NotEmpty(t, target.SourceCluster(), "SourceCluster must never return the empty sentinel") + }) + } +} + +// IsLocalSource is a NAME test, not a claim about the physical cluster — a provider named +// "default" is free to carry a kubeConfig and point at a remote cluster. It exists only to supply +// the pre-discovery default for SourceClusterReachable, so it must stay a pure derivation of +// SourceCluster and never be read as "this GitTarget is definitely watching the operator's own +// cluster". +func TestIsLocalSource_IsANameTest(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + ref *ClusterProviderReference + want bool + }{ + {name: "omitted ref is local", ref: nil, want: true}, + {name: "empty name is local", ref: &ClusterProviderReference{Name: ""}, want: true}, + { + name: "explicit default is local", + ref: &ClusterProviderReference{Name: DefaultClusterProviderName}, + want: true, + }, + {name: "any other name is not local", ref: &ClusterProviderReference{Name: "prod-eu"}, want: false}, + { + // Near-miss names must not be treated as the default; the comparison is exact. + name: "a name that merely contains \"default\" is not local", + ref: &ClusterProviderReference{Name: "default-eu"}, + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + target := &GitTarget{Spec: GitTargetSpec{ClusterProviderRef: tc.ref}} + + assert.Equal(t, tc.want, target.IsLocalSource()) + }) + } +} diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index 70363237..f9c991ed 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -48,6 +48,162 @@ func (in *AgeRecipientsSpec) DeepCopy() *AgeRecipientsSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AllowedNamespaces) DeepCopyInto(out *AllowedNamespaces) { + *out = *in + if in.Names != nil { + in, out := &in.Names, &out.Names + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllowedNamespaces. +func (in *AllowedNamespaces) DeepCopy() *AllowedNamespaces { + if in == nil { + return nil + } + out := new(AllowedNamespaces) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterProvider) DeepCopyInto(out *ClusterProvider) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterProvider. +func (in *ClusterProvider) DeepCopy() *ClusterProvider { + if in == nil { + return nil + } + out := new(ClusterProvider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterProvider) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterProviderList) DeepCopyInto(out *ClusterProviderList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterProvider, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterProviderList. +func (in *ClusterProviderList) DeepCopy() *ClusterProviderList { + if in == nil { + return nil + } + out := new(ClusterProviderList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterProviderList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterProviderReference) DeepCopyInto(out *ClusterProviderReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterProviderReference. +func (in *ClusterProviderReference) DeepCopy() *ClusterProviderReference { + if in == nil { + return nil + } + out := new(ClusterProviderReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterProviderSpec) DeepCopyInto(out *ClusterProviderSpec) { + *out = *in + if in.KubeConfig != nil { + in, out := &in.KubeConfig, &out.KubeConfig + *out = new(meta.KubeConfigReference) + (*in).DeepCopyInto(*out) + } + if in.AllowedNamespaces != nil { + in, out := &in.AllowedNamespaces, &out.AllowedNamespaces + *out = new(AllowedNamespaces) + (*in).DeepCopyInto(*out) + } + if in.QPS != nil { + in, out := &in.QPS, &out.QPS + *out = new(int32) + **out = **in + } + if in.Burst != nil { + in, out := &in.Burst, &out.Burst + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterProviderSpec. +func (in *ClusterProviderSpec) DeepCopy() *ClusterProviderSpec { + if in == nil { + return nil + } + out := new(ClusterProviderSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterProviderStatus) DeepCopyInto(out *ClusterProviderStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterProviderStatus. +func (in *ClusterProviderStatus) DeepCopy() *ClusterProviderStatus { + if in == nil { + return nil + } + out := new(ClusterProviderStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterResourceRule) DeepCopyInto(out *ClusterResourceRule) { *out = *in @@ -617,10 +773,10 @@ func (in *GitTargetSpec) DeepCopyInto(out *GitTargetSpec) { *out = new(GitTargetPlacementSpec) (*in).DeepCopyInto(*out) } - if in.KubeConfig != nil { - in, out := &in.KubeConfig, &out.KubeConfig - *out = new(meta.KubeConfigReference) - (*in).DeepCopyInto(*out) + if in.ClusterProviderRef != nil { + in, out := &in.ClusterProviderRef, &out.ClusterProviderRef + *out = new(ClusterProviderReference) + **out = **in } } diff --git a/charts/gitops-reverser/README.md b/charts/gitops-reverser/README.md index 293cea85..21ea71ea 100644 --- a/charts/gitops-reverser/README.md +++ b/charts/gitops-reverser/README.md @@ -194,6 +194,11 @@ nodeSelector: | `attribution.enabled` | Run audit ingress and name mirrored-resource commit authors from matching kube-apiserver audit facts | `false` | | `attribution.ttl` | How long an attribution fact is retained waiting for the matching watch event to join it | `10m` | | `attribution.grace` | Bounded per-event wait for a matching audit fact before a watch event ships as the committer | `3s` | +| `attribution.clusterAnnotationKey` | Audit-event annotation naming the `ClusterProvider` each event belongs to. Empty keeps audit routes named (`/audit-webhook/`). Set it only for a control plane emitting **one shared audit stream** for several logical clusters: it enables the bare `/audit-webhook`, which resolves the source cluster per event. An event with no annotation, or naming a provider that does not exist, is rejected (counted and logged) and never credited to a fallback | `""` | +| `clusterProvider.createDefault` | Render and own a `ClusterProvider` named `default` — the source cluster a `GitTarget` mirrors from when it omits `spec.clusterProviderRef`. The **operator never creates one**, so without this you commit the object yourself. Chart-owned: turning it off makes Helm delete the provider it created, and a `GitTarget` referencing a missing provider is held unready (`ClusterProviderNotFound`). The `quickstart` values never create one | `true` | +| `clusterProvider.default.kubeConfig.secretRef.name` | Secret (release namespace) holding a kubeconfig for the rendered `default` provider. Empty means the operator's **own in-cluster** cluster; a name points `default` at a **remote** cluster instead — the name is a convention, not a claim about which cluster it is | `""` | +| `clusterProvider.default.kubeConfig.secretRef.key` | Key within that Secret. Empty reads `value` then `value.yaml` | `""` | +| `clusterProvider.default.allowedNamespaces` | Deny-by-default policy (`names` and/or `selector`) for which **control-cluster** namespaces may reference this provider from a `GitTarget`. The default empty selector admits every namespace | `{selector: {}}` | | `servers.admission.enabled` | Install the validate-operator-types admission webhook that captures CommitRequest authors (a form of author attribution). Enabled by default; a no-op until `queue.redis.addr` is set | `true` | | `rbac.create` | Create the manager ClusterRole and its binding | `true` | | `rbac.watchTypes.mode` | Which types a `WatchRule` may read. `any` grants cluster-wide read on everything — convenient, but the reverser can then read every Secret in the cluster. `selected` grants read on `rbac.watchTypes.selected` only, so the reverser cannot list or watch Secrets (it keeps `get` on named Secrets it is pointed at). See [`docs/rbac.md`](../../docs/rbac.md) | `any` | @@ -230,8 +235,11 @@ See [`values.yaml`](values.yaml) for complete configuration options. ### Audit Webhook URL Contract -When `attribution.enabled=true`, `https://:9444/audit-webhook` receives audit events from -kube-apiserver. The operator extracts a minimal attribution fact from each (auditID, user, verb, +When `attribution.enabled=true`, `https://:9444/audit-webhook/` +receives audit events from kube-apiserver — audit routes are **named**, including +`/audit-webhook/default`. The bare `/audit-webhook` is rejected with **400** unless +`attribution.clusterAnnotationKey` is set, which turns it into the shared-stream endpoint that +resolves each event's source cluster from that annotation. The operator extracts a minimal attribution fact from each (auditID, user, verb, resourceVersion, GVR, namespace, name, UID, status, timestamps) into the Redis attribution index (populated only when audit attribution is enabled). When a Redis endpoint is configured it also stores each GitTarget's watch resume cursors, so reconnects resume a normal watch from the last processed @@ -243,12 +251,11 @@ configured-author: mirrored-resource commits are authored by the configured comm is optional here: set it for warm-restart resume cursors, or leave it empty and watches cold-replay from scratch on restart. -Cluster ID path segments are rejected. - ## Custom Resource Definitions (CRDs) This chart automatically manages the following CRDs: +- **`clusterproviders.configbutler.ai`** - Source-cluster connections and namespace access policy - **`gitproviders.configbutler.ai`** - Git repository connectivity and credentials - **`gittargets.configbutler.ai`** - Branch/path and optional encryption configuration - **`watchrules.configbutler.ai`** - Namespaced watch rules diff --git a/charts/gitops-reverser/templates/clusterprovider-default.yaml b/charts/gitops-reverser/templates/clusterprovider-default.yaml new file mode 100644 index 00000000..3d74332a --- /dev/null +++ b/charts/gitops-reverser/templates/clusterprovider-default.yaml @@ -0,0 +1,28 @@ +{{- if .Values.clusterProvider.createDefault }} +--- +# The "default" ClusterProvider — the source cluster a GitTarget mirrors FROM when it omits +# spec.clusterProviderRef. The operator never creates this object; the chart renders and OWNS it, so +# clusterProvider.createDefault=false makes Helm delete it again. "default" is only a name: omitting +# kubeConfig means the operator's own in-cluster cluster, and setting it points the same name at a +# remote cluster. allowedNamespaces is deny-by-default; the chart default (an empty selector) admits +# every namespace — tighten it via clusterProvider.default.allowedNamespaces. +apiVersion: configbutler.ai/v1alpha3 +kind: ClusterProvider +metadata: + name: default + labels: + {{- include "gitops-reverser.labels" . | nindent 4 }} +spec: + {{- with .Values.clusterProvider.default.kubeConfig }} + {{- if .secretRef.name }} + kubeConfig: + secretRef: + name: {{ .secretRef.name }} + {{- with .secretRef.key }} + key: {{ . }} + {{- end }} + {{- end }} + {{- end }} + allowedNamespaces: + {{- toYaml .Values.clusterProvider.default.allowedNamespaces | nindent 4 }} +{{- end }} diff --git a/charts/gitops-reverser/templates/deployment.yaml b/charts/gitops-reverser/templates/deployment.yaml index 35d0fa07..3d8a43a5 100644 --- a/charts/gitops-reverser/templates/deployment.yaml +++ b/charts/gitops-reverser/templates/deployment.yaml @@ -83,6 +83,9 @@ spec: - --author-attribution={{ .Values.attribution.enabled }} - --author-attribution-ttl={{ .Values.attribution.ttl }} - --author-attribution-grace={{ .Values.attribution.grace }} + {{- if .Values.attribution.clusterAnnotationKey }} + - --author-attribution-cluster-annotation-key={{ .Values.attribution.clusterAnnotationKey }} + {{- end }} {{- if .Values.servers.admission.enabled }} - --admission-webhook - --admission-webhook-bind-address={{ .Values.servers.admission.bindAddress }} diff --git a/charts/gitops-reverser/values.schema.json b/charts/gitops-reverser/values.schema.json index f085fa48..9d0d64ce 100644 --- a/charts/gitops-reverser/values.schema.json +++ b/charts/gitops-reverser/values.schema.json @@ -274,7 +274,56 @@ "properties": { "enabled": { "type": "boolean" }, "ttl": { "$ref": "#/$defs/duration" }, - "grace": { "$ref": "#/$defs/duration" } + "grace": { "$ref": "#/$defs/duration" }, + "clusterAnnotationKey": { + "type": "string", + "description": "Audit-event annotation naming the ClusterProvider each event belongs to. Empty (default) keeps audit routes named (/audit-webhook/); set it only for one shared audit stream carrying several logical clusters, which enables the bare /audit-webhook endpoint." + } + } + }, + + "clusterProvider": { + "type": "object", + "additionalProperties": false, + "description": "Rendering of the 'default' ClusterProvider — the source cluster a GitTarget mirrors from when it omits spec.clusterProviderRef. The operator never creates one.", + "properties": { + "createDefault": { + "type": "boolean", + "description": "Render and own a ClusterProvider named 'default'. Off means Helm deletes the one it created; you then commit the object yourself." + }, + "default": { + "type": "object", + "additionalProperties": false, + "description": "Spec of the rendered 'default' ClusterProvider.", + "properties": { + "kubeConfig": { + "type": "object", + "additionalProperties": false, + "description": "Which cluster 'default' names. An empty secretRef.name means the operator's own in-cluster cluster; a Secret name points it at a remote cluster instead.", + "properties": { + "secretRef": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Secret (in the release namespace) holding the kubeconfig. Empty means in-cluster." + }, + "key": { + "type": "string", + "description": "Key within the Secret. Empty reads 'value' then 'value.yaml'." + } + } + } + } + }, + "allowedNamespaces": { + "type": "object", + "description": "Deny-by-default control-cluster namespace policy (names and/or selector) for which namespaces may reference this provider from a GitTarget.", + "additionalProperties": true + } + } + } } }, diff --git a/charts/gitops-reverser/values.yaml b/charts/gitops-reverser/values.yaml index e673aeda..67e671ca 100644 --- a/charts/gitops-reverser/values.yaml +++ b/charts/gitops-reverser/values.yaml @@ -213,6 +213,43 @@ attribution: # Bounded per-event wait for a matching audit fact before a watch event ships as the committer. # Larger values raise attribution hit-rate at the cost of commit latency. grace: "3s" + # Audit-event annotation naming the ClusterProvider each event belongs to. Audit routes are + # normally NAMED (/audit-webhook/, including /audit-webhook/default), and this is empty. + # Set it only for a control plane that emits ONE shared audit stream for several logical + # clusters: it enables the bare /audit-webhook endpoint, which reads this annotation from every + # event, so one batch may fan out to several source clusters. An event with no annotation, or + # naming a ClusterProvider that does not exist, is rejected (counted and logged) and never + # credited to a fallback. See docs/configuration.md. + clusterAnnotationKey: "" + +# The source cluster a GitTarget mirrors FROM is a ClusterProvider, and a GitTarget that omits +# spec.clusterProviderRef references one named "default". The operator NEVER creates a +# ClusterProvider, so that object has to come from somewhere: either you commit it yourself, or the +# chart renders it from here. This is a rendering convenience, not runtime behavior — it is not +# limited to attribution (every mirror needs a provider) and "default" is just a name, so the +# provider it renders may be in-cluster or remote. +clusterProvider: + # Render and OWN a ClusterProvider named "default". Chart-owned: turn this off and Helm deletes + # the provider it created on the next upgrade, so ownership never silently splits — plan that + # switch together with committing your own object, because a missing provider holds its + # GitTargets unready via the "provider not found" path. The `quickstart` values never create one. + createDefault: true + # Spec of the rendered "default" ClusterProvider. + default: + # Which cluster "default" names. Leave secretRef.name empty for the operator's OWN in-cluster + # cluster; set it to a Secret (in the release namespace) holding a kubeconfig to make "default" + # mirror a REMOTE cluster instead. Immutable once created: to repoint the name, delete and + # recreate. key is optional — empty reads "value" then "value.yaml". + kubeConfig: + secretRef: + name: "" + key: "" + # Which namespaces may reference this provider from a GitTarget — deny-by-default authorization + # in the CONTROL cluster (names OR selector), not a filter on the source cluster. The chart + # default is an empty selector, which matches EVERY namespace, so a single-cluster install works + # out of the box; tighten it to a names list or a label selector in a real install. + allowedNamespaces: + selector: {} # RBAC configuration rbac: diff --git a/cmd/main.go b/cmd/main.go index ad721c6f..cc2c4cc5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -24,6 +24,7 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -143,11 +144,14 @@ func main() { RuleStore: ruleStore, EventRouter: nil, // Will be set below SensitiveResources: cfg.sensitiveResources, - // Resolve a GitTarget.spec.kubeConfig into a rest.Config by reading its Secret from the - // config plane. The manager client bypasses its cache for Secrets, so a rotated - // kubeconfig is seen without a Secret informer. + // Resolve a source cluster (named by a GitTarget.spec.clusterProviderRef) into a + // rest.Config: look up the ClusterProvider by name, read its kubeConfig Secret from the + // operator namespace, and build the client. The manager client bypasses its cache for + // Secrets, so a rotated kubeconfig is seen without a Secret informer. Per-provider qps/burst + // override the global --source-cluster-qps/-burst defaults passed here. SourceClusters: watch.NewSecretSourceClusterResolver( - mgr.GetClient(), cfg.kubeConfigSafety, float32(cfg.sourceClusterQPS), cfg.sourceClusterBurst), + mgr.GetClient(), os.Getenv("POD_NAMESPACE"), cfg.kubeConfigSafety, + float32(cfg.sourceClusterQPS), cfg.sourceClusterBurst), } // Initialize EventRouter with all dependencies. The streaming-snapshot resync @@ -236,6 +240,13 @@ func main() { auditHandler, err := webhookhandler.NewAuditHandler(webhookhandler.AuditHandlerConfig{ MaxRequestBodyBytes: cfg.auditMaxRequestBodyBytes, FactRecorder: attributionIndex, + // Gate every /audit-webhook/ route on the ClusterProvider existing. The audit + // server already requires a CA-signed client cert (RequireAndVerifyClientCert), so this + // only decides which named source clusters an authenticated apiserver may post for. + ProviderResolver: clusterProviderExistence{reader: mgr.GetClient()}, + // Empty leaves the bare /audit-webhook endpoint disabled (400); set, it demultiplexes a + // shared stream per event by this annotation. + ClusterAnnotationKey: cfg.clusterAnnotationKey, }) fatalIfErr(err, "unable to build audit handler") @@ -250,7 +261,15 @@ func main() { ctrl.Log.WithName("attribution"), ) setupLog.Info("author attribution enabled: matched audit facts name the commit author", - "redisAddr", cfg.redisAddr, "grace", cfg.attributionGrace.String()) + "redisAddr", cfg.redisAddr, "grace", cfg.attributionGrace.String(), + "clusterAnnotationKey", cfg.clusterAnnotationKey) + if cfg.clusterAnnotationKey == "" { + setupLog.Info("audit routes are named: post to /audit-webhook/; " + + "the bare /audit-webhook endpoint is disabled") + } else { + setupLog.Info("shared audit stream enabled on the bare /audit-webhook endpoint: each event's "+ + "ClusterProvider is read from its annotation", "annotationKey", cfg.clusterAnnotationKey) + } case cfg.redisAddr != "": setupLog.Info("configured-author mode: author attribution disabled; commits use the configured "+ "committer identity", "redisAddr", cfg.redisAddr) @@ -271,12 +290,20 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "GitProvider") os.Exit(1) } + if err := (&controller.ClusterProviderReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + OperatorNamespace: os.Getenv("POD_NAMESPACE"), + KubeConfigSafety: cfg.kubeConfigSafety, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ClusterProvider") + os.Exit(1) + } if err := (&controller.GitTargetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - WorkerManager: workerManager, - EventRouter: eventRouter, - KubeConfigSafety: cfg.kubeConfigSafety, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + WorkerManager: workerManager, + EventRouter: eventRouter, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "GitTarget") os.Exit(1) @@ -370,6 +397,7 @@ type appConfig struct { authorAttribution bool attributionFactTTL time.Duration attributionGrace time.Duration + clusterAnnotationKey string branchBufferMaxBytes int64 sensitiveResources types.SensitiveResourcePolicy sshHostKeys git.SSHHostKeyConfig @@ -480,6 +508,13 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { "Bounded per-event wait for a matching audit fact to arrive before a watch event ships as the "+ "configured committer (duration string; default 3s). Larger values raise attribution hit-rate "+ "at the cost of commit latency.") + fs.StringVar(&cfg.clusterAnnotationKey, "author-attribution-cluster-annotation-key", "", + "Audit-event annotation naming the ClusterProvider each event belongs to. Setting it enables the "+ + "bare /audit-webhook endpoint for a SHARED audit stream carrying several logical clusters: the "+ + "source cluster is resolved per event, so one batch may fan out to several providers. An event "+ + "with no annotation, or naming a provider that does not exist, is rejected (counted and logged) "+ + "rather than credited to a fallback. Empty (the default) leaves the bare endpoint disabled, and "+ + "every producer must post to /audit-webhook/.") branchBufferMaxSizeStr := os.Getenv("BRANCH_BUFFER_MAX_SIZE") if branchBufferMaxSizeStr == "" { branchBufferMaxSizeStr = defaultBranchBufferMaxSizeStr @@ -581,6 +616,13 @@ func validateAuditConfig(cfg appConfig) error { if cfg.redisDB < 0 { return fmt.Errorf("redis-db must be >= 0, got %d", cfg.redisDB) } + // The annotation key only has a receiver to configure when the audit ingress is running at all; + // silently ignoring it would look like annotation routing was enabled when nothing serves it. + if strings.TrimSpace(cfg.clusterAnnotationKey) != "" && !cfg.authorAttribution { + return errors.New( + "author-attribution-cluster-annotation-key requires author-attribution to be enabled; " + + "without it there is no audit ingress to route") + } if strings.TrimSpace(cfg.redisAddr) == "" { if cfg.authorAttribution { return errors.New("redis-addr is required when author-attribution is enabled") @@ -815,6 +857,22 @@ func buildServerTLSConfig(tlsOpts []func(*tls.Config)) *tls.Config { return serverTLS } +// clusterProviderExistence adapts the manager client to the audit handler's AuditProviderResolver: +// it reports whether a ClusterProvider (a named source cluster) exists, gating remote +// /audit-webhook/ routes. The read is cached; during startup it blocks until the cache syncs. +type clusterProviderExistence struct{ reader client.Reader } + +func (c clusterProviderExistence) ProviderExists(ctx context.Context, name string) (bool, error) { + var cp configbutleraiv1alpha3.ClusterProvider + if err := c.reader.Get(ctx, client.ObjectKey{Name: name}, &cp); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + func buildAuditServerTLSConfig(cfg appConfig, tlsOpts []func(*tls.Config)) (*tls.Config, error) { serverTLS := buildServerTLSConfig(tlsOpts) diff --git a/cmd/main_audit_server_test.go b/cmd/main_audit_server_test.go index b05c6922..09547907 100644 --- a/cmd/main_audit_server_test.go +++ b/cmd/main_audit_server_test.go @@ -45,7 +45,10 @@ func TestParseFlagsWithArgs_Defaults(t *testing.T) { assert.Equal(t, "valkey:6379", cfg.redisAddr) assert.False(t, cfg.redisInsecure) assert.True(t, cfg.authorAttribution) - assert.Equal(t, 15*time.Minute, cfg.attributionFactTTL) + // A LITERAL on purpose, not queue.DefaultAttributionFactTTL: this pins the default the flag + // help and docs/configuration.md promise. Asserting the constant against itself would pass + // while they drifted — which is exactly what happened (code 15m, docs 10m). + assert.Equal(t, 10*time.Minute, cfg.attributionFactTTL) assert.Equal(t, 3*time.Second, cfg.attributionGrace) assert.False(t, cfg.zapOpts.Development) assert.Equal(t, []string{"secrets"}, cfg.sensitiveResources.Entries()) diff --git a/config/clusterprovider-default.yaml b/config/clusterprovider-default.yaml new file mode 100644 index 00000000..7cecd6d4 --- /dev/null +++ b/config/clusterprovider-default.yaml @@ -0,0 +1,13 @@ +# The "default" ClusterProvider for the SUT / config-dir install, mirroring what the Helm chart +# renders when clusterProvider.createDefault is true. A GitTarget's clusterProviderRef defaults to +# {name: default}, and the reconcile-time hard gate REQUIRES the referenced ClusterProvider to +# exist, so local mirroring needs this object — the operator never creates it. The empty selector +# admits every namespace (matching the chart default); tighten in a real install. +apiVersion: configbutler.ai/v1alpha3 +kind: ClusterProvider +metadata: + name: default +spec: + # no kubeConfig => the operator's own in-cluster config (optional for any provider name) + allowedNamespaces: + selector: {} diff --git a/config/crd/bases/configbutler.ai_clusterproviders.yaml b/config/crd/bases/configbutler.ai_clusterproviders.yaml new file mode 100644 index 00000000..2135e940 --- /dev/null +++ b/config/crd/bases/configbutler.ai_clusterproviders.yaml @@ -0,0 +1,318 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: clusterproviders.configbutler.ai +spec: + group: configbutler.ai + names: + kind: ClusterProvider + listKind: ClusterProviderList + plural: clusterproviders + singular: clusterprovider + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=="Validated")].status + name: Validated + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + description: |- + ClusterProvider is the cluster-scoped, read-side peer of GitProvider: it names a SOURCE cluster + a GitTarget mirrors FROM, and is the home for that cluster's connectivity credential + (spec.kubeConfig), namespace-access authorization (spec.allowedNamespaces), and per-cluster + status. Its NAME is the cluster's identity everywhere: the /audit-webhook/ ingress route + and the attribution fact-index key. No name is special: "default" is merely the name + GitTarget.spec.clusterProviderRef defaults to when omitted. Whether a provider is the operator's + own cluster or a remote follows from spec.kubeConfig (omitted = in-cluster), not from its name, + so "default" may just as well name a remote cluster. + + Security model: + - ClusterProvider is cluster-scoped and requires platform-admin permissions to create. + - A GitTarget may reference it only from a namespace its spec.allowedNamespaces admits + (deny-by-default), enforced at admission AND before any watch starts. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of ClusterProvider. + properties: + allowedNamespaces: + description: |- + AllowedNamespaces is the deny-by-default policy for which namespaces may reference this + provider from a GitTarget. Empty (or omitted) means no namespace may reference it. + properties: + names: + description: Names is an explicit allow-list of namespace names + that may reference this provider. + items: + type: string + type: array + x-kubernetes-list-type: set + selector: + description: |- + Selector is a label selector matched against Namespace labels; a namespace whose labels + match may reference this provider. ORed with Names. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + burst: + description: |- + Burst overrides the operator's outgoing kube-client burst for this cluster. Omitted, the + operator-wide --source-cluster-burst applies. Ignored when kubeConfig is omitted. + format: int32 + minimum: 1 + type: integer + kubeConfig: + description: |- + KubeConfig names the SOURCE CLUSTER this provider represents and the credentials to reach + it (Flux's meta.KubeConfigReference, embedded verbatim). OMITTED means the operator's own + in-cluster cluster, for any provider name. IMMUTABLE. The referenced Secret is + resolved from the operator's namespace — the credential for a cluster never has to live on + that cluster. When secretRef.key is empty the resolver reads "value" then "value.yaml" + (Flux's order). Only secretRef is honored (configMapRef is rejected); unsafe kubeconfigs + (exec auth, insecure-skip-tls-verify) are rejected with a Validated=False reason unless the + operator opts in via flags. + properties: + configMapRef: + description: |- + ConfigMapRef holds an optional name of a ConfigMap that contains + the following keys: + + - `provider`: the provider to use. One of `aws`, `azure`, `gcp`, or + `generic`. Required. + - `cluster`: the fully qualified resource name of the Kubernetes + cluster in the cloud provider API. Not used by the `generic` + provider. Required when one of `address` or `ca.crt` is not set. + - `address`: the address of the Kubernetes API server. Required + for `generic`. For the other providers, if not specified, the + first address in the cluster resource will be used, and if + specified, it must match one of the addresses in the cluster + resource. + If audiences is not set, will be used as the audience for the + `generic` provider. + - `ca.crt`: the optional PEM-encoded CA certificate for the + Kubernetes API server. If not set, the controller will use the + CA certificate from the cluster resource. + - `audiences`: the optional audiences as a list of + line-break-separated strings for the Kubernetes ServiceAccount + token. Defaults to the `address` for the `generic` provider, or + to specific values for the other providers depending on the + provider. + - `serviceAccountName`: the optional name of the Kubernetes + ServiceAccount in the same namespace that should be used + for authentication. If not specified, the controller + ServiceAccount will be used. + + Mutually exclusive with SecretRef. + properties: + name: + description: Name of the referent. + type: string + required: + - name + type: object + secretRef: + description: |- + SecretRef holds an optional name of a secret that contains a key with + the kubeconfig file as the value. If no key is set, the key will default + to 'value'. Mutually exclusive with ConfigMapRef. + It is recommended that the kubeconfig is self-contained, and the secret + is regularly updated if credentials such as a cloud-access-token expire. + Cloud specific `cmd-path` auth helpers will not function without adding + binaries and credentials to the Pod that is responsible for reconciling + Kubernetes resources. Supported only for the generic provider. + properties: + key: + description: Key in the Secret, when not specified an implementation-specific + default key is used. + type: string + name: + description: Name of the Secret. + type: string + required: + - name + type: object + type: object + x-kubernetes-validations: + - message: exactly one of spec.kubeConfig.configMapRef or spec.kubeConfig.secretRef + must be specified + rule: has(self.configMapRef) || has(self.secretRef) + - message: exactly one of spec.kubeConfig.configMapRef or spec.kubeConfig.secretRef + must be specified + rule: '!has(self.configMapRef) || !has(self.secretRef)' + qps: + description: |- + QPS overrides the operator's outgoing kube-client query-per-second throttle for this + cluster's watches and discovery. Omitted, the operator-wide --source-cluster-qps applies. + Ignored when kubeConfig is omitted (the in-cluster client is not per-provider). + format: int32 + minimum: 1 + type: integer + type: object + x-kubernetes-validations: + - message: spec.kubeConfig is immutable; delete and recreate the ClusterProvider + to point a name at a different cluster + rule: has(self.kubeConfig) == has(oldSelf.kubeConfig) && (!has(self.kubeConfig) + || self.kubeConfig == oldSelf.kubeConfig) + - message: spec.kubeConfig.configMapRef (workload-identity auth) is not + yet supported; use secretRef + rule: '!has(self.kubeConfig) || !has(self.kubeConfig.configMapRef)' + - message: spec.kubeConfig.secretRef.name must not be empty + rule: '!has(self.kubeConfig) || !has(self.kubeConfig.secretRef) || size(self.kubeConfig.secretRef.name) + > 0' + status: + description: status defines the observed state of ClusterProvider. + properties: + conditions: + description: |- + Conditions report the provider's readiness: Validated (kubeconfig inputs are safe and + resolvable, asserted without a network dial) plus the aggregated Ready and the kstatus + Reconciling/Stalled pair. Runtime reachability/discovery health and a last-audit-event + timestamp are deferred until authenticated remote ingest wires them from the watch engine. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index d0a7c698..eb1310be 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -53,6 +53,10 @@ spec: name: ProviderReady priority: 1 type: string + - jsonPath: .status.conditions[?(@.type=="ClusterProviderReady")].status + name: ClusterProviderReady + priority: 1 + type: string - jsonPath: .status.conditions[?(@.type=="Ready")].message name: Status priority: 1 @@ -96,6 +100,40 @@ spec: Immutable: delete and recreate the GitTarget to change its destination. minLength: 1 type: string + clusterProviderRef: + default: + name: default + description: |- + ClusterProviderRef names the SOURCE cluster this GitTarget mirrors FROM, by referencing a + cluster-scoped ClusterProvider by name. The ClusterProvider is the home for that cluster's + connectivity credential, namespace-access authorization, and author-attribution mode. This + DEFAULTS to {name: "default"} — a user-created provider by that conventional name, which may + be in-cluster or remote — so a target that omits it persists with a concrete, jumpable ref + rather than an implicit nil. The operator never creates that provider; a GitTarget naming one + that does not exist is held unready. Immutable: a folder's source cluster is part of what the + folder means; delete and recreate to change it. + properties: + group: + default: configbutler.ai + description: API Group of the referent. + enum: + - configbutler.ai + type: string + kind: + default: ClusterProvider + description: |- + Kind of the referent. + Optional because this reference currently only supports a single kind (ClusterProvider). + enum: + - ClusterProvider + type: string + name: + description: Name of the referent. + minLength: 1 + type: string + required: + - name + type: object encryption: description: Encryption defines encryption settings for Secret resource writes. @@ -162,86 +200,6 @@ spec: required: - provider type: object - kubeConfig: - description: |- - KubeConfig names the SOURCE CLUSTER this GitTarget mirrors FROM: the kubeconfig - determines both the cluster and the credentials to reach it. Omitted means the cluster - the operator runs in, the single-cluster default that behaves exactly as before. Its - Secret is read from the GitTarget's OWN namespace, on the cluster the operator runs in — - the credential for a cluster never has to live on that cluster. When SecretRef.Key is - empty the resolver reads "value" then "value.yaml" (Flux's order). Immutable: the source - of a folder's content is part of what the folder means; delete and recreate to change it. - Only kubeConfig.secretRef is honored in v1alpha3 (configMapRef is rejected at admission); - unsafe kubeconfigs (exec auth providers, insecure-skip-tls-verify) are rejected by the - controller with a legible Validated=False reason unless the operator opts in via flags. - properties: - configMapRef: - description: |- - ConfigMapRef holds an optional name of a ConfigMap that contains - the following keys: - - - `provider`: the provider to use. One of `aws`, `azure`, `gcp`, or - `generic`. Required. - - `cluster`: the fully qualified resource name of the Kubernetes - cluster in the cloud provider API. Not used by the `generic` - provider. Required when one of `address` or `ca.crt` is not set. - - `address`: the address of the Kubernetes API server. Required - for `generic`. For the other providers, if not specified, the - first address in the cluster resource will be used, and if - specified, it must match one of the addresses in the cluster - resource. - If audiences is not set, will be used as the audience for the - `generic` provider. - - `ca.crt`: the optional PEM-encoded CA certificate for the - Kubernetes API server. If not set, the controller will use the - CA certificate from the cluster resource. - - `audiences`: the optional audiences as a list of - line-break-separated strings for the Kubernetes ServiceAccount - token. Defaults to the `address` for the `generic` provider, or - to specific values for the other providers depending on the - provider. - - `serviceAccountName`: the optional name of the Kubernetes - ServiceAccount in the same namespace that should be used - for authentication. If not specified, the controller - ServiceAccount will be used. - - Mutually exclusive with SecretRef. - properties: - name: - description: Name of the referent. - type: string - required: - - name - type: object - secretRef: - description: |- - SecretRef holds an optional name of a secret that contains a key with - the kubeconfig file as the value. If no key is set, the key will default - to 'value'. Mutually exclusive with ConfigMapRef. - It is recommended that the kubeconfig is self-contained, and the secret - is regularly updated if credentials such as a cloud-access-token expire. - Cloud specific `cmd-path` auth helpers will not function without adding - binaries and credentials to the Pod that is responsible for reconciling - Kubernetes resources. Supported only for the generic provider. - properties: - key: - description: Key in the Secret, when not specified an implementation-specific - default key is used. - type: string - name: - description: Name of the Secret. - type: string - required: - - name - type: object - type: object - x-kubernetes-validations: - - message: exactly one of spec.kubeConfig.configMapRef or spec.kubeConfig.secretRef - must be specified - rule: has(self.configMapRef) || has(self.secretRef) - - message: exactly one of spec.kubeConfig.configMapRef or spec.kubeConfig.secretRef - must be specified - rule: '!has(self.configMapRef) || !has(self.secretRef)' path: description: |- Path within the repository to write resources to, relative to the repository @@ -323,16 +281,9 @@ spec: - message: spec.path is immutable; delete and recreate the GitTarget to change its destination rule: self.path == oldSelf.path - - message: spec.kubeConfig is immutable; delete and recreate the GitTarget - to change the cluster it mirrors - rule: has(self.kubeConfig) == has(oldSelf.kubeConfig) && (!has(self.kubeConfig) - || self.kubeConfig == oldSelf.kubeConfig) - - message: spec.kubeConfig.configMapRef (provider auth) is not yet supported; - use secretRef - rule: '!has(self.kubeConfig) || !has(self.kubeConfig.configMapRef)' - - message: spec.kubeConfig.secretRef.name must not be empty - rule: '!has(self.kubeConfig) || !has(self.kubeConfig.secretRef) || size(self.kubeConfig.secretRef.name) - > 0' + - message: spec.clusterProviderRef is immutable; delete and recreate the + GitTarget to change the cluster it mirrors + rule: self.clusterProviderRef == oldSelf.clusterProviderRef status: description: status defines the observed state of GitTarget properties: diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index da17b9a4..f78a624b 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -1,6 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - bases/configbutler.ai_clusterproviders.yaml - bases/configbutler.ai_clusterwatchrules.yaml - bases/configbutler.ai_commitrequests.yaml - bases/configbutler.ai_gitproviders.yaml diff --git a/config/kustomization.yaml b/config/kustomization.yaml index 37c17d75..cdde3c56 100644 --- a/config/kustomization.yaml +++ b/config/kustomization.yaml @@ -9,6 +9,7 @@ resources: - deployment.yaml - certs - webhook +- clusterprovider-default.yaml images: - name: gitops-reverser newName: gitops-reverser diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b1f41a80..5091f084 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -39,13 +39,8 @@ rules: - apiGroups: - configbutler.ai resources: - - clusterwatchrules - - gitproviders - - gittargets - - watchrules + - clusterproviders verbs: - - create - - delete - get - list - patch @@ -54,6 +49,7 @@ rules: - apiGroups: - configbutler.ai resources: + - clusterproviders/status - clusterwatchrules/status - commitrequests/status - gitproviders/status @@ -63,6 +59,21 @@ rules: - get - patch - update +- apiGroups: + - configbutler.ai + resources: + - clusterwatchrules + - gitproviders + - gittargets + - watchrules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - configbutler.ai resources: diff --git a/config/samples/clusterprovider.yaml b/config/samples/clusterprovider.yaml new file mode 100644 index 00000000..bc9bf92c --- /dev/null +++ b/config/samples/clusterprovider.yaml @@ -0,0 +1,31 @@ +# The conventionally named "default" provider, here in-cluster. Omitting kubeConfig means "the +# cluster the operator runs in" — an option open to ANY provider name, not a property of this one. +# GitTarget.spec.clusterProviderRef defaults to {name: default}, so a single-cluster install needs +# no ClusterProvider of its own beyond this one (the chart renders it when +# clusterProvider.createDefault is true; the operator never creates one). allowedNamespaces is +# deny-by-default: only the listed/selected namespaces may bind it. +apiVersion: configbutler.ai/v1alpha3 +kind: ClusterProvider +metadata: + name: default +spec: + # no kubeConfig => in-cluster. Set it here instead to make "default" mirror a remote cluster. + allowedNamespaces: + names: + - team-a +--- +# A remote source cluster. Its name is the cluster's identity everywhere: the +# /audit-webhook/prod-eu-1 audit route and the attribution fact-index key. kubeConfig is immutable +# and its Secret is resolved from the operator's namespace (never from the source cluster). +apiVersion: configbutler.ai/v1alpha3 +kind: ClusterProvider +metadata: + name: prod-eu-1 +spec: + kubeConfig: + secretRef: + name: prod-eu-1-kubeconfig + allowedNamespaces: + selector: + matchLabels: + tier: trusted diff --git a/docs/INDEX.md b/docs/INDEX.md index d2ae5389..38e1878f 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -2,7 +2,7 @@ > Rebuilt 2026-07-11, when the tree went from 180 documents to 117. -There are 117 markdown files here. **About 35 of them bind.** This page names +There are 136 tracked markdown files here. **About 35 of them bind.** This page names those. If a document is not on this page, it is either a user guide (see [`README.md`](README.md)) or history you can safely not read. @@ -66,15 +66,14 @@ says what we support and refuse** — and then its kustomize field taxonomy, the write boundary, the orchestrator/expansion line, and how secrets are handled. -Ten other open items: +Nine other open items: | Doc | Open question | |---|---| -| [`config-plane-split.md`](design/config-plane-split.md) | remote-cluster mirroring via an inline, immutable `GitTarget.spec.kubeConfig` (Flux's `meta.KubeConfigReference`) — **redesign of #220's #1, built** | | [`watch-and-catalog-architecture.md`](design/watch-and-catalog-architecture.md) | the target three-layer watch model — **needs a human call before building** | | [`metrics-observability-plan.md`](design/metrics-observability-plan.md) | the watch-stage metrics do not exist yet | | [`reconcile-triggering.md`](design/reconcile-triggering.md) | which controllers still fail to wake up | -| [`multi-cluster-audit-ingestion-implications.md`](design/multi-cluster-audit-ingestion-implications.md) | §5's `SourceCluster` CRD is **superseded** by [`config-plane-split.md`](design/config-plane-split.md); the rest (per-cluster audit ingestion) is still open | +| [`multi-source-audit-ingress-hardening.md`](design/multi-source-audit-ingress-hardening.md) | how independent sources authenticate to a named audit route, when annotation routing is trustworthy, and how multi-provider ingestion remains fair | | [`release-image-reuse-plan.md`](design/release-image-reuse-plan.md) | PRs 2–5 unstarted | | [`e2e-coverage-gaps-and-improvements-plan.md`](design/e2e-coverage-gaps-and-improvements-plan.md) | tests A/B/C still proposals | | [`e2e-finish-plan.md`](design/e2e-finish-plan.md) | remaining e2e harness work | @@ -93,7 +92,7 @@ three open RBAC items. Five more ideas sit beside them. ## History — [`finished/`](finished/) -Eighteen shipped plans and closed investigations. **Nothing here binds.** Read one +Twenty-two shipped plans and closed investigations. **Nothing here binds.** Read one only when you want to know *why* something is the way it is; the answer to *what it is* always lives in `spec/`. diff --git a/docs/architecture.md b/docs/architecture.md index 5138b0d7..f56fabc0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -13,6 +13,13 @@ operator's short-lived coordination state.** Read the [Ground Rules](#ground-rul together. The later sections give the reference detail behind each piece. If a detail here ever disagrees with the source, the source wins; deeper design records live under [docs/design/](design/). +Source and destination connections deliberately have different scopes. A namespaced `GitProvider` is a +team's Git write boundary: its credential, branch policy, and targets usually belong together. +`ClusterProvider` is cluster-scoped because it represents one shared source identity whose client, +discovery surface, watch state, and attribution partition must remain consistent across namespaces. +`allowedNamespaces` then explicitly controls which control-cluster namespaces may reference that shared +source; it does not grant source-cluster RBAC or select source namespaces. + *** ## Ground Rules @@ -24,9 +31,10 @@ API. State is ingested by **watch**; these paths never treat Git as authority. W with a newer remote commit, the operator fetches the new remote state, resets its local clone, and replays its retained writes from the API. -**Watch is the only object-state source.** Each `GitTarget` opens one Kubernetes watch per claimed -`(GVR, scope)` with `sendInitialEvents=true`. Every Git write derives from persisted state the watch -observed. Audit never defines *what* changed — it only, optionally, explains *who* caused it. +**Watch is the only object-state source.** Each `GitTarget` names one `ClusterProvider` and opens one +Kubernetes watch per claimed `(GVR, scope)` against that source, with `sendInitialEvents=true`. Every +Git write derives from persisted state the watch observed. Audit never defines *what* changed — it only, +optionally, explains *who* caused it. **Sensitive resources are never written in plaintext.** Core Secrets and configured sensitive resource types must be encrypted before they touch the Git worktree. If encryption cannot be configured, the write @@ -43,10 +51,10 @@ attribution, CommitRequest author capture, and HA. Attributed-author mode requir require it as the shared store across replicas. **Audit is an optional attribution lookup.** When attribution is enabled, kube-apiserver posts audit -events to `/audit-webhook`; the operator extracts a minimal attribution fact (auditID, user, verb, -resourceVersion, GVR/namespace/name/UID, status, timestamps) into a Redis attribution index keyed for a -join, and a resolver attaches the commit author to a watch event by matching a fact within a bounded -grace window. A missing, late, or absent fact never blocks state capture; it only changes the author. +events to `/audit-webhook/` (or a configured annotation-routed shared endpoint). +The operator stores a minimal fact under that source-provider partition and joins it to a watch event +within a bounded grace window. A missing, late, or absent fact never blocks state capture; it only +changes the author. **Behavior is deterministic and proven by tests.** Given the same observed Kubernetes state, configuration, and Git base, the operator makes the same materialization decisions. Ordering, attribution fallbacks, @@ -99,7 +107,7 @@ optional and only add *who* did something: |---|---|---|---| | **Discovery** (CRD/APIService) | *what types exist* | yes | the API surface — served GVRs, scope, preferred version, subresources; rules resolve against it | | **Watch** (per claimed `(GVR, scope)`) | *what changed* | yes | the object body, ordered per type; the only object-state source, with deletes reconciled via `sendInitialEvents` replay + mark-and-sweep | -| **Audit webhook** (`/audit-webhook`) | *who changed mirrored state* | no | a post-persist attribution fact, joined to the watch event by resourceVersion ([Optional Attribution](#optional-attribution)) | +| **Audit webhook** (`/audit-webhook/`) | *who changed mirrored state* | no | a post-persist attribution fact, partitioned by source provider and joined to the watch event by resourceVersion ([Optional Attribution](#optional-attribution)) | | **Validating admission webhook** (`/validate-operator-types`) | *who issued a command* | no | the submitter of a `CommitRequest`, captured at admission and keyed 1:1 by UID ([CommitRequest Finalize](#commitrequest-finalize)) | With both optional sources off, the product still mirrors state correctly — every commit is simply authored @@ -148,9 +156,10 @@ capture audit and admission with **no** resulting watch event. ## Configuration Model -You configure GitOps Reverser entirely through five CRDs (group `configbutler.ai`, version -`v1alpha3`). `WatchRule` and `ClusterWatchRule` choose which Kubernetes resources enter the pipeline. -`CommitRequest` can ask for the current window to be saved. `GitTarget` chooses the branch and path. +You configure GitOps Reverser through six CRDs (group `configbutler.ai`, version `v1alpha3`). +`WatchRule` and `ClusterWatchRule` choose which Kubernetes resources enter the pipeline. +`CommitRequest` can ask for the current window to be saved. `GitTarget` joins one source cluster to one +Git destination. `ClusterProvider` supplies the source connection and authorization boundary; `GitProvider` supplies the repository, credentials, commit settings, and push policy. ```mermaid @@ -158,9 +167,11 @@ graph LR WR[WatchRule] -->|targetRef| GT[GitTarget] CWR[ClusterWatchRule] -->|targetRef| GT CR[CommitRequest] -->|targetRef| GT + GT -->|clusterProviderRef| CP[ClusterProvider] GT -->|providerRef| GP[GitProvider] style GP fill:#e8f4fd,stroke:#2196f3 + style CP fill:#e8f4fd,stroke:#2196f3 style GT fill:#e8f4fd,stroke:#2196f3 style WR fill:#fff3e0,stroke:#ff9800 style CWR fill:#fff3e0,stroke:#ff9800 @@ -172,7 +183,8 @@ graph LR | `WatchRule` | namespaced | which resources in *this* namespace route to a GitTarget | | `ClusterWatchRule` | cluster | which cluster scoped or cluster wide resources route to a GitTarget | | `CommitRequest` | namespaced | a one shot "save the open window now" signal | -| `GitTarget` | namespaced | one materialization destination `(provider, branch, path)` | +| `GitTarget` | namespaced | one materialization from a source provider to `(provider, branch, path)` | +| `ClusterProvider` | cluster | one source-cluster connection plus namespace access policy | | `GitProvider` | namespaced | a Git repo + credentials + commit/signing config | ### WatchRule / ClusterWatchRule @@ -227,18 +239,20 @@ How attribution and finalization interact is described under * **Source**: [api/v1alpha3/gittarget_types.go](../api/v1alpha3/gittarget_types.go) * **Controller**: [internal/controller/gittarget_controller.go](../internal/controller/gittarget_controller.go) -One materialization destination: `(provider, branch, path)`. Key fields: +One materialization from a source provider to a Git destination: `(cluster provider, provider, branch, path)`. +Key fields: * `spec.providerRef`: a `GitProvider` in the same namespace (`group`/`kind` default to `configbutler.ai`/`GitProvider`, the only accepted values). +* `spec.clusterProviderRef`: a cluster-scoped source `ClusterProvider`; it defaults to `{name: default}`. * `spec.branch`: immutable branch, validated against `GitProvider.spec.allowedBranches`. * `spec.path`: immutable, required path under the repo (`MinLength=1`; `.` means repo root and must be chosen explicitly). * `spec.encryption`: optional SOPS/age encryption settings for sensitive resources. -`providerRef`, `branch`, and `path` are immutable so a target cannot silently orphan an old -materialization. The controller also rejects path overlaps between GitTargets sharing a provider and -branch. +`providerRef`, `clusterProviderRef`, `branch`, and `path` are immutable so a target cannot silently +orphan an old materialization or change its source cluster. The controller also rejects path overlaps +between GitTargets sharing a provider and branch. Status has a kstatus-compatible summary layer plus domain conditions: @@ -248,6 +262,8 @@ Status has a kstatus-compatible summary layer plus domain conditions: * `Validated` and `EncryptionConfigured` explain control-plane health. * `StreamsRunning` explains the source side: every tracked type is past initial replay and routing live events. +* `ClusterProviderReady` and `SourceClusterReachable` distinguish valid source configuration from a + source API the data plane can currently reach. * `GitPathAccepted` explains the target side: the selected Git path is safe for the operator to materialize. * `status.streams` is a bounded count summary, not a per-type list. @@ -279,6 +295,21 @@ ecosystems is the credentials Secret, not a foreign repository object. The crede Kubernetes native, Flux, and Argo CD Secret key dialects (see [design/git-credentials-interop.md](finished/git-credentials-interop.md)). +### ClusterProvider + +`ClusterProvider` is the read-side peer of `GitProvider`. A `GitTarget` references it by the immutable +`spec.clusterProviderRef`; omission defaults to the conventional name `default`. The name is only a +defaulting convention: a provider without `spec.kubeConfig` uses the operator's in-cluster client, while +any name (including `default`) may instead carry a remote kubeconfig resolved from the operator namespace. + +Its cluster scope is intentional. Several namespaces may mirror one source cluster, but the source +identity must not vary by target because it keys source clients, discovery, watches, and attribution. +`spec.allowedNamespaces` is therefore a deny-by-default control-cluster policy, enforced on every +reconcile before watches start — so tightening it also stops an already-existing `GitTarget`, which an +admission-time check could not do. It guards which tenant may cause the operator to export a shared source; it +does not expand that source credential's Kubernetes permissions. The `ClusterProvider` validates its +connection inputs, while a `GitTarget` projects provider readiness and live source reachability. + *** ## Common Flows @@ -291,7 +322,7 @@ flowchart TD subgraph K8S["Kubernetes API server"] WATCH["WATCH + sendInitialEvents replay
(per claimed GVR + scope)"] DISC["Discovery: CRDs / APIServices"] - AUDIT["/audit-webhook (optional)"] + AUDIT["/audit-webhook/<provider> (optional)"] end subgraph PERGT["Per GitTarget: internal/watch + internal/reconcile"] @@ -350,9 +381,9 @@ Following the ConfigMap edit: the remote moved). Separately, the audit path (only when attribution is enabled): kube-apiserver POSTs audit events to -`/audit-webhook`; [AuditHandler](../internal/webhook/audit_handler.go) extracts a minimal attribution -fact and writes it to the Redis attribution index with a short TTL. That index is read only by the -resolver in step 3; it never creates or repairs object state. +`/audit-webhook/`; [AuditHandler](../internal/webhook/audit_handler.go) extracts a minimal +attribution fact and writes it to that provider's Redis partition with a short TTL. That index is read +only by the resolver in step 3; it never creates or repairs object state. **And if the watch had been lost?** A delete that happened while no watch was running is reconciled on the next watch (re)connect: the `sendInitialEvents` replay plus **mark-and-sweep** removes any Git file @@ -514,19 +545,22 @@ per-mutation change log. * **Attribution index**: [internal/queue/attribution_index.go](../internal/queue/attribution_index.go) * **Resolver (grace window join)**: [internal/watch/author_resolver.go](../internal/watch/author_resolver.go) -Attribution runs **only when attribution is enabled** (`--author-attribution`, the default); Redis — always -required — is its state store. The Kubernetes API server POSTs audit `EventList` -payloads to a **single** HTTP endpoint, `/audit-webhook`; there is no supplementary body endpoint and no +Attribution runs only when `--author-attribution=true`; Redis is then its required state store. A normal +source posts audit `EventList` payloads to `/audit-webhook/`. The bare +`/audit-webhook` endpoint is enabled only with `--author-attribution-cluster-annotation-key`, for a +trusted control plane that puts a provider name in each event. There is no supplementary body endpoint or body joiner, because watch — not audit — carries the object body. The handler applies an intrinsic accept gate (StageResponseComplete, a mutating verb, success, non-dry-run, a changed resourceVersion, and the -`/scale` subresource only), extracts a minimal attribution fact, and writes it to the Redis attribution -index with a short TTL. +`/scale` subresource only), then writes the minimal attribution fact to the provider's Redis partition. | Endpoint | Role | |---|---| -| `/audit-webhook` | Audit source (kube-apiserver) for the optional attribution index | +| `/audit-webhook/` | One source provider's audit stream; the provider must exist | +| `/audit-webhook` | Shared stream only when annotation routing is configured; each event names its provider | -Cluster ID path segments are rejected; multi cluster routing is not modeled yet. +The handler accepts a client certificate signed by the audit CA, but it does not yet bind that certificate +to a named provider. Do not treat named routes as an isolation boundary for independently administered +remote sources that share a client credential; provider-bound ingress authentication remains outstanding. ### Optional, but never casual @@ -539,11 +573,10 @@ to clearly record that the operator made a change than to assert an actor we are Two engineering choices follow directly from that stance: -* **mTLS on the audit ingress is on by default.** An attribution fact names a human or a service account, - so the channel that delivers it must be trustworthy. The audit server requires and verifies a client +* **mTLS on the audit ingress is on by default.** The audit server requires and verifies a client certificate (`tls.RequireAndVerifyClientCert` against a configured CA; `--audit-insecure` defaults to - `false`), so nothing can **impersonate the kube-apiserver** and inject forged facts. Disabling - verification is an explicit, deliberate opt-out. + `false`). This authenticates membership in that CA's client set. It is not yet a sender-to-provider + binding, so multi-source deployments must not share a credential across independently trusted sources. * **Tests pin the behavior.** Because a misattribution is a real harm, the attribution and resolver paths carry unit and e2e tests that prove the concrete cases — strong match, weak/last-key match, deletes whose audit RV differs from the watch RV, missing/late/expired facts, service-account vs human actor, @@ -559,14 +592,15 @@ The fact is the smallest thing needed to name an author, not an object log: | `user` / `impersonatedUser` | author candidate (human *or* service account) | | `verb`, `subresource` | explain the write | | `responseStatus.code`, `dryRun` | reject failures and non-persistent requests (at the handler gate) | -| GVR, namespace, name, UID | exact join keys | +| source provider, GVR, namespace, name, UID | source partition plus exact join keys | | response object resourceVersion | exact watch-event match | | stage timestamp | recency | -The index writes the fact under several join keys, strongest first: exact `(GVR, ns, name, uid, rv)`, -then `(GVR, ns, name, uid)` (for deletes whose watch RV differs from the audit RV), then -`(GVR, ns, name, rv)` (when UID is absent). Each key carries the same short TTL (minutes, not hours); -old facts are never needed for correctness because watch owns state. +The index writes the fact under several join keys, all prefixed by the `ClusterProvider` name: exact +`(provider, GVR, ns, name, uid, rv)`, then `(provider, GVR, ns, name, uid)` (for deletes whose watch RV +differs from the audit RV), then `(provider, GVR, ns, name, rv)` (when UID is absent). Each key carries +the same short TTL (minutes, not hours); old facts are never needed for correctness because watch owns +state. ### The resolver and its grace window @@ -912,7 +946,7 @@ hydrates only touched files into buffers for the commit, and flushes only change entry in the document's kustomization chain is written back to that entry (comment-preserving, only fields the entry already declares); the source manifest keeps its bytes. Anything the inversion cannot express falls back to the plain in-place patch. See - [gitops-api/finished/images-and-replicas-edit-through.md](design/support-boundary/finished/images-and-replicas-edit-through.md). + [images-and-replicas edit-through design](design/support-boundary/finished/images-and-replicas-edit-through.md). * **Deletes:** use the manifest identity index, so a moved manifest can still be deleted even when it is not at the canonical path. * **Field patches** (currently `/scale` → parent `spec.replicas`) are intentionally narrow: they only @@ -1000,20 +1034,25 @@ admission by the validating webhook, not derived from the audit attribution inde Controllers watch their dependencies so dependents reconcile quickly after spec changes: -* `GitTargetReconciler` watches `GitProvider`, `WatchRule`, `ClusterWatchRule`, and the encryption - `Secret`; it resolves the GitTarget's claimed `(GVR, scope)` watch set and derives the `Synced` condition - + materialization summary. +* `GitTargetReconciler` watches `GitProvider`, `ClusterProvider`, `Namespace`, `WatchRule`, and + `ClusterWatchRule`. Provider readiness/spec changes and namespace-label changes promptly re-check + source authorization; rules re-declare the claimed `(GVR, scope)` set. It deliberately does **not** + watch encryption Secrets, so their recovery is picked up by periodic reconciliation without retaining + every Secret value in the control-plane cache. * `WatchRuleReconciler` / `ClusterWatchRuleReconciler` watch `GitTarget` and `GitProvider`, populate the RuleStore, and trigger the rule-change reconcile. * `GitProviderReconciler` validates reachability and manages the signing key lifecycle. +* `ClusterProviderReconciler` validates source-connection inputs and projects its readiness to dependent + targets. It also removes the retired fact-purge finalizer from pre-release objects during an upgrade. * `CommitRequestReconciler` runs with `MaxConcurrentReconciles=1` and attributes/attaches as above; its optional `AuthorLookup` is the command-author cache populated by the `/validate-operator-types` webhook (wired whenever the admission webhook is enabled — independent of `--author-attribution` — and nil otherwise, so the request finalizes as the committer). -Dependency watches use generation change predicates to avoid queueing again on status only heartbeats. -`GitProvider`, `GitTarget`, and `CommitRequest` carry immutability constraints where a spec change would -orphan a materialized subtree or invalidate an in-flight finalize. +Dependency watches use narrow predicates to avoid status-only heartbeat churn: a `ClusterProvider` also +admits a `Ready` transition, and a `Namespace` admits only label changes. `GitProvider`, `GitTarget`, and +`CommitRequest` carry immutability constraints where a spec change would orphan a materialized subtree or +invalidate an in-flight finalize. *** @@ -1038,7 +1077,7 @@ flowchart TD Hq -->|no| J[Configured-author: no attribution index; audit webhook skipped] I --> K[Setup + register Watch Manager] J --> K - K --> L[Register GitProvider + GitTarget + CommitRequest controllers] + K --> L[Register GitProvider + ClusterProvider + GitTarget + CommitRequest controllers] L --> M[Add cert watchers + health checks] M --> N[mgr.Start] ``` @@ -1046,11 +1085,12 @@ flowchart TD Redis is optional in configured-author mode. When `--redis-addr` is set, the cursor store is wired and a Redis readiness gate keeps the pod not-ready until Redis is reachable; watches resume from their last stored resourceVersion after a restart. When `--redis-addr` is empty, the cursor store is skipped and -watches cold-replay from scratch on restart instead. With `--author-attribution` on (the default), a -non-empty `--redis-addr` is required: the attribution index is built on the Redis connection, the audit -HTTP handler is wired with the fact extractor, the watch manager gets the author resolver, and the audit -ingress is added to `/readyz`. With `--author-attribution=false` (configured-author) no attribution index is -built and the audit webhook is skipped entirely; every commit is committer-authored. +watches cold-replay from scratch on restart instead. The binary's `--author-attribution` flag defaults to +on, which requires a non-empty `--redis-addr`: the attribution index is built on the Redis connection, the +audit HTTP handler is wired with the fact extractor, the watch manager gets the author resolver, and the +audit ingress is added to `/readyz`. The Helm chart deliberately passes `--author-attribution=false` by +default, so a first install runs configured-author with no attribution index or audit webhook and every +commit is committer-authored. *** @@ -1089,7 +1129,10 @@ Current limitations: `sendInitialEvents` replay or LIST + mark-and-sweep. * **Per-watch and per-attribution metrics are not yet emitted** (see [Observability](#observability)). * **No pull request creation;** the operator writes directly to branches. -* **No multi cluster routing;** cluster ID path segments on `/audit-webhook` are rejected. +* **Audit ingress uses a shared-CA trust boundary.** Named `/audit-webhook/` routes and the + annotation-routed shared endpoint both require a client certificate signed by the audit CA, but that + certificate is not bound to one provider. This is an accepted privileged-control-plane assumption, not + tenant isolation; see [SECURITY.md](../SECURITY.md#shared-audit-ingress-trust-model). * **`deletecollection`** is reconciled by the watch (each item arrives as its own `DELETED`, or the mark-and-sweep reconciles them on replay). * **A fail-safe placement skip has no dedicated status condition.** A resource the writer refuses to diff --git a/docs/attribution-setup-guide.md b/docs/attribution-setup-guide.md index 42c4e27a..426ab2ac 100644 --- a/docs/attribution-setup-guide.md +++ b/docs/attribution-setup-guide.md @@ -23,6 +23,42 @@ On a managed platform, either front it with a self-managed control plane or stay mode. The [audit webhook connectivity design](facts/audit-webhook-api-server-connectivity.md) has the full reasoning on hosting. +## Source clusters — the `ClusterProvider` + +Each source cluster a `GitTarget` mirrors FROM is named by a cluster-scoped **`ClusterProvider`**, +the read-side peer of `GitProvider`. `GitTarget.spec.clusterProviderRef` **defaults to +`{name: "default"}`** — a provider you create by that conventional name, which the chart can render +for you with `clusterProvider.createDefault: true`. `default` is only the name an omitted reference +points at: that provider may omit `spec.kubeConfig` (the operator's own cluster) or set it to mirror +a remote one. + +The provider's **name is the cluster's identity everywhere** — the attribution fact-index key and +the `/audit-webhook/` audit route — so a fact from one cluster can never name +the author of an object watched on another. A provider also carries a deny-by-default +`spec.allowedNamespaces` policy: a `GitTarget` may reference it only from an admitted namespace +(enforced on every reconcile, before that target's watches start — so tightening the policy also +stops a `GitTarget` that already exists). + +**No provider, no streaming.** A `GitTarget` may mirror a source cluster only through an *existing* +`ClusterProvider` — `default` included — and the operator **never creates one**. If you set +`clusterProvider.createDefault: false` without committing your own `default`, a `GitTarget` that +references it is held `NotReady` (`ClusterProviderNotFound`) and never falls back to an implicit +in-cluster identity. Commit the object yourself, or point such targets at another `ClusterProvider`. + +**Remote source clusters.** Create a `ClusterProvider` with a `spec.kubeConfig.secretRef` (the +kubeconfig Secret lives in the operator namespace) for the outbound *watch* connection, and — for +attribution — configure that cluster's apiserver to POST audit events to `/audit-webhook/` +(the provider's name). The audit server already requires a CA-signed client certificate +(`RequireAndVerifyClientCert`); the remote apiserver presents the **cert-manager-issued audit client +certificate** the chart mints, exactly as the local apiserver does, and the operator accepts the +named route only for a `ClusterProvider` that exists. (Binding a distinct client certificate to each +provider is a future hardening; today the trust boundary is CA-level.) Remote attribution needs a +self-managed control plane — EKS/GKE/AKS are not supported (see *When it fits*). The current model is +in [the architecture guide](architecture.md#optional-attribution); the remaining multi-source hardening +work is in [multi-source audit-ingress hardening](design/multi-source-audit-ingress-hardening.md). See +[SECURITY.md](../SECURITY.md#shared-audit-ingress-trust-model) for the accepted shared-credential trust +assumption and its limits. + ## Prerequisites - GitOps Reverser installed and producing configured-author commits. @@ -44,11 +80,17 @@ helm upgrade gitops-reverser \ oci://ghcr.io/configbutler/charts/gitops-reverser \ --namespace gitops-reverser \ --reuse-values \ - --set attribution.enabled=true + --set attribution.enabled=true \ + --set queue.redis.addr=valkey.example.internal:6379 helm get notes gitops-reverser -n gitops-reverser ``` +Use the address of the Redis/Valkey service you prepared in the prerequisites. For an authenticated +instance, also set `queue.redis.auth.existingSecret` (and, if needed, +`queue.redis.auth.existingSecretKey`). The chart rejects attribution without a Redis address rather +than starting an audit receiver that cannot retain facts. + With `attribution.enabled=true` the chart additionally deploys the audit receiver, its Service, and (via cert-manager) the audit TLS materials — a root CA Secret and a kube-apiserver client-cert Secret. diff --git a/docs/ci-overview.md b/docs/ci-overview.md index 4948d011..edf6cd97 100644 --- a/docs/ci-overview.md +++ b/docs/ci-overview.md @@ -94,7 +94,11 @@ step merges those exact digests into a multi-arch manifest tagged with the semve | --- | --- | --- | | Multi-arch image (`linux/amd64`, `linux/arm64`) | `ghcr.io/configbutler/gitops-reverser` | cosign keyless signature, SLSA build provenance attestation, SPDX SBOM attestation | | Helm chart | `oci://ghcr.io/configbutler/charts/gitops-reverser` | cosign keyless signature | -| `install.yaml`, `sbom.spdx.json` | GitHub release assets | each signed directly (`.sigstore.json`) and SLSA-attested (`.intoto.jsonl`), also uploaded as release assets | +| `crds.yaml`, `install.yaml`, `sbom.spdx.json` | GitHub release assets | each signed directly (`.sigstore.json`) and SLSA-attested (`.intoto.jsonl`), also uploaded as release assets | + +The plain-manifest installer ships as **two** files: `crds.yaml` is applied first, then +`install.yaml` (which contains a custom resource that cannot be created before its CRD exists). +Each is signed and attested independently, so either can be verified on its own. Verify an image (also embedded in every release's notes): diff --git a/docs/configuration.md b/docs/configuration.md index b373c049..a59d2b73 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -6,6 +6,7 @@ steps in the [root README](../README.md). The short version: - `GitProvider` defines where and how to push +- `ClusterProvider` defines the Kubernetes source cluster a target mirrors from - `GitTarget` defines which branch and repository path to write into - `WatchRule` defines which namespaced resources should produce Git writes - `ClusterWatchRule` does the same for cluster-scoped or cross-namespace watching @@ -25,13 +26,41 @@ attribution later only when you need named Kubernetes users or service accounts The usual flow is: 1. Create a `GitProvider` for repository access and commit behavior. -2. Create a `GitTarget` that points at that provider plus a branch and repository path. -3. Create one or more `WatchRule` or `ClusterWatchRule` objects that point at that target. -4. Create a `CommitRequest` only when you want to flush an open window before the normal timer. +2. Create a `ClusterProvider` for the source cluster, including the `default` provider when a target + omits its source reference. +3. Create a `GitTarget` that points at the Git provider, source cluster, branch, and repository path. +4. Create one or more `WatchRule` or `ClusterWatchRule` objects that point at that target. +5. Create a `CommitRequest` only when you want to flush an open window before the normal timer. That means one repository connection can back multiple targets, and one target can be fed by multiple watch rules. +## Why the two provider types have different scopes + +`GitProvider` and `ClusterProvider` are both named connections, but their scope follows what they +identify and who normally owns their credentials—not a desire to make the API symmetric. A Git +destination is normally a team's write boundary, so a namespaced `GitProvider` keeps the repository +credential and its consumers together. A source cluster is a shared physical identity: its client, +discovery surface, watch state, and attribution partition must mean the same thing to every target +that uses it. That makes `ClusterProvider` cluster-scoped. + +| Object | Scope | What it represents | Why | +|---|---|---|---| +| `GitProvider` | Namespace | A Git destination and the credentials allowed to write it | A repository destination is normally owned by one team. Keeping the provider and its Secret in that team's namespace makes the ownership boundary direct. | +| `ClusterProvider` | Cluster | One physical Kubernetes source cluster | A source cluster can feed targets in several namespaces, while its connection, watch state, and attribution identity must stay the same everywhere. | + +There is no default `GitProvider`: the operator cannot infer a safe repository, branch, or write +credential. `GitTarget.spec.clusterProviderRef` instead defaults to the conventionally opinionated +name `default`. That is a convenient, concrete reference—not a claim that `default` is always the +local cluster. + +`ClusterProvider.spec.allowedNamespaces` is the control-cluster authorization boundary for that +shared source connection: it determines which namespaces may contain `GitTarget`s that reference +the provider. It does not select namespaces in the source cluster or grant permissions there. If a +platform later needs a shared, platform-owned Git destination, that should be a separate +cluster-scoped Git-destination concept with an explicit ownership model, rather than changing the +meaning of the namespaced `GitProvider`. + ## `GitProvider` `GitProvider` defines the Git remote, credentials, allowed branches, push strategy, and commit @@ -313,6 +342,88 @@ want custom `spec.commit` behavior because the starter values do not currently e For the platform-facing behavior behind "valid signature" versus "verified badge", see [commit-signing.md](commit-signing.md). +## `ClusterProvider` + +`ClusterProvider` names the Kubernetes cluster a `GitTarget` mirrors **from**. It is the read-side +peer of `GitProvider`: a target has one source cluster and one Git destination. + +`default` is the conventionally opinionated provider name, not an operator-generated object and not +a synonym for the local cluster. Its only special behavior is that a `GitTarget` which omits +`spec.clusterProviderRef` references a user-created `ClusterProvider` named `default`. That provider +may omit `spec.kubeConfig` to use the operator's in-cluster configuration, or set it to mirror a +remote cluster. + +For a remote source cluster, create a provider with a kubeconfig Secret. The Secret is resolved from +the operator's namespace; it is connection material for the operator, not a per-target setting. + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: ClusterProvider +metadata: + name: prod-eu-1 +spec: + kubeConfig: + secretRef: + name: default-source-kubeconfig + allowedNamespaces: + names: [team-a] + selector: + matchLabels: + gitops.configbutler.ai/source-access: "true" +``` + +`allowedNamespaces` is evaluated against namespaces in the **control cluster**, where +`GitTarget`s live. In this example, a `GitTarget` in `team-a`, or in a control-cluster namespace +with the shown label, may reference `prod-eu-1`. It has no effect on which namespaces are read from +the source cluster; that remains entirely the source connection's Kubernetes RBAC. `names` and +`selector` are ORed, and an omitted policy admits no control-cluster namespace. + +`spec.kubeConfig` and `GitTarget.spec.clusterProviderRef` are immutable: changing either would silently +make an existing materialization mean a different source cluster. Rotate credential *contents* in the +referenced Secret instead. `qps` and `burst` optionally tune a remote provider's client; the +`ClusterProvider` conditions validate its configuration, while the consuming `GitTarget` reports the +live source reachability and stream state. + +### Creating and managing the `default` provider + +The operator **never creates a `ClusterProvider`**, and never re-creates one you delete. If a +`GitTarget` references a provider that does not exist — including `default` — the target is held +unready through the ordinary "provider not found" path. That is deliberate: a source cluster is a +connection with credentials and an authorization policy, so it is yours to declare, review, and roll +back like any other resource under GitOps. + +There are two supported ways to get one, and both are fully declarative: + +- **Commit it yourself.** The object above is ordinary YAML. Put it in the repository that manages + this install. This is the recommended path once you are past a first trial. +- **Let the chart render it.** The chart can create and own a `ClusterProvider` named `default`, + including its `allowedNamespaces`, from a single value — see + [charts/gitops-reverser/README.md](../charts/gitops-reverser/README.md). Turn that value off to + manage the object yourself. Helm then deletes the provider it created on the next upgrade, so + ownership never silently splits between Helm and you. Because a missing provider holds its targets + unready, plan that switch together with committing your own object. + +The chart value is a rendering convenience, not runtime behavior: with it off, nothing in the +operator brings the object back. + +The chart renders the `default` provider by default, including when its optional `quickstart` starter +resources are enabled. It gives the starter `GitTarget` a declared in-cluster source without adding a +source reference to its manifest. Turn `clusterProvider.createDefault` off only when you manage that +provider yourself. + +Use another provider name when a target needs a different source cluster: + +```yaml +spec: + clusterProviderRef: + name: prod-eu-1 +``` + +The provider name is deliberately stable. It is the source-cluster identity used for watches and, +when audit attribution is enabled, for joining an audit event to the corresponding watch event. +Changing a target's source cluster changes what its folder means, so `clusterProviderRef` is +immutable. + ## `GitTarget` `GitTarget` decides where inside the repository resources are written. @@ -320,6 +431,8 @@ For the platform-facing behavior behind "valid signature" versus "verified badge The important fields are: - `spec.providerRef`: which `GitProvider` backs this target +- `spec.clusterProviderRef`: which `ClusterProvider` supplies resources; omit it to reference the + user-created `default` provider - `spec.branch`: which allowed branch to write to - `spec.path`: required relative path inside the repository; use `.` only when you deliberately want the repository root @@ -339,6 +452,8 @@ metadata: spec: providerRef: name: example-provider + # Omit clusterProviderRef to reference the user-created ClusterProvider named "default". + # clusterProviderRef: {name: prod-eu-1} selects a different source provider. branch: main path: live-cluster ``` @@ -357,6 +472,12 @@ and age details, see [sops-age-guide.md](sops-age-guide.md). `spec.providerRef` references a `GitProvider` in the same namespace as the `GitTarget`. Its `group` and `kind` default to `configbutler.ai` / `GitProvider`, so in practice you only set `name`. +`spec.clusterProviderRef` references a cluster-scoped `ClusterProvider`. It defaults to +`{name: default}` when omitted. That is intentionally different from `providerRef`: a source cluster +is a shared physical identity, while a Git destination and its credential normally belong to the +target's namespace. The default name can represent either an in-cluster or remote source according +to the `ClusterProvider` the user created. + The most useful status fields are: - `Ready`: true when the target is valid, the Git path is accepted, and watched streams are running. @@ -681,13 +802,51 @@ Progress and outcome are reported through kstatus-compatible **conditions** (no ## Audit ingestion settings Object state comes from Kubernetes **watch**, not from audit. Audit is an optional attribution lookup: -kube-apiserver posts audit events to a single HTTP path, `/audit-webhook`, and the operator extracts a -minimal attribution fact from each (auditID, user, verb, resourceVersion, GVR, namespace, name, UID, +kube-apiserver posts audit events to a named path, `/audit-webhook/`, and the +operator extracts a minimal attribution fact from each (auditID, user, verb, resourceVersion, GVR, namespace, name, UID, status, timestamps) into a Redis attribution index keyed for the join. A resolver attaches the commit author to each watch event by matching a fact (by resourceVersion/UID) within a bounded grace window. The same Redis connection also stores per-watch resume cursors, so short reconnects can resume a normal watch from the last processed resourceVersion when the apiserver can still serve that history. +Named ingress is currently authenticated to the shared audit CA and gated on the provider name existing; +it does **not** yet bind a particular client certificate to that provider. Do not use one shared audit +client credential to attribute several independently administered source clusters. A deployment that +needs that boundary should keep sources isolated until provider-bound ingress authentication is shipped. + +### Route a shared audit stream by event annotation + +Most audit streams represent one source cluster and must use a named route, including +`/audit-webhook/default`. Some control planes emit one shared stream for several logical clusters. +For that shape, the bare `/audit-webhook` endpoint is available only when the configuration model's +annotation key is set: + +```yaml +attribution: + clusterAnnotationKey: example.io/source-cluster +``` + +When this option is set, the receiver reads `example.io/source-cluster` from each event. Its value is +the name of the `ClusterProvider` that owns the event, so events in the same batch may route to +different source clusters. + +**The bare endpoint never guesses a source cluster.** Rejection happens at two levels: + +| Situation | Result | +|---|---| +| A request reaches `/audit-webhook` while `clusterAnnotationKey` is unset | The whole request is rejected with **400**. The bare endpoint is not enabled, so a producer posting to it is misconfigured. | +| An event carries no annotation, or names a `ClusterProvider` that does not exist | That **event** is rejected: it produces no attribution fact and is never credited to a fallback provider. The request still returns 200, so correctly-annotated events in the same batch are kept. | + +The second row is a per-event rejection rather than a per-request one on purpose. A shared stream is +heterogeneous by definition, so failing the whole batch would discard events that routed correctly and +leave the apiserver retrying a batch that can never succeed. Rejected events are counted and logged, so +a producer that is not stamping the annotation is visible rather than silent — if that count rises, +point the producer at `/audit-webhook/` instead. + +Use an annotation that the producing control plane sets consistently as source metadata. This is +routing metadata only: it keeps the audit fact and the watch event in the same source-cluster +partition, so a user from one logical cluster can never be credited for a matching object in another. + Valkey/Redis is **optional in configured-author mode**: when `--redis-addr` is set, watch resume cursors are stored so restarts pick up where they left off; when left empty, watches cold-replay from scratch on restart instead. When author attribution is enabled (`--author-attribution=true`), a non-empty diff --git a/docs/design/multi-cluster-audit-ingestion-implications.md b/docs/design/multi-cluster-audit-ingestion-implications.md deleted file mode 100644 index 71f2d2a2..00000000 --- a/docs/design/multi-cluster-audit-ingestion-implications.md +++ /dev/null @@ -1,327 +0,0 @@ -# Multi-Cluster Audit Ingestion: Implications and Configuration Mapping - -> **design** — open, not yet built. Index: [`../INDEX.md`](../INDEX.md) -> -> **Partly superseded.** §5's dedicated `SourceCluster` CRD proposal is replaced by -> [`config-plane-split.md`](config-plane-split.md), which puts remote-cluster -> connectivity **inline on `GitTarget`** (Flux's idiom) rather than in a separate -> onboarding CRD. That proposal's load-bearing rationale — fusing per-cluster -> **audit identity** with kube-API connectivity — no longer holds: the -> `/audit-webhook/` path was removed, so multi-cluster is now purely a -> kube-API story. The rest of this document (per-cluster **audit ingestion**, -> routing, fairness, quotas) remains an open, separate workstream. - -## 1. Purpose - -This document describes what changes are implied by supporting audit ingestion from multiple clusters, how this maps to -current `WatchRule` / `ClusterWatchRule` usage, and what new cluster connectivity model is needed for initial -reconcile and CRD discovery. - -## 2. Current Baseline (Today) - -1. Audit ingress already supports a path contract with cluster identity: -`/audit-webhook/` (`internal/webhook/audit_handler.go:231`, `charts/gitops-reverser/README.md:225`). -2. `WatchRule` is namespaced and scoped to resources in its own namespace (`api/v1alpha3/watchrule_types.go:57`). -3. `ClusterWatchRule` is cluster-scoped and can target both cluster-scoped and namespaced resources via `scope` -(`api/v1alpha3/clusterwatchrule_types.go:59`, `api/v1alpha3/clusterwatchrule_types.go:113`). -4. There is no first-class CRD yet for remote kube-apiserver connectivity (needed for seed reconcile and dynamic GVR -planning per source cluster). - -## 3. Core Implications of Multi-Cluster Ingestion - -### 3.1 Identity and Routing - -- `clusterID` becomes a first-class identity dimension in every event and derived key. -- Rule matching must evaluate `{clusterID, gvr, namespace, name, operation}`. -- Dedupe keys and replay tools must include `clusterID` to avoid cross-cluster collisions. - -### 3.2 Rule Semantics - -- Current rule model is cluster-agnostic; multi-cluster requires source-cluster selection. -- Without a source selector, one rule could unintentionally match events from all clusters. - -### 3.3 Security Model - -- Audit ingress needs authenticated cluster identity, not just path parsing. -- Per-cluster policy and quotas are required (fairness and blast-radius control). -- Remote kube-api credentials become sensitive assets requiring rotation and least privilege. - -### 3.4 Reconciliation and Discovery - -- Initial snapshot reconcile must run against each source cluster API. -- New CRD installation discovery is per source cluster, not global. -- Failures are per cluster and should not block processing for healthy clusters. - -### 3.5 Operational/HA Impact - -- Metrics/alerts require `clusterID` labels with cardinality controls. -- Backpressure, dead-letter, and lag need per-cluster visibility. -- Noisy cluster isolation is required to protect global throughput. - -## 4. Mapping to Existing Rule Model - -### 4.1 Recommendation for V1 (Pragmatic) - -Use `ClusterWatchRule` as the primary multi-cluster policy object with an optional source-cluster selector. -Keep `WatchRule` for single-cluster/local management-cluster use. - -Why this is pragmatic: - -1. `ClusterWatchRule` already supports both `Cluster` and `Namespaced` resource scopes. -2. It avoids redefining namespace semantics for remote clusters in a namespaced CRD. -3. It gives one place to express global policy and source-cluster targeting. - -### 4.2 Proposed Field Additions - -### A. `ClusterWatchRule` (recommended now) - -Add optional source selector: - -```yaml -spec: - source: - clusterIDs: ["prod-eu-1", "prod-us-1"] # empty/omitted = all onboarded clusters -``` - -### B. `WatchRule` (optional future extension) - -If needed later, add a constrained source selector: - -```yaml -spec: - source: - clusterIDs: ["dev-eu-1"] -``` - -For initial multi-cluster rollout, avoid this extension unless there is a hard requirement for namespace-scoped, -team-owned cross-cluster policies. - -### 4.3 Mapping Examples - -| Current usage | Multi-cluster equivalent | -|---|---| -| `WatchRule` in namespace `team-a` for `configmaps` | Keep as-is for local cluster, or migrate to `ClusterWatchRule` with `scope: Namespaced` and `source.clusterIDs` | -| `ClusterWatchRule` for CRDs/nodes | Add `source.clusterIDs` to scope clusters explicitly | -| "Watch all namespaces" policy | `ClusterWatchRule` + `scope: Namespaced` + optional namespace filters + `source.clusterIDs` | - -### 4.4 Example Translation (WatchRule -> ClusterWatchRule) - -Current namespaced rule: - -```yaml -apiVersion: configbutler.ai/v1alpha3 -kind: WatchRule -metadata: - name: configmaps-team-a - namespace: team-a -spec: - targetRef: - name: team-a-target - rules: - - operations: [CREATE, UPDATE, DELETE] - apiGroups: [\"\"] - apiVersions: [\"v1\"] - resources: [configmaps] -``` - -Multi-cluster equivalent: - -```yaml -apiVersion: configbutler.ai/v1alpha3 -kind: ClusterWatchRule -metadata: - name: configmaps-team-a-multi-cluster -spec: - targetRef: - name: team-a-target - namespace: team-a - source: - clusterIDs: [\"prod-eu-1\", \"prod-us-1\"] - rules: - - scope: Namespaced - operations: [CREATE, UPDATE, DELETE] - apiGroups: [\"\"] - apiVersions: [\"v1\"] - resources: [configmaps] -``` - -### 4.5 GitTarget Provenance Model (Important) - -By default, a single `GitTarget` should represent **one source cluster provenance**. - -Why: - -1. Audit trails remain clear ("this file came from cluster X"). -2. Drift/debug workflows stay understandable. -3. Operational blast radius is reduced. - -Recommendation: - -1. Do not mix multiple source clusters into one `GitTarget` by default. -2. If mixing is ever needed, require explicit opt-in and enforce path partitioning per cluster (for example -`clusters//...`). - -## 5. Proposed New Connectivity Model - -Multi-cluster audit ingestion is not enough by itself. You also need control-plane connectivity to each source cluster -for: - -1. Initial reconcile (list current resources) -2. CRD discovery / GVR planning -3. Periodic drift checks and orphan detection - -### 5.1 New CRD Proposal: `SourceCluster` - -Introduce a dedicated cluster onboarding CRD, e.g. `SourceCluster`: - -```yaml -apiVersion: configbutler.ai/v1alpha3 -kind: SourceCluster -metadata: - name: prod-eu-1 -spec: - clusterID: prod-eu-1 - auditIngress: - authMode: mTLS # or BearerToken - credentialSecretRef: - name: sourcecluster-prod-eu-1-audit - kubeAPI: - mode: Direct # Direct | Proxy | Disabled - kubeconfigSecretRef: - name: sourcecluster-prod-eu-1-kubeconfig - qps: 20 - burst: 40 - reconcile: - enabled: true - limits: - maxEventsPerSecond: 500 -status: - conditions: [] -``` - -### 5.2 Why Separate `SourceCluster` from Rules - -- Rules express "what to capture". -- `SourceCluster` expresses "where and how to connect". -- Separation improves security, ownership, and rotation workflows. - -### 5.3 Connectivity Modes for Kube API - -1. `Direct`: central controller connects directly to remote kube-apiserver using kubeconfig/credential secret. -2. `Proxy`: source-cluster agent or gateway exposes constrained API for list/discovery operations. -3. `Disabled`: no kube-api access for this source; audit ingest only (no snapshot/discovery from that cluster). - -For initial reconcile and CRD discovery, `Direct` or `Proxy` is required. - -### 5.4 Deployment Topology Options - -#### Option A: One GitOps Reverser per source cluster (simplest app model) - -Model: - -1. Deploy one controller per cluster (often inside that same cluster). -2. Configure kube-api endpoint at deployment level (default in-cluster, optional override). -3. No cluster-specific selection fields required in rule CRDs for that deployment. - -Implications: - -- Pros: lowest application complexity, clear ownership boundaries, simpler CRD model. -- Cons: fleet-level aggregation requires external orchestration; more deployments to manage. - -Suggested deployment-level config knobs: - -- `kubeAPI.mode=inCluster|override` -- `kubeAPI.server=https://...` (used when `override`) -- `kubeAPI.authSecretRef=...` -- `cluster.identity=` (used in emitted metadata) - -#### Option B: Central hosting cluster with multi-cluster ingestion/control plane - -Model: - -1. One or more controllers run in a hosting cluster. -2. `SourceCluster` defines audit auth + kube-api connectivity for each source cluster. -3. Rules may target one or more `clusterIDs`. - -Implications: - -- Pros: centralized operations, single control plane, easier global governance. -- Cons: higher product complexity (cluster-aware routing, auth, fairness, isolation). - -#### Option C: Hybrid - -1. Local per-cluster deployments for most teams. -2. Central deployment only for selected clusters/use-cases. - -This can share the same CRDs, with `SourceCluster` used only where centralized mode is enabled. - -## 6. Event and Reconcile Flow with `SourceCluster` - -```text -Source Cluster A/B/C - -> POST /audit-webhook/ - -> authenticate via SourceCluster credentials - -> normalize event (clusterID required) - -> rule match (ClusterWatchRule + source selector) - -> enqueue to durable bus - -> writer lease by repo+branch partition - -> git commit/push - -Startup / rule change - -> list active SourceClusters - -> create per-cluster discovery + snapshot jobs - -> publish reconcile events tagged with clusterID - -> same write pipeline + status projection -``` - -## 7. Security and RBAC Implications - -1. `SourceCluster` CRUD should be restricted to platform admins. -2. Secrets for remote kube-api and audit auth should be namespace-local and tightly RBACed. -3. Add validation that `spec.clusterID` is unique and immutable. -4. Enforce explicit allow-listing of accepted `clusterID`s at ingress. - -## 8. Failure Modes and Required Behavior - -1. Source cluster audit down: continue ingest from other clusters. -2. Source cluster kube-api unreachable: skip snapshot/discovery for that cluster; keep live ingest if available. -3. Bad credentials for one cluster: mark that `SourceCluster` degraded; do not block others. -4. Per-cluster event spike: apply per-cluster quota and backpressure before global degradation. - -## 9. Suggested Implementation Paths - -### Path A (Per-cluster deployment first) - -1. Add deployment-level kube-api connectivity config (`inCluster` default + optional override server/auth). -2. Keep `WatchRule`/`ClusterWatchRule` semantics unchanged within each deployment. -3. Enforce one-cluster-per-`GitTarget` provenance at deployment level. -4. Add fleet docs/automation for managing many deployments. - -### Path B (Central multi-cluster control plane) - -1. Add `SourceCluster` CRD and controller (status + connectivity checks). -2. Add ingress auth that maps request to registered `SourceCluster`. -3. Add `source.clusterIDs` selector to `ClusterWatchRule`. -4. Add cluster-aware matching in rule compiler and event identity. -5. Add per-cluster snapshot/discovery workers using `SourceCluster.kubeAPI` credentials. -6. Add per-cluster lag/quota metrics and alerts. -7. Optionally evaluate `WatchRule` source selector later. - -### Recommendation - -If you want to reduce rewrite risk, start with **Path A** and keep the application model simple. -If centralized governance is the immediate requirement, implement **Path B** directly. - -## 10. Direct Answer to "ClusterWatchRule only?" - -For the first multi-cluster version: **yes, that is the cleanest approach**. - -- Use `ClusterWatchRule` with optional `source.clusterIDs`. -- Keep `WatchRule` unchanged for local/simple namespaced use. -- Revisit extending `WatchRule` only if multi-tenant namespace-owned cross-cluster policy becomes a strong requirement. - -Direct answer to your deployment thought: - -1. Yes, allowing in-cluster deployment (local kube-api by default) is sensible and should be first-class. -2. Yes, allowing deployment-level kube-api override is useful for a one-reverser-per-cluster model. -3. In centralized mode, `SourceCluster` is still required for remote connectivity/auth. diff --git a/docs/design/multi-source-audit-ingress-hardening.md b/docs/design/multi-source-audit-ingress-hardening.md new file mode 100644 index 00000000..9c6fd428 --- /dev/null +++ b/docs/design/multi-source-audit-ingress-hardening.md @@ -0,0 +1,74 @@ +# Multi-source audit-ingress hardening + +> **design** — open, not yet built. Index: [`../INDEX.md`](../INDEX.md) +> +> This is deliberately narrow: `ClusterProvider` source connectivity, provider-name fact partitioning, +> and reconcile-time namespace authorization are shipped. This document decides how several source +> control planes may safely share one audit ingress, and how their ingestion is kept fair. + +## Current boundary + +When author attribution is enabled, a normal source posts an audit `EventList` to +`/audit-webhook/`. The server requires a certificate signed by the configured audit CA and +checks that the named `ClusterProvider` exists. It does **not** bind the peer certificate to that +provider. A holder of a shared, CA-signed client certificate can therefore submit facts for any existing +provider route. + +That is acceptable only inside one privileged control-plane trust domain where every holder of the +credential is authorized to act for every provider. It is not a tenant-isolation or independently +administered-source boundary. The current user-facing limit is documented in +[`../architecture.md`](../architecture.md#optional-attribution) and +[`../../SECURITY.md`](../../SECURITY.md#shared-audit-ingress-trust-model). + +The bare `/audit-webhook` route is different: it is enabled only when +`--author-attribution-cluster-annotation-key` is set, then resolves every event to a provider from that +annotation. It is suitable only if the upstream control plane stamps the annotation and an untrusted +writer cannot forge it. + +## Decision to make before supporting independent sources + +Choose one of these authentication contracts; do not describe named paths alone as authentication. + +### A. Provider-bound mTLS — default for independent control planes + +Give every `ClusterProvider` a client-certificate identity. The handler maps the verified peer +certificate to exactly one provider and requires it to match the route name. The design must specify: + +- how the binding is declared and where the public identity is stored; +- safe overlap during client-certificate rotation; +- how deletion or recreation immediately revokes the old identity; and +- audit logs and metrics for a missing, unknown, or mismatched identity without logging credentials. + +This keeps the named route and makes the provider name both routing and authenticated sender identity. + +### B. Annotation-routed shared ingress — for a trusted shard/control plane + +Use one authenticated producer and route individual audit events by an annotation the producer itself +stamps (for example, a virtual-cluster shard identity). The design must prove that normal source users +cannot set or alter that annotation, reject events with a missing or unknown provider, and document the +trust boundary of the shared credential. This is a different deployment model, not a shortcut for +independent clusters. + +## Fairness and limits + +Neither current route applies a provider-specific ingestion limit. Before supporting many providers on +one instance, define a bounded per-provider policy: maximum request/event rate, memory/Redis work bound, +shed response and retry behaviour, and metrics that distinguish one noisy provider from a global outage. +The policy must preserve the existing correctness rule: a dropped or late fact can only fall back to the +configured committer; it must never attach an author from another provider. + +## Acceptance criteria + +- An end-to-end remote round trip proves that a mutation from a named provider becomes a commit authored + by that actor. +- Identical object identities and resource versions from two providers never cross-credit an author. +- Under provider-bound mTLS, a valid certificate for provider A cannot record facts on provider B's route; + rotation overlaps safely and deletion/recreation rejects the retired identity. +- Under annotation routing, an event with a missing, unknown, or untrusted annotation records no fact. +- A provider exceeding its limit cannot prevent another provider from being ingested, and the resulting + fallback/shed outcome is observable. + +## Explicitly out of scope + +This work does not add managed-control-plane support, admission attribution, workload identity for source +cluster watches, or a durable HA delivery queue. Those remain separate product decisions. diff --git a/docs/facts/audit-webhook-api-server-connectivity.md b/docs/facts/audit-webhook-api-server-connectivity.md index 648b62f0..48872822 100644 --- a/docs/facts/audit-webhook-api-server-connectivity.md +++ b/docs/facts/audit-webhook-api-server-connectivity.md @@ -2,7 +2,9 @@ > **reference** — durable background. Index: [`../INDEX.md`](../INDEX.md) -GitOps Reverser relies on the Kubernetes [audit webhook backend](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#webhook-backend): the kube-apiserver POSTs audit events to an HTTPS endpoint using a kubeconfig file written on the control-plane node(s). This works best on clusters you fully control — k3s, k3d, Talos, Kamaji. Managed platforms (EKS, GKE, AKS) restrict access to apiserver configuration; running on them requires switching to a self-managed control plane or a platform that does expose it. +GitOps Reverser relies on the Kubernetes [audit webhook backend](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#webhook-backend): the kube-apiserver POSTs audit events to an HTTPS endpoint using a kubeconfig file written on the control-plane node(s). + +> **The path names the source cluster.** Audit routes are `/audit-webhook/` — the `ClusterProvider` whose facts this stream carries. The examples below use `/audit-webhook/default`, matching a `GitTarget` that omits `spec.clusterProviderRef`; substitute your provider's name. The bare `/audit-webhook` is the shared-stream endpoint for one audit feed carrying several logical clusters, and it is rejected with **400** unless `attribution.clusterAnnotationKey` is set — see [configuration.md](../configuration.md). This works best on clusters you fully control — k3s, k3d, Talos, Kamaji. Managed platforms (EKS, GKE, AKS) restrict access to apiserver configuration; running on them requires switching to a self-managed control plane or a platform that does expose it. ## The problem @@ -131,7 +133,7 @@ spec: clusterIP: 10.43.200.200 # webhook kubeconfig -server: https://10.43.200.200:9444/audit-webhook +server: https://10.43.200.200:9444/audit-webhook/default ``` **TLS**: Use `insecure-skip-tls-verify: true` for development. For production, pre-generate a certificate with a SAN for this IP and include its CA in `certificate-authority-data`. @@ -150,7 +152,7 @@ server: https://10.43.200.200:9444/audit-webhook Run the audit consumer with `hostNetwork: true` or a `hostPort`, so the apiserver can reach it at a loopback or node IP address. ```yaml -server: https://127.0.0.1:9444/audit-webhook +server: https://127.0.0.1:9444/audit-webhook/default ``` **TLS**: A certificate with `localhost` or the node IP as a SAN is straightforward to provision. @@ -169,7 +171,7 @@ server: https://127.0.0.1:9444/audit-webhook Route audit events to a stable external URL: a LoadBalancer service, a NodePort, or a tunnel (e.g. Cloudflare Tunnel, ngrok). ```yaml -server: https://audit.example.com/audit-webhook +server: https://audit.example.com/audit-webhook/default ``` **TLS**: Standard — use a publicly trusted certificate or any cert your apiserver trusts. @@ -209,7 +211,7 @@ spec: nodePort: 30444 # webhook kubeconfig -server: https://127.0.0.1:30444/audit-webhook +server: https://127.0.0.1:30444/audit-webhook/default ``` **TLS**: Issue a cert with `localhost` or the node IP as a SAN. Works naturally with cert-manager if you provision the cert before writing the kubeconfig (two-phase setup), or use a pre-generated cert. @@ -228,7 +230,7 @@ server: https://127.0.0.1:30444/audit-webhook Run the audit receiver as a [static Pod](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/) — a manifest placed in `/etc/kubernetes/manifests/` (or the kubelet's configured staticPodPath). Kubelet starts static pods independently of the API server, before the rest of the cluster is up. With `hostNetwork: true` the pod binds directly to the node's network stack at `127.0.0.1`. ```yaml -server: https://127.0.0.1:9444/audit-webhook +server: https://127.0.0.1:9444/audit-webhook/default ``` **TLS**: The pod can mount a cert from a host path, provisioned during cluster bootstrap alongside the other control-plane certs. @@ -268,7 +270,7 @@ socat TCP-LISTEN:9444,fork TCP:10.43.200.200:9444 [Kamaji](https://kamaji.clastix.io/) runs tenant control planes as pods inside a parent cluster. The tenant kube-apiserver is itself a workload, and its audit webhook kubeconfig can point directly at any service in the parent cluster using standard in-cluster DNS — because the parent cluster's CoreDNS *is* available to the tenant apiserver pod. ```yaml -server: https://gitops-reverser.gitops-reverser.svc.cluster.local:9444/audit-webhook +server: https://gitops-reverser.gitops-reverser.svc.cluster.local:9444/audit-webhook/default ``` **TLS**: Normal in-cluster TLS with cert-manager. The parent cluster's service infrastructure handles it. diff --git a/docs/finished/clusterprovider-fact-purge.md b/docs/finished/clusterprovider-fact-purge.md new file mode 100644 index 00000000..577e9b58 --- /dev/null +++ b/docs/finished/clusterprovider-fact-purge.md @@ -0,0 +1,181 @@ +# Fact purge for `ClusterProvider`: decision record + +> **finished** — shipped or closed. Kept for context only; **nothing here binds**. For current +> behaviour see [`../architecture.md`](../architecture.md). Index: [`../INDEX.md`](../INDEX.md) +> +> Prompted by a real failure: `helm uninstall` stranded the `default` `ClusterProvider` in +> `Terminating` forever, which then blocked reinstalling. This document asked whether +> purge-on-delete was the right mechanism. **Outcome: option D — no finalizer, and no purge.** + +## The decision + +**No finalizer, and no purge.** The `ClusterProvider` controller takes no finalizer; it only sheds +the retired one so objects created by older operators can still be deleted. + +Two steps got here. First, a finalizer is the wrong *mechanism*: purge-on-delete needs a living +operator, which `helm uninstall` does not provide. Second — and this is what settled it — the purge +itself is not earning its keep. The exact-key join is +`(cluster, group/resource, object UID, resourceVersion)`, and a re-provisioned cluster mints fresh +object UIDs, so a stale fact cannot match on that key at all. Combined with a TTL that clears +everything within minutes, and with the fact that repointing a provider name is an operator/automation +action rather than a routine one, the residual risk does not justify a mechanism. + +The rest of this document is the analysis that led there, kept because the reasoning matters more +than the conclusion. + +### The one caveat worth knowing + +The UID argument covers the exact key and the uid-keyed `:last` pointer. It does **not** cover the +**rv-only escape hatch** (`factKeyRV`), which is written when an audit fact carries no UID at all and +read as a last resort. resourceVersions are cluster-scoped integers, so a fresh cluster restarts low +and *can* collide with a stale fact from a previous incarnation. + +Reaching a wrong author through it needs all of: a no-UID fact recorded under the old incarnation, a +new object whose uid-keyed lookups miss, an RV collision between the two clusters, and all of it +inside the TTL. The result is already classed `AttributionWeak`, affects one commit's author (never +Git content), and self-clears. Accepted knowingly rather than overlooked. + +## What is being protected + +When author attribution is enabled, each mutating audit event is reduced to a small **fact** in Redis +(who did it, to which object, at which resourceVersion). A live watch event later looks the fact up +and the commit is authored by that user instead of the configured committer. + +Facts are keyed by **source cluster**, and the source cluster is a `ClusterProvider` **name** +([`attribution_index.go`](../../internal/queue/attribution_index.go)): + +``` +:author:v1:audit:cluster:::object:: +``` + +That name partitioning is what stops a fact from one cluster authoring a commit for a matching object +in another — an invariant covered by `TestAttributionIndex_CrossClusterIsolation` and +`TestAttributionIndex_RVOnlyHatchIsClusterScoped`. + +Two properties matter for this decision: + +- **Facts expire on their own.** They carry a TTL and nothing else deletes them. They are never object + state; a miss just means "absent". +- **A provider name can be reused for a different physical cluster.** `spec.kubeConfig` is immutable, + and the CEL rejection message tells you so explicitly: *"delete and recreate the ClusterProvider to + point a name at a different cluster"*. Delete-and-recreate is therefore the **supported** way to + repoint a name — not an exotic edge case. In practice it is an operator/automation action, not + something that happens on its own. + +## The actual risk + +Put those together and the hazard is precise: + +> A provider name is deleted and recreated against a **different cluster**, and a watch event from +> the new cluster joins a fact left over from the old one — crediting a user who never touched it. + +The window is bounded by the fact TTL. Outside that window there is no hazard at all, because the +facts are gone by themselves. + +This is a **misattribution** bug, not a data-loss one: Git content is unaffected, only the commit +author is. It is nonetheless the thing this whole feature exists to get right, so it is worth +addressing — but it is worth addressing *proportionately*. + +> **Note — the documented TTL is wrong.** `DefaultAttributionFactTTL` is **15 minutes**, but the +> `--author-attribution-ttl` flag help and [`configuration.md`](../configuration.md) both say +> "default 10m". Chart installs are unaffected in practice because `values.yaml` sets `ttl: "10m"` +> explicitly, but a non-chart install gets 15m while the docs promise 10m. Worth fixing regardless of +> which option below is chosen. + +## What the finalizer costs + +The finalizer (`configbutler.ai/clusterprovider-fact-purge`, now retired to +`LegacyClusterProviderFinalizer` and only ever removed) held the object until the controller had +scanned and deleted every fact under that provider name. + +The cost is that **a finalizer is a promise only a living operator can keep**: + +- **`helm uninstall` removes the operator and the `ClusterProvider` together.** Nothing is left to + detach the finalizer. The object strands in `Terminating` permanently, and the next + `helm install --wait` then fails on it: `resource ClusterProvider//default not ready. status: + Terminating`. Recovering needs a manual `kubectl patch` of the finalizer. Verified on a live + cluster, and the reason the narrow fix shipped. +- **If the operator is simply down when someone deletes a provider**, the finalizer blocks the delete + until it comes back — and if the object is force-removed to unblock, the purge never happens and the + facts leak anyway. So the finalizer does not even reliably deliver the guarantee it exists for. + +In the default configuration the trade was worse still: with no Redis there are no facts, so the +finalizer guarded *nothing* while still stranding the object. That is what the shipped fix removes. + +## Options + +### A. Unconditional finalizer (the old behaviour) + +Take the finalizer always; purge on delete. + +- **Pro** — simple, one code path. +- **Con** — strands the object on uninstall *even when there is nothing to purge*. This is the bug + that was hit. **Rejected.** + +### B. Conditional finalizer — *shipped, then superseded* + +Take the finalizer only when a `FactPurger` is wired (i.e. attribution is on). + +- **Pro** — small, safe, and removes the hazard for the chart default and the quickstart, which is + where it actually bit. No semantic change to the attribution path. +- **Con** — only moves the problem. With attribution **enabled**, `helm uninstall` can still strand + the provider, and the operator-is-down case is untouched. A partial fix. + +### C. Purge on adopt — *the right way to keep a purge, if we wanted one* + +Drop the finalizer. Instead, when a provider name is observed for the **first time**, purge anything +left under it. "First time" is detectable precisely as `status.observedGeneration == 0`: empty on a +freshly created object, and never reset by an operator restart, so a running provider is never +re-purged. + +- **Pro** — same guarantee ("a recreated name starts clean"), and it is the guarantee that actually + matters, because stale facts are only dangerous once the name is *in use again*. +- **Pro** — works when the operator was **down** during the delete, which the finalizer cannot do at + all. Strictly more robust for the stated goal. +- **Pro** — no finalizer, so no liveness hazard: uninstall, reinstall and force-delete all behave. +- **Con** — facts for a name that is never recreated linger until their TTL rather than being deleted + promptly. This is storage hygiene, not correctness, and the TTL already bounds it. +- **Con** — a bug in the "is this newly adopted?" test would purge **live** facts and silently degrade + attribution to committer-authored commits. This is the one real risk and it needs a test that pins + "a steady-state reconcile never purges". +- **Con** — needs a migration step: existing objects already carry the finalizer, so the controller + must actively strip it, or they will strand exactly as before. + +### D. No purge at all — rely on the TTL — **CHOSEN** + +Delete the mechanism entirely and accept the window. + +- **Pro** — simplest possible; no finalizer, no adopt hook, nothing to get wrong. +- **Con** — leaves a misattribution window on a supported workflow (repointing a name). Accepted: + the exact key includes the object UID, which a re-provisioned cluster does not reproduce, so the + window only exists for the narrow rv-only case described at the top. + +### E. Purge on adopt **and** keep the finalizer as best-effort + +Both: purge on adopt for correctness, finalizer for prompt cleanup. + +- **Pro** — facts also disappear promptly in the common case. +- **Con** — reintroduces the whole liveness hazard for a benefit the TTL already provides. The + finalizer is the expensive half and the least reliable half. **Not recommended.** + +## Outcome — option D + +Chosen over C on the grounds that the purge protects against something the key shape already +prevents. Option C's adopt-purge would have been the right way to *keep* a purge; it just turned out +not to be worth keeping one. + +What was implemented: + +1. **No finalizer is ever taken.** `AddFinalizer` is gone. +2. **The retired finalizer is shed**, including on an object already stuck in `Terminating` — that is + the only way an object stranded by an older operator becomes deletable again, so the shed runs + *before* the deletion check rather than after it. +3. `ClusterFactPurger` and its wiring are removed. `AttributionIndex.PurgeClusterFacts` remains + (exported and tested) so a future purge has something to call. + +`PurgeClusterFacts` was deliberately left in place. If the rv-only caveat above ever shows up in +practice, purge-on-adopt (option C) is the way to reintroduce it, and the mechanism is still there. + +## Status + +Implemented. The rv-only collision is a known, accepted residual risk. diff --git a/docs/design/config-plane-split.md b/docs/finished/config-plane-split.md similarity index 98% rename from docs/design/config-plane-split.md rename to docs/finished/config-plane-split.md index 43ef46ac..08afe39f 100644 --- a/docs/design/config-plane-split.md +++ b/docs/finished/config-plane-split.md @@ -1,8 +1,9 @@ # Separating the config plane from the watched cluster -> **design** — **built** (this PR). Index: [`../INDEX.md`](../INDEX.md) -> Supersedes the `SourceCluster` CRD proposal in -> [`multi-cluster-audit-ingestion-implications.md`](multi-cluster-audit-ingestion-implications.md) §5. +> **finished** — shipped or closed. Kept for context only; **nothing here binds**. For current behaviour see [`../spec/`](../spec/). Index: [`../INDEX.md`](../INDEX.md) + +> Shipped 2026-07-17 as [#249](https://github.com/ConfigButler/gitops-reverser/pull/249). Supersedes +> the now-retired `SourceCluster` CRD proposal. > Redesign of feature #1 from the closed multi-tenant PR (#220), shipped on its own. ## One sentence @@ -112,8 +113,7 @@ wrapper today. ### Why inline, not a dedicated `SourceCluster` CRD -[`multi-cluster-audit-ingestion-implications.md`](multi-cluster-audit-ingestion-implications.md) -§5 proposed a dedicated `SourceCluster` CRD fusing **audit-ingress identity** and +An earlier design proposed a dedicated `SourceCluster` CRD fusing **audit-ingress identity** and **kube-API connectivity** into one onboarding object. Its load-bearing rationale is gone: `main` **removed** the `/audit-webhook/` path — the handler now rejects any cluster-id segment @@ -662,10 +662,9 @@ No `observedDestination` / `retargetingTo` — those belong to #6. - `spec.kubeConfig` is a new optional field. Existing installs are unaffected: absent → local cluster → today's behavior, byte for byte. -- The stale [`multi-cluster-audit-ingestion-implications.md`](multi-cluster-audit-ingestion-implications.md) - should have its §5 `SourceCluster` CRD proposal marked superseded by this doc, - and its INDEX line updated from "there is still no CRD for remote cluster - connectivity" to point here (the connectivity model is now inline, by choice). +- This design intentionally closed the older `SourceCluster` CRD proposal. The later + [`ClusterProvider` attribution design record](multi-cluster-author-attribution.md) explains why + the source connection was subsequently given a cluster-scoped home. ## Future shape (designed-in, not built) @@ -933,11 +932,11 @@ YAML applied to the cluster. The source-cluster corner is its own gated leg 4. **Least-privilege ClusterRole shape on the remote.** Ship a documented broad read ClusterRole, or require the operator to grant per-type read and degrade unobservable cells gracefully (ties into - [`watch-and-catalog-architecture.md`](watch-and-catalog-architecture.md) §1.7)? + [`watch-and-catalog-architecture.md`](../design/watch-and-catalog-architecture.md) §1.7)? 5. **`GitProviderReady` projection trigger and `Ready` gating.** Wire a `Watches(&GitProvider{})` so a provider going un-ready promptly re-reconciles its GitTargets (best, but adds an edge to the set that - [`reconcile-triggering.md`](reconcile-triggering.md) tracks), or lean on the + [`reconcile-triggering.md`](../design/reconcile-triggering.md) tracks), or lean on the 5-minute periodic reconcile (simpler, laggier)? And does `SourceClusterReachable=Unknown` (pre-first-discovery) hold `Ready` at `Unknown`, or is `Ready` allowed to settle on the other axes first? (Proposal: `Watches`, and diff --git a/docs/finished/documentation-triage.md b/docs/finished/documentation-triage.md index f2c47561..99f174fd 100644 --- a/docs/finished/documentation-triage.md +++ b/docs/finished/documentation-triage.md @@ -42,7 +42,7 @@ live invariant.** They are not finished at all. The worst case: > **`docs/finished/current-manifest-support-review.md`** is the live specification > of `internal/manifestanalyzer`. It is cited by section name from eight Go files, -> and it is a cited input to the *active* gitops-api workstream. It holds the +> and it is a cited input to active downstream work. It holds the > non-negotiable rules — *a GitTarget makes an all-or-nothing claim on a folder*; > *never partially materialize a multi-doc file*; *refuse the folder rather than > prune unwatched KRM*. It is filed under "finished." @@ -110,7 +110,7 @@ from the code: ### Still-open design (9) — `docs/design/` `e2e-coverage-gaps-and-improvements-plan.md`, `e2e-finish-plan.md`, -`metrics-observability-plan.md`, `multi-cluster-audit-ingestion-implications.md`, +`metrics-observability-plan.md`, `multi-source-audit-ingress-hardening.md`, `reconcile-triggering.md`, `release-image-reuse-plan.md`, `sensitive-resource-diagnostics-follow-up.md`, `watch-and-catalog-architecture.md`, `stream/residual-e2e-flakes-2026-06-19.md`. @@ -124,7 +124,7 @@ from the code: ### Still wanted (8) — `docs/future/` -`idea-application-editing.md` (the product seed of the gitops-api workstream — +`idea-application-editing.md` (the product seed of a future application-editing workstream — still holds the branch/session grouping strategies nothing else covers), `ha-gittarget-distribution-plan.md` (cited three times by `architecture.md`), `least-privilege-remaining-work.md`, `design-commit-request-phase-2.md` (verified @@ -244,7 +244,7 @@ Superseded, closed investigations, archaeology, stale. Extract the three code-ci facts first (noted above). Git keeps everything; nothing is lost that a `git log --follow` cannot recover. -### Phase 5 — the gitops-api consolidation +### Phase 5 — documentation consolidation Separately proposed in [expansion-boundary-and-corpus-organisation.md](../design/support-boundary/expansion-boundary-and-corpus-organisation.md): @@ -268,7 +268,7 @@ invariant into one, and de-duplicate the `--dry-run=server` evidence table. **Should `docs/design/manifest/` survive at all?** Its three subsystems — `internal/typeset`, `internal/git/manifestedit`, `internal/manifestreport` — have all shipped, so most of its 19 files are archaeology. But it is also where the -*reasoning* behind the manifest model lives, and the gitops-api workstream is built +*reasoning* behind the manifest model lives, and downstream work is built directly on top of it. My instinct is that two or three docs graduate to `spec/` (`contextual-namespace-and-kustomize-folder-editing.md`, `version2/gittarget-new-file-placement-rules.md`, diff --git a/docs/finished/multi-cluster-author-attribution.md b/docs/finished/multi-cluster-author-attribution.md new file mode 100644 index 00000000..c46b9eeb --- /dev/null +++ b/docs/finished/multi-cluster-author-attribution.md @@ -0,0 +1,474 @@ +# `ClusterProvider` multi-cluster author-attribution design record + +> **finished** — shipped or closed. Kept for context only; **nothing here binds**. For current +> behaviour see [`../architecture.md`](../architecture.md), [`../configuration.md`](../configuration.md), +> and [`../../SECURITY.md`](../../SECURITY.md). Index: [`../INDEX.md`](../INDEX.md) +> +> This design produced the cluster-scoped `ClusterProvider`, immutable +> `GitTarget.spec.clusterProviderRef`, provider-name-partitioned attribution facts, reconcile-time +> namespace authorization, and named audit routes. The remaining multi-source ingress work is tracked +> separately in [`../design/multi-source-audit-ingress-hardening.md`](../design/multi-source-audit-ingress-hardening.md). + +## Final implemented shape + +`ClusterProvider` is the cluster-scoped read-side peer of `GitProvider`. A `GitTarget` references a +provider by immutable name; an omitted reference defaults to `default`, which is only a convention — +any provider name may use the in-cluster client or a remote kubeconfig. That provider name partitions +the source client, discovery and watch state, and author-attribution facts. Its +`spec.allowedNamespaces` policy is deny-by-default and is checked on every reconcile before watches +start. + +Audit ingestion uses `/audit-webhook/` and stores facts in that provider's partition. The +current server verifies that the sender is signed by the audit CA and that the named provider exists; +it does **not** bind an individual certificate to a provider. This is an accepted shared-control-plane +trust boundary, not tenant isolation. The outstanding provider-bound identity, annotation-routing +trust, and ingress-fairness decisions are deliberately kept out of this archived design. + +## Design history + +The material below is the working record that led to the shipped shape. It retains alternatives that +were rejected or deliberately not implemented — especially the earlier reserved-`default`, admission +webhook, finalizer, and per-provider-certificate proposals — and must not be read as the current API +contract. + +## Problem + +Author attribution is a **join keyed by `(group/resource, object-uid, resourceVersion)`** with no +cluster dimension: + +- **Write side** — the local apiserver POSTs audit events to `/audit-webhook`; the handler + ([`audit_handler.go`](../../internal/webhook/audit_handler.go)) records a minimal fact per + accepted mutation via `RecordFact` + ([`attribution_index.go`](../../internal/queue/attribution_index.go)), keyed + `…:author:v1:audit::object::` (plus a `:last` pointer and an rv-only + `:rv:` hatch). +- **Read side** — a live watch event calls `AuthorResolver.ResolveAuthor(ctx, gvr, uid, rv, + exactCapable)` ([`author_resolver.go`](../../internal/watch/author_resolver.go), + [`target_watch.go`](../../internal/watch/target_watch.go) `attachAuthor`), reading the fact back + by `(group/resource, uid, rv)`. + +Config-plane-split broke the symmetry: a remote `GitTarget` now **watches a remote cluster** (remote +UIDs/RVs), but the audit webhook is **local-only** (`validateAuditWebhookPath` rejects any cluster-id +segment). A remote watch event looks up `(gr, uid, rv)`, finds no fact, and ships as the committer. + +Two things are missing, and both need a **name for the cluster**: an *ingress* a remote apiserver can +reach tagged with which cluster it is, and a *cluster dimension in the keyspace* so a fact from +cluster A never joins a watch event from cluster B. + +## Identity model — the name is the key; the UID only authenticates + +`GitTarget.SourceClusterID()` (`//`) and every use of that string as a +data-plane key are **removed** (see *What we remove*). It fused two identities and leaked a Secret +reference. The replacement is **one identity, the provider name**, with the UID confined to the auth +layer: + +| Concern | Value | Why | +|---|---|---| +| **External** — the audit route | `ClusterProvider.metadata.name` | admin-chosen, DNS-safe, known before install (bakeable into apiserver params at cluster-creation time), immutable | +| **Internal** — the fact-index cluster key, GVR scoping, the `clusters` map key | **the same name** | unique because cluster-scoped; on the audit path already; known to the watch side with no lookup; **local keys by `default`** — no `""` special case | +| **Authentication** — which incarnation is talking | `ClusterProvider.metadata.uid` | binds the per-provider client cert to one incarnation; revoked on delete (see *Authenticated ingress*) | + +**Why the name is the key, not the UID (a reversal from v1, and a pushback on the review's +"UID-keyed facts").** Two reasons: + +1. **UID does not deliver the incarnation safety it seems to.** The audit *route* is name-based, so a + delayed/retried batch from a *deleted* cluster still hits `/audit-webhook/prod-eu-1`, resolves to + the **current** provider, and lands under whatever key we choose. UID-in-the-key does not stop + that; only **rejecting the stale sender** does (auth revocation) — plus a **fact purge on + delete**. Those close it regardless of key shape. +2. **The name needs no translation.** The audit path already carries it; the watch side already knows + its provider. UID-keying would add a `name → uid` lookup on *both* the write and read paths for no + gain, and make Redis keys opaque. + +(The concern that first surfaced this — an implicit-local cluster has no object and therefore no UID — +is now moot, since local is a shipped object with a name; but the two reasons above stand on their own, +and are why keying by name survives even now that every source has a UID.) + +So the UID is used where it is actually load-bearing — **binding the authenticated sender to a +physical incarnation** — and nowhere else. `git.Event.SourceClusterID` / +`ResolvedTargetMetadata.SourceClusterID` become **`SourceCluster`** carrying the provider *name* +(the shipped local provider's name for local); the target-scoped GVK→GVR resolution +config-plane-split added is unchanged in shape. + +## Decision + +1. **Add a cluster-scoped `ClusterProvider`** — the read-side peer of `GitProvider`; a `GitTarget` + references one by name (`spec.clusterProviderRef`), as it already references a `GitProvider`. +2. **Identity = the provider name**, everywhere; UID only authenticates (above). Retire + `SourceClusterID`. +3. **The provider carries a namespace-access policy** (`spec.allowedNamespaces`, **deny-by-default**); + a `GitTarget` may reference it only from an allowed namespace — enforced at admission *and* before + any watch starts. +4. **Authenticated ingress is a prerequisite**, not a late add: a per-provider client cert bound to + the provider, `cert-provider == path-provider`, revoked on delete. Remote paths are refused until + this exists. +5. **The fact index gains a cluster dimension** keyed by provider **name** (the local provider + included); a finalizer purges a provider's facts on delete. +6. **Local is the reserved `default` `ClusterProvider`** (kubeConfig omitted = in-cluster); + `clusterProviderRef` **defaults to `{name: "default"}`** (concrete, jumpable — never `nil`). CEL + enforces *named `default` iff no kubeConfig*, so name-uniqueness makes it a singleton (no webhook). + The chart ships it (`watchLocal: true`); `watchLocal: false` → not-found → `NotReady`. No `isDefault`. +7. **v1 = audit attribution on self-managed clusters only.** Remote `attribution.mode` defaults to + **`None`**; **`Admission` is deferred entirely** (it does not unlock managed clusters); provider + `kubeConfig` is **immutable**; workload identity and mutable repointing are deferred. + +``` +GitTarget ─ spec.clusterProviderRef ─▶ ClusterProvider (source: READ + attribution, authorized per namespace) + ─ spec.providerRef ────────▶ GitProvider (destination: WRITE) +``` + +## Why this is config-plane-split's planned step + +Config-plane-split argued against a dedicated CRD, but its reasoning was conditional: *"A dedicated +CRD's remaining benefits — reuse across many `GitTarget`s, platform-admin RBAC ownership, a home for +connectivity status — are real but not needed for the first version … its load-bearing rationale is +gone: `main` removed the `/audit-webhook/` path … Multi-cluster is now purely a kube-API +story."* Re-adding attribution restores the audit story and makes all three deferred benefits needed. +It pre-drew the shape: a **sibling** `sourceClusterRef` naming a `ClusterConnection`/`SourceCluster` +CRD "that only platform admins may create … referenced by name from the `GitTarget`." This is that +object. + +## Does an object keep the Flux vision? Mostly — one item deferred + +Moving the kubeconfig from inline into a `ClusterProvider` embeds the **same** `meta.KubeConfigReference` +type, so the Flux-shaped roadmap is preserved: + +| config-plane-split capability | On a `ClusterProvider` | +|---|---| +| Embed `meta.KubeConfigReference` verbatim; a Flux kubeconfig Secret works unchanged | **Preserved** — same embedded type | +| `value`→`value.yaml` key order; reject-not-strip `exec`/insecure-TLS | **Preserved** — same resolver, run by the provider reconciler; verdict on the provider | +| Workload identity (`configMapRef`, `provider: generic`→cloud) | **Preserved, deferred** — in the type; a v1 CEL guard rejects it (config-plane-split already deferred it) | +| ServiceAccount impersonation (`serviceAccountName`) | **Preserved, more Flux-faithful** — a sibling of `kubeConfig` on the provider, as in a Flux Kustomization | +| Target-scoped GVK→GVR (no union) | **Preserved** — keyed by provider name | +| Split `Validated`/`SourceClusterReachable` conditions | **Preserved, re-homed** onto the provider | + +The object **unlocks** what config-plane-split parked for lack of a home — per-provider `qps`/`burst` +(off the global `--source-cluster-qps/-burst` flags), per-provider attribution mode, and the status +surface — which is exactly the *"real second property"* it said a wrapper lacked. **One thing is now +deferred, not unlocked:** mutable rotation. v1 promised it; the review is right that a mutable +*endpoint* silently retargets and misattributes (below), so **`kubeConfig` is immutable in v1**; only +Secret *contents* rotation stays transparent (the resolver re-reads — as config-plane-split already +did). Immutability sits on both `GitTarget.spec.clusterProviderRef` (which cluster a folder sources +from = folder identity) and `ClusterProvider.spec.kubeConfig` (which physical cluster a name means). + +## Local — the reserved `default` provider, and a defaulted ref + +Local is a **first-class `ClusterProvider`** named **`default`**: `kubeConfig` is optional, and +**omitted means in-cluster** (the operator's own cluster via in-cluster config — Flux's "the cluster I +run in"). The chart **ships it** by default (`attribution.watchLocal: true`, named `default`). + +**The ref is *defaulted*, never `nil`.** `GitTarget.spec.clusterProviderRef` carries a schema default of +`{name: "default"}`, so a target that omits it persists with a concrete ref to the `default` provider — +there is no implicit-`nil` sentinel. This is deliberately chosen over "`nil` means local" for one +forward-looking reason: **every reference is always populated and jumpable**, so a "follow reference" +(F12) traversal over the object graph never hits an implicit hop with nowhere to land. It also makes +`kubectl get gittarget -o yaml` self-describing — the source cluster is always shown, even for local. + +**`default` is reserved for the in-cluster cluster, enforced by per-object CEL** — no validating webhook +needed for this (a simplification over the v3 draft, which used one). The rule is *named `default` **iff** +`kubeConfig` is absent*: + +```go +// +kubebuilder:validation:XValidation:rule="(self.metadata.name == 'default') == !has(self.spec.kubeConfig)",message="the ClusterProvider named 'default' is the in-cluster provider and must omit kubeConfig; every other provider must set kubeConfig" +``` + +Kubernetes name-uniqueness makes `default` a singleton for free, so "exactly one in-cluster provider" +falls out — the cross-object count the v3 webhook did is gone. The bare `/audit-webhook` path, the ref +default, and "the operator's own cluster" all coincide on `default`. + +**Turning local watching off** stays explicit: `watchLocal: false` ships no `default` provider, and a +`GitTarget` (defaulted or explicit) referencing `default` is then a clear `NotReady` — the *same* +"referenced ClusterProvider not found" path as any missing remote, not a special case. + +**We still reject an `isDefault` flag.** `default` is a **fixed reserved name on an immutable object**, +not a movable pointer: because `clusterProviderRef` is immutable (a folder's source *is* its identity) +and `kubeConfig` is immutable, nothing about `default` can be re-pointed to silently retarget the +folders bound to it. A mutable/movable default (`isDefault`) would reintroduce exactly that +silent-retarget hazard, so it stays rejected. + +Because local is a named object it keys facts by its **name** (`default`) like every other provider (no `""` +special case) and gets the same status surface; the bare `/audit-webhook` path resolves to it (the +single in-cluster provider), so the local apiserver's config stays simple. + +## Authorization — the namespace is a *policy on the provider*, not the Secret's location + +**The confused-deputy this must close is data export, not credential read.** A cluster-scoped provider +holds a credential that may read a lot of a remote cluster. Any `GitTarget` that references it causes +the operator — using *its* credential — to mirror that cluster's state into the `GitTarget`'s +destination. So a tenant who can create a `GitTarget` **and their own `GitProvider`** could reference a +platform provider and **export whatever the operator can read on the remote into a repo they control**. +Pinning the kubeconfig Secret to the operator namespace does **not** help: the tenant never touches the +Secret. (The v1 draft's claim that Secret-pinning "closes the confused-deputy by construction" was +wrong and is retracted.) + +**Fix: a namespace-access policy on the provider, deny-by-default.** + +```yaml +spec: + allowedNamespaces: # deny-by-default: empty = no namespace may reference this provider + names: [team-a, team-b] # explicit list, and/or + selector: {matchLabels: {tier: trusted}} # a label selector on namespaces +``` + +Enforced in **two** places: a validating admission webhook rejects a `GitTarget` whose namespace is +not allowed by its referenced provider, and the watch manager **refuses to start watches** for a +target that fails the check at reconcile (defense in depth against a policy that tightened after the +`GitTarget` was created). Denial is tested explicitly. + +This is config-plane-split's "RBAC on the object" model made real: platform admins create providers +and decide who may reference them; tenants reference by name and are bounded by the policy. + +## Authenticated ingress — per-provider client identity, before any remote path + +Path is **routing, not authentication.** A name is guessable; on its own, anyone who can reach +`:9444/audit-webhook/` could inject false author facts. Today the server already does +`RequireAndVerifyClientCert` against a CA ([`main.go`](../../cmd/main.go): `buildAuditServerTLSConfig`), +but the chart issues **one** client cert with `commonName: kube-apiserver` +([`audit-certificates.yaml`](../../charts/gitops-reverser/templates/audit-certificates.yaml)) and the +handler **never reads the peer certificate** — so every source is indistinguishable and the path is the +only (unauthenticated) discriminator. That is fine for one local apiserver; it is not enough the moment +a second cluster can POST. + +**One audit server, not one per cluster** (a listener per cluster is unscalable and pointless). The +per-cluster distinction is the **client identity**: + +- Each `ClusterProvider` has its **own client credential** — a cert whose subject/SAN (or a pinned + SPKI fingerprint) maps to that provider. The apiserver on the source cluster presents it. +- The handler **reads `r.TLS.PeerCertificates`**, maps it to a provider, and **requires + `cert-provider == path-provider`** — a forged path without the matching cert is rejected. +- The credential is **bound to the provider incarnation (UID)** and **revoked on delete**, so a + delayed batch from a deleted/recreated cluster fails auth instead of misattributing (the P0.3 fix; + the fact-purge finalizer is the belt to this suspenders). + +This is a **prerequisite**, sequenced *before* remote path routing — the operator must never accept a +remote fact it cannot attribute to an authenticated provider. The cert issuance/rotation/revocation +flow is genuinely new design (the chart's cert-manager path mints the *local* client cert but cannot +reach a remote apiserver's filesystem), and is an explicit build step, not an afterthought. + +**Answer to "separate TLS per webhook or just path?"** — one server, path for routing, a per-provider +**client cert** for authentication, required to agree. Never path-only. + +## API shape + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: ClusterProvider # cluster-scoped +metadata: + name: prod-eu-1 # identity: /audit-webhook/prod-eu-1 AND the fact-index key +spec: + kubeConfig: # OPTIONAL: omit for the in-cluster (local) provider. IMMUTABLE in v1. + secretRef: # Flux's meta.KubeConfigReference; secretRef pinned to the operator ns + name: prod-eu-1-kubeconfig + # configMapRef: {...} # RESERVED — Flux workload identity; v1 CEL guard rejects it + # serviceAccountName: ... # RESERVED — Flux remote impersonation; future sibling + allowedNamespaces: # authorization, DENY-BY-DEFAULT (empty = none) + names: [team-a] + attribution: + mode: None # None (default) | Audit. Admission is deferred (not in the enum) + qps: 20 # outgoing kube client throttle (was --source-cluster-qps) + burst: 40 + ingressLimits: # INCOMING audit protection — distinct from qps/burst + maxEventsPerSecond: 500 # per-provider ceiling; excess is shed, counted, not queued unbounded +status: + observedGeneration: 1 + conditions: # Validated, Reachable, DiscoveryHealthy (per cluster, not per GitTarget) + - type: Ready + lastAuditEventTime: "..." # NOT "AuditIngestionActive": a quiet cluster is not unready +``` + +```go +// GitTarget names its source by a ref. Inline spec.kubeConfig is removed. +type GitTargetSpec struct { + // … providerRef (GitProvider, write side), branch, path (unchanged, immutable) … + // ClusterProviderRef names the SOURCE cluster. DEFAULTS to {name: "default"} (the in-cluster + // provider) — a concrete, jumpable ref, never nil. IMMUTABLE. + // +kubebuilder:default={name: "default"} + // +optional + ClusterProviderRef *ClusterProviderReference `json:"clusterProviderRef,omitempty"` +} +// +kubebuilder:validation:XValidation:rule="self.clusterProviderRef == oldSelf.clusterProviderRef",message="spec.clusterProviderRef is immutable" +``` + +The shipped local provider is the same kind, reserved-named `default`, with `kubeConfig` omitted: + +```yaml +kind: ClusterProvider +metadata: {name: default} # chart-shipped when attribution.watchLocal=true; RESERVED name +spec: + # no kubeConfig => in-cluster. CEL: (name == "default") == !has(kubeConfig) + allowedNamespaces: {...} # who may bind local (chart sets a sensible default) + attribution: {mode: None} # or Audit, fed by the local apiserver on the bare /audit-webhook path +``` + +The cluster-scoped `secretRef` needs a namespace Flux's type lacks (pinned to the operator ns) — the +*only* departure from verbatim-Flux reuse; the embedded type is otherwise unchanged. + +## Ingress, keyspace, and the recorder API + +- **Routing.** `ServeHTTP` switches on the path: bare `/audit-webhook` → the **`default`** (in-cluster) + provider; `/audit-webhook/` → the named provider **iff** the peer cert authenticates it; + unknown/unauthorized → 404/403. `validateAuditWebhookPath` relaxes to "accept a segment naming an + *authenticated* provider." +- **Keyspace.** A `cluster:` infix on `factKeyExact/Last/RV`, using the resolved provider name — + the local provider included, so there is no `""` special case. The rv-only hatch becomes + `(cluster, group/resource, rv)` — **the correctness fix** (RV is not globally unique, so the no-UID + hatch was cross-cluster-ambiguous without this). Facts are ephemeral (15-min TTL, + [`DefaultAttributionFactTTL`](../../internal/queue/attribution_index.go)); a **delete finalizer purges + `cluster::*`** so a recreated name starts clean. +- **Recorder API (explicit).** `RecordFact(ctx, providerName string, event auditv1.Event)` — the + current interface has **no** cluster argument ([`audit_handler.go`](../../internal/webhook/audit_handler.go) + `AuditFactRecorder`); the handler resolves the authenticated provider and threads its **name**. + `ResolveAuthor`/`LookupAuthorResolution` gain the same `providerName` (the local provider's name for + a `nil`-ref target); `attachAuthor` passes it. + +## Timing — batch-max-wait vs the grace window is a stated prerequisite + +Exact attribution needs the audit fact to arrive within the resolver's grace window +([`DefaultAttributionGraceWindow`](../../internal/watch/author_resolver.go) = 3s). The apiserver's +`--audit-webhook-batch-max-wait` **defaults to 30s**, so at low mutation volume a fact can arrive after +the commit and the event ships as committer. This is **not new** — the local path relies on the same +relationship, and the repo already handles it: e2e sets `--audit-webhook-batch-max-wait=1s` +([`start-cluster.sh`](../../test/e2e/cluster/start-cluster.sh)) and the chart NOTES / setup guide make +low max-wait the freshness knob. The multi-cluster design makes it **explicit and per-provider**: a low +`batch-max-wait` on the source apiserver is a documented prerequisite, and the grace is +**per-`ClusterProvider` configurable** for links that are laggier than local. (A missed fact degrades +to a committer commit — never wrong, just less rich — so this is a freshness SLO, not a correctness +bug.) + +## Status — a quiet cluster is not unready + +Owned by the provider: `Validated` (inputs), `Reachable` (discovery reached the API), +`DiscoveryHealthy` (types + followability, per cluster), aggregated `Ready`, with `observedGeneration`. +Attribution health is a **`lastAuditEventTime` timestamp**, *not* an `AuditIngestionActive` condition — +a normally-quiet cluster with no recent mutations must not read as unready. If a readiness signal is +wanted, it is `Unknown` until the first event, never `False` for silence. `GitTarget` **projects** a +one-line `ClusterProviderReady` (the `GitProviderReady` pattern, with a `Watches(&ClusterProvider{})` +trigger), so per-cluster detail lives once on the shared object, not copied onto every target. + +## Admission attribution — deferred entirely from v1 + +The v1 draft sold a remote `ValidatingWebhookConfiguration` as the *"easy, identical-identity"* fallback +for managed clusters. That is **wrong on both counts**, and it is removed from the v1 enum: + +- **Not identical.** A `ValidatingWebhookConfiguration` configures the webhook **server URL + CA**, not + a client identity for the *apiserver*. For the apiserver to authenticate *to* the webhook (so we can + trust and attribute the caller) needs an `AdmissionConfiguration` via `--admission-control-config-file` + — an **apiserver flag**. +- **Not a managed-cluster unlock.** That flag is the *same class* of thing managed control planes hide. + The product's own [attribution-setup-guide.md](../../docs/attribution-setup-guide.md) already scopes + attribution to self-managed control planes and lists EKS/GKE/AKS as **not supported**. Admission does + not change that. + +So admission attribution buys us an *unauthenticated* callback on exactly the clusters where audit also +fails, plus a fail-open webhook that risks blocking tenant writes, plus a weaker (`:last`-only, no +post-write RV) join. The honest managed-cluster answer is a **source-side agent** (runs in the cluster, +authenticates outward) — a separate, larger future feature. Admission stays out of v1; if it ever +returns it needs its own authenticated-source design, and command authorship (`/validate-operator-types`, +config-plane-only) is untouched regardless. + +## CRD invariants (to specify before coding) + +- `clusterProviderRef` / `providerRef`: typed local references; `clusterProviderRef` **defaults to + `{name: "default"}`** and is immutable. +- `attribution.mode`: enum `{None, Audit}`, default `None`; `configMapRef` rejected by CEL. +- `kubeConfig`: **optional (omitted = in-cluster), immutable**; `secretRef` namespace pinned (operator + ns); **CEL `(name == "default") == !has(kubeConfig)`** — the reserved `default` is the sole in-cluster + provider (name-uniqueness gives the singleton; **no validating webhook** for it); no `isDefault` field. +- `qps`/`burst`: positive, bounded defaults; `ingressLimits.maxEventsPerSecond` required with a bounded + default. +- `allowedNamespaces`: deny-by-default; names and/or selector; semantics enforced at admission + reconcile. +- Status: `observedGeneration`, kstatus-style `Ready` aggregation, `lastAuditEventTime`, no + quiet-cluster-unready condition. + +## What we remove (and it must not be left in place) + +The `SourceClusterID` string is **not** something to keep — it is deleted, not adapted: + +- **`GitTarget.SourceClusterID()`** ([`gittarget_types.go`](../../api/v1alpha3/gittarget_types.go)) — + deleted. Resolution is `spec.clusterProviderRef.name` → the provider (→ its uid only for auth). +- **The `//` string as any data-plane key** — gone. `clusters + map[string]*clusterContext` and `clusterIDForGitTarget` key by **provider name** (`default` for the + in-cluster one; no `""` sentinel and no implicit-`nil` case — the ref is always populated). +- **`git.Event.SourceClusterID` / `ResolvedTargetMetadata.SourceClusterID`** → renamed **`SourceCluster`**, + carrying the provider *name*. +- **Inline `GitTarget.spec.kubeConfig`** — removed (unreleased; no migration). +- **The v1-draft `cluster:` fact infix and any `name→uid` fact-key lookup** — never built; the + infix is the name. +- **The "mutable kubeConfig rotation" idea** — not built in v1 (immutable); Secret-contents rotation + stays. + +## Minimal safe v1, and what is deferred + +**v1 (build):** `ClusterProvider` (kubeConfig optional = in-cluster, immutable) + a **shipped local +provider** (`clusterProviderRef` defaults to `{name: "default"}`) + immutable source identity + **namespace authorization** + +**name-keyed facts** + **authenticated audit ingress** + per-provider ingress limits + name-based status +with `lastAuditEventTime`. + +**Deferred (explicitly not v1):** admission attribution; workload identity (`configMapRef`); +ServiceAccount impersonation; mutable kubeConfig / endpoint repointing; an `isDefault`/movable default +(the ref defaults to the reserved `default` provider instead); managed-control-plane support (needs a +source-agent). + +## Build order + +1. **`ClusterProvider` CRD + reconciler** — cluster-scoped; kubeConfig **optional (omitted = + in-cluster)** + immutable; the **`(name == "default") == !has(kubeConfig)`** CEL; `attribution.mode` + (`None` default); `allowedNamespaces`; validate kubeconfig (exec/TLS reject) → `Validated`; + `observedGeneration`. +2. **Retire `SourceClusterID`; re-home the engine onto the provider name** — delete + `SourceClusterID()`; key `clusters` by name; rename the `git` carriers to `SourceCluster` (name); + delete inline `kubeConfig`; **default `clusterProviderRef` to `{name: "default"}`** so a ref-less + target resolves to the reserved in-cluster provider. Behavior-preserving. +3. **Namespace authorization** — a validating webhook rejecting a `GitTarget` in a non-allowed + namespace; reconcile-time refusal to start watches; denial tests. (Before any remote data flows.) + *(The in-cluster singleton is CEL from step 1 — no webhook needed for it.)* +4. **Authenticated ingress** — per-provider client credential contract (subject/SAN or SPKI), handler + reads the peer cert, `cert-provider == path-provider`, incarnation-binding + revoke-on-delete. + (Before routing accepts remote paths.) +5. **Ingress routing + name-keyed facts** — relax `validateAuditWebhookPath` to authenticated + providers; `RecordFact(ctx, providerName, event)`; `cluster:` infix; thread `providerName` + through the read path; delete-finalizer fact purge; per-provider `ingressLimits`. Unit-prove a + single-provider install's keyspace matches a bare install. +6. **Status + projection** — `Reachable`/`DiscoveryHealthy`/`lastAuditEventTime` on the provider; + `ClusterProviderReady` projected onto `GitTarget` with a `Watches(&ClusterProvider{})` trigger; + per-provider grace override. +7. **Chart + docs** — **ship the local `ClusterProvider` by default** (`watchLocal: true`, sensible + `allowedNamespaces`); issue/rotate per-provider client certs; document the remote apiserver + audit-config recipe (name + low `batch-max-wait`); extend the attribution setup guide. + +## Test plan + +Reuse config-plane-split's kcp harness (`test/e2e/kcp_workspace_test.go`) where a real remote is needed. + +- **Unit** — a single-provider install's keyspace matches a bare install; cross-cluster isolation incl. + the rv-only hatch; recorder threads the provider name. +- **Local binding & defaulting** — a `GitTarget` omitting `clusterProviderRef` persists with + `{name: "default"}` and binds to the `default` provider; with `watchLocal:false` (no `default`) it is + `NotReady` via the ordinary "provider not found" path; CEL rejects a non-`default` provider without a + kubeConfig **and** a `default` provider *with* one; the ref is always populated (jumpable), never nil. +- **Authorization** — a `GitTarget` in a non-allowed namespace is rejected at admission and never + starts watches; allowed namespace works; tightening the policy stops an existing target. +- **Authentication** — cert/path mismatch is rejected; a provider's cert authenticates only its own + path; cert rotation continues ingestion; revocation on delete stops it. +- **Recreation/repoint** — stale audit delivery to a recreated name fails auth (not misattributed); + the fact purge ran; `kubeConfig` mutation is rejected (immutable). +- **Timing** — with `batch-max-wait` above the grace, a low-volume mutation degrades to committer (not + wrong); with the e2e's 1s max-wait it attributes exactly. +- **Isolation** — a noisy provider hitting `ingressLimits` sheds its own excess without starving other + providers' ingestion, Redis, or the commit queue. +- **e2e (kcp)** — remote author round-trip (`/audit-webhook/`, authored by the real user); the + three-workspace same-`(ns, ConfigMap, name)` non-leak centerpiece; `ClusterProviderReady` projection + and recovery. + +## Open questions + +1. **Namespace-authorization surface.** `allowedNamespaces` (names + selector) on the provider vs a + `ReferenceGrant`-like object each namespace opts in with. The former is one object for the admin; + the latter is standard cross-namespace-consent shape. Recommendation: `allowedNamespaces` + deny-by-default now; revisit ReferenceGrant if per-namespace self-service consent is wanted. +2. **Per-provider client-credential mechanism.** mTLS client cert (subject/SAN map, or SPKI pin) vs a + per-provider bearer token in the audit webhook kubeconfig. Cert fits the existing mTLS setup; token + is simpler for some operators. Pick one contract, incl. rotation/revocation. +3. **`:last`-as-weak** — only relevant if admission ever returns; leave the read policy exact-only for v1. +4. **Ingress backpressure policy** — shed vs buffer-with-bound when a provider exceeds `ingressLimits`, + and how that surfaces (a condition? a metric only?). diff --git a/docs/future/ha-gittarget-distribution-plan.md b/docs/future/ha-gittarget-distribution-plan.md index fff060ef..64e67de8 100644 --- a/docs/future/ha-gittarget-distribution-plan.md +++ b/docs/future/ha-gittarget-distribution-plan.md @@ -1,469 +1,287 @@ -# High Availability and GitTarget Distribution Plan +# High Availability and Durable Delivery Plan Status: **proposed** (not started) -> **Reconciliation note (watch-first rewrite).** This plan predates -> [watch-first ingestion](../finished/watch-first-ingestion-architecture.md), which removed the -> audit-as-correctness pipeline. The **branch-ownership core is unchanged and remains the HA target**: -> the `BranchWriteShard` model, the shard-ownership leases (HA-0 → HA-2), durable per-shard write -> queues, and the `PushAtomic` compare-and-swap fence. What changed is the **ingress half**: object -> state now comes from a per-GitTarget Kubernetes **WATCH** (`sendInitialEvents` replay + -> mark-and-sweep), not from a shared canonical audit stream with per-type sequencing. So "Current -> Shape", "Audit Ingress Fan-In", and "Resource-Type Sequencing Queues" below describe the retired -> model; in the new model the durable write-shard queues (HA-1) are fed by the watch -> [`EventRouter`](../../internal/watch/event_router.go), and audit is only an optional -> [attribution lookup](../../internal/queue/attribution_index.go) that names the commit author. -> -> **Redis stays a hard dependency for HA.** It already holds the attribution facts; for multi-pod it -> additionally holds the per-`(GVR, scope) → RV` watch **resume cursors** (so a failover resumes a -> watch instead of cold-replaying), the **branch-shard ownership leases**, and the **durable write-shard -> queues**. Committer-only single-replica operation is the only mode that runs without Redis. - -## Goal - -Make GitOps Reverser run safely with multiple pods while preserving the most -important write invariant: - -> At any moment, only one pod should own work for a given Git branch, and every -> push must still be protected by remote-branch compare-and-swap semantics. - -The cluster's Kubernetes state should be observable from multiple -`gitops-reverser` pods, and `GitTarget`s should be distributable across those -pods. The write path must still serialize every write that can touch the same -branch. The safety model is at-least-once delivery plus idempotent replay, not -exactly-once processing. - -## Current Shape - -The current implementation is intentionally single-active: - -- `AuditConsumer` (retired with watch-first) used Redis consumer groups under - leader election to drain the canonical audit stream. Object state now comes from - a per-GitTarget watch on the leader instead — see - [`watch.Manager`](../../internal/watch/manager.go) and - [`target_watch.go`](../../internal/watch/target_watch.go). -- [WorkerManager](../../internal/git/worker_manager.go) also participates in - leader election. It creates in-process [BranchWorker](../../internal/git/branch_worker.go) - instances keyed by `(GitProvider namespace, GitProvider name, branch)`. -- [BranchWorker](../../internal/git/branch_worker.go) serializes commits and - pushes for that branch key inside one pod. This protects the branch locally, - but does not by itself protect a multi-pod deployment. -- [GitTargetEventStream](../../internal/reconcile/git_target_event_stream.go) - buffers live events while snapshots are reconciling and deduplicates events - per target, but its state is in memory and pod-local. -- [watch.Manager](../../internal/watch/manager.go) is also leader-elected, so - discovery, informer lifecycle, and snapshot emission happen on one active pod. - -This means Redis/Valkey already helps with ingress durability and future -failover, but the Git write ownership boundary is still process-local. - -## Audit Ingress Fan-In - -All pods can safely serve the audit webhook and append to the same canonical -audit stream, as long as webhook handling stays a **producer-only** path: - -- each pod receives `/audit-webhook` traffic; -- each pod performs request decode, validation, audit body joining, and - canonical event preparation; -- each pod appends accepted events to a shared Redis/Valkey stream (the - `RedisAuditQueue` that backed this — retired with watch-first; in the new model - the durable handoff is the per-shard write queue fed by the watch event router); -- no ingress pod routes directly to a local `BranchWorker`. - -Redis stream append is atomic across producers, so multiple pods can `XADD` to -the same stream. The Redis-backed audit joiner is also compatible with multiple -ingress pods because body parking, decision claims, commit, and release all use -shared Redis keys keyed by audit ID. - -This does not create a new perfect ordering guarantee. The canonical stream -orders events by enqueue time, not by Kubernetes resource version or by a global -API-server sequence. With multiple API servers, webhook retries, load-balanced -HTTP requests, and optional additional audit sources, Kubernetes audit delivery -can already arrive slightly out of order. Multiple ingress pods can make that -arrival-order reality more visible, but they are not the fundamental source of -it. - -The design response should be: - -- treat the canonical audit stream as an at-least-once ingress log, not an - exactly-once ordered history; -- preserve event metadata such as deterministic event id, audit ID, stage - timestamp, user, object reference, operation, and object resource version when - available; -- keep ordering-sensitive Git writes serialized later by branch write shard; -- make replay and duplicate handling idempotent in the shard writer; -- use snapshot/reconcile as the correction path when audit ordering or delivery - produces an uncertain derived Git view. - -So the answer is "yes, all pods can push audit events to the same queue," but -that only solves ingress availability. It does not by itself make consumers, -branch workers, or snapshots active/active-safe. - -## Resource-Type Sequencing Queues - -An optional layer between audit ingress and branch-shard writes is a set of -small **resource-type queues**: - -```text -ResourceTypeQueue = API group + resource type -``` - -This is deliberately narrower than "API group." Kubernetes `resourceVersion` -ordering is only meaningful for objects from the same API group and resource -type when served by kube-apiserver, per the Kubernetes -[resource versions](https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions) -rules. For example, two `apps/deployments` resource versions can be ordered, but -`apps/deployments` and `apps/replicasets` cannot be ordered just because both -are in the `apps` group. For extension API servers, numeric ordering is only -safe when both resource version strings parse as decimal numbers; otherwise -equality-only comparison is the safe fallback. - -The useful shape is: - -1. Audit ingress appends to the canonical stream, or directly to a - resource-type stream after lightweight GVR extraction. -2. A resource-type sequencer consumes events for one group/resource. -3. It holds a short reorder window, sorts comparable arrived events by - `metadata.resourceVersion`, and coalesces stale updates for the same - `(namespace, name)`. -4. It fans the resulting event to every active `GitTarget` whose `WatchRule` or - `ClusterWatchRule` matches that resource type. -5. The fan-out still writes to branch-shard queues, where Git ordering and - branch ownership are enforced. - -This can improve local ordering and reduce redundant writes for noisy resource -types. It also gives a clean fan-out point: one Kubernetes mutation can be -sequenced once, then delivered to many target-specific branch queues. - -The reorder window must stay bounded. Resource versions are orderable, but they -are not a promise of contiguous per-type integers that lets a client prove "RV -123 is missing, so wait until it arrives." Gaps may be normal. The sequencer can -delay briefly to let near-simultaneous out-of-order deliveries settle; after the -window expires, it should emit what it has and rely on idempotent replay plus -snapshot/reconcile for correction. - -Queue key choice should probably be **group/resource** rather than full GVR. -Different served versions of the same group/resource represent the same -underlying objects, while `WatchRule` planning and routing can still retain the -observed API version in the event payload. If implementation convenience starts -with GVR queues, the design should still normalize high-water marks and -deduplication at the group/resource level where Kubernetes comparison rules -apply. - -This layer is not a prerequisite for branch-safe HA. It is a later refinement if -audit ordering noise or fan-out cost becomes visible in practice. - -## Desired Model - -Move from "one leader owns everything" to "many pods may ingest and route, but -each write shard has exactly one owner." - -The first durable shard should be a **branch write shard**: - -```text -BranchWriteShard = canonical Git remote identity + branch -``` - -Using only `GitProvider namespace/name + branch` is not sufficient forever, -because two `GitProvider` objects may point at the same repository and branch. -That collision is already tracked in [TODO.md](../TODO.md). HA should resolve -that at the same time by normalizing branch ownership around the remote identity -that actually receives the push. - -Each `GitTarget` maps to exactly one branch write shard, while one branch write -shard may serve many `GitTarget`s that write to different paths. - -`GitTarget`s can still be spread across pods. The distribution rule is that -their **write owner** is selected by branch shard, not by target name alone. A -pod may own many branch shards, and a branch shard may own many targets. - -## Why Not One Queue Per GitTarget First? - -A per-`GitTarget` queue is attractive because it isolates target backlogs and -matches the user's mental model. It is not sufficient as the first HA primitive: -two targets can write different paths on the same branch, and two independent -target queues could then produce two independent push loops. - -The safer shape is: - -- route events with target identity preserved; -- partition durable write queues by branch write shard; -- optionally add per-target subqueues or priority lanes inside the shard later; -- let exactly one branch owner coalesce, commit, and push all target events for - that branch. - -That gives the desired spread across pods without violating the branch-push -invariant. - -## Delivery and Fencing Model - -The durable write path should assume **at-least-once** delivery: - -- Redis/Valkey streams retain work until it is acknowledged after the Git write - path reaches a durable terminal point. -- Crash recovery may replay already-seen events. -- Replayed events must be safe through deterministic event ids, content hashes, - current remote state, and existing `PushAtomic` conflict handling. - -The branch-owner lease is an ownership and coordination mechanism, not the final -correctness fence for Git itself. Git remotes do not provide a native fencing -token that can reject a push because a Redis or Kubernetes lease changed while -the network operation was in flight. A lease check immediately before `git push` -would still be a time-of-check/time-of-use race. - -Therefore the true write fence remains the remote ref compare-and-swap performed -by [PushAtomic](../../internal/git/git_atomic_push.go): a push is valid only if -the remote branch is still at the expected root. The lease prevents duplicate -work and keeps one intended owner per branch shard; `PushAtomic` protects the -remote branch if a stale owner races during failover or a slow push. - -## First Work Item: Queue-Based Branch Ownership - -This is the first HA milestone because it moves branch work onto a durable -handoff before informer or snapshot work is spread across pods. HA-0 and HA-1 -do **not** by themselves unlock multiple active writer pods; they prepare the -write path while it is still leader-elected. HA-2 is the phase that makes -active writer distribution safe. - -### Phase HA-0: Make Same-Branch Ownership Explicit - -Introduce a small abstraction around the current branch key: - -- compute a canonical `BranchWriteShardID` from resolved `GitProvider` remote - identity and branch; -- keep the current `BranchWorker` behavior, but key worker creation by - `BranchWriteShardID`; -- expose the computed shard in logs, metrics, and possibly `GitTarget.status`; -- detect multiple `GitProvider`s that resolve to the same remote + branch and - make them converge onto the same shard; -- reject or clearly degrade configurations where two `GitTarget`s would write - overlapping paths on the same shard. - -Overlapping-path detection should start as controller reconciliation logic that -sets a degraded status condition before registering the target with the shard. -Admission-time validation is useful for simple literal paths, but it cannot be -the only guard if future target paths become templated or otherwise dynamic. -Runtime conflict detection in the branch owner should remain a defense-in-depth -check before committing a batch. - -This can still run single-pod. The purpose is to name the invariant in code -before distributing it. - -### Phase HA-1: Per-Shard Redis Work Queues - -Split the current single audit-consumer-to-local-worker handoff into durable -per-shard queues: - -1. The canonical audit stream remains the ingress queue from kube-apiserver. -2. The active consumer reads audit events, matches them against rules, sanitizes - the object, and produces one `git.Event` per matched `GitTarget`. -3. Instead of directly calling the local `GitTargetEventStream` / `BranchWorker`, - the router appends each write event to the Redis stream for that target's - `BranchWriteShardID`. -4. The still-leader-elected branch owner consumes that shard stream and owns the - in-memory `GitTargetEventStream`s plus the single `BranchWorker` for the - shard. - -HA-1 is a compatibility and durability step, not active/active writing. It can -coexist with the current direct handoff behind a feature flag or versioned -runtime mode: existing installs keep direct in-process routing, while the new -mode writes to `gitopsreverser.write.shard.v1.*` streams. The `v1` stream name -is intentionally versioned so payload changes can be introduced without -silently confusing old consumers. - -Once HA-2 adds shard leases, any active consumer may read audit events, match -them against rules, sanitize the object, and append the resulting write events -to the appropriate shard queue. A branch owner pod then consumes that shard -stream and owns the in-memory `GitTargetEventStream`s plus the single -`BranchWorker` for the shard. - -Suggested stream shape: - -```text -gitopsreverser.write.shard.v1.{shardID} -``` - -Payload should include enough context to route without re-reading mutable CRDs: -target namespace/name, target path, provider identity, branch, resource -identifier, operation, user, timestamp, sanitized object payload or tombstone, -and a deterministic event id for deduplication. - -This preserves event ordering per branch shard. Once HA-2 enables multiple -active consumers, the same partitioning also keeps ordering stable even if -several pods ingest audit events. It lets a remote outage stall only the -affected shard queue. - -### Phase HA-2: Lease Branch Shards - -Add ownership leases for branch write shards: - -- each shard has one owner pod and a short renewable lease; -- only the lease holder may consume that shard's write stream or attempt to push - that branch; -- when ownership changes, the new owner claims pending Redis messages, rebuilds - any required local clone state from the remote branch, and resumes; -- shutdown drains or hands off without acknowledging work that has not reached +## Scope and Definition of Done + +This plan updates the previous HA proposal for the current watch-first +architecture. Kubernetes WATCH is the source of mirrored object state. Audit is +optional attribution only; it is not a source of object state or a write queue. + +The first release is active/passive HA, not active/active scheduling: + +- Run at least two controller Pods. +- One elected Pod owns controllers, target watches, and Git branch workers. +- Other Pods remain ready to serve admission and audit endpoints, and can become + the active Pod after the leader fails. +- Losing one controller Pod must not silently drop a Kubernetes-to-Git state + change. The replacement may replay a change or create a no-op Git attempt. + +The durability contract is eventual state convergence: after recovery, Git +matches the watched Kubernetes state. It is not a promise of one Git commit for +every Kubernetes mutation, nor an exactly-once event history. + +This contract assumes the Kubernetes API, Git remote, and durable queue remain +available. A single Redis or Valkey Pod is therefore not sufficient for an +installation that also needs to survive loss of any one dependency Pod. + +## Current State + +The repository has useful foundations, but it is not HA today. + +- The Helm chart rejects a replica count greater than one. See + [validate-replica-count.yaml](../../charts/gitops-reverser/templates/validate-replica-count.yaml). +- The watch manager and worker manager declare that they need leader election, + but the controller-runtime manager does not enable it. See + [manager.go](../../internal/watch/manager.go) and + [worker_manager.go](../../internal/git/worker_manager.go). +- Each GitTarget runs per-GVR, per-scope watches with initial-event replay and a + mark-and-sweep resync. See [target_watch.go](../../internal/watch/target_watch.go). +- A live watch event currently goes through EventRouter and + GitTargetEventStream into an in-memory BranchWorker FIFO. See + [event_router.go](../../internal/watch/event_router.go) and + [git_target_event_stream.go](../../internal/reconcile/git_target_event_stream.go). +- Redis currently stores resume cursors and author-attribution data. A cursor is + written after the event enters the in-memory FIFO, but before the eventual Git + push. A Pod crash in that interval can make a replacement resume past work + that only existed in RAM. This is the blocker for lossless failover. +- BranchWorker keeps open commit windows, local commits, unpushed writes, and + CommitRequest outcomes in memory. Its local clone is disposable. +- Git pushes already use a remote reference compare-and-swap. PushAtomic remains + the final protection against a stale owner or an external remote update. See + [git_atomic_push.go](../../internal/git/git_atomic_push.go). +- The chart has a PDB and preferred Pod anti-affinity, but these only improve + placement; they do not make the data path durable or coordinate writers. + +## Target Architecture: HA v1 + +HA v1 retains one active data-plane owner for the whole release. This is the +smallest design that satisfies loss of one controller Pod without introducing +distributed watch ownership. + + Kubernetes API WATCH + | + v + active watch manager + | + v + durable branch-shard journal <---- Redis/Valkey, durable and HA + | + v + active branch worker + | + v + Git remote with compare-and-swap + +The standby controller does not run target watches or branch workers. It does +run the non-leader HTTP servers, so admission and audit traffic can use the +Service endpoints on either Pod. Audit writes attribution facts to the shared +store and never directly writes Git. + +Controller-runtime leader election owns the active/passive transition. The +leader lock must use a release-scoped Kubernetes Lease in the release namespace. +On lock loss, the old owner stops reading new durable work and cancels its +watches. A stalled old owner may still finish an already-started push; the +remote compare-and-swap rejects it if a newer owner has moved the ref. + +## Durable Write Journal + +Redis or Valkey becomes mandatory in HA mode. Add a versioned durable journal +under a branch-write-shard key, with Redis Streams used for delivery and +consumer-group recovery. + +### Shard identity + +The journal and worker key must be: + + BranchWriteShard = canonical remote identity + branch + +The current GitProvider namespace/name plus branch key is not sufficient: +multiple GitProvider objects can name the same repository and branch. The +canonical remote identity should normalize the resolved Git URL and be hashed +for key and Lease names. It must be exposed in logs, metrics, and target status. + +Every GitTarget maps to exactly one branch write shard. Targets sharing a remote +branch use one journal and one worker, even when they have different paths. +Overlapping paths must remain a reconciliation-time and writer-time refusal. + +### Journal record + +Introduce a versioned record that includes: + +- target UID, namespace, and name; +- source cluster, GVR, namespace scope, resource identity, operation, and + resource version; +- a deterministic idempotency key; +- the sanitized object or delete/field-patch payload; +- author attribution when available; +- the resolved branch-shard identity and target path; +- a kind for live event, snapshot member, snapshot start, or snapshot complete. + +The object payload must be encrypted before it is written to Redis when it +contains a sensitive resource. Today plaintext sensitive content is only +transient in the Pod before the Git writer encrypts it. A durable queue changes +that boundary. Queue encryption needs a Kubernetes Secret-backed key, rotation +plan, TLS in transit, and tests proving plaintext secret fields are absent from +Redis values. + +### Atomic handoff and acknowledgement + +For each watched GVR and scope, persist the journal record and its resume cursor +in one idempotent Redis operation. A Lua script or transaction must make a +successful enqueue and cursor advancement inseparable. The key layout must also +work in Redis Cluster, including its same-hash-slot constraints. + +The source watch may advance only after this durable handoff succeeds: + +1. WATCH receives an event. +2. The active leader derives a journal record. +3. It atomically records the event and the new cursor. +4. The branch worker consumes the record and may create local commits. +5. It acknowledges the record only after the corresponding state reaches the + remote Git branch successfully. + +If the leader dies before step 3, the cursor is not advanced and the watch +replays. If it dies after step 3 but before step 5, the consumer group reclaims +the unacknowledged record. If it dies after the remote push but before the +acknowledgement, replay is harmless because the write is idempotent against the +current remote tree. + +The worker must not rely on its local clone for recovery. A new owner fetches +the remote branch, replays retained journal records, and lets the existing +PushAtomic conflict/rebuild behavior settle external changes. + +### Snapshot and replay delivery + +Initial events and the list fallback currently produce an in-memory resync +request before storing the cursor. They need the same durable handoff as live +events. + +Do not store an unbounded full snapshot in one Redis value. Journal a snapshot +start marker, ordered per-object snapshot members, and a snapshot-complete marker +with the collection resourceVersion. The branch worker applies the scoped +mark-and-sweep only after it has received the complete marker. A failed or +superseded snapshot remains replayable; a new leader can also enqueue a fresh +complete replay after the retained journal tail. HTTP 410 Gone continues to mean +fresh replay, never loss of the old journal tail. + +## Implementation Phases + +### HA-0: Specify and expose branch ownership + +Before changing delivery: + +- Add BranchWriteShard resolution from normalized remote URL plus branch. +- Update WorkerManager, EventRouter, metrics, and GitTarget status to use and + report the shard identity. +- Detect multiple providers naming one remote branch and converge them on the + same shard. +- Add overlap checks for target paths on a shard. +- Add a feature gate for the durable delivery path. Existing single-Pod installs + retain the current direct in-memory path until migration is complete. + +### HA-1: Add the durable journal in single-active mode + +Add a journal package beside the existing Redis store: + +- publish idempotent work records; +- create consumer groups, claim abandoned pending records, and acknowledge only + remote-successful work; +- encode and encrypt sensitive payloads; +- atomically couple cursor advancement to durable publication; +- route both live events and snapshot/replay markers through the journal. + +Refactor EventRouter so it publishes durable work instead of directly calling a +local GitTargetEventStream. Refactor BranchWorker so a durable consumer, rather +than its process-local FIFO, owns the acknowledgement lifecycle. The commit +window may remain in memory because unacknowledged records reconstruct it after +failover; local commits must be treated as disposable. + +Ship and exercise this phase with one active Pod first. It removes the existing +crash-loss window independently of multi-Pod scheduling. + +### HA-2: Enable active/passive controller failover + +Enable controller-runtime leader election in the manager configuration and use a +release-specific Lease ID and namespace. Add explicit configuration for lease +durations rather than relying on undocumented defaults. + +- Controllers, WatchManager, journal consumers, and BranchWorkers must require + leadership. +- Admission, audit, metrics, health checks, and Redis readiness remain + non-leader services. +- On leadership loss, stop consumption before starting any new Git work; leave + unacknowledged records for the next leader. +- On startup, claim pending records, rebuild workers from remote Git, then + establish watches from their durable cursors or fresh replay. +- Persist CommitRequest terminal results in Kubernetes status. A failover must + not depend on an in-memory outcome map to decide whether a command completed. + +### HA-3: Release the supported Helm mode + +Only after HA-1 and HA-2 pass end-to-end fault tests: + +- remove the replica-count rejection; +- reject HA configuration without a Redis endpoint, queue-encryption key, and + TLS unless an explicitly documented trusted development exception is chosen; +- add leader-election and durable-queue values to the chart schema; +- add Lease RBAC and regenerate the chart RBAC artifact; +- use at least two replicas, RollingUpdate with maxUnavailable zero and + maxSurge one, and a PDB with minAvailable one; +- make hostname anti-affinity and topology spread required for the HA profile; + offer a zone-spread profile where clusters have multiple zones; +- document a supported Redis or Valkey durability topology. It must survive the + failure model being advertised, not merely pass a ping. + +The existing PDB can remain for single-Pod installs, but it must not be described +as HA by itself. + +### HA-4: Fault-injection acceptance suite + +Add e2e tests for each durable boundary: + +- kill the active Pod before durable publication; +- kill it after publication and cursor advancement, before local commit; +- kill it after local commit, before remote push; +- kill it after remote push, before journal acknowledgement; +- force a slow push and leader handoff to exercise a stale writer; +- force remote branch movement and verify replay plus PushAtomic; +- force expired watch cursors and verify replay plus mark-and-sweep; +- roll the deployment with queued, replay, delete, and sensitive-resource work. + +Every case must assert final remote Git state, no lost deletion, no divergent +ref, no permanently pending journal record, and a healthy replacement leader. +Run the normal repository validation sequence after implementation: task fmt, +task generate where needed, task vet, task lint, task test, and task test-e2e. + +## Optional Follow-on: Active/Active Branch Shards + +Do not make this a prerequisite for HA v1. It improves throughput and isolation, +but creates a second distributed-systems problem. + +When needed, assign each BranchWriteShard its own Kubernetes Lease. Any Pod may +publish durable records, but only the shard-Lease holder consumes that shard and +pushes its Git branch. The lease is coordination, not the final Git fence; the +remote compare-and-swap remains mandatory. + +Tracking and watching can remain leader-owned initially. Distributing target +watches or GVR scopes should happen only after the snapshot markers, replay +watermarks, deduplication, and completeness state are proven durable. A target +with several watched scopes must never sweep Git state until every required +scope has completed its authoritative replay. + +## Acceptance Criteria for Supported HA + +- A two-Pod controller deployment survives loss of the active Pod without + silently skipping Kubernetes state. +- No durable cursor can advance past work that is absent from the journal. +- No journal record is acknowledged before its state is recoverable from remote Git. - -This is the point where the deployment can safely run multiple active writer -pods. The Redis consumer group alone is not enough: consumer groups prevent two -pods from receiving the same queue entry, but they do not prevent two different -local branch workers from pushing to the same branch after independent routing. -The lease backend choice affects operational behavior and failover latency, but -not the final Git safety property: stale owners must still lose to the remote -ref compare-and-swap. - -### Phase HA-3: Durable Per-Target Stream State - -Move the correctness state currently held by `GitTargetEventStream` out of the -pod: - -- reconciliation state (`RECONCILING` vs `LIVE_PROCESSING`); -- buffered live events during snapshot/reconcile; -- processed content hashes; -- pending snapshot/reconcile delivery markers. - -This can be stored in Redis first. Some low-churn status markers may also belong -in `GitTarget.status`, but the hot dedup and buffering path should not depend on -Kubernetes status writes. - -Where the snapshot path needs a freshness boundary, prefer a collection -`resourceVersion` watermark over open-ended content-hash buffering: a snapshot is -authoritative as of the collection RV it was listed at, so only live events newer -than that RV (for the same group/resource) need to be replayed on top, and -anything at or below it is already reflected. Content hashes then become a -secondary idempotency guard rather than the primary buffering mechanism. See -HA-4 for the resume side of the same watermark. - -This makes branch-owner failover safe during an in-flight snapshot, not merely -during ordinary live-event processing. - -## Replicating Kubernetes State Across Pods - -After branch ownership is safe, the Kubernetes observation side can evolve in -two layers. - -### Phase HA-4: Active/Passive Snapshot State - -Keep one active `watch.Manager`, but persist enough state that a newly elected -leader can resume instead of starting from an empty in-memory contract: - -- pending and last-delivered rule-set hashes; -- per-target snapshot delivery status; -- tracked GVR completeness and degraded discovery state; -- last-seen resource hashes or resource versions used for deduplication. - -Persisting the per-`(group/resource[, namespace])` collection `resourceVersion` -lets a newly elected leader **resume the watch from that point** instead of -relisting cold, and gives snapshot reconciliation a precise watermark rather than -relying only on content hashes. Kubernetes -[consistent reads from the watch cache](https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions) -graduated to GA in 1.34 and is stable in 1.35+, so a consistent LIST now returns -a trustworthy collection `resourceVersion` cheaply from cache instead of forcing -a quorum read against etcd. HA can therefore lean on RV-based watermarks and -watch resume more than earlier client guidance allowed. Two constraints still -hold and must be coded for: RV remains comparable only within one group/resource -(never collated across resource types), and a persisted RV that has aged past the -API server's compaction horizon returns `410 Gone` and must fall back to a fresh -relist. - -This matches the low-risk path already sketched in -[design-snapshot-engine-evolution.md](../architecture.md#34-multi-pod--ha). - -### Phase HA-5: Active/Active Tracking Shards - -Only after the state model is durable, shard tracking across pods by -`(GVR, namespace)` or another explicit tracked-set key. Each tracking shard has -its own lease, informer lifecycle, and completeness state. - -Snapshots then become a fan-in problem: a single `GitTarget` may need state from -many tracking shards. Snapshot emission should therefore be a queued operation -that waits for all required shards to be synced before producing authoritative -absence/deletion facts. - -## Relationship To WatchRule Wildcards - -[WatchRule wildcard support](../spec/type-followability.md) increases the -number of GVRs a single target may watch. That stresses informer scale and -snapshot completeness, but it should not be the first distributed-systems -problem solved. - -Recommended ordering: - -1. Ship HA-0/HA-1 branch-shard queueing first, so wildcard events can be routed - to a durable per-branch stream instead of a local in-process worker. -2. Then implement wildcard resolver expansion and status visibility. -3. Gate "wildcard support is done" on snapshot robustness, especially per-GVR - list failure handling. -4. Only later shard the watch/tracking engine across pods. - -This lets wildcard work proceed without accidentally creating a world where -multiple pods can push the same branch. - -## Failure Cases To Design For - -- **Pod dies after dequeue, before push.** Redis pending-entry reclaim must make - the event visible to the new shard owner. -- **Pod dies after local commit, before push.** The new owner rebuilds from the - remote and replays retained/pending writes; local commits are disposable. -- **Pod dies after push, before ACK.** Replayed events must be idempotent via - deterministic event ids, content hashes, and remote-state replay. -- **Lease expires during slow push.** The stale owner may complete wasted work, - but the remote ref compare-and-swap must prevent it from overwriting a newer - owner. The owner should keep renewing during long operations to reduce churn, - but renewal is not the Git fence. -- **Graceful handoff during rolling deploy.** The old owner should stop reading - new shard work, finish or abandon in-flight work without premature ACKs, and - let the new owner resume from the shard stream without duplicate divergent - commits. -- **Remote branch moves externally.** Existing `PushAtomic` conflict handling - still applies, but replay must draw from the shard queue/state rather than only - local memory. -- **Two providers point to the same branch.** They must map to one - `BranchWriteShardID`; otherwise HA is unsafe even if each provider-local - worker is serialized. -- **Persisted resource version too old (HTTP 410 Gone).** A resume `resourceVersion` - that has aged past the API server compaction horizon must trigger a fresh relist - and reconcile rather than a hard failure. Watch bookmarks should be used to keep - the persisted RV recent and reduce how often this fallback fires. - -## Acceptance Criteria - -- Multiple pods can receive audit webhook traffic and append canonical audit - events. -- Official and additional audit payloads may land on different ingress pods and - still produce one canonical event decision per audit ID. -- Multiple pods can route matched events to write-shard queues. -- For a given canonical remote + branch, exactly one pod owns the branch worker - and pushes at a time. -- Killing the owner pod during queued, committed-but-unpushed, and post-push - windows does not lose events or produce duplicate divergent commits. -- Rolling deploys and voluntary lease handoff do not lose events or produce - duplicate divergent commits. -- Independent branch shards continue processing when one remote or branch is - slow, broken, or rate-limited. -- The README can remove the blanket "HA is not supported yet" statement only - after branch-shard ownership and failover are covered by e2e tests. - -## Open Decisions - -- What is the canonical remote identity: normalized URL, provider UID plus URL, - resolved host/repo pair, or an explicit `GitProvider.status.remoteID`? -- Should per-shard queues be created lazily per active shard, or should Redis - store all write events in one stream with `shardID` fields and consumer-group - partitioning? Redis Cluster topology is a factor: stream-per-shard can - distribute load naturally, while a single stream is simpler but can become a - hotspot. -- Should Redis or Kubernetes `Lease` objects own branch-shard coordination? - Either way, the lease is advisory for Git correctness; the remote ref - compare-and-swap is the push fence. -- How much of `GitTargetEventStream` state belongs in Redis versus - `GitTarget.status`? -- Should active/active audit consumers perform full rule matching, or should a - central matcher fan out to shard queues first? -- Do shard writers need a per-resource monotonicity guard using resource version - or timestamp, or is snapshot/reconcile correction enough for rare out-of-order - audit delivery? -- Is a resource-type sequencing layer worth the extra streams and dynamic CRD - lifecycle handling, or should branch-shard queues absorb audit events directly - until ordering noise becomes a measured problem? +- A stale or partitioned owner cannot overwrite a newer Git ref. +- Replays, queue claims, and post-push-before-ack crashes converge without + divergent commits. +- Sensitive resource content is never stored as plaintext in the durable queue. +- The chart prevents unsupported HA configurations and documents dependencies + that must themselves be highly available. +- README and chart documentation remove the single-Pod limitation only after + the fault-injection suite passes. diff --git a/docs/images/config-basics.excalidraw.svg b/docs/images/config-basics.excalidraw.svg index 9a1a5eaa..da704436 100644 --- a/docs/images/config-basics.excalidraw.svg +++ b/docs/images/config-basics.excalidraw.svg @@ -1,2 +1,2 @@ -eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO1da3PiSLL9Pr+io/fjrpkqVZVUmlx1MDAxYlx1MDAxMzewRbvltsDYYDe+sdGBhYxcdTAwMDVcdTAwMThcdTAwMThcdTAwMWVcdTAwMDa0Mf/9niyJN7axxz3bPdOz83BLqqqszDyZJ1Ml739+evfu/WjWj97/8u59NFxy6524MahP3v+Lrj9Eg2Hc6+KWZf487I1cdTAwMDehefJuNOpcdTAwMGZ/+fnn+/qgXHUwMDFkjfqdelx1MDAxOOVcdTAwMWXi4bjeXHUwMDE5jsaNuJdcdTAwMGJ79z/Ho+h++L/072L9Pvq137tvjFx1MDAwNrnlXCJcdTAwMDdRI1x1MDAxZfVcdTAwMDbpWlEnuo+6oyFm/z/8+d27/5h/407coFx1MDAxNY9kp9g+OK9cdTAwMWU6SWnSu1x1MDAxZFx1MDAwNl/ikzMz1Dw038IgXG5H9W6zXHUwMDEzLW9NcV1KmWMrf/HF3Vx1MDAxOe5yJXNqcWVcdTAwMTI3Rne4amuds6S0XHUwMDFjqZnLbWYtnriL4ubdiNTi2IuL6aq/vGOLK8PRoNeOjnpcdTAwMWTsXHUwMDExov2DR/S/pWA39bDdXHUwMDFj9MbdxvKZ29sodN3lM7dxp3MxmpmZoX8o7v3G/FeZuNbG9cdGYcHmXTdcdTAwMWGSppdq6PXrYTwyymDLXHUwMDFkkHR9v2GM8u+lTFx1MDAwM5jTJ6t0x53O4nLcbUSk6/d1trZat5GtNrfo0lxcXCK78vtS9iiiiTlUblx0mzO9uLN0RinszavFXtc4pnBcdTAwMTRcdTAwMTMuV8tcdTAwMDfioVx1MDAwN9dcdTAwMWGZWW/hntFS/SRaYdPtVl1vzbNG0XS02NeKY7r3XHUwMDE3Xd7r1347L+StT/f1g4fu4fH7xXO/Zz8t1TfuN+qpPNyxLeFYjFx1MDAwYqGXXHUwMDEyd+Jue1O3nV7YXm7hp1x1MDAxNZ1twGS3NFswWduMQYhj6ZytmMO4Vo7D1CZC3Fx1MDAxZFxi4Vxc51xcaWmHM1x1MDAxN5rfgVx1MDAwZuBKM64s2+GOUo4l3lx1MDAxNi6jQb077NdcdTAwMDew4HdcdTAwMGVcdTAwMTm+XHUwMDFiMmtPZ9hwuXZd7bhbICBo2O5j0LCYK5Ww8PdrsPGc+zovcN+lN5JcdTAwMTdi993hL+9cdTAwMWHRbX3cWTVjrzu6iFx1MDAxM1x1MDAxMp27OduW3LK4cCSTSq899KF+XHUwMDFmd8hcZnpt4nwnbpJK3ofYQzR4v6qXUYxcdTAwMWO0eGDU6y/vhpixXHUwMDFld6OBv0/a6Vxy4mbcrXcqT+2kPlx1MDAxZfXOo2G6l9FgXHUwMDFjreoq+jhcdTAwMDdcdTAwMGLPWepcdGR3vUaQP5Fi1jxkV3e3R1x1MDAwN61cdMTfN1x1MDAwMVx1MDAwMqY5bTOmmHaZ0kuskuJcdTAwMDSTOWFxrS1XcVtcdHdcdTAwMDfUJcvJXHUwMDE1LC+db1x1MDAwMXZb5/RqvnxjrP+jrlx1MDAxYfr29jvHufXHU6PrXHUwMDAwx47NrVWPzvCvtlx1MDAxM+ZcdTAwMTL/wraVllr+ebnxvDb8ML383DnP8/C2XHUwMDFl5ouqf+qu5MZ/7Z42XHUwMDFk3L7+1K9bX4bVON+p1K7cpHozvV1fZb5+fTDoTfadt1cp11vWteyNWzdcdTAwMTfXZ7fDwVx1MDAxN7e437x75vKXXHUwMDA0wydcdTAwMTC/W3t75HLX0TkhXHUwMDAxdlx1MDAwN76ilS020O4+h3bbzUlXgyYzV9vC0nxcdTAwMWLt3MlRQlfcZVq4tivfmFx0/3VSu9g/tXNbKtey1S5o28J5XGbaXFwqh0nk9m8vtc9QMlx1MDAxZVxmon5vd2JHVbaaVVbSylfP7M/k083MvmMjb5PXW7Og/GX06fN5kJxOTy4uP5/ps+E2yuP7enMzp7syt1x1MDAwMLFcXIZ1XHUwMDAzcsvNuUpJhznCcblYgedcdTAwMWPkXHUwMDAyhN9SrpLc1dJcdTAwMTHW0s5cdTAwMGKQP/7Is1x1MDAxON+J3780zuX+OJdaXHUwMDBizZm9q7p19KPVLVx1MDAwN31zuXTdV5W3X1x1MDAwNefDUX00punf96NuI+4214yY6uu9fStCpDCnIVwiXHUwMDE22lGIsv7WcW4soVgob1x1MDAxYfZccsiMaGi9YmSgOFrjXHUwMDFkXHUwMDBio+HHrTRcdTAwMWNcdTAwMGVcdTAwMDD0VOAnkKZb7FS0PtTuLk9nx8VcdTAwMGZnvWrrS7Q3g4bNUOzaysZcdTAwMGZKura1mVXtXHUwMDFjX70vt1x1MDAwMPeDQ79cctDUXHUwMDFm59COYFx1MDAwZWIj24E/XHUwMDA0y8fwp5XLhEZcdTAwMTn1dlxmOvXMm8Z0/Kn08aRd4OwoOnNF+bbGd1NSk1j3ZLpfrIfxYfHq42BcdTAwMWNcdTAwMDSl1mG/Vlx1MDAxOfTFt8h0n97/XHUwMDEzTFcxZFdcdTAwMWJEl7BcdTAwMDL64KxjkvNnMclVTlx1MDAwYupcYmuB8LraP/lBdV+MTPtcdTAwMDVUlyubSe3oXVx1MDAxOERq3Lq8SIIuilx1MDAxYVx1MDAwN9xHv1x1MDAwNoVflez2up3ZXHUwMDAxaOZt3Lyv94ffXHUwMDFj5X0mXHUwMDAxblLeR7fzNsS31lx1MDAxNZ2jRN/2bvX5MXdPruujy85exFx1MDAxN3w0Z1x1MDAwM7BcdTAwMTaziUQ5671qIUTOXHUwMDE1iqMqQrTmtrNd31xuJ0evXHUwMDE3XHUwMDE2PcVcdTAwMWTM97Enflx1MDAxMN+5XHUwMDExXHUwMDE3qHdegHrmWpYrV1/QrFa4cvPqXHUwMDAy9Jqh0tHiOyO+zlxyiL6DwkmLRiTErW647CZU7JbVbS5cInHDnVx1MDAxYseJxFcmvsnRx8NEXX26nv7W+ei0vpw3Wr/d7k18ba2AJ+1cbm65jsNcXHdcdTAwMWRtXGZcdTAwMDFNO6haTKFobWFcciDNUe0457U7ekk/WO9cdTAwMWUo03+c9Vwibtqu3k17XHUwMDFk64n2XHUwMDEykGlcdTAwMDOBf2LnuDZo+9HgTvqiPyyWRvnmzD1cdTAwMTL/7c7xd8Ond2tvXHUwMDBmPu1cdTAwMDDNllx1MDAxMODTkIpcYvFcdTAwMDbW3aexXHUwMDBl98ppLVx1MDAxObNcdTAwMTA0QKa13lx1MDAwNvtcdTAwMGY2PZfpOcS7L2HTlnCl4DvZtHb5o9C2UVx1MDAwNzPBXlfSflUyPepcdTAwMWTc9jqNaHDQiVx1MDAxZqKDsDNcdTAwMWWuMd9vhFM/k1s3OfVzu3pcdTAwMWJqfXVih1d+M6j07XN+3Fx1MDAwZr9cZm9cbqd7UWuHcnGahJlcdTAwMTJSbFx1MDAwNFx1MDAwMFBra5nKmd7RVUY9XHLOvSRcZju49aOP/CDXczMugkD+XHUwMDA1QUApkGSXWbuCgJJq8+ri0Fx1MDAxNOMgaFx1MDAwZf/OyLVcYlx1MDAxYlxyy7ZsxfFcdTAwMTNKXHUwMDAzpZVcZm9D4tNWyG5AdFTYcFe7hV+FXFw/TTY2XGLBXHUwMDFh1rSrckim4GOWJVx1MDAxZGXpXHKsOdRvlsp1tWUxtl3F0plGJYEjprVtc7WDWbOcTf4gXHUwMDE01ONKVFDu/lj7eyXbw33ptfVcdTAwMTi91lxcK0tyZ9fLW4c9ij5cdTAwMGXLaODE+VxuL2+Vo19cdTAwMDO/fi/eJO/Ln94tPcb8YfHzv/+18+lHvZT+Oth20OV8W4Ds1Iejo979fTzCRs9IyK1IOKpcdTAwMGZGh3FcdTAwMWEz1qyXnS/eJ2lcdTAwMWLaXHUwMDEwmvjDcsiDrsstjcLJ4ra2hV55rFmnXHUwMDAwIXLUKNL4XHUwMDFiXHUwMDFiwE5tueVcIohiz0v19Fx1MDAxYul1qVx1MDAxY+FcbpBwpGlpWXpZMa9cYlx1MDAwNVx1MDAxMlx1MDAwZaKtXHUwMDFjMFwiRqdcbrb9lpSVp7h0XHUwMDE31begXHUwMDAxkVfvbVx1MDAwNrCoc9Ob7FWMPF01PVx1MDAxNSBtQfpHkVx1MDAwMM+B4+jNXHUwMDAwqXPYnutcblugJLG3y1x1MDAxMYXh1G1cdTAwMTJcdTAwMWGFr1xyerzjiCpcdTAwMDOhkWCMZGLgXHUwMDA1TPhcdTAwMDV05O9cdTAwMTVcIo/+cIjkXHUwMDAyUUlKhMlcdTAwMWQxXHUwMDEy1eZW6Fxc9v9gfc2+xlx1MDAwMZdvIUY+7qhm+LaL/lx1MDAxOUHy6bdcdTAwMDUr4VxiQZzZlqNcdTAwMTlioONcIlxcqmUjd1x1MDAxMY/4lk/sXHUwMDE1XHUwMDEz947UJISFLCrh6Fx1MDAwZdeuXHUwMDA1UrUlXHUwMDA0XG5cdTAwMGZUh1Ij1VCp6Njqv1x1MDAxNVx1MDAxNIeFwoeD+771eZLPO43u9NOXm9Knvd54cicnJTbHXHUwMDEx+Fx1MDAxNeNcdTAwMWKnXHUwMDEwNFxuNNtFUMzej2zFRNvNLY/jK/7jdee711x1MDAwN0TvXHUwMDA1h/YtSlF8Z+BcdTAwMTOPXHUwMDFm2udAlEOfwryq9fpI4KOcyV0h5XLZV/Rnruqj8O58vPq24NtoyaxtYrP/skPqt2m5uEX38Kr0SZ39du9fWyimO1x1MDAwN4e/7dVyVVaOS26hXHUwMDEyZFxuReBcdTAwMDbJ0bSuTdnK5vbqSdzVs7pKXG7BpMS/XHUwMDEx2X4g+vWILryk5SotrWyld36i5jx6XHUwMDBlX7rK4UrwN3yZ8pXfTqTxgumVXFzzinhxXHUwMDFjjyr1QTN65Fx1MDAxM59vNF7skPpt4oVzqftfPtSns/7NSWt0O5PTs+PKfof76UtcdTAwMWWqloWmY2xcdTAwMWKvY7WVXHUwMDEz3FbMlqieUHdvXHUwMDFmedIyXHUwMDA3Lm1b9o/D/X88YHzYP2BYyrElc3f2h1x1MDAxNHvi6Fx1MDAwM7dcdTAwMTlcdTAwMTKA++ZcdTAwMDHjq31cYpNcdTAwMDZcZupG/8GAcTboPcSNb++tz3MhY1vut1x0XHUwMDFhVpuL2+bJ5ZdiXFyI3N5dVPiYr+59kFx1MDAwM0VPTq5cdTAwMWWMWotcdTAwMWOgmkQkspP+2pLf5EeAt7ehXHUwMDFifu/fx1x1MDAxZu9cdTAwMGVcdTAwMWEvOMqhXURwZIFdb3ps9XivWdj0skHpNzzB/NxJjt5xvX9WrX+5qMmBvJxcZmp++bL7l/tWb/cu90nnmuVcXK64xcEkmdZcdTAwMWLf3bv8OVD++FbvXHKR+XH/dC6UxVHPb/8uXG6q6bbPYSy/4bFManS/vW/1mvHoIFx1MDAxY0SNb+/g8jN5bzNcdTAwMDPv2Mjb5F/Z9lx1MDAxZSafK3fs6k5cdTAwMWVcXFxcJKybXHUwMDFjqr1QTr9hZv5cdTAwMTVcdTAwMDLXYv1jPVDxnFrR7Y7vXHUwMDE0pJ1bzdyu2PEq41x1MDAwN8znMj1cdTAwMDdz/1x1MDAwNWW+trlrs91lvs1cdTAwMWZPta5lw2LM+vY+U7iIXHUwMDAwju+rXGbfXHUwMDE0+W3gXHUwMDFj13hn1r8oXHUwMDBi/lu9PL3yuYiScK9jUqhvcvNjkCY5r+OZvlx1MDAwNVxc+T6B7Xg1Sd8tLbvwbMdBycdcdTAwMWaZo5nnlMNcdTAwMWNcdTAwMTdZXHUwMDFmXHUwMDExQWrXtjcg9uJzU39ccn598lx1MDAwMniDdzFp7/5tOpZ6tCrXUisuV19Jf+tF+Z9wXCKL34S2jpRcdTAwMTKhinj9xrmpI20r/IdJ0VA2d6SIwkiuutDXOJH1tFxunzpwwFx1MDAxOVx1MDAwMiDqKtSuXGLe0nE2v/FcdTAwMTU5R6z+KqItXFwjkXMpXHUwMDFkm0mXUdLYgWuJNM20dFx1MDAxZIFkjvjwgm+L/j4l8qfdXHUwMDEwfslZXHUwMDAzOjtnayF29dss/fgrN+CE4u3rfofcc2dccl71yu1Nz1x1MDAxYVx1MDAxYzzqoubulncu59uC45tcdTAwMWQ1eMnJJ8txXa1cdTAwMDRqXVx1MDAxN9FMbr/kf+VJg6fJwLu1k1x1MDAwNpy+NeauQ0U300pvn1x04zmuoEJLcHpRb4NcdGz76ZufNPgpM877er9/gYBcdTAwMWQttlx1MDAwYlx1MDAxOMaNjOPNj1x1MDAwN6fXRpE5XHUwMDE4sXIp6DWiQrd+09n0xPdcdTAwMGZxNDncXHUwMDE5cegvykRmdyYjLNG6/69/WFx1MDAxYef9fXxcdTAwMWZVVvnXz8OH5j+n9ytcdTAwMWaIpsH+5b9bXHUwMDAysKxXz09pMP34y9r0/3NTXHUwMDFmRlxiyWdcdTAwMWaL1vXsUN5cXE3HYcLi+sdzXHUwMDE2er2HU9FcdTAwMTCNmVx1MDAxMsFMPYT34UPQyk+CIzdp3Iex/7HRv/543ju78GVw5Dfrx5f9a+uOzf/cuO90XHUwMDFh7OQh8lhcdTAwMWNcdTAwMWPlJ77XTP+JXHUwMDBm7+tX0+HZxcn4xlJcdTAwMWS/JT/5R3lcdTAwMWRcdTAwMWV/YPWj9N7p51x1MDAxM35zXFx1/ftL6/pKPVxcXHUwMDFml2P/uDisf86Pwu7l8LrC4uvP152be7d9jbWusUalXHUwMDEyJKet/CxIys2gUlx1MDAxZUPWWalcdTAwMTAkwUxOSlx1MDAxNT/xvVx1MDAwMu5cdTAwMDdJyVx1MDAwYpq4Oi4meVx1MDAxNczy0yCWVuC1ue9cdTAwMDXstFW1glx1MDAxNo33x0Wvmlx1MDAwNCxIijMpi60qw/jpaavJSl6tWfRcbpi/mlx1MDAxNI8w/lx1MDAwMve9pvK9Mo3nxVYwXHUwMDFmzzCeXHUwMDE1LyTD/DOMn522atPAXHUwMDBiMb42XHUwMDBlPN9ayFQpZ9farHSRn1x1MDAxNo/kLPBcdTAwMDJZ8crWaatsXHUwMDE1K03M2Vx1MDAxY2PtiZkzljxoNWlPXG5zsqJH40OML0vf8yeYXHUwMDEzcuVcdTAwMTlkXHUwMDE32CdPr9VmQaXjQdZxsVKYlY6gqyOpsFx1MDAxN8xTxjw+w5hcdTAwMTn2k1x1MDAwNF5VpdfaPLsmgkpVlpMqzWNcdTAwMDVcdTAwMTXsseWPsU9cdTAwMGV5OY3BPLRcdTAwMGX2WFx1MDAxNdh7XHUwMDEza4xLlaYqxlx1MDAxOFx1MDAwZlx1MDAxYlx1MDAxNCuhwnjSXHUwMDAxdOFj39VZyfPxXFxAuoKPmecs/Dz1obvTVsEqJjWsXHUwMDEzjIMkL4LqhJeOJNlcdTAwMGby+rBFMFx0KkWvWGlDjppVvEj3XHUwMDAzvUCvNchZmFx1MDAxOFu0XG6kV1x0OSzct4qtprlWTKpcbrq0YH/YLyCZXHUwMDE46a2YNJvFVlx1MDAxOTpcbizSXHUwMDExdDgrVnzYv8bN3nFcdTAwMWbyjGm/pWqAdaHDXG7pPYBMflx1MDAxMrTgXHUwMDFmrepcdTAwMThrY85cdTAwMDD7LSRBpWDmLFVcblx1MDAwMvpKjM+1fFXxalx1MDAxOIO9JSHmrMKnwomR6VxiPtNcIlx1MDAxYlx1MDAwNFgznGIsxuexz7zEPlx1MDAxM9iflSptle7Tn8BeXHUwMDE4XHUwMDBm+7ea03Jiril6XHUwMDBl+ppAXyx9jryZfJ/8qI19XHUwMDFifcBupM9AmXUqd55ZJylcdTAwMDM3eYb7XHUwMDFjcs3XoTlZaut2Klx1MDAwZsVcdTAwMDN6LpZcdTAwMThbUOnaXHUwMDA1Wlx1MDAxM/7oY780N10rQ4fk94S7XHUwMDAy2crCPJNiXHUwMDEy8Fx1MDAxNDdcdTAwMDXo4zrIdFx1MDAwNDvnycdl4DVcdTAwMDXmJL2Tj1xuY3Mv1W+pUoVt8lxme5xCXHUwMDE3ycI+XHUwMDE1+J5cdTAwMDef8DBnXHUwMDFi9jF7XGJcdTAwMDR8imwhSnRcdTAwMWa+XHUwMDBiXHUwMDE5XHT/XHUwMDFj9pvCn7M9kn3JJ0PYL5zN91x1MDAxM1x1MDAxMD6BP4omwDx8XHUwMDFl+21cdTAwMTVg86qEXHUwMDFmT2DL7H4oS3RcdTAwMWYyXHUwMDA1rbaozP2Y7FepXHUwMDE5vFx1MDAxNFx1MDAwYlx1MDAwMYefM/xcdTAwMTn+USX7ymJC91x1MDAxMTNaXHUwMDA1lV5rT4pcdTAwMDZcdTAwMWJcdTAwMDZDZHNcdTAwMGV9JHiOXHUwMDAzQ1NjS69p5ixcdTAwMDE70Fx1MDAwM099O0/jjZ6AXHUwMDBio1x1MDAwN+x3ml7L7Eu6aZWnqeyFzC5ZTLgwssO38pm+Q2H0bWJPXlx1MDAxNSn2XHUwMDEwlr0mjVc03txH7IL8MtONKHkh9jaPXamvYJ1pwCZkV8TjKk/t5SMmNbx5PCSMmvvAXHUwMDA1xuN+XHUwMDE5z1x1MDAxMlx1MDAxZWmfXHUwMDA1ZvxcdTAwMDL2NDLTNZqzXHUwMDFkkE/OXGKvmFOY+ENcdTAwMThcdTAwMDdcdTAwMWUwXHUwMDBmJz+HzLyYhNNFjF+On2E85VxiQb6G8ZC5OYVcdTAwMWNZ3ihk14A7+Fx1MDAwNeVccshJsZNwP4PdYWPCONZBrJnvs0hzXCJGXHUwMDAzZyrVXHUwMDEz2Vx1MDAxOLEsQcz0gPGL1IdcdTAwMDJjr8DYXHUwMDEz8SjTM/ytivFHlJfSa7DZhOJoiXJNUrOWtmsjlmXxXHUwMDBmuqd4n8ZpXHUwMDFm98vIRVXjy4hXXHUwMDE441x1MDAxYnuSbuZ+k8U38lWyJ/ZcdTAwMTkgdjc8xDBaU1x1MDAxNlN8KORcdTAwMTWK7Ut8VVx1MDAxMN+SJvlcdTAwMWTwQPGPclxijSeZroOA9Fx1MDAwMGzTeIOFpMDKSSHNJ2Z98rc8S/NcdTAwMTdyKOJShi/EkvyUfFx1MDAxMHOy1G9riI9cdTAwMWQvIPy0Qr7ELOGnQPaUJqdWKFx1MDAxZjVnND7FXHUwMDE35deymb9UOcH4gPQ0XHLSmFx1MDAwMb+lmE75t0x+S36NvEfxiOYsqDSGXHUwMDE5v4M9JyRcdTAwMTP8PlCLeJXQnNhcdTAwMDd0R3HI+Fx1MDAxMPwqoHxcdTAwMDSsUP6FXHUwMDBmqMDEuFx1MDAwMuyB55KCiYtcdTAwMDHFrkJAc0r4XHUwMDEw5ChcdTAwMWJcdTAwMWYgXHUwMDFlYOZMQmbGm9hVW72WlFx1MDAxNrmY5oQ9vcJcIqan60BcdTAwMDetNM7Dp5PF2q32PEdMilWsjfhcdTAwMWJcdTAwMTjeXHUwMDEyXHUwMDEwTlWJcvp8P5Rj0nVUOlx1MDAxZbIsclRNlVxuXHUwMDEz4ijQK+HU+D9iZarLIDb5XHI+1Zyl+kU8r1x1MDAxMHaIM/nEXHUwMDA3MvtQjFx0KDfg2TuPOFx1MDAxYnxVkJ+bnOu1gb1cdTAwMDJk80XKN8Dpklx1MDAwMmyf+YdHOCBcdTAwMWVcdTAwMTBcdTAwMDLb516md7Il8Vx1MDAxNWC3JlP/XHUwMDAyp1wiXHUwMDFkwSchh1VKeVx1MDAwMnyqMEvH+9j7ZVx1MDAxMJjcU51cdTAwMTi+Y3yhhnyW+TxxRvBcdTAwMTDEcWYwY/JvaC14zFx1MDAwMkdcdTAwMTRcdTAwMTfpXHUwMDFhfDQhXHUwMDFjhWQ3a8mLaFx1MDAxZcIzYlx1MDAxMK1j9EQ69tU8T6TcXHUwMDA0OitcdTAwMDQreE/5juGBxD1ac07XNj6R5uRQLHnepbeINbHZLzhIXqS2MjZNeeNRykVSfmN8XHUwMDA3tupcdTAwMDRZnDP8NuWivrxqM4q9XHUwMDAyXFyHp3YjXnLiXHUwMDExXHUwMDE3Ruwlf5+RPyMmXHUwMDExXHUwMDE3SuMwYlx1MDAxYe1cdTAwMTexRkFHmFx1MDAwZnJcIuavjjd+USlcdTAwMTPXMjFcdTAwMTNx1lroIzZcXFx1MDAwN35KXHUwMDFjZnktxVx1MDAwMPnc4lx1MDAxYeFcdTAwMGacqd1cXJnPcIhMXHUwMDFl4naWydOU61x1MDAxMsNccpIsL2A/xGvBQVspt8/2Qzk1k4cwXHUwMDBmP6N4vFx1MDAxY08xQ1x1MDAwMS8yzXWku+Y8505WZVq5tlwie8avPcpcdTAwMGKU/0g/+ZU5ja0ymcrEV1x1MDAxOeHS+C7GX8Woe+47w1x1MDAxYtQ+fnJyXFzyXHUwMDBlP5xcdTAwMWb5XHUwMDBmZ82ePlx1MDAxNajnXHUwMDEy+evK24dBtOzPcNeit+v2yklsam6cR6NBXHUwMDFjPWw/tdp93P+3vLymvn35r5D5XHUwMDFi1bdlK+XpXHUwMDA1+NR5YDhcdTAwMGVyXHUwMDFh1Vx1MDAwMcXqZJrlPcSyNq9cdTAwMTi8LJ+lmjjlzOZZ4MLwSeTQcHbZWjxXX8rut8pt3yq3y8qP9T+P4nzz7OPhXeO4mclSyGJ728TOXHJZkLfyVPdmsqw9u7ZGqerzoFqdmDVak4dQXFx3z5q/PuW3jmSWetZv06fW/HbvXHUwMDBmqF/jty//Ovt789s2rlx1MDAxN1xyXHUwMDA3q1BdabhcdHJkXHUwMDFilWJBmN6Bl/FcdTAwMGKqcdLeXHUwMDAyuCDlYOI/lFx1MDAxM2tJ0dTOxl+QyynnXHUwMDA0qGdCqoHBMYnHIL6muYd4WmL6XHUwMDE3XHUwMDFl8cRwZp5JOVFCvVx1MDAxOdTObJ7bs5qVU+1sckxC/Vx1MDAxY6qt2/g5yOq9cGp6XHUwMDAwM1N7TonX4Vx1MDAxOcxTQOxv0p4sjFeGN5t+XHUwMDEw1ZyUq5vKyEV5mbjCUZ74XHUwMDE05YKstkVNmFx1MDAxNINdulmL0YxifpuVZyZGN31vOql9Pu/5x9f9m+PJTtzXrGk/POKzxtW0XHUwMDAz/XdcdTAwMWH3l7DNeVx1MDAxYrbK7EG/9qLhmZqLelx1MDAwM4hcdTAwMDHgojLN5aFcIpuYPs9lbVa6RL6/rFHPQqY1SVx1MDAxYnm/2ixcdTAwMTl9+pPShelcdTAwMDcgX5JemlSHIz+H6b4r5Vx1MDAxOXRD/GBcdTAwMDJuIFPdXCJcdTAwMDdWXG5Nw2G9miym48leMlx1MDAxZI98VVx0MtuAw7FJyr0v4Vx1MDAwN+C+ZN8glVulnDuVXHUwMDE3/KZpeHRy6Fx1MDAxN8FjTP2S2lx1MDAxZbmP6qbVcaZHwlx1MDAxMMP4XHUwMDA1/OFcdTAwMTL2Qc1OXFzBms9Hts/Gz1JcdTAwMTnx32rTcCbEJ9iE6i/MSdyhVc34XHUwMDE18WLqXHUwMDA3gUubnlx09XtCntU7qGGqrGI4TCFJazBwU3BcYvBi4mzgSMRdU25YrCD+Uo2WNFx1MDAxM9OjYlx1MDAxM+KPs5JHNTjZXHT1TVx1MDAwNcyQekamX0J4qqVcdTAwMWPecF7iS4VsPshpeoFccj+onKzF0nPDY89cdTAwMGa34jX5325cdTAwMWZq3Vx1MDAxY3cmu3NR2fXbjHwpXHRgK+J9qb6DiW/6haSzzF7EzZPzq6LX2YmBV8RcdTAwMDfYvXOFOT2j24T4cJv8k+pbcFGqVUmOquGhXHUwMDE30M0mzs5bJ8dBvD9cdTAwMTdyubPyf1x1MDAxMvJcdTAwMThcdTAwMTcyT63llL1fXHUwMDAxvyanvPz98veWU/5Ir99wXG7qUqZ9+TawXFww/Vx1MDAxNcSyqenLm14w4YbeXHUwMDBiXHUwMDE0iDPDV6mHSv2ZXHTlXHUwMDE2qsepZ5ReQ1x1MDAxY6WaXHUwMDE5+VwioXpcdTAwMWX+iFx1MDAxYbIs0trct6iXXFw0PZm26UNR31x1MDAxOLVccltcdTAwMTlcdTAwMWak/Vx1MDAwMtNcdTAwMDej9c384PmmXHUwMDE2ovdcdTAwMDa0PmrQadq3NO9cdTAwMGWQW8ifqX9cXEguUSunfdpcdTAwMGZcdTAwMDH19E1NS7tcdTAwMDSnMnF8Zvq1U8S59edaa88ly+eM7NOihzrN8DNT/1JcdTAwMWRcdTAwMDd526aPlNY+vjA9NapfTL+I5lxuuOlzzUysyt5cdTAwMTGg3qReNvVcdTAwMDHoXHUwMDE5k4/zoniRrocrXHUwMDEzqn1LXHUwMDE0d5fPzLJcdTAwMWVcXDZcdTAwMGbFM9OjJPm9tHeaN71cdTAwMWRcdTAwMTNfY9NcdTAwMTeeZn1cdTAwMTbUVKZvlr1zXHUwMDAxT+3Uplx1MDAwMe9cdTAwMTeRZ0xuymQ1XFwhjVG1tD+akD9Qr5j6PoZcdTAwMGZbac8xW9fcQ31eqc7mXFwjXHUwMDFir+bj0zXmuqXe1Dz+rD5TXp2HamprqY8y+Udi8kTqM/Px0/U1sn1cXPbijFx1MDAwZrh+XHUwMDFjkH/RO5L4NJGfzlIs/vPROCZsV1li5Vxmxs44Nn9q8Vx1MDAwZfX3n37/f9bh1+IifQ==ns: defaultyour-repoonly-configmapsto-folder-live-clusterWatchRuleGitTargetGitProvidergit-credsSecret \ No newline at end of file +eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO1da3PiSLL9Pr+io/fjrtl6SqW5sXFcdTAwMDNcdTAwMWLajdtcdTAwMDJjg934xkZcdTAwMDdcdTAwMTYy5mFgeFx1MDAxONDG/vd7siRAPGxjt3ume6Z7dnpsqapUlZkn82SqSvufX969ez+eXHUwMDBmwve/vntcdTAwMWbOgnq31Vx1MDAxONan7/9B11x1MDAxZsLhqNXv4Zawv4/6k2FgW96Nx4PRr//853192Fx0x4NuPVxiM1x1MDAwZq3RpN5cdTAwMWSNJ41WP1x1MDAxM/Tv/9lcdTAwMWGH96P/pb+L9fvwX4P+fWM8zKxcdTAwMWVyXHUwMDEwNlrj/jB+Vthccu/D3niE0f9cdTAwMGa/v3v3XHUwMDFm+zfutFx1MDAxYfTEI9Utdlx1MDAwZc6rh25UmvZvR/6X1smZ7WpcdTAwMWItljBcZoNxvdfshqtbM1xcV0plWOpcdTAwMGZf3p3jLtcqo5dXpq3G+Fx1MDAwZVdcdTAwMWRjMkIp4SrDPO4wsWxxXHUwMDE3tpp3Y1x1MDAxMovrLC/GT/31XHUwMDFkW15cdTAwMTmNh/1OeNTvYo2Y2t94SP+sJnZTXHUwMDBmOs1hf9JrrNrc3oaB563a3La63Yvx3I5cZvlDcO83xr9Kpis2rj/WXHUwMDBiXHUwMDBmbN71wlx1MDAxMUl6JYb+oFx1MDAxZbTGVlx1MDAxOGy1XHUwMDAymt2g0LBK+fdqTkOos0Ba6U263eXlVq9cdTAwMTGSrN/X2drTeo3kaVx1MDAwYo2u1CWTK/9dzT1cZmlgXHUwMDBlkVx1MDAwYulwZpZ3VsaopLN5tdjvWcOUrmbS43rVoDXKwbTGdtRbmGe4XHUwMDEyP00tv2l2adNbs6xxOFx1MDAxYi/XlTJM7/6ix/uD2m/n+az4dF8/eOhcdTAwMWRcdTAwMWW/X7b7b/LTSnyTQaNcdTAwMWXPh7uOkK5gXFxKs5pxt9XrbMq221x1MDAwZjqrJfySktlcdTAwMDZMds9mXHUwMDBiJmuLsVxicbmbMdJ1udSuo7TeRIi3XHUwMDAzIVxcsYxcdTAwMDZMlOtoR+hcdTAwMWT4XHUwMDAwrlxm41o4XHUwMDE4XmtXyLeFy3hY741cdTAwMDb1ITT4g0OG74bMWutcdTAwMDRcdTAwMWJcdTAwMWU3nmdcXG9cdTAwMGJcdTAwMDS4p7n7XHUwMDE4NKAgx1XMuK+Bxm7rNUpy5UimUjB93npXxkhGiMX3Rr++myO0XHUwMDFj9EZcdTAwMDcpOdz2e+OLVkRz517GcVx1MDAxNFx1MDAxN4JLLEFps9boQ/2+1SU9mLWhs91Wk2TyPsAqwuH7tGDGLVx1MDAwNKFlg3F/sLpcdTAwMWJgxHqrXHUwMDE3XHUwMDBlXHUwMDBi+8Sd/rDVbPXq3crTa6lPxv3zcFx1MDAxNK9mPJyEaXmFXHUwMDFmXHUwMDE3eOFcdTAwMTmhn1x1MDAwMHcv1/CzJ0rOm4fs6u726KA9xVx1MDAwMvaNgZ5cdTAwMTJcdTAwMTnjMKaZ8Zg2K7iS6KDEjFx1MDAxNNxcdTAwMTjhae5o6e1Gu0rBeWV/S7w7JmPSIfON4f63um6Y29tcdTAwMWZcdTAwMWPq4uujo+cqLV2Hi7RNL1xcwHbMXFy4XHUwMDAwwaTjaKOM+v3C43lt9GF2+bl7nuXBbT3IXHUwMDE29eDUS4XHf+xcdTAwMWU27ty5/jSoiy+jaivbrdSuvKh6M7tdf8ri+fXhsD/dd9x+pVxcb4tr1Z+0by6uz25Hwy9ecb9x91xm525cdTAwMWHgr1x1MDAwZue7pbdHOPdcXJORXG5gd2ErRjtyXHUwMDAz7d5zaHe8jPJcZpgy84wjheHbaFx1MDAwN2OgmK65x4z0XHUwMDFjT70xXHUwMDE5/vNEd7l/dOcgX55w9C5oO/LR6M6VdpmSXCLV76vD++useSu823A4XGZcdTAwMDf93aFcdTAwMWSJWTqqpMLKN4/tz8TTzdi+YyFvXHUwMDEz19tzv/xl/OnzuVx1MDAxZp3OTi4uP5+Zs9E2ylv39eZmTPdUZlx0YrVy61x1MDAxNuTCy3haK5e50vW4TMFzXHUwMDAxcmkyoIWeVtxcdTAwMDOJl2Kl5yXIXHUwMDFmb/Isxnfi90+Nc7U/zpUx0nDm7EpwXfNogstB3zyuPO9VXHUwMDE57jfB+WhcXFx1MDAxZk9o+PeDsNdo9ZprSozl9d65lVx1MDAwMUKY25AhXHUwMDBinDBAZn/rujdCalx1MDAxNqibhnNcdTAwMDMyI1x1MDAxYsaklFxmXHUwMDE0h2u8Y6k0/LhcdTAwMTWGgyGAXHUwMDFlT/hcdKSZNjuV7Vx1MDAwZrW7y9P5cfHDWb/a/lx1MDAxMu7NoKGzXGainHbwg1aeIzajqpPh6ftqXHUwMDBicD859NtcdTAwMDBNfz2HdiVz4Vx1MDAxYtlcdTAwMGX8wVk+hj+jPSZcctKot2PQsWXeNGaTT6WPJ508Z0fhmSfLtzW+m5LawLon0/1cIlx1MDAxZSaHxauPw4nvl9qHg1plOJDfI9N9ev1PMF3NXHUwMDEwXVx1MDAxZFx1MDAxMF3CXG7og7uOSc6fxSTXXHUwMDE5I6kobCTcq3K8n1T39ch0XkB1uXaYMq7ZhUGExq3LyyDoIalxwX3Ma1D4Tcluv9edXHUwMDFmgGbetpr39cHou6O8z1x1MDAwNMBNyvvoct6G+NZ6sntcdTAwMTSZ2/6tOT/m3sl1fXzZ3Yv4go9mXHUwMDFjXHUwMDAwVjCHSJS7Xq6WUmY8qTmyXCJ4a+642/mtdDP0hmFZVdzBfFx1MDAxZmvxk/gulLhEvftcdTAwMDLUM09cYk+l39GkM1xctXl1XHR6w5DpXHUwMDE4+YNcdTAwMTFf98bQa1x1MDAxNSmMbIRS3pqGx25cdTAwMDLNblnd4TKUN9y9cd1QfmPiXHUwMDFiXHUwMDFkfTyM9NWn69lv3Y9u+8t5o/3b7d7E1zFcdTAwMWF4Mp7kwnNd5nnraGNwaMZF1mJcdTAwMTNFsYU1gDRDueOC1+6oJf1kvXugzHw964XfdDyzm/a64tHKsVFcdTAwMGXuXHUwMDFhxV9cdTAwMDO+3fz0ucJxbdgphMM7VZCDUbE0zjbn3pH8o1x1MDAwYsdfQaefXHUwMDFjN3d4Om+Nzlx1MDAwN8G4P/54O3347eTq5u5ccmi6fUNHXHUwMDE0S64g+1U0fbdW9qDpLpyEkFx1MDAxMjRcdTAwMWR+l3j2hlx1MDAwYvGediGw2lxm7I8xXHUwMDAxX1x1MDAwNI5uVqb6k6S/2JF4LyHpQnqwop0k3Xh88+oyXFw7SK+ZZK/LlL8pR1x1MDAxZvdcdTAwMGZu+91GODzotlx1MDAxZcKDoDtcdTAwMTmtXHUwMDEx6u+Eqj9cdTAwMTOyN6n6c6t6XHUwMDFixn514lx1MDAwNFeFpl9cdTAwMTk45/x4XHUwMDEwfFx1MDAxOd3kT/di7C6F+Di2My2V3HBcdTAwMDBg7GLFXHUwMDEwmNlRrEaaXHUwMDBlKr/iITso+6NNfnL2hVx1MDAxYZdOIPtcdTAwMDInoDW4t8fELiegld68utyOxTh4n8t/MM4ug0ZDOMLRXHUwMDFjPyHj0Ear4DYgmi5cdTAwMDJ2XHUwMDAz/qSDhpcuQn5cdTAwMTPO/jSJ2SBcdTAwMDRrWDOeziCYguZcdKFcXJ3idTHWXFwqYyvteUZcYsa2k2PaLalcdTAwMTVwxIxxXHUwMDFjrndcdTAwMTB2lnHIXHUwMDFlpIZ4PIXEzNtcdTAwMWZrf61ge7gva1x1MDAxN4+xdsONXHUwMDE2iru73lx0u+xR9HFoxlx1MDAwMCfuN3gnrN1XbflcdTAwMWH0W5tJweqndyuLsb8sf/73P3a2ftRK6c/BtoGuxttcdTAwMDJktz5cdTAwMWFcdTAwMWb17+9bYyz0jCa55Vx0x/Xh+LBcdTAwMTX7jDXtJTuX91x02pY2XHUwMDA01v+wXGbioOdxYZCPXHTuXHUwMDE4R5pUs2adXHUwMDFjhMxQ/cngf1hcdTAwMDBW6qgtXHUwMDEzgVx1MDAxN3t+Vk+/6F6flSs9XHRcdTAwMTKOMK2EMKtEPDUpkHBcdTAwMTBt7YJcdTAwMTEx2qywbbckrCz5pbuwvlx1MDAwNVxyTDl9b9OBhd2b/nSvZOTpbOwpXHUwMDA36UiSP5JcdTAwMDRYXHUwMDBlXGbHbDpIk8HyPE86XHUwMDEyKYmznY5odKdcIpZESlxmXHJJvmPzK1x1MDAwM6FRYIykYuBcdTAwMDVM+Fx1MDAwNXTkr+VcIo++2kVyXHSvpFx1MDAxNNzkXHUwMDBlXHUwMDFmiWxzy3WuyorQvmHfYt/M9+AjXHUwMDFmN1TbfdtEf1x1MDAwZif59EuIlDuCXHUwMDEzZ45wXHKDXHUwMDBmdD24S72qXHUwMDBmL/1cdTAwMTHfsom9fOLenpomIVx1MDAxMEVcdTAwMTVcZt3lxlx1MDAxMyBVW5NA4oHsUFx1MDAxOYRcdTAwMWFKXHUwMDE1XUf/UU5xlM9/OLhcdTAwMWaIz9Ns1m30Zp++3JQ+7fVcIpW7XHUwMDE5pbA4XHUwMDBlx69cdTAwMTnf2NxgkKA5XHUwMDFlnGLy2mXLJzpeZrXRX/Ofb1Hfvd4h5l5wXHUwMDFjQFCI4jtcdTAwMWSfTL3L3vJ7XHUwMDBlVfKFfNVe4EdcdTAwMWNcdTAwMWbFTO5JpV5SbVxcmWZSybiqj4O780n6JcT3UZJZW8Rm/WXHrN+m5OJcdTAwMTW9w6vSJ332233hWiCZ7lx1MDAxZVx1MDAxY/62V8lVi1xmV1xcIFx1MDAxM2RcdTAwMWFJ4Fx1MDAwNskx9Fxch6KVw530XHUwMDA23/RcdTAwMTZgraRkSuFveLafiH49ovMvKbkqYbSjzc7Db+6jL2mUp12uJX/D3f3feFx1MDAxM1HsL5hJxZpX+Ivj1rhSXHUwMDFmNsPxXHUwMDBm5S92zPpt/IV7aVx1MDAwNl8+1Gfzwc1Je3w7V7Oz48p+Z1x1MDAwNuiAXHUwMDEwZcvS0O64jbe8RmQkdzRz6MhcdTAwMTfy7u2dVEZlwKVcdTAwMWTh/Dwz8PVcdTAwMGXjw/5cdTAwMGVD0JFN5u2sXHUwMDBmafbEjlxu7jBcdTAwMDRcdTAwMDDvzVx1MDAxZMY3O19cdTAwMTM7XGaqRn+lwzhcdTAwMWL2XHUwMDFmWo3v763Pcy5je95v4zREh8vb5snll2IrXHUwMDFmev27MP8xW917f1xikp6MSu+3WvNcdTAwMWOgmkQkklx1MDAwM1x1MDAwNEao7/Js4e1t4Fx1MDAwNT/6yfvj3U7jXHUwMDA1O0SMXHUwMDA3XHUwMDBmjiiw602Po1x1MDAxZq81S4deNmjzhlx1MDAxYqOf2yHSP65cdTAwMGbOqvUvXHUwMDE3NTVUl9NhrVC+7P3pjlx1MDAwMO5e5T7h3LCMxzVcdTAwMTdcdTAwMWNMklx1MDAxObNxot/jz4Hy51x1MDAxMcA3RObH/cO51IIjn9/+ylx1MDAwNeV02/swVkeDhFxyjd73d1x1MDAwNLDZXHUwMDFhXHUwMDFmXHUwMDA0w7Dx/e2HfibubUbgXHUwMDFkXHUwMDBieZv4qzq5h+nnylx1MDAxZLu6U1x1MDAwN1x1MDAxN1x1MDAxN1x1MDAxMetFh3ovlNO3a1x1MDAxNodcdTAwMWK4ketnXHUwMDAwQcUzOiXbXHUwMDFkx1x1MDAxZpSTSUduT+54lfFcdTAwMTPmizk9XHUwMDA388JcdTAwMGLSfONwz2G703yHP1x1MDAxZWo94UBjTHx/p1x1MDAxZi5CgOPHSsM3p/w2cG7VeHc+uChL/lu9PLsqcFx1MDAxOUbBXtukkN9kXHUwMDE22yBtcF7HM1x1MDAxZDFMXHUwMDFke2A7Xk3ScahVXHUwMDE1nu3YKPl4k1x1MDAwNZp5RrvM9Vx1MDAxMPXhXHUwMDExlPFcdTAwMWNnXHUwMDAzYi/eN/Xn4NcnL4A3eFx1MDAxN1PO7u/0XGL9aFZulNFcXKVfSX/vSfnvsCOL31x1MDAwNI5cdLWWgVx1MDAwZXn9xr2pI2xr/Icp2dBcdTAwMGV3lVxmg1ClTehb7Mh6WoRPbTjgXGZcdTAwMGVcdTAwMTB5XHUwMDE1clc4b+W6m0eHZcaV6W9cdTAwMWNt4Vx1MDAxYYGcK+U6THmMgsZcdTAwMGVcXCuEaWaU50pcdTAwMDRz+IdcdTAwMTdcdTAwMWNZ+uukyJ92Q/gle1xyaO+cY6TcVW9cdTAwMTPm8VduwFx0+dvXfZ3uub1cdTAwMDaveuX2pntcclx1MDAwZVx1MDAxZTVRe3fLOlfjbcHxzbZcdTAwMWG8ZOeTcD3PaIlcXNeDN1PbL/lfudPgaTLwbm2nXHUwMDAxpyPM3HMp6WZGm+09YTzDNUQoJKdcdTAwMTf1XHUwMDBlmMC2nf4+O1xyxKR6+OmmUPk8ODk+PfrYa1xmL93J3jVD18jMMmshPrPmXHUwMDEwlVx1MDAxMVx1MDAxOf30gVx1MDAxMLF2poy9ccnQXGL652l/+Kc4VFb66pIhpZWGKya2Ulx1MDAxNrpcdGN+3CF6QjDDU+dcdL950XBgmvMvXHUwMDFmXHUwMDFl6qIxbTeuJ52Dz6ftyndxTMuhr7O95EXoXHUwMDEz0Ny9yi1obpdcdTAwMTOMNFx1MDAxOeZpLlx1MDAxZFx1MDAwZlx1MDAxY2WjmqCM+zQotd2Ry1x1MDAxODyXUp548z1Ae4Dyz1NKONs/11x1MDAwMFx1MDAwM+DIXHUwMDFhnG3uYenKXHUwMDEzNUMpXdrupd74q6BaeMKk3lx1MDAxML+iltBcYm/rk+53V0x4Lupt1lx1MDAxN7aW8TZcdTAwMDVcdTAwMDb/9O52Xuw+5Ie/XHUwMDFkys7wQ3h04V7uVWAwmq2dw1rfXHUwMDE1pJmTsXVcdTAwMDVcdTAwMDFcbuc6aVx1MDAwZvDzo2HfXHUwMDA25+WX1Fx1MDAxNJTjMe05W8cu7U3v8Vf9Wmi4XHUwMDAxnqJAb/T1X1xuWq/C+Y/02bBxf/ZlzHJHXHUwMDE3wbEuXHUwMDFmR/2HSq+z47NhO8IpXHUwMDA3jpmLcKroo7tcdTAwMWJcdTAwMDextPYyq29cdTAwMWEz/lx1MDAwN3yb6C9cdTAwMTVRz1+ANMcz9LFcdTAwMDS99SUwO5WnPrON/JeLt//MNoxIfdU2uaP4RPBcdTAwMGa58+XRub9NNH2a1z950tJcdTAwMDGEjYFcdTAwMWb06JBAilx1MDAxM9uynqtcdTAwMDBwh7lCXCJcbmpudiCc9twzT7qOwzlcdTAwMWTY3Fx1MDAwMXDmZlxckGlcdTAwMTcqoXNcYs5Lzlx1MDAxMf2lXHUwMDAwfrtcdTAwMWLgL6jtaTpCpDy2XHUwMDBi9tJ54vO7TNIp4dRbuz86vr5tae9RM6U/21x1MDAwNrpcdTAwMWFvK+L+/kctXHUwMDBmWMZ1XU5fXHIhlLps+1xmkcx4XHUwMDE0Yil1lVxmrNfdso+9Kn1P51x1MDAwNu82qo2O/UZcdTAwMDFcdTAwMTSstTRsV6XPYbBFh1x1MDAwZYi6jlx1MDAxMsxsW+2bV/p+SXT1vj5cdTAwMThcXICihcvlXHUwMDAylK1GXHUwMDEyJ1x1MDAxNlx1MDAxZlx1MDAwMoivjUN7XHUwMDA0KnXJ7zfCfK9+0900zPdcdTAwMGatcHq4891cdTAwMDL9oYKJXZ3lgCvs7k9cdTAwMDRXynl/37pcdTAwMGYr6UTon6OH5t9n96kvzLVeyTKB0nr1/NQmePjx17Xh/+emPlxuXHUwMDAx2rOPRXE9P1Q3V7NJXHUwMDEwsVb94zlcdTAwMGJy/YdT2ZCNuZb+XFw/XHUwMDA098GD385O/SMvatxcdTAwMDetwsfG4Prjef/soqD8o0Kzfnw5uFx1MDAxNnds8XvjvtttsJOHMMda/lF2Wsg1439bh/f1q9no7OJkciN0t9BWn1xuR1lcdTAwMTNcdTAwMWN/YPWj+N7p51x1MDAxM35zXFz1XG73l+L6Sj9cXFx1MDAxZpdbhePiqP45O1x1MDAwZXqXo+tcbmtdf77u3tx7nWs861x1MDAxYc+oVPzotJ2d+1G56VfKXHUwMDEzzHVeyvuRP1fTUqVcdTAwMTBcdTAwMTVyedz3o1LOb+LqpFx1MDAxOGW1P8/O/JZcdTAwMTJ+rsNcdTAwMGI5n522q8JvU//CpJirRj7zo+JcXKliu8rQf3babrJSrtYs5vJcdTAwMTi/XHUwMDFhXHUwMDE1j9D/XHUwMDAy93NNXciVqT8vtv1Ff4b+rHihXHUwMDE4xp+j//y0XZv5uVx1MDAwMP1rXHUwMDEzP1dcdTAwMTDLOVXKybVcdTAwMGUrXWRnxSM193O+quTK4rRdXHUwMDE2xUpcdTAwMTNjNid49tSO2VLcbzdpTVx1MDAxYWOyYo76XHUwMDA36F9WhVxcYYoxMa8sw9wl1snja7W5X+nmMNdJsZKfl44gqyOlsVx1MDAxNoxTxjhcdTAwMDWGPnOsJ/JzVVx1MDAxZF/r8OSa9CtVVY6qNI7wK1hjuzDBOjnmy6lcdTAwMGbGoedgjVWJtTfxjEmp0tTFXHUwMDE2+kNcdTAwMDfFSqDRn2RcdTAwMDBZXHUwMDE0sO7qvJQroJ1PsoKN2XZcdTAwMDI/z1xukN1pOy+KUVxyz/EnfpSVfnXKS0eK9If5XHUwMDE2oFx1MDAwYn/qV4q5YqWDedRE8VwiXlx1MDAwZuRcdTAwMDK51jDP/NTqop0nuSrMQ+C+KLab9loxqmrIUkD/0J9Pc2Ikt2LUbFx1MDAxNttlyMhcdTAwMTckI8hwXqxcdTAwMTSg/1x1MDAxYbdrx33MZ0LrLVV9PFx1MDAxNzKskNx9zKlcdTAwMTD5bdhHuzrBszGmj/XmI7+St2OWKnlcdHlF1ubaXHUwMDA1XcnV0Fx1MDAwN2uLXHUwMDAyjFmFTVx1MDAwNVM7pyPYTJt04OOZwVxmfdE/i3VmXHUwMDE11lx1MDAxOUH/rFTp6HidhSn0hf7Qf7s5K0f2mqZ2kNdcdTAwMTTyYnE7smayfbKjXHUwMDBl1m3lXHUwMDAxvZE8fW2fU7nL2edEZeAmy3CfY16L59CYLNZ1J55cdTAwMGb5XHUwMDAzatdS6JvX8bPz9EzYY1x1MDAwMeulselaXHUwMDE5MiS7J9zlSVdcdTAwMDLjTIuRz2Pc5CGPaz+RXHUwMDEx9JwlXHUwMDFiV36uKTEmyZ1sVFqd52L5lipV6CbLsMZcdTAwMTlkXHUwMDExLfVTge3lYFx1MDAxMzmM2YF+7Fx1MDAxYXxcdJtcIl3IXHUwMDEy3YftYo6Ef1x1MDAwZf3NYM/JXHUwMDFhSb9kk1x1MDAwMfRcdTAwMTfMXHUwMDE36/FcdJ/AXHUwMDFmeVx1MDAxM2BcdTAwMWU2j/W289B5VcGOp9Blcj9QJbqPOfntjqws7Jj0V6lZvFx1MDAxNPM+h50z/Fx1MDAwZfuokn5VMaL78Fx1MDAxOe28jq91pkWLXHKLIdI5hzxcIrTjwNDM6jLXtGOWgFx1MDAxZMiBx7adpf5WTsCFlVx1MDAwM9Y7i68l+iXZtMuzeO75RC+JT7iwc4dtZVx1MDAxM3lcdTAwMDfSytv6nqwuku8hLOea1F9Tf3tcdTAwMWa+XHUwMDBi81eJbGQpXHUwMDE3YG1cdTAwMGLfXHUwMDE121xunjPz2ZT0XG5/XFzlsb5cbvBJjdzCXHUwMDFmXHUwMDEyRu194Fx1MDAwMv1xv4y2hEdaZ55Zu4A+7ZzpXHUwMDFhjdnxySbnhFeMKa3/IYxcdTAwMDNcdTAwMGZcdTAwMTiHk51jzrxcdTAwMThcdTAwMDWzpY9f9Z+jP8VcYkm2hv6Yc3OGeSRxI59cXFx1MDAwM+5gXHUwMDE3XHUwMDE0NzBP8p2E+zn0XHUwMDBlXHUwMDFkXHUwMDEzxvFcdTAwMWP4msU6izQmfDRwpmM5kY7hy1wi+MxcdTAwMWMwflx1MDAxMduQb/XlW33CXHUwMDFmJXKGvVXR/4jiUnxccjqbklx1MDAxZi1RrIlqYqW7XHUwMDBlfFni/yB78vexny7gflx1MDAxObGoam1cdTAwMTn+XG59XG5WnySbhd0k/o1slfSJdfrw3Y1cdTAwMWN8XHUwMDE4PVNcdTAwMTVjfGjEXHUwMDE18u0rfFXg36Im2Vx1MDAxZPBA/o9iXGL1pzld+z7JXHUwMDAx2Kb+XHUwMDE2XHUwMDBiUZ6Vo3xcdTAwMWNP7PPJ3rIsjl+IofBLXHS+4EuyM7JBjMliu63BP3ZzPuGnXHUwMDFk8Fx1MDAxNWZcdD950qeyMbVC8ag5p/4xvii+lu34pcpcdPr7JKeZXHUwMDFm+1xm2C35dIq/ZbJbsmvEPfJHNGZexz7M2lx1MDAxZPQ5pTnB7n299FdcdTAwMTGNiXVAduSHrFxywa58ikfAXG7FX9iA9q2Py0NcdTAwMWZoXHUwMDE35a1f9Ml35X1cdTAwMWFTwYYwj7K1XHUwMDAx4lx1MDAwMXbMKGC2v/VdtfS1qLSMxTQm9JnLL316/Fx1MDAxY8igXHUwMDFk+3nYdLR8druziFx1MDAxMdNiXHUwMDE1z4b/9S1v8Vx0p7pEMX2xXHUwMDFlijHxc3TcXHUwMDFmc1nGqJou5afEUSBXwqm1f/jKWJZ+y8Y32FRzXHUwMDFly1x1MDAxN/68QtghzlQgPpDoh3yMT7FcdTAwMDFt73LE2WCrkuzcxtxcXFx1MDAwN9jLY25cdTAwMDVcdTAwMTnzXHJwuihcdTAwMGbdJ/aRI1x1MDAxY1x1MDAxMFx1MDAwZlxigO3zXFxcInfSJfFcdTAwMTVgt6Zi+1x1MDAwMqdcIlx1MDAxOcEmMVx1MDAwZlGKeVx1MDAwMmwqP4/7XHUwMDE3sPZL37expzq1fMfaQlxy8SyxeeKM4CHw48xixsbfQCx5zFx1MDAxMkfkXHUwMDE36Vx1MDAxYWw0XCJcdTAwMWNcdTAwMDWkN7HiRTRcdTAwMGXhXHUwMDE5PoieY+VEMi7oRZyIuVx0ZJb3U3iP+Y7lgcQ92lx1MDAwYk7XsTZcdTAwMTHH5ECueN5lbulrWna94CBZXHUwMDE568rqNOaNRzFcdTAwMTeJ+Y21XHUwMDFk6KrrJ37O8tuYi1x1MDAxNtRVh5HvleA6PNZcdTAwMWLxkpNcdTAwMWNxYfhesvc52TN8XHUwMDEycaHYXHUwMDBmw6fReuFrNGSE8TBP+Px0f2tcdTAwMTeVMnEt6zPhZ8VSXHUwMDFlLct1YKfEYVbXYlxmkM0tr1x1MDAxMf7AmTrN1HiWQyTzIW4nbJymWFx1MDAxN1luXHUwMDEwJXFcdTAwMDHrIV5cdTAwMGJcdTAwMGXajrl9slx1MDAxZYqpyXxcYvOwM/LHq/7kMzTwouJYR7JrLmLuND2n1LXU3Fx1MDAxM36do7hA8Y/kk02NaXWVzKlMfJVcdTAwMTEure2i/1VcdTAwMGJ5z313dIPcp1x1MDAxMJ1cdTAwMWOXcodcdTAwMWbOj1xuXHUwMDBmZ82+OZXI51wi9a9UXHUwMDA1c1x1MDAxOK52YnFP0G5cdTAwMTQnVaOnWsd5OFx1MDAxZbbCh+1W6Xrk/p+Jfk1++/JvUP+F8tuyiHl6XHUwMDFlNnXuW46DmEZ5QLE6nSVxXHUwMDBmvqzDK1x1MDAxNi+rtpRcdTAwMTPHnNm2XHUwMDA1LiyfRFxmXHLml+1lu/pq7oV2uVNcdTAwMTDlTllcdTAwMTda5u9HrWzz7OPhXeO4mcwln/j2jvWdXHUwMDFic0HcylLem8xlre3aM0rVXHUwMDAy96vVqX1Ge/pcdTAwMTDI695Z819P2a1Lu3eetdu41Zrd7v2pxNfY7cu/w/ij2W1cdTAwMDfXi5aDVSivtNxcdTAwMDQxsoNMMS9t7SCX8Fx1MDAwYspx4tpcdTAwMDK4IMVg4j9cdTAwMTRcdTAwMTNrUdHmztZeXHUwMDEwyynm+MhnXHUwMDAyyoHBMYnHwL/GsYd4WmTrXHUwMDE3OeKJwdy2iTlRRLVcdTAwMTnkzmxcdTAwMTHbk5yVU+5sY0xE9Vx1MDAxY8qtO/jZT/K9YGZrXHUwMDAwc5t7zojXoVxyxsnD9zdpTVx1MDAwMv215c22XHUwMDFlRDknxeqmtvOiuExcXOEoS3yKYkGS21wiJ4yK/i7ZrPloRj6/w8pz66ObhdxsWvt83i9cdTAwMWNfXHUwMDBmbo6nO3FfXHUwMDEzs0FwxOeNq1lcdTAwMTfy7zbuL6Gb81x1MDAwZXSV6IM+cNvI2ZyLalx1MDAwM/BcdTAwMDHgoiqO5YEmndg6z2VtXrpEvL+sUc1CxTlJXHUwMDA3cb/aLFl5XHUwMDE2pqVcdTAwMGJbXHUwMDBmQLwkuTQpXHUwMDBmR3xcdTAwMGXidVfKc8iG+MFcdTAwMTTcQMWyRVxmrOSblsPmaqpcdTAwMTj3J32puD/iVcVPdFx1MDAwM1x1MDAwZcemMfe+hFx1MDAxZID7kn79eN465tzxfMFvmpZHR4eFXCJ4jM1fYt0j9lHelO5nayRcZj6MX8BcdTAwMWUuoVx1MDAxZuTsxFx1MDAxNcRiPNJ90n9cdTAwMWXPXHUwMDEx/602LWeCf4JOKP/CmMRcdTAwMWTa1YRfXHUwMDExL6Z6XHUwMDEwuLStmVC9J+BJvoNcdTAwMWOmyiqWw+SjOFx1MDAwN1x1MDAwMzdcdTAwMDWHXHUwMDAwLybOXHUwMDA2jkTcNeaGxVxu/C/laFEzsjUqNiX+OC/lKFx1MDAwNyc9Ib+pgFx1MDAxOVLNyNZLXGJPtZjDW85LfCmfjId52lpgo+BXTtZ86bnlseeHW/6a7G+3XHK1b467092xqOxcdTAwMTU6jGwp8qEr4n2xvP1pwdZcdTAwMGJJZom+iJtH51fFXFx3J1x1MDAwNl7hXHUwMDFmoPfuXHUwMDE1xsxZ2UbEhztkn5TfgotSrkrzqFpcdTAwMWV6XHUwMDAx2Wzi7Lx9cuy39udCXHUwMDFld1P/L8OPcSHbai2m7H3Y4zUx5eUnSX60mPI1tX7LKahKXHUwMDE51+U7wHLe1lfgy2a2Lm9rwYRcdTAwMWJ6L5AnzlxmW6VcdTAwMWEq1WemXHUwMDE0WyhcdTAwMWanmlF8XHJ+lHJmxIuI8nnYI3LIsoxz84KgWnLR1mQ6tlx1MDAwZUV1Y+Q2LNXfj+tcdTAwMDW2XHUwMDBlRs+344Pn21xciN5cdTAwMWLQ85GDzuK6pX13gNhC9kz143x0iVxcOa7TfvCppm9zWlolOJX143Nbr53Bz623a6+1i1bt7NxnxVx1MDAxY/I0y89s/kt5XHUwMDFj5tuxdaQ49ylIW1Oj/MXWi2gsn9s619z6quRcdTAwMWRcdTAwMDHyTaplU1x1MDAxZIDa2HiclcWL+Hm4MqXct0R+d9VmntTgknHIn9lcdTAwMWElzT9cdTAwMTfXTrO2tmP9a8vWhWdJnVx1MDAwNTmVrZsl71xcwFO7tZnPXHUwMDA3RcRcdTAwMTlcdTAwMWKbkrlarlx1MDAxMPuoWlxcXHUwMDFmjchcdTAwMWWoVkx1XHUwMDFmy4dFXFxzTJ5r7yE/r1TnXHUwMDBirpH014v+8TNcdTAwMTaypdrUwv+k25TT41BOLVbyKJN9RDZOxDaz6D9bf0ayjst+K+FcdTAwMDNeoeWTfdE7ktZppD6dxVj8+6N+TDqeXHUwMDE2MnXaaqdcdTAwMWZbtFq+Q/3vL//9f8v8zqcifQ==ns: your-ns-1your-repoonly-configmapsto-folder-live-clusterWatchRuleGitTargetGitProvidergit-credsSecretdefaultClusterProvider \ No newline at end of file diff --git a/docs/images/config-cluster.excalidraw.svg b/docs/images/config-cluster.excalidraw.svg index 8b60c158..5539c4f8 100644 --- a/docs/images/config-cluster.excalidraw.svg +++ b/docs/images/config-cluster.excalidraw.svg @@ -1,2 +1,2 @@ eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO1da3PiSJb93r+iovbjtOl86tFcdTAwMWJcdTAwMTNcdTAwMWKUhV1UWWBssFx1MDAwYm9MVGAhY/F0YzCgif7ve25cboNcdTAwMDBhY5vumt62Z7rHlpSpzHvvOffcVErz758+fPg4mt2FXHUwMDFmf/3wMZxcdTAwMDaNbtRcdTAwMWM2Jlx1MDAxZn+m41x1MDAwZuHwPlx1MDAxYfRxSpi/71x1MDAwN+NhYK68XHUwMDFkje7uf/3ll15j2Fx0R3fdRlx1MDAxMOZcdTAwMWWi+3Gjez9cdTAwMWE3o0EuXHUwMDE49H6JRmHv/n/o36VGL/zn3aDXXHUwMDFjXHJzy5tcdTAwMWOEzWg0XHUwMDE4JvdcbrthL+yP7tH7/+LvXHUwMDBmXHUwMDFm/m3+jTNRk+6oykdqOL1q11x1MDAwZdTXQVRsW8FpW5qm5qLHKVxmw2DU6Le64fLUXHUwMDE0xy3JXHUwMDE3f89oPu7iz0nUXHUwMDFj3ZpL5OLYbVx1MDAxOLVuRzgoXHUwMDE1W1x1MDAxY0y6/fXD8sj9aDjohIeDLiaBe/9cdTAwMTdcdTAwMGbpP8s7XzeCTms4XHUwMDE495vLa25uwsB1l9fcRN3u+WhmeoaBYZmPa/1fzlx1MDAwNyjWjm9rhVx1MDAxYrZu++E9mXI568FdI4hGNHfOljOg0d1cdTAwMTWbxur/Wo5pXGJ/XHUwMDE1yez9cbe7OFx1MDAxY/WbIVx1MDAxOfPj1W8rd+s353d7dNnSXHUwMDFmcn7k9+XYw5A65oopx+KOvTixXGY2x1o/WFx1MDAxYfRN3LmOtl1pL2dcdTAwMTXde1xinJHp8lx1MDAwNsFcdTAwMTcubU/jKqxcdTAwMDdVOrBW4mZcdTAwMTROR4tJpcKu21x1MDAxYVxmXHUwMDFmerXvv33nTev4MrqbOFx1MDAwN+2Pi+t+n/+2tN34rtlIxsNti7tC29pcdTAwMTZsOeJu1O+sXHUwMDFitjtcYjrLKfyUMthcdTAwMWFcYrJHs1x1MDAwMYKVyZj4d5RcXIl/qTbiXzB3M/6F3m/4j4aN/v1dY1xip/zFITDMhsDK1Y+xbtvStTiTOiPYrVxyXHUwMDA0PFx1MDAwNjtnQtlCaW6/JtyfikhuuyngPVx1MDAxZpHLXHUwMDAwo8AyiaLRu+uGXHUwMDA3fVx1MDAxOFx07lxmwlx1MDAwM5Fy56A/Oo/i0MTUytGjRi/qkv2dlVx1MDAxZfPdqEWm+Fx1MDAxOGDwYcqwMMgoQrZYXFwwXHUwMDFh3C3PXHUwMDA26LFcdTAwMTH1w2Fxl1x1MDAwNDFcdTAwMThGrajf6FZ3mUJjPFx1MDAxYZyF98kkRsNxmDZW+PlcdTAwMTFcdTAwMWQ8J/RcdTAwMTNo/f7ZXHUwMDFhXUWfrm5vWOO3auP09OSqcLdzyrJT2COb2U7OsZGkLK0lOIUtXHUwMDAx/Yhg7to5lv5Z8uhcdTAwMDLQ7tL0+8lnXHLddG5u/uJgvn97PrO1pW3QbFZCUylqXVx1MDAwN7llu8rmWr9cbuSvy2nhXVU3XHUwMDBi+dOjVqN+XHUwMDFhz6RcdTAwMWZPOuepnPZzdrfzxu4nWTmyXHUwMDE4XHUwMDE38dcv49KR6uUvw9W7PN6/MVx1MDAxY1x1MDAwZSa79ts8iupcdTAwMTX95ZO4XHUwMDFjn1x1MDAwZiPmNkfXfb5bvzvkYFx1MDAxMF5K3r0pXHUwMDA3Z1tvh1x1MDAxY2y7LLdcbmlHPlx1MDAwN2krXHUwMDAzwu85eVx1MDAxYoxHL8jJMDnn2lx1MDAxMjxcdTAwMGKvTyRlV0ibOWk9taekLCT+5y1JuTrwkVx1MDAwZX94XHUwMDFlfibrrefh9VHvJ/V+0Ye/PZTbN6czp3Z/3Fx1MDAxMl/HXHUwMDA1e7Zz6pXaRfdcXHCNXCIlpZRmXHUwMDA2kTnNuNJcdTAwMGXTjqVUKscuhTR7T8Ovwu/szWlY25blKGFlolqp9aNcdTAwMGJUI0M4toBP/7ws7J3c8HL/Iep49mmtc9I8+lx1MDAxY366+9FZ+Gv/Wlx1MDAxY4Qn/Vb+oF60j0fRoG1d7C9cdTAwMGIrLVlcdTAwMWHPr8/C2dbbIVx1MDAwYiPZ5uwt2Lb5c9hOL1i8J+RnXHUwMDAxXHUwMDFkv6RIXHUwMDE2loJcdTAwMWHKTsju1iUhbmluucyyXrUo9ES0WijbxZuq5Hy3ezhseuFN1I+ApP79XHUwMDBmz83PpMX13PzEXHUwMDA09pOmXHUwMDBi01K3fFdoTOXp6WGXXHUwMDFk5EU1KuycpjlbK5FcdTAwMTFcYu/ZeF/gbbA9LPJcdTAwMTKIbOFmYVqmyp11TKNcdTAwMWFylLLVn1hcdTAwMTRfz/yC60++Ryff70a317WKw+s7XHUwMDE3xW8oXp/sN1x1MDAxOFx1MDAxZdSuy/1aYTo9OIFcdTAwMDGc8Pwy2ls6drTUeyqKs623QzrmzGarXHUwMDE4tjeXpjlnm6B9z7rbgMt3z7paKiYszTJcdTAwMDEqtutlV1lcdTAwMDKwlntPuraU6k1J9zhcdTAwMWGFjbPwbsB/eLZ9JrutZ9uske8nzWp75HxcdTAwMWKXXd+vdU9cdTAwMWV61eNv375fb6Iz6jVaaylWSb61XHUwMDEy5kwhsz4pl/Uyglx1MDAxNrhNXHUwMDFke1x1MDAxNreZmPx/jV2xO3YtXHUwMDAxwKAgyVpx1qk4X8OurbV2pSteVer+IXr5ftRcdTAwMTiNqfuPd2G/XHUwMDE59VsrPkzM9dG+dlx1MDAxY2nbUjiyXHUwMDE5SnnjNF12XHUwMDFkaHbDXHUwMDFhXHUwMDE2l6G85va1bYepJz73wGm4XCJcYlx1MDAxNj7Dr1x1MDAxYvkxXHUwMDE4XHUwMDAyysmAn4BS57D5XHUwMDEwXHUwMDA3lW7x6+3BkKubXHUwMDA39c3r7Vx1MDAwNCXOXFx3XHUwMDA1P4I5qfXgJWRy7spPXHUwMDE2grZd8lx1MDAwZahHZy1cdTAwMDAld1x1MDAwN1x1MDAxNHe1XHI201bWc1ptOdtcdTAwMTDFtUZcdTAwMDUqpdz3mrCW3Hb1XHUwMDFmXHUwMDA2KetGXHUwMDA2PLixmzJkgVx1MDAxNVx1MDAwNpw5N7Z9LaRmgbpuWtdcdTAwMTZcdTAwMTey6Th/MKSa51Z8evX9rPn9yFx1MDAwYjrX7Pj4vvxtJ0jZrpOTtnJsR7ma89UtXHUwMDBlnGucXFw+XeGbulLzXGZw8Xc0bUeTelx0mlx1MDAxME+O48qNMo+0Jd+qLVx1MDAwNVx1MDAxN47FJNh+31x0ynX067TlTmiSQbMpLIhpjt9cdTAwMWNEXHUwMDFk7lx1MDAxNtxcdTAwMDSUk0TArl3H1kHTbeg/XHUwMDE4TTfup9r38lk5dq+rVfHle88/rcSbaNq66cBae0YppNjAzftGg9fip3CRXHKglyyqcFx1MDAwN4LHcfTGnlx1MDAwMtLrauuTS1cooSGh3NfAajG8XHUwMDE3ranUfVx1MDAxNdyefVx1MDAwYif+oPrl9qxZXHUwMDFm9fp/xqOIJ/t9w5rKk/1aklV57b40XHUwMDEzXHUwMDE3RzezwcVvX09uivtbq5FSLFH2prWabK9sMETWXHUwMDA2hiWSXHUwMDEyalhcdTAwMDbb+4aF19DB0Vx1MDAwYtKpZMx2XHUwMDAw/SzYa759yyyTwL1mr4L9PiNyXHUwMDE5YIsn/9XwPu3BXHUwMDFms0rzTMLc3K+wOur9rNCcnZa79nHtqNuf1lx1MDAwN7VcdTAwMDGr2Tf9+s5cdTAwMWF4LWNbmXXlu9Z9ITg/71x1MDAwZU6UfiBD/JMldeX2wtF1UWwy/VcrXHUwMDFj/0Ok7tObINay7VxuaLTFc5aS0nY4UzpVi1DouCrnXGJXSXo7QSgmnFxyLHHt5Cwt0FS5XHUwMDEyRpBcdTAwMTlJj1x1MDAwM4SWXHUwMDA2aTvC4Vx1MDAwZXO12Fx1MDAxZGt/ryRY2VVcdTAwMTKLbZLYkcrRqDaz4CeeWFx0dbnj6le+TvJUoelIV7+q0LxcdTAwMWJE64J7+duHZcCYP1x1MDAxNr//6+fMq7dHqTm7XHUwMDExn8v+NvDYbdyPXHUwMDBlXHUwMDA3vV40wkRPaZBcdTAwMWJEOGpcZkefooQyVpw3f7Frl1xyXGIm3Vx1MDAwN4Z+XHUwMDBlWE5Dt1xiXHUwMDFiJbtUliXt1Og/tlx1MDAxYURcdTAwMGZcImfZXHUwMDBls7igwt5xXHUwMDFkoTZcdTAwMDJcdTAwMDRcdTAwMWP2/KCe3rGYXHUwMDFhXHUwMDE0yzGlLOYysIOtpGunljFcdTAwMTaj0jlcdGsyV0J3uVx1MDAxNFx1MDAxMZtRS7bKXHUwMDEzK92GjVxyYGDI6XPr9Fx1MDAxNXavXHUwMDA3k510/tNcdTAwMDXU0/Qoc4pzm2smLYGqcYVcdTAwMWa50DmukWOkXHUwMDAza7jpJ1x0XHUwMDBiglx1MDAwNL9cbma5lrY5reVkaFx1MDAwZi7dXHUwMDFjsy1cdTAwMTflrHJcdTAwMWRLOS94LPT34sezN/Oj4NpmluNkrWtLtpVcdTAwMWalsFFxSGffpcPrn1x1MDAxNO2XXHUwMDFmt1x1MDAwNak5uVx1MDAxMZ7/cfTogmAsTYOHhHGc1FVcdFx1MDAxMSnQo4Y6cl1LMdCo9Tp2fLo+Wlx1MDAxYpPAqFx1MDAwNFx1MDAwMlxy4snWXHUwMDE5Y1x1MDAxMjnOXHUwMDE5XHUwMDA3s4PUUUFy/aPI8elcdTAwMWQ7T5GjK1ROXHUwMDAyXHUwMDE4UrhISshAa+Ro54Tt4r/cZVx1MDAxY39tLpG4Okfr4kJy2Fx0qTmDXHUwMDFjXHUwMDFkmeOWXHKJbbuMXHUwMDFlbyw7eefGXHUwMDE1bjx/MzdaNtK45FlcdTAwMGbRXHUwMDExwtuo0XWZYzPN9v1q5n+IdNxcdTAwMWGi9LNcdTAwMTGcf1x1MDAwNjPurNHAQiB0YVmcKyhcdTAwMTWecmJaOFrmwTlhXHUwMDE0XHUwMDEyR79SOD69wWdtUMxFJnHBx9JhznLvY1o3XG56fYHW6lx1MDAxYyXFJl3/SdT49Fx1MDAwMvnT1OjmXHUwMDA0xDqUI+fQvmvfXFxwNDQ9lIhGxcFdxTZ1o8shni2oa8e2lCtERl2tJSVtXHUwMDBi2Fx1MDAxM1x1MDAwMpnGZe+Ly1u4sfpmboSyYEA6z9xcdTAwMTDBra1rzkJp1J37f0fOLGu9as15v+y4LUrp52AzQP9cZn7cWaWBibi0LFx1MDAwMVx1MDAxNuJKubYlUlx1MDAxNz0ykZ1cdTAwMDIxc9xNlbZfemQ5WouDUoXedulccpxNdlQ51JoouaGptMtAkj+KXHUwMDFkq4NcdTAwMTOv1pfWeHp2OVx1MDAxOE1/q1x1MDAxZD/Io012zHrxyMmRXHUwMDFhZzZcdTAwMTSHXHUwMDAz8b62XcXWOfNcbqiF9LDl5SP7rVx1MDAwZtdCJrnkf1x1MDAxM/67esleXHUwMDE1XHUwMDAxg1tO5otcbnxzXXH58pFr1p7UXHUwMDFlv0jzXHUwMDA3v4inbVx1MDAxN5SlXHUwMDE0f8knaZbhPX9cbnbYXHUwMDFk34/C4WVjXHUwMDE03J6N07tIfsxTvJWxrz+y2z7Y/Ty8k6eiMT64LI3sk4PDRnhed1x1MDAxYuzTzltupMVyNipcdOEyVJRi9Vx1MDAxMbtkXCKnXFxlaUl73Gy++Szi/f2m1/LDza76aOtWXHUwMDFjXHUwMDA3ikRILTL3i4qtj/1cdTAwMDSTtm1J/bq3J7JZ47mtOPZDcO/cncTTu0t2XHUwMDE0ti9bX6y7wz9ha8uT/f7Bb1x1MDAxYluW9ZJnoE8gPNt6uyR9h74ksFx1MDAxNdz2M+BOKaH37TTPI7r1goyvXHUwMDA1vZqYfjSRwu6mXHUwMDBleMQuhFx1MDAxOZfacvf+zFBw8bbtNPSy7qB/XHUwMDEztfzG3Y9/0/iZnJj1pnHG4Pf0lvHFMFx1MDAxOHa7br44eVx1MDAxOFx1MDAwNEe3J932w3An9FrIvohLjnLWkUqv7lx1MDAxM1DMzdGjVcXFNvi6dlx1MDAwZXpcdTAwMGJyXHUwMDFlRY1cdTAwMGK+X2LlXb8/i+bb3dFMj3GE66RFbWpcdTAwMDeAXHUwMDEy60dcdTAwMTf6XHUwMDFkOthcdTAwMTJK8n2uVCRcbttcdTAwMTapb4y+K2wz3NWZvlx1MDAwNMLHJ05451/zxpGyz09rncrnz7XLnfbHKSmfkNfqOXn9/lx1MDAwNuNLYVx1MDAxYu1cdTAwMGVbS7mM25kviOjtz15cdTAwMDRTXHUwMDE200Lu/+XjV+fgv9JcdTAwMWKMT4v553fNgWmlXHUwMDAzS1n22idlhZ1ztKvoIz+WwiQ3n3xylLq4xFKOwzT+yPyYXHUwMDBlekFcdTAwMTHLk1x1MDAwN9eus+fS9f9cdTAwMGbO2tk4e8nyPpSLq4S1+SGOj2Z/ztasSdtJXHUwMDFjtf/1/ddcdTAwMDNwz1x1MDAxYkO2hSn9XHUwMDFjbETosr9cckjubX3/aT39YeVRI3NJrdLnzOhjXG5yc4F/c0va3neCME7bXG5cdTAwMWSpXHUwMDFkxVx1MDAxNHybsX1P5X7c7o/wc734rTZon1x1MDAxN6zReOJcdTAwMTdddnyyS0XAmeXklFx1MDAwNsNJS3NMb1x1MDAxOa3mXHUwMDExp61yYEDL0cJcIpRkrOFcdTAwMGJGL287UnAtpWBZuz/eXHUwMDBi/G2c19tdW3CHOXCCytxcdTAwMTScfrqyRm5cdTAwMWFFm8vZXrfkJ1x1MDAxNVx1MDAwMbftN1VcdTAwMDTH0eh0OHiImmlcdP9cdTAwMWZYXGZkjnM/lbz6fHF+cnx4fNqbhbp4rb9cdTAwMWVcdTAwMTfqtV1wa9s6Z6FcYmDMtSVLf1jKRKbDXHTUQC5tqrPBWFx1MDAxYrh1XHUwMDFkXHUwMDAzW1x1MDAwYrKHOag0M8qCd9hug+3gXHUwMDA1sGXwXHUwMDAybazL3Kuqt1byQjtcXNiOtXfYcqadN32WXHUwMDE3cKg2hq3wh7/p9lx1MDAxY2jXR7lcdTAwMWbIXjvl8Oz4wCtcdTAwMTXOWt9cdTAwMWLWuD2bnWa8pJNcdTAwMDVZJ0crrfCCJek9hdVqQ+qcUihcdTAwMDJdVIOQsTJjM9E7ZF9cdTAwMGbZu90hS7vaxFxup6ZcdTAwMTBrP/HVXHUwMDE0zjhTSu5ze3lcdTAwMDJZpexXvfz294PsT/NiXHUwMDA1XHUwMDBlvztcdTAwMWbBjlx1MDAwYvmPwImaazNOjo1Cs0kodchcdTAwMWY0w0K/cd1dt+nHhyicfMr8f0GiXHUwMDFmep5nWMOskiyr192XSpbFysde1Fx1MDAwYqvpdcBf7lx1MDAxZlr/mPa6S/NEr1xch0GANWpnJ6ZywK+/rnT/39eN+9BSP59+Lomr2Sd1fTlcdTAwMWRcdTAwMDcxi1x1MDAxYZ/PWOBcclx1MDAxZU5kUzZnWvoz/Vx1MDAxMPSCXHUwMDA3v52f+Idu3OxcdTAwMDVR8XPz7urz2eD0vKj8w2KrcXxxdyVu2ePfzV6322RfXHUwMDFlQo9F/mF+UvRayT/Rp17jcnp/ev5lfC10t9hWX4uHeSc4PmKNw+Tcybcv/Pq45lx1MDAxNntcdTAwMTfi6lI/XFxcdTAwMWRXouJx6b7xLT9cbvpcdTAwMTf3V1VcdTAwMTZdfbvqXvfczlx1MDAxNe51hXtUq1x1MDAxNVH0fH3SLjC/feaX2q2WX/XH5WpHl2qTqVx1MDAxZuWn/kxp/M2rns/S1/pxpVVq11x1MDAxZa9lpSjPS5Ga+tVgdtFeXFzXWI692K50iqLSqehi5PzjMMq3Tj9/um1cdTAwMWW35mMpzIpeIT5pd3C/i/WxMNiDlc5cdTAwMWbHsnLtyj3KtVwi92u1iblHe/JcdTAwMTDIq/5p65//TGFuXHUwMDE4rqxDoIpjXCL1eVxyKuLPwtEwXG5cdTAwMWY2r0qnvt2/qPSauH3555r+RnHrw/f5XHUwMDE5xaBfrYwx1lm54MeI1Um5WoyT2PDjsue3cHRcXIrz2p8hliMlfK/DXHUwMDExx+ykXVx1MDAxM36b2lx1MDAxN8clr1x1MDAxNvvMj0szpVx1MDAxMNOI88L0pN1iZa/eKnlcdTAwMDX0X4tLh2h/jvNeS1x1MDAxN71cbrXnpbb/2J6hPcUnQ/9cdTAwMTSbs5N2fep7XHUwMDAx2tfHvldcdTAwMTSLMVUr82NcdTAwMWRWPs9PS4dq5nu+qnpcdTAwMTVx0q6IUpXivjXGvSemz0hxv92iOVx1MDAwMVN1VvKofYD2XHUwMDE1VfSKXHUwMDEz9IlxXHUwMDAxXHUwMDFiMyUxT55cdTAwMWOrz/xq18NYxyXgqnxcYltcdTAwMWQqjbmgn1xu+ikytJlhPrHv1XRyrMPnx6RfralKXFyjflx1MDAwNDDY8tvFMebJMV5ObdBcdTAwMGbdXHUwMDA3c6xJzL2Fe1x1MDAwMKctXHJcdTAwMGWYkVx1MDAwZkrVQKM92Vx1MDAwMLYoYt61Wdkr4jqfbIVcdTAwMTgz11x0/D4twnbgXG5Riuu4jz/247z0a1x1MDAxM14+VOQ/jLdcYl/4XHUwMDEzv1ryStVcdTAwMGXGUVx1MDAxN6XzZD6wXHUwMDBi7FrHOFx1MDAwYlx1MDAxM+OLdoHsqjBcdTAwMGWB88JwXGKOleKahi1cdTAwMDX8XHUwMDBm//k0JkZ2K8UtcFhcdTAwMDU28lx1MDAwNdlcYjaclapF+L/OzdxxXHUwMDFl41x1MDAxOdN8yzVfXHUwMDEw/5SqZHdcdTAwMWZjKsZ+u244XHUwMDEw90afPuZbiP1qwfRZrlx1MDAxNiTsXHUwMDE1m5hrXHUwMDE3ddWro1xy5lx1MDAxNlx1MDAwN+izhphcbiZmTIeImTb5wMc9gynaon1cdTAwMWXzzCvMM4b/XHUwMDE5cWAyz+JcdTAwMDT+Qnv4v92aVmJzTNN1sNekbDiajlE0U+xTXHUwMDFjdTBvY1x1MDAwZviN7EncjPtUbz1zn7hcdTAwMDLc5MGviLO49nhcdTAwMWbqkyW+7iTjIT6g68Dx8LdO7l2ge1wiXHUwMDFli5gv9U3HKrAhxT3hrkC+XHUwMDEy6GdSin2e4KZcdTAwMDB7XFz5c1x1MDAxYsHPeYpx5XstiT7J7lx1MDAxNKPS+Nyrz3NMXHK+yTPMcVxuW8RcdTAwMGL/VFx1MDAxMXtcdTAwMWViwkOfXHUwMDFk+MfMwZeIKfKFLNN5xC7GSPjn8Fx1MDAxZvKTP58j+ZdiMoD/gtnjfHzCJ/BHbFx1MDAwMswj5jHfdlx1MDAwMT6vKcTxXHUwMDA0vpyfXHUwMDBmVJnOY0x+uyOrj3FM/qvWXHJeSlx1MDAwNZ8jzlx1MDAxOf5GfNTIv6pEeZM4o13QybHOpGSwYTBEPqc8XHUwMDFh4zpcdTAwMGVcZk2NL72W6bNcZuzAXHUwMDBlPIntPLU3dlx1MDAwMi6MXHUwMDFkMN9pcmzuX7JNuzJNxl6Y+2XOXHTnZuyIrfzc3oE09jbck9cl4lx1MDAxZcKy16L2lM9n5jy4XHUwMDBi41dz28iyXHUwMDE3YG6P3JXEXG7uM/VcdTAwMTlpXHUwMDAy4uNcdTAwMWFP/FVcdTAwMDQnNb1HPiSMmvPAXHUwMDA12uN8XHUwMDA111x1MDAxMlx1MDAxZWmeXHUwMDA1ZuJcdTAwMDL+NGOmY9Rnx6eYnFx1MDAxMV7RpzT8Q1x1MDAxOFx1MDAwN1x1MDAxZdBcdTAwMGanOMeYeSlcdTAwMGWmXHUwMDBijl+2n6E95VxiSbGG9lx1MDAxOHNrinHM80ZhflxmuENcXFDewDiJO1x09zP4XHUwMDFkPiaM4z7gmsd5lqhPcDRwplx1MDAxMzuRj8FlMTjTXHUwMDAzxs+TXHUwMDE48o2/fONP8NHczoi3XHUwMDFh2lx1MDAxZlJeSo7BZ1x1MDAxM+LRMuWauC6WvuuAy+b8XHUwMDA321x1MDAxM98nPF3E+VxuclHNxDL4XG5tisafZJvHuJnzXHUwMDFixSr5XHUwMDEz8/TB3U1cdTAwMGZcdTAwMWNG91SlXHUwMDA0XHUwMDFmXHUwMDFheYW4fYmvKvgtblHcXHUwMDAxXHUwMDBmxH+UQ6g9jenK98lcdTAwMGXANrU3WIhcdTAwMGKsXHUwMDEyXHUwMDE3knxi7k/xlmdJ/kJcdTAwMGVcdTAwMDUvzfHFSF9SXGaiT5bEbVx1MDAxZPzY9XzCTzvgS8xcdTAwMTJ+XG7kT2VyapXyUWs216doT/m1YvovV794pFx1MDAxYtFm6iecgbglTqf8W6G4pbhG3iM+oj5cdTAwMGI64TBcdTAwMTN38OeExoS49/WCr2LqXHUwMDEz84DtiIdMXGYhrnzKR8BcbuVfxID2XHLHXHUwMDE14Fx1MDAwZlxcXHUwMDE3XHUwMDE3XGYv+sRdXHUwMDA1n/pUiCGMo2JigHSA6TNcdTAwMGWYaW+4q54+XHUwMDE2l1x1MDAxN7mY+oQ/vcKC05P7wFx1MDAwNu2E51x1MDAxMdPx4t7tzmOOmJRquDf41ze6xSecQjP7rcV8KMck99FJe4xlkaPqulxcmJBGgV1cdKcm/sGViS39yOQ3xFRrrsfB51XCXHUwMDBlaaZcIumBuX+IY3zKXHK49tYjzYZYlVx1MDAxNOcm53pcdTAwMWRgr4CxXHUwMDE1ZaI3oOniXHUwMDAyfD+PXHUwMDBmj3BAOiBcdTAwMDC2z7y53cmXpFeA3bpK4lx1MDAwYpqKbISYxDhEOdFcdIgpqlx1MDAxN6h90dRcdTAwMTa+yT21idE7Jlx1MDAxNurIZ/OYJ81cYlx1MDAxZFx1MDAwMlx1MDAxZWdcdTAwMDYzJv9cdTAwMDZioWNcdTAwMTY4XCJepGOI0ZhwXHUwMDE0kN/EUlx1MDAxN1E/hGdwXHUwMDEw3cfYiWxcXNSPeVwi0SawWcFP4T3RO0ZcdTAwMDeS9mg/arqOiYkkJ1x1MDAwN3Kp8y68XHUwMDA110RmvtAgeZn4yvg00Y2HiVx1MDAxNkn0jYlcdTAwMWT4quvPec7o20SLXHUwMDE21WWHXHUwMDEx90poXHUwMDFknviNdMlcdTAwMTePtDC4l+J9RvFcZk5cIi2U8DA4jeZcdTAwMGKu0bBcdTAwMTH6wzjB+en2Ji5Qc5aTscBuhNu5PVwio3VcdTAwMTCnpGGWx1x1MDAxMlxmUMwtjlx1MDAxMf6gmTqtVH9GQ8zHQ9pOmDxNuS422iCe51x1MDAwNcyHdC00aDvR9vP5UE6dj4cwjzgjPl62J87QwItKclx1MDAxZNmu9ZhzJ+kxpY6lxj7X11x1MDAxZeVcdTAwMDXKf2SffKpP46v5mCqkV1x1MDAxOeHSxC7aX0aoe3rd+2vUPsX4y3HZ+3R0dlh8OG1ccpxcdTAwMTOJei5WT9W3XHUwMDBlvaT5bH2bXFy1Ut/u/OGn19S3L/+q1F+tvu3geMnksirpc8Px4JpcdTAwMGVcdTAwMTR3QZpcdTAwMWHMm/M0acWkRkNOJS6jPELcUo9Lplx1MDAwNjHrIeBEwq5cdTAwMGZdXHUwMDE4UC2BXFxN+Vx1MDAwMHGaYJjyXWzqQI/ybTAz1yS5JaZcdTAwMWFcdTAwMTc1XGJ75Mi59udUg1x1MDAxOKzGVFx1MDAxN1ON0sHv/lxcN1x1MDAwN1NTS82Mhp9SfsQ16KdcdTAwMDBcZrVoTlx1MDAwMu210Vx1MDAxZqauJu1OnNfSZlxcxG/EuYd5ykuEqXmNXHUwMDAwbVx1MDAxZJf8LNusxDoj7HRYZWZivVX0ppP6t7NB8fjq7vp4krk+UFx1MDAxN9O74JDPmpfTLuzfbfYu4JuzXHUwMDBlfDX3XHUwMDA38lx1MDAxM3So0a5UYyFHIaerhFx1MDAxM1x1MDAwM00+MfXyRX1Wvlx1MDAwMG9e1Kn2U4m264A/a62ysWdxUj43dVx1MDAxNXiH7NKielx1MDAwNjxcdTAwMTck865WZrBccvHsXHUwMDA0XHUwMDFjq1x1MDAxMtuCS1BcdTAwMWZcdTAwMWEt4NVVKWlP/lJJe+C+6s99g1xcyCaJhrlAXHUwMDFjQEOQf/1k3DrRLsl4kSdaRo/En4ol5Fx1MDAwM6NcdTAwMDNcdTAwMTPfg0NIf6bbmVqTXbR9fo54uIB/UPtcdTAwMTDnisf+yPfz9rNkjPjfWsvkXHUwMDFl1F/wXHTpWPRJXHUwMDFj3K7N81x1MDAxNOlcdTAwMGKqq6FJTO1JdXPA57pcdTAwMTFasMaqJlx1MDAxN1x1MDAxNOJEy1wix4OLoS8o9yHXkFx1MDAwNkhybKl65lx1MDAxYq1cdTAwMWK3YlPrs1x05eFZ2aNahvxcdTAwMDSdWEWGpdrb1J2Ep3qihYx2oLxTmPeHcZo1lWbRr35ZWSs8M3rg7NPGeiTFX3ZcZrWvj7uT7DWrilvsMIql2IevKH8m9vYnRbPuQjab+4s0Tnx2WfK6mVx1MDAxOHhcdTAwMDU/wO/dS/TpXHUwMDE528akKzpcdTAwMTSfVCcgp5Pmp3HUTD4/h23WcXbW/nLsR7vnXHUwMDE0l9upXHUwMDBmXGZtyynmqsWzh99/+v3/XHUwMDAwYijXZyJ9example-namespace-2ToMainAllCrdDefinitionsGiteaRepo1ToTestClusterWatchRuleAllConfigMapsClusterWatchRuleGitProviderGitTargetGitTarget \ No newline at end of file + @font-face { font-family: "Comic Shanns"; src: url(data:font/woff2;base64,d09GMgABAAAAABWYAAsAAAAAJtwAABVMAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAABmAAgSQRCAqyUKouC0YAATYCJANIBCAFlRgHIBuzITOjwsYBgIK2NKKIoOz/ktwYA2tAq0mwZBYHTmOoQ2G3w+dMMRcKHRgTRPDjJ9ZKa031r22z7WrbjSuTLgzTVO+7IySZhX+eY+9972cerwZPW0QITGBUPuKdURTIrDl1U3P6vVUN2OEXKU0BcTrhD7f/KakZsfNzcsPTNv/du4K7hpajRdGeETEMsHDhIsN1uL/Mdr+W4SplOhg+5eAObVozrdRKCxRY3+4xPWGF9j5D09jtuT0whNgQBOc+DrHA91t7v0/X9tSzNXUm1yZ2JrNIBcQPOLv9D5BOPQP6/xsQIPT12aZA3Rt1qSYv/BjTVsj+tfarX7R9GtEkQaiERIt778SQY7GdPWQRtQaNUCiB0DTdJST5dKppieadEDKR7KFFoM9WP82tV0iCQiIsVlw1w3ReXjqrN/0BfQp/ncKARqMqTsV9aQCUezPzp306l+udcXxGmI3YCFHu9fl/yyDghk7AwceuBICp86m1C4DSXKZFwH9fvCRQF3Pa2wjX2DRHATjfKexqE/NLppTaAFDsdS1NJycB5J3O1fZRGtp6JuaW1r4xEMvBqfrCmUqtFcXR7G9P+/onP/GxD1qyaMGcmqoxo0ZCuKzXvzx3V5wTR6cu+geIYzalQdE48SrDBRRSIb3oFhuvgHz1USlm250anQTYVMwY49dHZoB3M0H2bplqX/WP/TdOvU0Xi07k46JB/040lSKnAuFms0m4hFr+xRC+vJU7joXKpc3sSiL09N4MToQwoXU4HKQMWe8Q5Xg6p7IKJISA0gnUaZ9KKtwCMUNYKUV4DGG6n+LG/j4VCSZOxMffResrlXh64PK8tQnICggL8Szq6gi9cf7km250I44IC2tgTDhSQxGN6mF0TqQyTbNqkzoQVpB6T9VReWt9R1+iVuJjrfWVuEKlEcROpdFsFBL8GNAWwnvXcaX+k1nuKmbQ0ZbBnGoGzdBDTB0TOGcmVxe1Mn3FDZ+2vmgWdu6eEKKFOlMIXam/rPjDt7qRmwFjjHMOVCYDzt25KaVslUf0rW8r2y4llca0Riil7hfZ75zWDyRP20FuC8wUWCDEw4uKa6jMzo4+OLJswNChalRz+h9vaLdpNmeYblIq/6ajCQpiRKpaaaYQX/33EkA2izRCmHRdCB3WWmszjnsi+uCKteU6afJ1f0nJrB22UwajDFKOBMX+ouc33oq87w8qlwuKlZymthdlJtzFd3Obsdlvb89TkafS33PGIpW8vUggsLNU9EY/or4091t0ptSnvs2DVEWk20VvZO0uBDe2czKFMy2bbpjKC+7b0rHe5v1PKV/brcIjLFT/JOFZLlTMSSPOlLq/FP7UkpVURjkmOmukle8oslUEOP6cVfmZijp/Uio/1QSOwTzcOXkp/UkndQvgEuXgpx5tJ5BoLQRQpqdanQw5WVCzPNtJbryWNkcOjnUw0Vw3M4EAgqbSt4L9VXjJIQApP1RyU+zjASVjR5Az9b8LWublz9RbWxS75ca0Cx12c4R3VsRfu0sWsTrhhXOglQw75Ix74tvFvZnHTgdHmZZZcS5UqQX+VKeKE/O3YOTr4ZEzHLRR6vYe4sqmJT/XD0RemrSbV+gcQ+C+BU52FjMyX6mYQEW4KOPW46sw18vTG6A3bqn8tuPipGNmRNw5tt9dXbVtoTDCBiFEKn39QdPqlyvmdtzS0N16fMsLKxOe22Z7XtlbSmyeu5fVAERfnwoMGZMq6bzisNBGO3U3AeHvY9cM4v1SpsMsP93UdFibz1TUraDlW/5li8w8Z30bIJkVnnWE8J5SN5SrcDvEByeu2x0EGRWXxzT7FV6Zi13UESbNzsqEZ4DsdPJCS4We7zfG3TyXzKgARdbrVNRp7RWGmcpN0ggUgjkErtfQ4qSsBGorh1ArHUOEeexaKfJyst4zzWrZEW6aFfJdjm4PmyKGvIiTfAHBVGsmNCJL+4LKaC30gRTDCmz/fMWTPkpIptM2XgDkU8HvQ4ouiBBYhzjIPtCLgxfhSPz27GOBIlpqRB9hwWUOCG2S35nnbLtKrpYrg95B28cpzyrVDFs0AGTt1NG+XGXhFMZeFRpYnj5TyqNuQH/h/ib/P/vt1eSZ3ThOILNevyMsviKga7vfjdDtR5kpWNd64sFpaiwq1bumIuoi39pQV3imFFp3p9O0T7dUTHnQN6Zb4vYnU/U52IuLG4bFjf0H51So7eyE0VAbWMPkKgyNGoPFQfWmrUVAnCQRSYk/Sl8+mpDo0Cd2OJpbbe9f2UZE3Z66/7WG6LxGZyf4QqgBCXO4ubOxAuJ2cJafykM7BGITSxvC6pfm7hYlOBPRV0JXVjJRNrZHjDAQaEXwIt+Zkfcq19IimMKb8g8T0YOam55ODcjpgyN9G/PbbPj4YkuX1uoTrg4S4Dh91BdwqKifjv4YbBHnAijn2mlGhNL2wP9oRg658oGhIaNB8r529r1MTzqZglw2dNtZfttSbw7rKytsqBG0oqyv6xqNjStP6vJXe7r8JVtbXbXvszHb5/OxHfH2AnuU1zaL0yRO5n8i6Z9z61AJs8aRHVArtH/wA44Vi/AfxeCHfmDHt9TX5oj3JTKrD9TrnjD8Cz1ju3Zdo3/Ocy+AmknDjarcBNlRiFjsUQkDQFWR2S/NLfEIJnF8KNoZ6+/jmVjpwTeYeeEOkxl5VqseSSvRSrQFLOeDfUWoM3KyulyElSYu5LnBi8xsM9AL9AAmjU7BDZkCSuKHZ5ke2Vp3atosbImtKCTxQUwrx41wv+Pp2LIW4vBwPxRj09ooQ9ujrWHl8XQ+yKRMb95eUZ0TNPa1JWJdhDMt7KvsKkRY4mkxhnPgC7aSyWu0h1vpOnE5gmL06R1VlhaexWSNXNbqqdtJMamhUlFpn6kks8aRHtYhpFmQW3/Mw0LInXoWXW3b7r/Tt2ZGNSv3n3k0ZNs+U7ap+JmjAr9/xWcUaDdfRE8ke7ZClL4ti+wjBavO3oco+zxATaL06hsKR0lCVuuQMQ3NSdwbQ2qToJE/amt/KoXoy9PWjY8JxYkGhf7NDMXODkmbmw7pmQUSJIKoGnqYEi+sU6EoFoiF0zFwGJKGGFS8kqX2iJqp+jlN+u+c5zRUROcUhj5viaRRBp8eZWykYcdnIcPsTe7T2NUp+Ag9EPwX8zSdcuFpCiM9nJeYyseEKA/286Z6Lm6MRGCtq+6ctK1pdc+caPWPLXXlF83OhA2Pe1VffsU7SpKy2oWML0A+QMbBhy25HYfPqti/qzcp61Ob/A/I4pvWChAV/tod5FX108zULZGldZNNx71oTlWMCuA4kJYICBJ+R1LSHQobyAdQnZmSrrNi/TgI2Z/lI5Wjxb+EmS38qaFErospZ/P3ppTn+PIPlntSEujlzC5mUXYqQ/l7nh6294BcXEoNki0mNiI5A9SjsdxYphNTcQAgt5197TA9hSglVd++KxAI7u1VoihahnSGQxnayFEWTmGtlMtyZxy37665RoZ59MusDfkoS5OkzFK/4L5iFvaBRXKVCTrdzfG2lCwH0U1vuzThB1hPHJ1cmNksQ41ml8mKFSr5c+Fg7b7jjtnlrFVJcjRtkHCznmXISgVOtiiv0FKyR6WtsSlh3kyn2y8wJ8NSJfbWVyRlJt2eFcvHdps9m8Hwj/7/aIg8a4+rOM7cb4v1Noba5pQGVhkcWhp459uqsO5IL8Sp0zH8kilaBFoK4NVwOAOdh9YgKuqHiEBXbA6YtCYkC1uKuPVBIn/NhdpGtRS5RQtEHLwcYcpmwqk8uoru2s7NVbWX5bMbiY4xSg2HEDiGHppz85gnPI8ezwzkmlXPljmMcEUTxXg+nuFM5eeLC8WxncvLHetPI5Evp7tT1VRP0gtIQl3Y8mkrGTXLKSFQxErIztSwqjhZYURb1U7NNtVU8GzOkFSHr1xGM5V7fGTCuAsChPztr9WmKnO7/MSY4RmOu7osS9PgQXl5LYztchy275qZ+YxGNy9maV87WlkZFmpWRl42F4LbvVNornLzlxpa1tJ30cRQp/LDJktRICnjgc51xUMQoRvfNTtqb0nSNV+qfUmuppvClUcG4fj9XNvXnVvtKArcQQPyCqv8IiILG4Lfga3HwxaEesnJsKDX8X4F1sQMkU/BUBQNPnnX5bLeD83TDAcN4UC9/YlQt2+dhOEyrDscDqsD5GRWv8znknHYvtShGh0pE4DACaE60h+B2xhjcjHLJUHwX2zN6d1NMKwpiMxokNY9LcqakuG1Ragj6qonBk/6FtWmRa7FZczVDuj0ugGK6nLaKUoUTRUYyq2pdaM4f7FYucy7OGjm4WxtptHSpL4hOXWm+zRjMAu8IhXNUx2lp+K16UMoWh0f6MmNaK6NVM7wi+PLWhelcnJ5i0ZAM3L/SxRVaLiOmI6kQyXR44FCn80JjX34dFBUDLYtXTSJUy6cqIPg2JnTYXlACbYv52xIm+5oj+TGYZXZcV4p1Lr89jfJb9zDY/MNeT0ayXMkEU/MsSc2BZ7BvUBucNvclGZSRORaQ0ci0NetGp2hxLAW4sp7ASyvVtoyU0KNEFqcSh873hRsuYHIIAVyGjj5zZCjObBMpxMZmQ/Sti1oAqg9IkTE2nuGQBuoAE3sVm9SkWXjdCrNzjIpUcWx5jdvRASp+6GN1KIBKjUnZUmWa+ycb6FXMhyMTISTodvi0CyazIl+6jDk/lYcp+uIsRqPF7egbWFP0Gb7F3mMwXJMuD/CzCZDQ4JSCVw0wv4SoHBxgoYk6Rwa3RumPDVOyEyKM2pKYRp0WiBnDNGjqH3uPh6B3HNKK7MrlDQpl4cITgpBDIVVZwSz09dpV0MMk/ac5RDI97JyojkyJ9OVJOrC0z9zCrmcZYJRN9iynuDJkOKI5Iy0logHcdHpbKlTWn9RQKDwijYnsD3C0IJGNrOm5dkRpIzRs8FUod70/bmIwPujKX4QJHQxmURDmpN+CBAR763T4OCPdT8/ICLr4tlgDI2TWsTllDljAwqM8Rmf9sdzPNG1KKk3FItQNGbxNphPU97bQfFxce3RcgJPb9IdeB51e3vtfmUSHo5q0jW2oGytZOgI3Chq37U3PDM36D7LGD0PckCWjDVoz10Q6q4zRnQFsrNiYmiBZwYFXIgLuGMlCMxpV2eMTkvwIXCRXyEprXmWMeDwVCM7/eSfte2DcXz5w646BFUlRTQ/EY4xa/42WzkMhYiwwZlpD61PAgmCYYLlHhoi+meneQjrhuuQJRDXxGWRLpojlOL5fRIFcpNTGcrP/XQlIx9Zvn7deKeySC7YFIIu4Wqh4PJUOvOWe9fK/5tThmMoK6U6epH0mB8JrrEd9c9LuF8Kes9ftcTxvUWa6U+P6wg8LyQtO6iJNiKyKgRsJc8rZ+oIIgtpW4a45bogRVxEqP4Fxz/Xa9irmJam7LEMIwr9JVEiSX9+ekdUqX/BM090mnbgKESzJVJGcjKt+iiOqVufuHirewMU49ZfBuMknxd5O8Lk00fsx0NwPyTfEerPaCmZbqWgeKzTOmkryzqQ3Oysjz7BFQGnuCzSgXgN6jF14meNDWTR0JM52riqDiLi9+8glYj3Du+Qlz6w7KDJ3ZKjc2c87U3Jl3tXGhpnLpbSgyuyl2hdM2Jr5PJWp3vG0ZxnsZQZ2DhxEFiI/e8qL/Y/UY13JAQEgAvzUiHGnVzivfQhsGXD7K6fBdWrc86K9Z/0gddSk0rL7TAcY+asdWM4e2hbhmc3EOcUNVzh5CZsbAARFeiYNsMm4lwy42N4INBLLRqaUapZXqfc1nOpWRD44OVWDC/I4DikEG3cV3ebY79phSZIF9ARMepfcvwzIPgBKwACCSBAoWZYva+XAAAwHDxCViKx4Gz9l1qA0SCq/pt+w40gof6t6uAKYKn/YRVcA/zqH1oGdwHHaDJw0duF0XA1kOq/mIM1IKT+jZ7B+SBSUv6BC4Gz/qMrCkQa+myqJq17vqiG1pd2nt4bxdNpn3R6ok7PTep7/L2LBv9WpvW/Br30BGrjrui95yPNdFqoF37otTaUd9R08j317F7q/772+VBWoP5717vQE39vJDlAYHfQ9bU/dVdctscn1Wr0zntd5ACA+45Tx58ekn5ldOATgGd5ng4AvMRs0fp34JXB6QCiGADBv4oNWJkFHilJyJcn2fohYg1+nAW0IsjPx+9dTFrQm7qWoo6JgYEGWkhbXt9GW0EXLa7BAVT8wrmAYUAIGAAYARE2XaFgDGJH4P8lVrT8xZo0Z7EBDiXHBsOxIeI01HC2dYAxkw7hrMgYBgMWsySQ2DSMvZS4tLikLpOBbEDt1RsBRJUaqr8n9REWh8EeWSm5f5cr2a9Rv1DomNeRgq7WucdxFsI3gmviyQctZ3uIJJIriAqB+X6m5GXRtkAT6Hh6YepESWuIwmwo99KPzgVDI7hoSSS/kemWJobaj2AEY560J9TN5Hqx6BqtrlIGhGhG6n/2TMfUGjnrAEdCLY4qzZmt5ve1pyOeY4gdbwqSyxcBC0EHMs45OU28qMUAknsmEUzGU4RMGBBPqi2hxOJlYpl0VWWQokjvavJabo10EOMPJpaSs4uafzA+FUclgwZrPpRN95JJ6dBWIkQDSQflkfrNbTW/dAupaOY1R/gadoYpj25uoCMUKJNtByXNMrwdCIgicnKSxiTP4InE0iBzAv9h5rC9ualB4uLGP1i2lzWdDLP49r1glKwPZvMUJn8gbgCbxehRO3gMU8qGFU64m4hJoOV1/kSsw9KnuYVEEG+e6dofc9CMwz6S+r5uB0tdn2XvbWyIihzNK81h9/VNCmEmyyG8JqKLATgMQyHIaygRmovDK5cTFgZrfFh+xQ1aJq4kRINAuB/djw/TxxNQZAarBoHfhwfuQmtbIP0cY+a/8jZpnRg2RJKdUGh7oZGJtZcmAksidzcLUT/I+AVkTMkUz80H4TnRQv5aUYuRqGT8kOjpqr3ivg6QNczZyKEinlbCli+hCSqtQtpVkHmTIoZosVMN/MVqsuy5KkoPxLYfVGeQ6ncJ6h4Fssk2oBR2dxWjAzcEZaRAn16IlzkiWhbogQ+iyN0wg39DydpQZs4boTw5qCP+uVR1WGQna2KdLaaxyVmKPaBdel6RkMvBfa7sdgGTfKRL0YmE7Q3sUeeU4IVSeQlFYZKZkjLCmfhaAUEdC+4aBpaFUHDr3iA0IkJbkjszavAYRgCO2qyBfG1cq09k9HLhuoA6b8Id0HUCT+TMIT8niJHrRyTOwtHfguExFhHyQHgCzkAc0zpStqxh4OOif8c+LgE=); }example-namespace-2ToMainAllCrdDefinitionsGiteaRepo1ToTestClusterWatchRuleAllConfigMapsClusterWatchRuleGitProviderGitTargetGitTarget \ No newline at end of file diff --git a/hack/e2e/install-plain-manifests.sh b/hack/e2e/install-plain-manifests.sh index f4b4c341..9ae0f800 100755 --- a/hack/e2e/install-plain-manifests.sh +++ b/hack/e2e/install-plain-manifests.sh @@ -30,6 +30,13 @@ KUBECTL="${KUBECTL:-kubectl}" tmpdir="$(mktemp -d)" trap 'rm -rf "${tmpdir}"' EXIT +# CRDs ship as their own file and must exist (and be Established) BEFORE the bundle is applied: +# dist/install.yaml contains the reserved `default` ClusterProvider, a custom resource kubectl +# cannot map if its CRD is created in the same apply. This mirrors the documented two-step user +# flow (kubectl apply -f crds.yaml, then -f install.yaml), so the e2e exercises it for real. +"${KUBECTL}" --context "${CTX}" apply -f dist/crds.yaml +"${KUBECTL}" --context "${CTX}" wait --for=condition=Established --timeout=60s -f dist/crds.yaml + cp dist/install.yaml "${tmpdir}/install.yaml" # Patch Redis address diff --git a/hack/generate-audit-webhook-kubeconfig.sh b/hack/generate-audit-webhook-kubeconfig.sh index 76b94030..ff69944f 100755 --- a/hack/generate-audit-webhook-kubeconfig.sh +++ b/hack/generate-audit-webhook-kubeconfig.sh @@ -7,7 +7,10 @@ set -euo pipefail CTX="${CTX:-k3d-gitops-reverser-test-e2e}" NAMESPACE="${NAMESPACE:-gitops-reverser}" AUDIT_CLUSTER_ID="${AUDIT_CLUSTER_ID:-kind-e2e}" -AUDIT_WEBHOOK_SERVER_URL="${AUDIT_WEBHOOK_SERVER_URL:-https://127.0.0.1:30444/audit-webhook}" +# Audit routes are NAMED (/audit-webhook/); "default" is the provider a +# GitTarget references when it omits spec.clusterProviderRef. The bare /audit-webhook is the +# shared, annotation-routed endpoint and 400s unless the cluster annotation key is configured. +AUDIT_WEBHOOK_SERVER_URL="${AUDIT_WEBHOOK_SERVER_URL:-https://127.0.0.1:30444/audit-webhook/default}" AUDIT_TLS_SERVER_NAME="${AUDIT_TLS_SERVER_NAME:-gitops-reverser-audit.${NAMESPACE}.svc}" WAIT_RETRIES="${WAIT_RETRIES:-60}" WAIT_SECONDS="${WAIT_SECONDS:-2}" diff --git a/internal/audit/outcome/outcome.go b/internal/audit/outcome/outcome.go index 5ea296e2..e0d18d31 100644 --- a/internal/audit/outcome/outcome.go +++ b/internal/audit/outcome/outcome.go @@ -80,6 +80,14 @@ const ( OlderThanHighWater Outcome = "older_than_high_water" // NonNumericRV — an RV that is not a uint64 (aggregated apiservers). NonNumericRV Outcome = "non_numeric_rv" + // MissingClusterAnnotation — an event on the shared (annotation-routed) /audit-webhook endpoint + // that carries no source-cluster annotation, so it names no ClusterProvider. Never credited to a + // fallback; a rising rate means a producer is not stamping the annotation. + MissingClusterAnnotation Outcome = "missing_cluster_annotation" + // UnknownClusterProvider — the source-cluster annotation names a ClusterProvider that does not + // exist. Dropped rather than guessed: a wrong source cluster would let a user from one logical + // cluster author a matching object in another. + UnknownClusterProvider Outcome = "unknown_cluster_provider" // WriteError — a redis/enqueue failure; the event never reached the log. WriteError Outcome = "write_error" @@ -95,7 +103,8 @@ func (o Outcome) Category() Category { return Held case NotNeeded, NilEvent, Stage, ReadOnlyOrUnknownVerb, FailedRequest, DryRun, UnchangedResourceVersion, MalformedAdditional, NonScaleSubresource, - ShallowDropped, RVLessEmptyHighWater, OlderThanHighWater, NonNumericRV: + ShallowDropped, RVLessEmptyHighWater, OlderThanHighWater, NonNumericRV, + MissingClusterAnnotation, UnknownClusterProvider: return Dropped case WriteError: return Error diff --git a/internal/audit/outcome/outcome_test.go b/internal/audit/outcome/outcome_test.go index 9860dee7..bb1faa12 100644 --- a/internal/audit/outcome/outcome_test.go +++ b/internal/audit/outcome/outcome_test.go @@ -7,8 +7,14 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" + + "github.com/ConfigButler/gitops-reverser/internal/telemetry" ) +const auditEventsMetric = "gitopsreverser_audit_events_total" + func TestOutcomeCategory(t *testing.T) { cases := map[Outcome]Category{ Queued: Stored, @@ -26,6 +32,8 @@ func TestOutcomeCategory(t *testing.T) { RVLessEmptyHighWater: Dropped, OlderThanHighWater: Dropped, NonNumericRV: Dropped, + MissingClusterAnnotation: Dropped, + UnknownClusterProvider: Dropped, WriteError: Error, } for o, want := range cases { @@ -39,7 +47,163 @@ func TestOutcomeCategory(t *testing.T) { func TestRecordNilGuard(t *testing.T) { // With telemetry uninitialized (AuditEventsTotal == nil) Record is a no-op and must not panic. + // This test must stay ahead of the tests below: InitTestExporter wires the global + // instrument permanently, so the nil branch is only reachable before the first call. assert.NotPanics(t, func() { Record(context.Background(), nil, Queued) }) } + +// TestGvrParts_LabelValues pins how an objectRef is split into the group/version/resource +// labels. These land on a counter, so a mis-split silently forks one logical series into +// two (or merges two unrelated ones) and every audit dashboard/alert keyed on them lies. +func TestGvrParts_LabelValues(t *testing.T) { + tests := []struct { + name string + ref *auditv1.ObjectReference + wantGroup string + wantVersion string + wantResrc string + }{ + { + name: "grouped apiVersion splits on the slash", + ref: &auditv1.ObjectReference{APIGroup: "apps", APIVersion: "apps/v1", Resource: "deployments"}, + wantGroup: "apps", + wantVersion: "v1", + wantResrc: "deployments", + }, + { + // Core-group events carry a bare "v1", so the group label is the (empty) APIGroup + // rather than "unknown" — core resources must not be bucketed with malformed ones. + name: "bare apiVersion falls back to APIGroup for the group label", + ref: &auditv1.ObjectReference{APIGroup: "", APIVersion: "v1", Resource: "configmaps"}, + wantGroup: "", + wantVersion: "v1", + wantResrc: "configmaps", + }, + { + // APIVersion is authoritative when it carries a group: a disagreeing APIGroup must + // not win, or the same GVR would be counted under two different group labels. + name: "slashed apiVersion wins over a disagreeing APIGroup", + ref: &auditv1.ObjectReference{APIGroup: "other", APIVersion: "apps/v1", Resource: "deployments"}, + wantGroup: "apps", + wantVersion: "v1", + wantResrc: "deployments", + }, + { + // An empty APIVersion means no usable identity at all, so the group/resource that + // *are* present are deliberately discarded — a partial series would be misleading. + name: "empty apiVersion discards the rest of the objectRef", + ref: &auditv1.ObjectReference{APIGroup: "apps", APIVersion: "", Resource: "deployments"}, + wantGroup: "unknown", + wantVersion: "unknown", + wantResrc: "unknown", + }, + { + name: "missing resource is labelled unknown", + ref: &auditv1.ObjectReference{APIGroup: "apps", APIVersion: "apps/v1", Resource: ""}, + wantGroup: "apps", + wantVersion: "v1", + wantResrc: "unknown", + }, + { + // Cut takes only the first separator, so anything past it stays in the version. + name: "extra slashes stay in the version", + ref: &auditv1.ObjectReference{APIVersion: "a/b/c", Resource: "things"}, + wantGroup: "a", + wantVersion: "b/c", + wantResrc: "things", + }, + { + name: "leading slash yields an empty group", + ref: &auditv1.ObjectReference{APIGroup: "apps", APIVersion: "/v1", Resource: "things"}, + wantGroup: "", + wantVersion: "v1", + wantResrc: "things", + }, + { + name: "trailing slash yields an empty version", + ref: &auditv1.ObjectReference{APIVersion: "apps/", Resource: "things"}, + wantGroup: "apps", + wantVersion: "", + wantResrc: "things", + }, + { + name: "nil objectRef", + ref: nil, + wantGroup: "unknown", + wantVersion: "unknown", + wantResrc: "unknown", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + group, version, resource := gvrParts(&auditv1.Event{ObjectRef: tt.ref}) + assert.Equal(t, tt.wantGroup, group, "group") + assert.Equal(t, tt.wantVersion, version, "version") + assert.Equal(t, tt.wantResrc, resource, "resource") + }) + } + + // A nil event is defensive: Record is called from paths that terminate on a nil event + // (the NilEvent outcome), and it must still produce a full, bounded label set. + group, version, resource := gvrParts(nil) + assert.Equal(t, "unknown", group) + assert.Equal(t, "unknown", version) + assert.Equal(t, "unknown", resource) +} + +// TestVerb_NilEventAndPassthrough pins the verb label. Unlike gvrParts it has no "unknown" +// sentinel: a nil event yields an empty label, which is the value dashboards must expect. +func TestVerb_NilEventAndPassthrough(t *testing.T) { + assert.Empty(t, verb(nil), "nil event must not panic and yields an empty verb label") + assert.Empty(t, verb(&auditv1.Event{}), "an unset verb is passed through as empty, not unknown") + assert.Equal(t, "create", verb(&auditv1.Event{Verb: "create"})) +} + +// TestRecord_EmitsLabelledSample covers the recording branch of Record (telemetry +// initialized) and pins the six labels it stamps, including that category is derived from +// the outcome rather than passed in — an alert on category="error" depends on that. +func TestRecord_EmitsLabelledSample(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + ctx := context.Background() + Record(ctx, &auditv1.Event{ + Verb: "update", + ObjectRef: &auditv1.ObjectReference{APIGroup: "apps", APIVersion: "apps/v1", Resource: "deployments"}, + }, WriteError) + + count, ok := telemetry.CollectInt64Sum(reader, auditEventsMetric, map[string]string{ + "outcome": string(WriteError), + "category": string(Error), + "group": "apps", + "version": "v1", + "resource": "deployments", + "verb": "update", + }) + require.True(t, ok, "expected a sample for the recorded write_error outcome") + assert.Equal(t, int64(1), count) +} + +// TestRecord_NilEventStillRecords guards the defensive path: the NilEvent outcome is +// recorded with no event at all, and must still emit one bounded, fully-labelled sample +// rather than being dropped from the census. +func TestRecord_NilEventStillRecords(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + Record(context.Background(), nil, NilEvent) + + count, ok := telemetry.CollectInt64Sum(reader, auditEventsMetric, map[string]string{ + "outcome": string(NilEvent), + "category": string(Dropped), + "group": "unknown", + "version": "unknown", + "resource": "unknown", + "verb": "", + }) + require.True(t, ok, "expected a sample for the nil-event outcome") + assert.Equal(t, int64(1), count) +} diff --git a/internal/controller/clusterprovider_controller.go b/internal/controller/clusterprovider_controller.go new file mode 100644 index 00000000..9e50c92c --- /dev/null +++ b/internal/controller/clusterprovider_controller.go @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + "sync" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/go-logr/logr" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/kubeconfig" +) + +// LegacyClusterProviderFinalizer is the fact-purge finalizer this controller USED to take. It is +// no longer added; it is only ever REMOVED, so an object created by an older operator can still be +// deleted after an upgrade instead of stranding in Terminating. +// +// It was dropped because purge-on-delete is a promise only a living operator can keep, and the two +// cases that matter break it: `helm uninstall` removes the manager and the ClusterProvider +// together, so nothing detaches the finalizer and the object strands forever (which then blocks +// reinstalling); and an operator that is simply down blocks the delete, then loses the purge anyway +// if the object is force-removed. Nor is the purge needed: facts are keyed by +// (cluster, group/resource, uid, resourceVersion) and expire on their own, so a re-provisioned +// cluster reusing a provider name produces different object UIDs and cannot join a stale fact on +// the exact key. See docs/finished/clusterprovider-fact-purge.md. +const LegacyClusterProviderFinalizer = "configbutler.ai/clusterprovider-fact-purge" + +// ClusterProviderReconciler reconciles a ClusterProvider object. It is the read-side peer of the +// GitProviderReconciler: it validates the cluster's connectivity inputs (spec.kubeConfig) without +// dialing, and owns the per-cluster status the watch engine and GitTargets project from. The +// in-cluster "default" provider has no kubeConfig and is trivially Validated. +type ClusterProviderReconciler struct { + client.Client + + Scheme *runtime.Scheme + + // OperatorNamespace is the namespace a remote provider's kubeConfig Secret is pinned to (the + // operator's own namespace). A cluster-scoped provider has no namespace of its own, so the + // credential for a cluster is always read from here — never from the source cluster. + OperatorNamespace string + + // KubeConfigSafety gates exec-auth and insecure-TLS kubeconfigs (reject-not-strip), matching + // what the watch engine's resolver enforces, so Validated agrees with what a watch would use. + KubeConfigSafety kubeconfig.SafetyPolicy + + firsts clusterProviderLogFirsts +} + +// clusterProviderLogFirsts keeps startup progress visible without turning every routine +// revalidation into default-level log noise. +type clusterProviderLogFirsts struct { + validationSuccess sync.Once +} + +// This reconciler reads providers, updates them to shed the retired finalizer, and writes status. +// It never creates or deletes one, so it takes neither verb. +// +kubebuilder:rbac:groups=configbutler.ai,resources=clusterproviders,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=configbutler.ai,resources=clusterproviders/status,verbs=get;update;patch + +// Reconcile validates a ClusterProvider's inputs and updates its status. +func (r *ClusterProviderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx).WithName("ClusterProviderReconciler") + log.V(1).Info("Starting reconciliation", "namespacedName", req.NamespacedName) + + var provider configbutleraiv1alpha3.ClusterProvider + if err := r.Get(ctx, req.NamespacedName, &provider); err != nil { + if client.IgnoreNotFound(err) == nil { + log.Info("ClusterProvider not found, was likely deleted", "name", req.Name) + return ctrl.Result{}, nil + } + log.Error(err, "unable to fetch ClusterProvider", "name", req.Name) + return ctrl.Result{}, err + } + + // This controller takes NO finalizer. It only sheds the legacy one, so an object created by + // an older operator can still be deleted after an upgrade. Deletion is otherwise ordinary: + // nothing has to happen before a ClusterProvider goes away. + shed, err := r.shedLegacyFinalizer(ctx, log, &provider) + if err != nil { + return ctrl.Result{}, err + } + if shed { + return ctrl.Result{}, nil + } + if !provider.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + return r.reconcileClusterProvider(ctx, log, &provider) +} + +// shedLegacyFinalizer removes the retired fact-purge finalizer if the object still carries one, and +// reports whether it did (in which case the update it wrote will re-trigger this reconcile). +// +// This is the upgrade path, and it must run BEFORE the deletion check: an object stuck in +// Terminating from an older operator is only recoverable by removing the finalizer, and a +// controller that returns early on deletionTimestamp would leave it stranded forever. +// +// A failed Update is returned as an error rather than swallowed: a stranded object gets no further +// events of its own, so dropping a transient conflict here would leave it in Terminating until an +// operator restart. The error is what makes the workqueue retry. +func (r *ClusterProviderReconciler) shedLegacyFinalizer( + ctx context.Context, + log logr.Logger, + provider *configbutleraiv1alpha3.ClusterProvider, +) (bool, error) { + if !controllerutil.RemoveFinalizer(provider, LegacyClusterProviderFinalizer) { + return false, nil + } + if err := r.Update(ctx, provider); err != nil { + log.Error(err, "remove retired ClusterProvider finalizer failed; will retry", "name", provider.Name) + return false, fmt.Errorf("remove retired finalizer from ClusterProvider %s: %w", provider.Name, err) + } + log.Info("removed the retired fact-purge finalizer", "name", provider.Name) + return true, nil +} + +// reconcileClusterProvider performs the main validation logic. +func (r *ClusterProviderReconciler) reconcileClusterProvider( + ctx context.Context, + log logr.Logger, + provider *configbutleraiv1alpha3.ClusterProvider, +) (ctrl.Result, error) { + log.V(1).Info("Validating ClusterProvider", + "name", provider.Name, + "inCluster", provider.IsInCluster(), + "generation", provider.Generation) + + r.setProgressingConditions(provider, ReasonChecking, "Validating cluster provider inputs...") + + valid, reason, message, err := r.validateProviderKubeConfig(ctx, provider) + if err != nil { + log.Error(err, "failed to read ClusterProvider kubeconfig Secret", "name", provider.Name) + return ctrl.Result{}, err + } + if !valid { + r.setCondition(provider, ClusterProviderConditionValidated, metav1.ConditionFalse, reason, message) + r.setStalledConditions(provider, reason, message) + // A failed status write is a real failure, not a verdict: propagate it so the invalid + // provider is retried rather than left reporting a stale status for a whole steady interval. + return r.updateStatusAndRequeue(ctx, provider) + } + + r.setCondition(provider, ClusterProviderConditionValidated, metav1.ConditionTrue, reason, message) + r.setReadyConditions(provider, message) + + if err := r.updateStatusWithRetry(ctx, provider); err != nil { + log.Error(err, "failed to update ClusterProvider status", "name", provider.Name) + return ctrl.Result{}, err + } + + r.firsts.validationSuccess.Do(func() { + log.Info("First ClusterProvider validation completed successfully", "name", provider.Name) + }) + return ctrl.Result{RequeueAfter: RequeueSteadyInterval}, nil +} + +// validateProviderKubeConfig is the legibility gate for spec.kubeConfig (feeds Validated). It +// reads and parses the kubeconfig Secret FROM THE OPERATOR NAMESPACE and applies the exec/TLS +// safety policy, but deliberately does NOT dial the cluster — reachability is a runtime signal the +// watch engine records on the Reachable condition. It returns ok=false with a typed reason and a +// legible message when an input is wrong; a non-NotFound read error is returned as err so the +// reconcile requeues rather than falsely reporting a bad input. An omitted kubeConfig is an +// in-cluster provider — trivially valid, regardless of its name. +func (r *ClusterProviderReconciler) validateProviderKubeConfig( + ctx context.Context, + provider *configbutleraiv1alpha3.ClusterProvider, +) (bool, string, string, error) { + if provider.IsInCluster() { + return true, ReasonInCluster, "in-cluster provider (no kubeConfig); the operator's own cluster", nil + } + if provider.Spec.KubeConfig.SecretRef == nil { + // A remote provider needs a Secret reference. configMapRef is CEL-rejected, so secretRef is + // the only supported path. + return false, ReasonKubeConfigInvalid, + "spec.kubeConfig.secretRef is required for a remote ClusterProvider", nil + } + ref := provider.Spec.KubeConfig.SecretRef + secretKey := k8stypes.NamespacedName{Namespace: r.OperatorNamespace, Name: ref.Name} + + var secret corev1.Secret + if getErr := r.Get(ctx, secretKey, &secret); getErr != nil { + if apierrors.IsNotFound(getErr) { + return false, kubeconfig.ReasonSecretNotFound, fmt.Sprintf( + "spec.kubeConfig.secretRef names Secret %s, which does not exist in the operator namespace", + secretKey), nil + } + return false, "", "", fmt.Errorf("read kubeconfig Secret %s: %w", secretKey, getErr) + } + + raw, usedKey, present := kubeconfig.ResolveKey(secret.Data, ref.Key) + if !present { + return false, kubeconfig.ReasonKeyNotFound, fmt.Sprintf( + "kubeconfig Secret %s has no kubeconfig under key %q (set spec.kubeConfig.secretRef.key)", + secretKey, describeKubeConfigKey(ref.Key)), nil + } + if rej := kubeconfig.Check(raw, r.KubeConfigSafety); rej != nil { + // A RejectionError is a validation VERDICT, not a reconcile error: surfaced as the + // Validated=False reason/message, reconcile requeues normally (err == nil). + //nolint:nilerr + return false, rej.Reason, fmt.Sprintf("kubeconfig Secret %s key %q: %s", secretKey, usedKey, rej.Message), nil + } + return true, ReasonValidated, fmt.Sprintf("kubeconfig Secret %s validated", secretKey), nil +} + +func (r *ClusterProviderReconciler) setReadyConditions( + provider *configbutleraiv1alpha3.ClusterProvider, + message string, +) { + r.setCondition(provider, ConditionTypeReady, metav1.ConditionTrue, ConditionTypeReady, message) + r.setCondition(provider, ConditionTypeReconciling, metav1.ConditionFalse, ConditionTypeReady, + "Reconciliation complete") + r.setCondition(provider, ConditionTypeStalled, metav1.ConditionFalse, ConditionTypeReady, + "ClusterProvider is not stalled") +} + +func (r *ClusterProviderReconciler) setProgressingConditions( + provider *configbutleraiv1alpha3.ClusterProvider, + reason, message string, +) { + r.setCondition(provider, ConditionTypeReady, metav1.ConditionFalse, reason, message) + r.setCondition(provider, ConditionTypeReconciling, metav1.ConditionTrue, reason, message) + r.setCondition(provider, ConditionTypeStalled, metav1.ConditionFalse, reason, + "Reconciliation is making progress") +} + +func (r *ClusterProviderReconciler) setStalledConditions( + provider *configbutleraiv1alpha3.ClusterProvider, + reason, message string, +) { + r.setCondition(provider, ConditionTypeReady, metav1.ConditionFalse, reason, message) + r.setCondition(provider, ConditionTypeReconciling, metav1.ConditionFalse, reason, + "Reconciliation is stalled") + r.setCondition(provider, ConditionTypeStalled, metav1.ConditionTrue, reason, message) +} + +// setCondition sets or updates one condition by type and pins observedGeneration. +func (r *ClusterProviderReconciler) setCondition( + provider *configbutleraiv1alpha3.ClusterProvider, + conditionType string, + status metav1.ConditionStatus, + reason, message string, +) { + provider.Status.ObservedGeneration = provider.Generation + provider.Status.Conditions = upsertCondition( + provider.Status.Conditions, + conditionType, + status, + reason, + message, + provider.Generation, + ) +} + +// updateStatusAndRequeue updates the status and requeues on the steady interval. +func (r *ClusterProviderReconciler) updateStatusAndRequeue( + ctx context.Context, + provider *configbutleraiv1alpha3.ClusterProvider, +) (ctrl.Result, error) { + if err := r.updateStatusWithRetry(ctx, provider); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: RequeueSteadyInterval}, nil +} + +// updateStatusWithRetry updates the status, re-reading the latest object on conflict. +func (r *ClusterProviderReconciler) updateStatusWithRetry( + ctx context.Context, + provider *configbutleraiv1alpha3.ClusterProvider, +) error { + return wait.ExponentialBackoff(wait.Backoff{ + Duration: RetryInitialDuration, + Factor: RetryBackoffFactor, + Jitter: RetryBackoffJitter, + Steps: RetryMaxSteps, + }, func() (bool, error) { + latest := &configbutleraiv1alpha3.ClusterProvider{} + key := client.ObjectKeyFromObject(provider) + if err := r.Get(ctx, key, latest); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + latest.Status = provider.Status + if err := r.Status().Update(ctx, latest); err != nil { + if apierrors.IsConflict(err) { + return false, nil + } + return false, err + } + return true, nil + }) +} + +// clusterProviderReconcilePredicate admits spec changes and the start of deletion. A deletion +// transition is needed only for the upgrade path: an older object can still carry the retired +// fact-purge finalizer, and this controller must shed it promptly rather than waiting for periodic +// reconciliation. The controller never adds a finalizer itself. +func clusterProviderReconcilePredicate() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return true + } + oldDeleting := e.ObjectOld.GetDeletionTimestamp() != nil + newDeleting := e.ObjectNew.GetDeletionTimestamp() != nil + return oldDeleting != newDeleting || e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() + }, + } +} + +// SetupWithManager sets up the controller with the Manager. +func (r *ClusterProviderReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For( + &configbutleraiv1alpha3.ClusterProvider{}, + builder.WithPredicates(clusterProviderReconcilePredicate()), + ). + Named("clusterprovider"). + Complete(r) +} diff --git a/internal/controller/clusterprovider_controller_test.go b/internal/controller/clusterprovider_controller_test.go new file mode 100644 index 00000000..5553714b --- /dev/null +++ b/internal/controller/clusterprovider_controller_test.go @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "time" + + meta "github.com/fluxcd/pkg/apis/meta" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +var _ = Describe("ClusterProvider Controller", func() { + const timeout = 10 * time.Second + const interval = 200 * time.Millisecond + + readyStatus := func(name string) func() metav1.ConditionStatus { + return func() metav1.ConditionStatus { + var got configbutleraiv1alpha3.ClusterProvider + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: name}, &got); err != nil { + return metav1.ConditionUnknown + } + c := findCondition(got.Status.Conditions, ConditionTypeReady) + if c == nil { + return metav1.ConditionUnknown + } + return c.Status + } + } + + It("validates the in-cluster 'default' provider and goes Ready", func() { + // The "default" ClusterProvider is created once in BeforeSuite (as an install ships it — the + // operator never creates one), so this spec asserts the existing object reconciles to Ready + // rather than creating its own; "default" is cluster-scoped and name-unique. + Eventually(readyStatus("default"), timeout, interval).Should(Equal(metav1.ConditionTrue)) + + var got configbutleraiv1alpha3.ClusterProvider + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "default"}, &got)).To(Succeed()) + Expect(findCondition(got.Status.Conditions, ClusterProviderConditionValidated).Status). + To(Equal(metav1.ConditionTrue)) + }) + + It("admits any name that omits kubeConfig — in-cluster is not reserved to 'default'", func() { + // kubeConfig is optional for EVERY provider: omitted means the operator's own cluster, + // whatever the object is called. "default" only names what an omitted clusterProviderRef + // points at; it makes no claim about which cluster that is. + provider := &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "local-extra"}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{ + AllowedNamespaces: &configbutleraiv1alpha3.AllowedNamespaces{Names: []string{"default"}}, + }, + } + Expect(k8sClient.Create(context.Background(), provider)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(context.Background(), provider) }) + + Eventually(readyStatus("local-extra"), timeout, interval).Should(Equal(metav1.ConditionTrue)) + }) + + It("does not forbid a 'default' provider that sets a kubeConfig", func() { + // The converse of the spec above: a provider named "default" MAY mirror a remote cluster. + // The existing "default" object makes this create collide on the name, so the assertion is + // precisely that the rejection is the name being taken — NOT a schema rule tying the name + // "default" to an absent kubeConfig, which no longer exists. + provider := &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{ + KubeConfig: &meta.KubeConfigReference{SecretRef: &meta.SecretKeyReference{Name: "kc"}}, + }, + } + err := k8sClient.Create(context.Background(), provider) + Expect(apierrors.IsAlreadyExists(err)).To(BeTrue(), "expected a name collision, got %v", err) + Expect(apierrors.IsInvalid(err)).To(BeFalse(), "the name must not constrain kubeConfig") + }) + + It("validates a remote provider with a valid kubeconfig Secret", func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "prod-eu-kc"}, + Data: map[string][]byte{"value": []byte(scValidKubeConfig)}, + } + Expect(k8sClient.Create(context.Background(), secret)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(context.Background(), secret) }) + + provider := &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "prod-eu-1"}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{ + KubeConfig: &meta.KubeConfigReference{SecretRef: &meta.SecretKeyReference{Name: "prod-eu-kc"}}, + }, + } + Expect(k8sClient.Create(context.Background(), provider)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(context.Background(), provider) }) + + Eventually(readyStatus("prod-eu-1"), timeout, interval).Should(Equal(metav1.ConditionTrue)) + }) + + It("stalls a remote provider whose kubeconfig Secret is missing", func() { + provider := &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "prod-us-1"}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{ + KubeConfig: &meta.KubeConfigReference{SecretRef: &meta.SecretKeyReference{Name: "absent-kc"}}, + }, + } + Expect(k8sClient.Create(context.Background(), provider)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(context.Background(), provider) }) + + Eventually(readyStatus("prod-us-1"), timeout, interval).Should(Equal(metav1.ConditionFalse)) + + var got configbutleraiv1alpha3.ClusterProvider + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "prod-us-1"}, &got)).To(Succeed()) + Expect(findCondition(got.Status.Conditions, ClusterProviderConditionValidated).Status). + To(Equal(metav1.ConditionFalse)) + Expect(findCondition(got.Status.Conditions, ConditionTypeStalled).Status). + To(Equal(metav1.ConditionTrue)) + }) +}) diff --git a/internal/controller/clusterprovider_controller_unit_test.go b/internal/controller/clusterprovider_controller_unit_test.go new file mode 100644 index 00000000..22eaf4d6 --- /dev/null +++ b/internal/controller/clusterprovider_controller_unit_test.go @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "errors" + "testing" + + meta "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/kubeconfig" +) + +const cpOperatorNS = "gitops-reverser-system" + +func clusterProviderWithKubeConfig(name, secretName, key string) *configbutleraiv1alpha3.ClusterProvider { + p := &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + } + if secretName != "" { + p.Spec.KubeConfig = &meta.KubeConfigReference{ + SecretRef: &meta.SecretKeyReference{Name: secretName, Key: key}, + } + } + return p +} + +func TestValidateProviderKubeConfig_AllScenarios(t *testing.T) { + tests := []struct { + name string + provider *configbutleraiv1alpha3.ClusterProvider + secretData map[string][]byte + safety kubeconfig.SafetyPolicy + wantOK bool + wantReason string + }{ + { + name: "in-cluster default (no kubeConfig) is valid", + provider: clusterProviderWithKubeConfig("default", "", ""), + wantOK: true, wantReason: ReasonInCluster, + }, + { + name: "missing Secret", + provider: clusterProviderWithKubeConfig("prod-eu-1", "absent", ""), + wantOK: false, wantReason: kubeconfig.ReasonSecretNotFound, + }, + { + name: "missing key", + provider: clusterProviderWithKubeConfig("prod-eu-1", "kc", "value"), + secretData: map[string][]byte{"elsewhere": []byte(scValidKubeConfig)}, + wantOK: false, wantReason: kubeconfig.ReasonKeyNotFound, + }, + { + name: "unparseable", + provider: clusterProviderWithKubeConfig("prod-eu-1", "kc", ""), + secretData: map[string][]byte{"value": []byte("not a kubeconfig")}, + wantOK: false, wantReason: kubeconfig.ReasonInvalid, + }, + { + name: "exec rejected", + provider: clusterProviderWithKubeConfig("prod-eu-1", "kc", ""), + secretData: map[string][]byte{"value": []byte(scExecKubeConfig)}, + wantOK: false, wantReason: kubeconfig.ReasonExecNotAllowed, + }, + { + name: "insecure TLS rejected", + provider: clusterProviderWithKubeConfig("prod-eu-1", "kc", ""), + secretData: map[string][]byte{"value": []byte(scInsecureKubeConfig)}, + wantOK: false, wantReason: kubeconfig.ReasonInsecureTLSNotAllowed, + }, + { + name: "valid via value.yaml fallback", + provider: clusterProviderWithKubeConfig("prod-eu-1", "kc", ""), + secretData: map[string][]byte{"value.yaml": []byte(scValidKubeConfig)}, + wantOK: true, wantReason: ReasonValidated, + }, + { + name: "exec allowed when opted in", + provider: clusterProviderWithKubeConfig("prod-eu-1", "kc", ""), + secretData: map[string][]byte{"value": []byte(scExecKubeConfig)}, + safety: kubeconfig.SafetyPolicy{AllowExec: true}, + wantOK: true, wantReason: ReasonValidated, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + builder := fake.NewClientBuilder().WithScheme(scScheme(t)) + if tc.secretData != nil { + builder = builder.WithObjects(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: cpOperatorNS, Name: "kc"}, + Data: tc.secretData, + }) + } + r := &ClusterProviderReconciler{ + Client: builder.Build(), + OperatorNamespace: cpOperatorNS, + KubeConfigSafety: tc.safety, + } + ok, reason, msg, err := r.validateProviderKubeConfig(context.Background(), tc.provider) + require.NoError(t, err) + assert.Equal(t, tc.wantOK, ok) + assert.Equal(t, tc.wantReason, reason) + assert.NotEmpty(t, msg) + }) + } +} + +// TestClusterProviderReconcile_InClusterDefault checks the reserved "default" provider validates +// and goes Ready with no kubeConfig Secret to read. +func TestClusterProviderReconcile_InClusterDefault(t *testing.T) { + provider := clusterProviderWithKubeConfig("default", "", "") + cl := fake.NewClientBuilder(). + WithScheme(scScheme(t)). + WithObjects(provider). + WithStatusSubresource(&configbutleraiv1alpha3.ClusterProvider{}). + Build() + r := &ClusterProviderReconciler{Client: cl, OperatorNamespace: cpOperatorNS} + + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: k8stypes.NamespacedName{Name: "default"}}) + require.NoError(t, err) + + var got configbutleraiv1alpha3.ClusterProvider + require.NoError(t, cl.Get(context.Background(), k8stypes.NamespacedName{Name: "default"}, &got)) + assert.Equal( + t, + metav1.ConditionTrue, + findCondition(got.Status.Conditions, ClusterProviderConditionValidated).Status, + ) + assert.Equal(t, metav1.ConditionTrue, findCondition(got.Status.Conditions, ConditionTypeReady).Status) + assert.Equal(t, got.Generation, got.Status.ObservedGeneration) +} + +// TestClusterProviderReconcile_TakesNoFinalizer pins the contract: this controller never makes a +// ClusterProvider undeletable. Nothing has to happen before one goes away — attribution facts are +// keyed by (cluster, group/resource, uid, resourceVersion) and expire on their own, so a +// re-provisioned cluster reusing a provider name mints different object UIDs and cannot join a +// stale fact. See docs/finished/clusterprovider-fact-purge.md. +func TestClusterProviderReconcile_TakesNoFinalizer(t *testing.T) { + provider := clusterProviderWithKubeConfig("default", "", "") + cl := fake.NewClientBuilder(). + WithScheme(scScheme(t)). + WithStatusSubresource(&configbutleraiv1alpha3.ClusterProvider{}). + WithObjects(provider). + Build() + r := &ClusterProviderReconciler{Client: cl, OperatorNamespace: cpOperatorNS} + + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: k8stypes.NamespacedName{Name: "default"}}) + require.NoError(t, err) + + var got configbutleraiv1alpha3.ClusterProvider + require.NoError(t, cl.Get(context.Background(), k8stypes.NamespacedName{Name: "default"}, &got)) + assert.Empty(t, got.Finalizers, "a ClusterProvider must never be held by this controller") +} + +// TestClusterProviderReconcile_ShedsLegacyFinalizer is the upgrade path. An object created by an +// older operator carries the retired fact-purge finalizer; after this upgrade nothing would ever +// remove it, so it would strand in Terminating forever and block reinstalling. The controller has +// to actively shed it — including while the object is ALREADY deleting, which is the state a +// stranded object is in. +func TestClusterProviderReconcile_ShedsLegacyFinalizer(t *testing.T) { + tests := []struct { + name string + deleting bool + }{ + {name: "live object carrying the retired finalizer", deleting: false}, + {name: "object already stranded in Terminating", deleting: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider := clusterProviderWithKubeConfig("prod-eu-1", "kc", "") + provider.Finalizers = []string{LegacyClusterProviderFinalizer} + if tt.deleting { + now := metav1.Now() + provider.DeletionTimestamp = &now + } + cl := fake.NewClientBuilder(). + WithScheme(scScheme(t)). + WithStatusSubresource(&configbutleraiv1alpha3.ClusterProvider{}). + WithObjects(provider). + Build() + r := &ClusterProviderReconciler{Client: cl, OperatorNamespace: cpOperatorNS} + + _, err := r.Reconcile( + context.Background(), + ctrl.Request{NamespacedName: k8stypes.NamespacedName{Name: "prod-eu-1"}}, + ) + require.NoError(t, err) + + var got configbutleraiv1alpha3.ClusterProvider + getErr := cl.Get(context.Background(), k8stypes.NamespacedName{Name: "prod-eu-1"}, &got) + if tt.deleting { + // Shedding the last finalizer on a deleting object lets it go immediately. + assert.True(t, apierrors.IsNotFound(getErr), + "a stranded object must be released, not left in Terminating") + return + } + require.NoError(t, getErr) + assert.Empty(t, got.Finalizers, "the retired finalizer must be shed") + }) + } +} + +// TestClusterProviderReconcile_ShedFinalizerUpdateFails pins the retry contract for the upgrade +// path. A stranded object gets no further events of its own, so if the finalizer-shedding Update +// fails the reconcile MUST return the error — that is the only thing that re-queues it. Swallowing +// the failure would leave the object in Terminating until the operator restarts. +func TestClusterProviderReconcile_ShedFinalizerUpdateFails(t *testing.T) { + provider := clusterProviderWithKubeConfig("prod-eu-1", "kc", "") + provider.Finalizers = []string{LegacyClusterProviderFinalizer} + now := metav1.Now() + provider.DeletionTimestamp = &now + + cl := fake.NewClientBuilder(). + WithScheme(scScheme(t)). + WithStatusSubresource(&configbutleraiv1alpha3.ClusterProvider{}). + WithObjects(provider). + WithInterceptorFuncs(interceptor.Funcs{ + Update: func( + _ context.Context, + _ client.WithWatch, + _ client.Object, + _ ...client.UpdateOption, + ) error { + return apierrors.NewConflict( + schema.GroupResource{Group: "configbutler.ai", Resource: "clusterproviders"}, + "prod-eu-1", errors.New("conflict"), + ) + }, + }). + Build() + r := &ClusterProviderReconciler{Client: cl, OperatorNamespace: cpOperatorNS} + + _, err := r.Reconcile( + context.Background(), + ctrl.Request{NamespacedName: k8stypes.NamespacedName{Name: "prod-eu-1"}}, + ) + require.Error(t, err, "a failed finalizer shed must requeue, not silently give up") + assert.Contains(t, err.Error(), "remove retired finalizer") + + // The object is still held, which is exactly why the retry has to happen. + var got configbutleraiv1alpha3.ClusterProvider + require.NoError(t, cl.Get(context.Background(), k8stypes.NamespacedName{Name: "prod-eu-1"}, &got)) + assert.Contains(t, got.Finalizers, LegacyClusterProviderFinalizer) +} + +// TestClusterProviderReconcile_InvalidStatusWriteFails checks that a failed status write on the +// INVALID path is propagated. The verdict itself (a bad kubeconfig) is not an error, but failing to +// record it is: without the error the provider would report a stale status for a whole steady +// interval before anything looked again. +func TestClusterProviderReconcile_InvalidStatusWriteFails(t *testing.T) { + provider := clusterProviderWithKubeConfig("prod-eu-1", "absent-kc", "") + cl := fake.NewClientBuilder(). + WithScheme(scScheme(t)). + WithObjects(provider). + WithStatusSubresource(&configbutleraiv1alpha3.ClusterProvider{}). + WithInterceptorFuncs(interceptor.Funcs{ + SubResourceUpdate: func( + _ context.Context, + _ client.Client, + _ string, + _ client.Object, + _ ...client.SubResourceUpdateOption, + ) error { + return errors.New("status write boom") + }, + }). + Build() + r := &ClusterProviderReconciler{Client: cl, OperatorNamespace: cpOperatorNS} + + _, err := r.Reconcile( + context.Background(), + ctrl.Request{NamespacedName: k8stypes.NamespacedName{Name: "prod-eu-1"}}, + ) + require.Error(t, err, "a failed status write must requeue rather than report success") + assert.Contains(t, err.Error(), "status write boom") +} + +// TestClusterProviderReconcile_NotFound checks a deleted provider reconciles to a no-op. +func TestClusterProviderReconcile_NotFound(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(scScheme(t)).Build() + r := &ClusterProviderReconciler{Client: cl, OperatorNamespace: cpOperatorNS} + res, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: k8stypes.NamespacedName{Name: "gone"}}) + require.NoError(t, err) + assert.Zero(t, res.RequeueAfter) +} + +// TestValidateProviderKubeConfig_NilSecretRef checks a remote provider whose kubeConfig has no +// secretRef is rejected (belt to the CEL configMapRef rejection). +func TestValidateProviderKubeConfig_NilSecretRef(t *testing.T) { + provider := &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "prod-eu-1"}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{KubeConfig: &meta.KubeConfigReference{}}, + } + r := &ClusterProviderReconciler{ + Client: fake.NewClientBuilder().WithScheme(scScheme(t)).Build(), + OperatorNamespace: cpOperatorNS, + } + ok, reason, msg, err := r.validateProviderKubeConfig(context.Background(), provider) + require.NoError(t, err) + assert.False(t, ok) + assert.Equal(t, ReasonKubeConfigInvalid, reason) + assert.NotEmpty(t, msg) +} + +// TestClusterProviderUpdateStatus_DeletedObject checks the status writer treats a vanished object +// as done (the NotFound branch of the retry loop) rather than erroring. +func TestClusterProviderUpdateStatus_DeletedObject(t *testing.T) { + cl := fake.NewClientBuilder(). + WithScheme(scScheme(t)). + WithStatusSubresource(&configbutleraiv1alpha3.ClusterProvider{}). + Build() + r := &ClusterProviderReconciler{Client: cl, OperatorNamespace: cpOperatorNS} + // The object was never created, so the retry loop's Get returns NotFound -> success, no error. + err := r.updateStatusWithRetry(context.Background(), clusterProviderWithKubeConfig("gone", "", "")) + require.NoError(t, err) +} + +// TestClusterProviderReconcile_RemoteInvalidKubeConfig checks a remote provider whose kubeconfig +// is missing goes Validated=False / Stalled, not Ready. +func TestClusterProviderReconcile_RemoteInvalidKubeConfig(t *testing.T) { + provider := clusterProviderWithKubeConfig("prod-eu-1", "absent-kc", "") + cl := fake.NewClientBuilder(). + WithScheme(scScheme(t)). + WithObjects(provider). + WithStatusSubresource(&configbutleraiv1alpha3.ClusterProvider{}). + Build() + r := &ClusterProviderReconciler{Client: cl, OperatorNamespace: cpOperatorNS} + + _, err := r.Reconcile( + context.Background(), + ctrl.Request{NamespacedName: k8stypes.NamespacedName{Name: "prod-eu-1"}}, + ) + require.NoError(t, err) + + var got configbutleraiv1alpha3.ClusterProvider + require.NoError(t, cl.Get(context.Background(), k8stypes.NamespacedName{Name: "prod-eu-1"}, &got)) + assert.Equal( + t, + metav1.ConditionFalse, + findCondition(got.Status.Conditions, ClusterProviderConditionValidated).Status, + ) + assert.Equal( + t, + kubeconfig.ReasonSecretNotFound, + findCondition(got.Status.Conditions, ClusterProviderConditionValidated).Reason, + ) + assert.Equal(t, metav1.ConditionFalse, findCondition(got.Status.Conditions, ConditionTypeReady).Status) + assert.Equal(t, metav1.ConditionTrue, findCondition(got.Status.Conditions, ConditionTypeStalled).Status) +} diff --git a/internal/controller/clusterprovider_predicate_test.go b/internal/controller/clusterprovider_predicate_test.go new file mode 100644 index 00000000..b6fb5e0d --- /dev/null +++ b/internal/controller/clusterprovider_predicate_test.go @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/event" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// provider builds a ClusterProvider with the given generation, optionally marked for deletion. +func providerAt(generation int64, deleting bool) *configbutleraiv1alpha3.ClusterProvider { + p := &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "prod-eu-1", Generation: generation}, + } + if deleting { + now := metav1.Now() + p.DeletionTimestamp = &now + p.Finalizers = []string{LegacyClusterProviderFinalizer} + } + return p +} + +// TestClusterProviderReconcilePredicate_AdmitsDeletionAndSpecChanges pins the predicate contract. +// The deletion case lets the controller promptly shed the retired fact-purge finalizer from an +// object created by an older operator. +func TestClusterProviderReconcilePredicate_AdmitsDeletionAndSpecChanges(t *testing.T) { + p := clusterProviderReconcilePredicate() + + tests := []struct { + name string + old *configbutleraiv1alpha3.ClusterProvider + new *configbutleraiv1alpha3.ClusterProvider + want bool + }{ + { + name: "deletion begins is admitted even with no generation bump", + old: providerAt(3, false), + new: providerAt(3, true), + want: true, + }, + { + name: "spec change is admitted", + old: providerAt(3, false), + new: providerAt(4, false), + want: true, + }, + { + name: "status-only update is filtered", + old: providerAt(3, false), + new: providerAt(3, false), + want: false, + }, + { + name: "repeated update while already deleting is filtered", + old: providerAt(3, true), + new: providerAt(3, true), + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := p.Update(event.UpdateEvent{ObjectOld: tt.old, ObjectNew: tt.new}) + assert.Equal(t, tt.want, got) + }) + } +} + +// TestClusterProviderReconcilePredicate_NilObjectsAreAdmitted keeps the predicate fail-open on a +// malformed event: dropping it would silently skip a reconcile, which is worse than one extra pass. +func TestClusterProviderReconcilePredicate_NilObjectsAreAdmitted(t *testing.T) { + p := clusterProviderReconcilePredicate() + assert.True(t, p.Update(event.UpdateEvent{ObjectOld: nil, ObjectNew: providerAt(1, false)})) + assert.True(t, p.Update(event.UpdateEvent{ObjectOld: providerAt(1, false), ObjectNew: nil})) +} diff --git a/internal/controller/condition_wait_test.go b/internal/controller/condition_wait_test.go index bbd15671..f5eea333 100644 --- a/internal/controller/condition_wait_test.go +++ b/internal/controller/condition_wait_test.go @@ -4,6 +4,7 @@ package controller import ( "context" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -13,23 +14,39 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// eventuallyConditionStatus blocks until the object at key publishes a condition of condType -// with the wanted status, then returns the matched condition for any further assertions -// (reason, message). getConditions extracts the freshly-fetched object's Status.Conditions — -// our CRD status structs don't share a conditions accessor, so the caller supplies a closure -// over its typed object. +// conditionWaitTimeout bounds every condition wait in this suite. Gomega's built-in default is 1s, +// which is not a wait for an ASYNC controller at all: the reconciler has to be triggered, run, and +// write a status subresource before the first condition even exists, and under a loaded envtest +// that regularly takes longer. A 1s ceiling made these waits pass or fail on machine speed — +// exactly the flake this suite kept seeing. Match the 10s the other specs already use. +const ( + conditionWaitTimeout = 10 * time.Second + conditionWaitPolling = 100 * time.Millisecond +) + +// eventuallyConditionStatusReason blocks until the object at key publishes a condition of condType +// that has settled on BOTH the wanted status and the wanted reason, then returns it for any further +// assertions (message). getConditions extracts the freshly-fetched object's Status.Conditions — our +// CRD status structs don't share a conditions accessor, so the caller supplies a closure over its +// typed object. // // This is the unit-test analog of the e2e verifyResourceCondition helper: it removes the -// create→async-reconcile race that bites specs which read a *dependency's* published status -// (e.g. a WatchRule mirroring its referenced GitTarget's Ready condition). A single synchronous -// Reconcile would otherwise observe an as-yet-unpopulated status. -func eventuallyConditionStatus( +// create→async-reconcile race that bites specs which read a *dependency's* published status (e.g. a +// WatchRule mirroring its referenced GitTarget's Ready condition). A single synchronous Reconcile +// would otherwise observe an as-yet-unpopulated status. +// +// The REASON is part of the wait on purpose. Waiting on status alone and then asserting the reason +// separately asserts a value that is still moving: a controller can publish the target status with +// a transient reason (a dependency it has not observed yet) and settle on the final reason a moment +// later. Folding it in makes the poll the assertion. +func eventuallyConditionStatusReason( ctx context.Context, key types.NamespacedName, obj client.Object, getConditions func() []metav1.Condition, condType string, - want metav1.ConditionStatus, + wantStatus metav1.ConditionStatus, + wantReason string, ) metav1.Condition { GinkgoHelper() var matched metav1.Condition @@ -37,8 +54,11 @@ func eventuallyConditionStatus( g.Expect(k8sClient.Get(ctx, key, obj)).To(Succeed()) cond := meta.FindStatusCondition(getConditions(), condType) g.Expect(cond).NotTo(BeNil(), "condition %q not published yet on %s", condType, key) - g.Expect(cond.Status).To(Equal(want)) + g.Expect(cond.Status).To(Equal(wantStatus)) + g.Expect(cond.Reason).To(Equal(wantReason), + "condition %q on %s is %s but has not settled on reason %q yet", + condType, key, wantStatus, wantReason) matched = *cond - }).Should(Succeed()) + }).WithTimeout(conditionWaitTimeout).WithPolling(conditionWaitPolling).Should(Succeed()) return matched } diff --git a/internal/controller/constants.go b/internal/controller/constants.go index 4d022875..f5146534 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -54,6 +54,21 @@ const ( // remote repository. ConditionTypePushed = "Pushed" + // ClusterProviderConditionValidated reports whether a ClusterProvider's inputs are safe and + // resolvable: the in-cluster "default" provider is trivially Validated; a remote provider is + // Validated once its kubeconfig Secret is present, keyed, and passes the exec/TLS safety + // policy. It is asserted WITHOUT a network dial — runtime reachability/discovery health are + // deferred until authenticated remote ingest wires them from the watch engine. + ClusterProviderConditionValidated = "Validated" + + // ReasonValidated is the Validated=True reason. + ReasonValidated = "Validated" + // ReasonInCluster is the Validated=True reason for the in-cluster "default" provider. + ReasonInCluster = "InCluster" + // ReasonKubeConfigInvalid is the Validated=False reason for a malformed or unsafe kubeconfig + // whose specific cause is carried in the message. + ReasonKubeConfigInvalid = "KubeConfigInvalid" + // MsgSnapshotCompleted is returned as the condition message when the initial // cluster snapshot has been successfully committed to Git. MsgSnapshotCompleted = "Initial snapshot reconciliation completed" diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index 1a1e3afb..32c7740c 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -21,6 +21,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -29,7 +30,6 @@ import ( configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/git" - "github.com/ConfigButler/gitops-reverser/internal/kubeconfig" "github.com/ConfigButler/gitops-reverser/internal/reconcile" "github.com/ConfigButler/gitops-reverser/internal/types" "github.com/ConfigButler/gitops-reverser/internal/watch" @@ -107,10 +107,6 @@ type GitTargetReconciler struct { Scheme *runtime.Scheme WorkerManager *git.WorkerManager EventRouter *watch.EventRouter - // KubeConfigSafety is the exec / insecure-TLS opt-in applied by the Validated gate when a - // GitTarget names spec.kubeConfig. It mirrors the resolver's policy so the controller's - // legibility verdict and the data plane's dial never disagree on what is safe. - KubeConfigSafety kubeconfig.SafetyPolicy } // +kubebuilder:rbac:groups=configbutler.ai,resources=gittargets,verbs=get;list;watch;create;update;patch;delete @@ -222,7 +218,7 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if declareErr := r.EventRouter.WatchManager.DeclareForGitTarget( ctx, gitDest, - target.SourceClusterID(), + target.SourceCluster(), gitPathWasRefused, ); declareErr != nil { log.V(1).Info("stream declaration skipped; surface not observable", @@ -251,15 +247,18 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // reachable by definition — so the default (used when no watch manager is wired, e.g. in // tests) is True/LocalCluster for a local target and Unknown only for a remote one. sourceReach := watch.SourceClusterReachableStatus{State: "True", Reason: "LocalCluster"} - if target.SourceClusterID() != "" { + if !target.IsLocalSource() { sourceReach = watch.SourceClusterReachableStatus{State: "Unknown", Reason: "AwaitingDiscovery"} } if r.EventRouter != nil && r.EventRouter.WatchManager != nil { - sourceReach = r.EventRouter.WatchManager.SourceClusterReachable(target.SourceClusterID()) + sourceReach = r.EventRouter.WatchManager.SourceClusterReachable(target.SourceCluster()) } providerStatus, providerReason, providerMessage := r.gitProviderReadiness(ctx, &target, providerNS) - r.projectSourceAndProvider(&target, sourceReach, providerStatus, providerReason, providerMessage) - streamsSettling = streamsSettling || sourceReach.State != "True" || providerStatus != metav1.ConditionTrue + cpStatus, cpReason, cpMessage := r.clusterProviderReadiness(ctx, &target) + r.projectSourceAndProvider(&target, sourceReach, providerStatus, providerReason, providerMessage, + cpStatus, cpReason, cpMessage) + streamsSettling = streamsSettling || sourceReach.State != "True" || + providerStatus != metav1.ConditionTrue || cpStatus == metav1.ConditionFalse if err := r.updateStatusWithRetry(ctx, &target); err != nil { return ctrl.Result{}, err @@ -305,17 +304,22 @@ func (r *GitTargetReconciler) evaluateValidatedGate( return false, fmt.Sprintf("Validated gate failed: %s", GitTargetReasonInvalidConfig), nil, nil } - // spec.kubeConfig legibility gate: read/parse the kubeconfig Secret and apply the exec/TLS - // safety policy WITHOUT dialing. Reachability is a runtime observation the data plane records - // on SourceClusterReachable, so this only ever fails on a directly-readable input problem. - kcOK, kcReason, kcMsg, kcErr := r.validateKubeConfig(ctx, target) - if kcErr != nil { - return false, "", nil, kcErr + // The source cluster's connectivity inputs (kubeConfig) are validated on the referenced + // ClusterProvider now, not here — the GitTarget only NAMES its source cluster. The + // ClusterProvider's readiness is projected onto the GitTarget as a separate condition. + + // Namespace authorization: a GitTarget may reference a ClusterProvider only from a namespace + // its spec.allowedNamespaces admits. Enforced HERE and only here, on every reconcile — which + // also covers a policy tightened after the GitTarget was created. Failing this gate returns + // before DeclareForGitTarget below, so an unauthorized target starts no watch and writes no Git. + authorized, authReason, authMsg, authErr := r.checkSourceAuthorization(ctx, target) + if authErr != nil { + return false, "", nil, authErr } - if !kcOK { - r.setCondition(target, GitTargetConditionValidated, metav1.ConditionFalse, kcReason, kcMsg) + if !authorized { + r.setCondition(target, GitTargetConditionValidated, metav1.ConditionFalse, authReason, authMsg) result := ctrl.Result{RequeueAfter: RequeueSteadyInterval} - return false, fmt.Sprintf("Validated gate failed: %s", kcReason), &result, nil + return false, fmt.Sprintf("Validated gate failed: %s", authReason), &result, nil } r.setCondition( @@ -323,7 +327,7 @@ func (r *GitTargetReconciler) evaluateValidatedGate( GitTargetConditionValidated, metav1.ConditionTrue, GitTargetReasonOK, - "Provider, branch, placement, and kubeconfig validation passed", + "Provider, branch, and placement validation passed", ) return true, "", nil, nil } @@ -397,14 +401,15 @@ func (r *GitTargetReconciler) evaluateWorkerWiringGate( // setBlockedDataPlane marks stream readiness as not-yet-evaluated when a control-plane gate // blocked the reconcile before watches could be declared. -// stopSourceClusterMirror tears down the data plane of a remote-source GitTarget that is no longer -// Validated, so a dead credential can never keep an active mirror running behind a Validated=False -// status. It is a no-op for a local GitTarget (nothing remote to stop) and idempotent across -// requeues while the target stays blocked. A later recovery re-declares the watches and re-snapshots. +// stopSourceClusterMirror tears down the data plane of a GitTarget that is no longer Validated, so +// a dead credential — or a source ClusterProvider that was deleted (including the reserved +// "default" for the LOCAL cluster) — can never keep an active mirror running behind a +// Validated=False status. It applies to local AND remote targets: a local target references the +// "default" ClusterProvider, so if that provider is gone the local mirror must stop too, never +// falling back to an implicit in-cluster identity that bypasses the authorization policy. It is +// idempotent across requeues while the target stays blocked; a later recovery re-declares the +// watches and re-snapshots. func (r *GitTargetReconciler) stopSourceClusterMirror(target *configbutleraiv1alpha3.GitTarget) { - if target.SourceClusterID() == "" { - return - } if r.EventRouter == nil || r.EventRouter.WatchManager == nil { return } @@ -1125,6 +1130,26 @@ func (r *GitTargetReconciler) SetupWithManager(mgr ctrl.Manager) error { handler.EnqueueRequestsFromMapFunc(r.gitProviderToGitTargets), builder.WithPredicates(predicate.GenerationChangedPredicate{}), ). + // React to the referenced (cluster-scoped) ClusterProvider becoming Ready/NotReady so the + // projected ClusterProviderReady condition and the namespace-authorization refusal re-run + // promptly instead of waiting for the periodic reconcile. A plain GenerationChangedPredicate + // would miss a Ready flip (a STATUS-only update), so this fires on spec changes OR a change + // in the provider's Ready condition status (plus create/delete). + Watches( + &configbutleraiv1alpha3.ClusterProvider{}, + handler.EnqueueRequestsFromMapFunc(r.clusterProviderToGitTargets), + builder.WithPredicates(clusterProviderReadyOrSpecChanged()), + ). + // React to a Namespace's LABELS changing: a ClusterProvider's allowedNamespaces selector is + // evaluated against namespace labels, so a label change can grant or revoke a GitTarget's + // authorization. Re-enqueue the GitTargets in that namespace so the reconcile-time refusal + // converges instead of waiting for the periodic reconcile. LabelChangedPredicate ignores the + // unrelated namespace churn (annotations, status). + Watches( + &corev1.Namespace{}, + handler.EnqueueRequestsFromMapFunc(r.namespaceToGitTargets), + builder.WithPredicates(predicate.LabelChangedPredicate{}), + ). // React to a GitTarget's WatchRule/ClusterWatchRule set changing so it re-reconciles and // re-Declares its watched-type set promptly (the R3 replacement for the deleted whole-target // rule-change resync). Without this a rule added after the GitTarget went Ready would not be @@ -1208,3 +1233,82 @@ func (r *GitTargetReconciler) gitProviderToGitTargets( } return requests } + +// clusterProviderToGitTargets maps a ClusterProvider event to every GitTarget that references it, +// across ALL namespaces (the provider is cluster-scoped). It re-enqueues dependents when the +// provider's Ready flips or its allowedNamespaces policy changes, so the projected +// ClusterProviderReady and the namespace-authorization refusal converge without waiting for the +// periodic reconcile. +func (r *GitTargetReconciler) clusterProviderToGitTargets( + ctx context.Context, + obj client.Object, +) []ctrlreconcile.Request { + var targets configbutleraiv1alpha3.GitTargetList + if err := r.List(ctx, &targets); err != nil { + logDependencyListError(ctx, err, "GitTargets", obj) + return nil + } + + var requests []ctrlreconcile.Request + for i := range targets.Items { + t := &targets.Items[i] + if t.SourceCluster() != obj.GetName() { + continue + } + requests = append(requests, ctrlreconcile.Request{ + NamespacedName: k8stypes.NamespacedName{Name: t.Name, Namespace: t.Namespace}, + }) + } + return requests +} + +// namespaceToGitTargets maps a Namespace label change to every GitTarget in that namespace, so the +// reconcile-time ClusterProvider authorization (which may match the namespace's labels via a +// selector) re-runs and grants/revokes the target promptly. +func (r *GitTargetReconciler) namespaceToGitTargets( + ctx context.Context, + obj client.Object, +) []ctrlreconcile.Request { + var targets configbutleraiv1alpha3.GitTargetList + if err := r.List(ctx, &targets, client.InNamespace(obj.GetName())); err != nil { + logDependencyListError(ctx, err, "GitTargets", obj) + return nil + } + requests := make([]ctrlreconcile.Request, 0, len(targets.Items)) + for i := range targets.Items { + t := &targets.Items[i] + requests = append(requests, ctrlreconcile.Request{ + NamespacedName: k8stypes.NamespacedName{Name: t.Name, Namespace: t.Namespace}, + }) + } + return requests +} + +// clusterProviderReadyStatus returns a ClusterProvider's Ready condition status ("" if absent), so +// a watch predicate can react to a Ready FLIP that arrives as a status-only update. +func clusterProviderReadyStatus(cp *configbutleraiv1alpha3.ClusterProvider) metav1.ConditionStatus { + if c := findCondition(cp.Status.Conditions, ConditionTypeReady); c != nil { + return c.Status + } + return "" +} + +// clusterProviderReadyOrSpecChanged is the ClusterProvider watch predicate for the GitTarget +// controller: fire on create/delete, and on an update when the SPEC changed (generation) OR the +// Ready condition status changed — the latter a status-only update GenerationChangedPredicate drops. +func clusterProviderReadyOrSpecChanged() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(event.CreateEvent) bool { return true }, + DeleteFunc: func(event.DeleteEvent) bool { return true }, + GenericFunc: func(event.GenericEvent) bool { return false }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldCP, ok1 := e.ObjectOld.(*configbutleraiv1alpha3.ClusterProvider) + newCP, ok2 := e.ObjectNew.(*configbutleraiv1alpha3.ClusterProvider) + if !ok1 || !ok2 { + return true + } + return oldCP.Generation != newCP.Generation || + clusterProviderReadyStatus(oldCP) != clusterProviderReadyStatus(newCP) + }, + } +} diff --git a/internal/controller/gittarget_source_cluster.go b/internal/controller/gittarget_source_cluster.go index 3b468fe7..f6a33049 100644 --- a/internal/controller/gittarget_source_cluster.go +++ b/internal/controller/gittarget_source_cluster.go @@ -12,10 +12,72 @@ import ( k8stypes "k8s.io/apimachinery/pkg/types" configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" - "github.com/ConfigButler/gitops-reverser/internal/kubeconfig" "github.com/ConfigButler/gitops-reverser/internal/watch" ) +// GitTargetReasonNamespaceNotAuthorized is the Validated=False reason when a GitTarget's namespace +// is not admitted by its referenced ClusterProvider's spec.allowedNamespaces. This is the SINGLE +// enforcement point for that policy: it runs on every reconcile, so a policy tightened AFTER a +// GitTarget was created stops that target's watches too. +const GitTargetReasonNamespaceNotAuthorized = "NamespaceNotAuthorized" + +// GitTargetReasonClusterProviderNotFound is the Validated=False reason when a GitTarget's +// referenced ClusterProvider does not exist. This is a HARD GATE: a GitTarget may mirror a source +// cluster ONLY through an existing ClusterProvider, "default" included. The operator never creates +// one, so a target whose provider was never declared is held NotReady and its data plane stopped +// rather than mirroring on an implicit local identity. +const GitTargetReasonClusterProviderNotFound = "ClusterProviderNotFound" + +// checkSourceAuthorization is the reconcile-time source-cluster gate. It first REQUIRES the +// referenced ClusterProvider to exist — a missing provider ("default" included) is a hard NotReady +// gate, so a GitTarget can never mirror a source cluster the operator was not configured for, and +// local mirroring is never an implicit bypass of the authorization policy. +// Then it enforces the provider's namespace-access policy. This is the ONLY place that policy is +// enforced, and it is enforced on every reconcile rather than only at admission — so a policy +// tightened AFTER a GitTarget was created stops that target's watches too. Its caller runs it +// inside the Validated gate and returns BEFORE DeclareForGitTarget, so an unauthorized GitTarget +// never starts a watch and never writes to Git. It returns authorized=false with a legible reason +// in either case; a non-NotFound read error is returned as err so the reconcile requeues. +func (r *GitTargetReconciler) checkSourceAuthorization( + ctx context.Context, + target *configbutleraiv1alpha3.GitTarget, +) (bool, string, string, error) { + providerName := target.SourceCluster() + var provider configbutleraiv1alpha3.ClusterProvider + if err := r.Get(ctx, k8stypes.NamespacedName{Name: providerName}, &provider); err != nil { + if apierrors.IsNotFound(err) { + return false, GitTargetReasonClusterProviderNotFound, fmt.Sprintf( + "referenced ClusterProvider %q was not found; a GitTarget may mirror a source cluster "+ + "only through an existing ClusterProvider. The operator never creates one: declare it "+ + "yourself, or let the chart render %q via clusterProvider.createDefault", + providerName, configbutleraiv1alpha3.DefaultClusterProviderName), nil + } + return false, "", "", fmt.Errorf("read ClusterProvider %q: %w", providerName, err) + } + + nsLabels := map[string]string{} + var ns corev1.Namespace + if err := r.Get(ctx, k8stypes.NamespacedName{Name: target.Namespace}, &ns); err != nil { + if !apierrors.IsNotFound(err) { + return false, "", "", fmt.Errorf("read namespace %q: %w", target.Namespace, err) + } + } else { + nsLabels = ns.Labels + } + + allowed, selErr := provider.AllowsNamespace(target.Namespace, nsLabels) + if selErr != nil { + return false, GitTargetReasonNamespaceNotAuthorized, fmt.Sprintf( + "ClusterProvider %q allowedNamespaces selector is invalid: %v", providerName, selErr), nil + } + if !allowed { + return false, GitTargetReasonNamespaceNotAuthorized, fmt.Sprintf( + "namespace %q is not permitted to reference ClusterProvider %q (spec.allowedNamespaces)", + target.Namespace, providerName), nil + } + return true, "", "", nil +} + const ( // GitTargetConditionSourceClusterReachable is the RUNTIME reachability of the source // cluster a GitTarget mirrors from: True (reason LocalCluster) when kubeConfig is omitted, @@ -33,48 +95,56 @@ const ( GitTargetReasonGitProviderNotReady = "GitProviderNotReady" // GitTargetReasonGitProviderReady is the GitProviderReady=True reason. GitTargetReasonGitProviderReady = "GitProviderReady" + + // GitTargetConditionClusterProviderReady projects the referenced ClusterProvider's Ready onto + // the GitTarget, so one `kubectl get gittarget` shows whether the SOURCE cluster's provider is + // healthy — distinct from SourceClusterReachable (the data plane's runtime reach) and + // GitProviderReady (the destination). It follows the GitProviderReady contract: only an + // EXPLICIT Ready=False downgrades the GitTarget; a not-found or not-yet-reported provider is + // Unknown and does not (so a single-cluster install that has not yet installed the "default" + // provider is not held down). + GitTargetConditionClusterProviderReady = "ClusterProviderReady" + // GitTargetReasonClusterProviderNotReady is the ClusterProviderReady=False/Unknown reason. + GitTargetReasonClusterProviderNotReady = "ClusterProviderNotReady" + // GitTargetReasonClusterProviderReady is the ClusterProviderReady=True reason. + GitTargetReasonClusterProviderReady = "ClusterProviderReady" ) -// validateKubeConfig is the legibility gate for spec.kubeConfig (extends Validated). It reads -// and parses the kubeconfig Secret from the config plane and applies the exec/TLS safety -// policy, but deliberately does NOT dial the cluster — reachability is a runtime observation -// the data plane records on SourceClusterReachable. It returns ok=false with a typed -// KubeConfig* reason and a legible message when an input is wrong; a non-NotFound read error is -// returned as err so the reconcile requeues rather than falsely reporting a bad input. An -// omitted kubeConfig is trivially valid — the local cluster. -func (r *GitTargetReconciler) validateKubeConfig( +// clusterProviderReadiness reads the referenced ClusterProvider's Ready condition and projects it, +// mirroring gitProviderReadiness. It returns Unknown — which does NOT downgrade Ready — when the +// provider cannot be observed (not found, or no Ready condition yet), so a not-yet-installed +// "default" provider never blocks a local GitTarget; only an explicit Ready=False downgrades. +func (r *GitTargetReconciler) clusterProviderReadiness( ctx context.Context, target *configbutleraiv1alpha3.GitTarget, -) (bool, string, string, error) { - if target.Spec.KubeConfig == nil || target.Spec.KubeConfig.SecretRef == nil { - return true, "", "", nil +) (metav1.ConditionStatus, string, string) { + name := target.SourceCluster() + var cp configbutleraiv1alpha3.ClusterProvider + if err := r.Get(ctx, k8stypes.NamespacedName{Name: name}, &cp); err != nil { + return metav1.ConditionUnknown, GitTargetReasonClusterProviderNotReady, + fmt.Sprintf("referenced ClusterProvider %q readiness not observed: %v", name, err) } - ref := target.Spec.KubeConfig.SecretRef - secretKey := k8stypes.NamespacedName{Namespace: target.Namespace, Name: ref.Name} - - var secret corev1.Secret - if getErr := r.Get(ctx, secretKey, &secret); getErr != nil { - if apierrors.IsNotFound(getErr) { - return false, kubeconfig.ReasonSecretNotFound, fmt.Sprintf( - "spec.kubeConfig.secretRef names Secret %s, which does not exist in this namespace", - secretKey), nil + c := findCondition(cp.Status.Conditions, ConditionTypeReady) + switch { + case c == nil: + return metav1.ConditionUnknown, GitTargetReasonClusterProviderNotReady, + fmt.Sprintf("referenced ClusterProvider %q has not reported readiness yet", name) + case c.Status == metav1.ConditionTrue: + return metav1.ConditionTrue, GitTargetReasonClusterProviderReady, + fmt.Sprintf("referenced ClusterProvider %q is Ready", name) + case c.Status == metav1.ConditionFalse: + msg := fmt.Sprintf("referenced ClusterProvider %q is not Ready", name) + if c.Message != "" { + msg = fmt.Sprintf("referenced ClusterProvider %q is not Ready: %s", name, c.Message) } - return false, "", "", fmt.Errorf("read kubeconfig Secret %s: %w", secretKey, getErr) - } - - raw, usedKey, present := kubeconfig.ResolveKey(secret.Data, ref.Key) - if !present { - return false, kubeconfig.ReasonKeyNotFound, fmt.Sprintf( - "kubeconfig Secret %s has no kubeconfig under key %q (set spec.kubeConfig.secretRef.key)", - secretKey, describeKubeConfigKey(ref.Key)), nil - } - if rej := kubeconfig.Check(raw, r.KubeConfigSafety); rej != nil { - // A RejectionError is a validation VERDICT, not a reconcile error: it is surfaced as - // the Validated=False reason/message, and reconcile requeues normally (err == nil). - //nolint:nilerr - return false, rej.Reason, fmt.Sprintf("kubeconfig Secret %s key %q: %s", secretKey, usedKey, rej.Message), nil + return metav1.ConditionFalse, GitTargetReasonClusterProviderNotReady, msg + default: + // Ready=Unknown is the provider saying it does not know yet, which is exactly the case the + // contract above refuses to downgrade on. Collapsing it into False would hold a GitTarget + // down on a provider mid-reconcile. + return metav1.ConditionUnknown, GitTargetReasonClusterProviderNotReady, + fmt.Sprintf("referenced ClusterProvider %q readiness is unknown", name) } - return true, "", "", nil } // describeKubeConfigKey renders the resolved-key hint for a "key not found" message. @@ -139,6 +209,8 @@ func (r *GitTargetReconciler) projectSourceAndProvider( sourceReach watch.SourceClusterReachableStatus, providerStatus metav1.ConditionStatus, providerReason, providerMessage string, + clusterProviderStatus metav1.ConditionStatus, + clusterProviderReason, clusterProviderMessage string, ) { reachStatus := conditionStatusFromString(sourceReach.State) r.setCondition( @@ -149,41 +221,44 @@ func (r *GitTargetReconciler) projectSourceAndProvider( sourceReach.Message, ) r.setCondition(target, GitTargetConditionGitProviderReady, providerStatus, providerReason, providerMessage) + r.setCondition(target, GitTargetConditionClusterProviderReady, clusterProviderStatus, + clusterProviderReason, clusterProviderMessage) switch { case providerStatus == metav1.ConditionFalse: // Destination-side: the provider's own periodic check is failing. Wait for it to // recover (progressing), which the Watches(&GitProvider{}) trigger promptly re-runs. - r.downgradeReady(target, metav1.ConditionFalse, providerReason, providerMessage, true) + r.downgradeReady(target, metav1.ConditionFalse, providerReason, providerMessage) + case clusterProviderStatus == metav1.ConditionFalse: + // Source-config side: the ClusterProvider explicitly reports itself not Ready (e.g. its + // kubeconfig stopped validating). Progressing — the Watches(&ClusterProvider{}) trigger + // re-runs this as the provider recovers. + r.downgradeReady(target, metav1.ConditionFalse, clusterProviderReason, clusterProviderMessage) case reachStatus == metav1.ConditionFalse: // Source-side: an otherwise-valid kubeconfig whose API server cannot be contacted. A // transient the data plane retries, so this is progressing, not stalled. - r.downgradeReady(target, metav1.ConditionFalse, sourceReach.Reason, sourceReach.Message, true) + r.downgradeReady(target, metav1.ConditionFalse, sourceReach.Reason, sourceReach.Message) case reachStatus == metav1.ConditionUnknown: // An unconfirmed source is not yet Ready; hold Ready at Unknown until first discovery. - r.downgradeReady(target, metav1.ConditionUnknown, sourceReach.Reason, sourceReach.Message, true) + r.downgradeReady(target, metav1.ConditionUnknown, sourceReach.Reason, sourceReach.Message) } } -// downgradeReady lowers the aggregate below Ready without ever raising it: it is called only on -// a source/provider problem, and rewrites Ready plus the kstatus Reconciling/Stalled pair to -// match. progressing=true means the condition is expected to clear on its own (Reconciling), -// false means it needs action (Stalled). +// downgradeReady lowers the aggregate below Ready without ever raising it: it is called only on a +// source/provider problem, and rewrites Ready plus the kstatus Reconciling/Stalled pair to match. +// Every such problem here is expected to clear on its own (a provider recovers, a source becomes +// reachable), so it is always PROGRESSING (Reconciling=True, Stalled=False) — a re-check via the +// Watches triggers converges it. A blocking, needs-a-human problem is handled by the setStalled* +// path in Reconcile instead, never here. func (r *GitTargetReconciler) downgradeReady( target *configbutleraiv1alpha3.GitTarget, readyStatus metav1.ConditionStatus, reason, message string, - progressing bool, ) { r.setCondition(target, GitTargetConditionReady, readyStatus, reason, message) - if progressing { - r.setCondition(target, GitTargetConditionReconciling, metav1.ConditionTrue, reason, message) - r.setCondition(target, GitTargetConditionStalled, metav1.ConditionFalse, ReasonProgressing, - "Reconciliation is making progress") - return - } - r.setCondition(target, GitTargetConditionReconciling, metav1.ConditionFalse, reason, "Reconciliation is stalled") - r.setCondition(target, GitTargetConditionStalled, metav1.ConditionTrue, reason, message) + r.setCondition(target, GitTargetConditionReconciling, metav1.ConditionTrue, reason, message) + r.setCondition(target, GitTargetConditionStalled, metav1.ConditionFalse, ReasonProgressing, + "Reconciliation is making progress") } // conditionStatusFromString maps the watch layer's "True"/"False"/"Unknown" onto the API type. diff --git a/internal/controller/gittarget_source_cluster_test.go b/internal/controller/gittarget_source_cluster_test.go index f5c41e5c..a04ecc19 100644 --- a/internal/controller/gittarget_source_cluster_test.go +++ b/internal/controller/gittarget_source_cluster_test.go @@ -4,20 +4,73 @@ package controller import ( "context" + "errors" "testing" - meta "github.com/fluxcd/pkg/apis/meta" + "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/event" configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" - "github.com/ConfigButler/gitops-reverser/internal/kubeconfig" + "github.com/ConfigButler/gitops-reverser/internal/types" + "github.com/ConfigButler/gitops-reverser/internal/watch" ) +func TestNamespaceToGitTargets(t *testing.T) { + gt := func(name, ns string) *configbutleraiv1alpha3.GitTarget { + return &configbutleraiv1alpha3.GitTarget{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}} + } + cl := fake.NewClientBuilder().WithScheme(scScheme(t)). + WithObjects(gt("a", "team-a"), gt("b", "team-a"), gt("c", "team-b")).Build() + r := &GitTargetReconciler{Client: cl} + + reqs := r.namespaceToGitTargets(context.Background(), + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "team-a"}}) + // A label change on team-a re-enqueues exactly the GitTargets IN team-a, so a selector-based + // authorization revocation converges without waiting for the periodic reconcile. + names := []string{} + for _, req := range reqs { + assert.Equal(t, "team-a", req.Namespace) + names = append(names, req.Name) + } + assert.ElementsMatch(t, []string{"a", "b"}, names) +} + +func TestClusterProviderReadyOrSpecChanged(t *testing.T) { + p := clusterProviderReadyOrSpecChanged() + cp := func(gen int64, ready metav1.ConditionStatus) *configbutleraiv1alpha3.ClusterProvider { + c := &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "default", Generation: gen}, + } + if ready != "" { + c.Status.Conditions = []metav1.Condition{{Type: ConditionTypeReady, Status: ready}} + } + return c + } + + assert.True(t, p.Update(event.UpdateEvent{ + ObjectOld: cp(1, metav1.ConditionUnknown), ObjectNew: cp(1, metav1.ConditionTrue), + }), "a Ready flip (status-only, same generation) must fire") + assert.False(t, p.Update(event.UpdateEvent{ + ObjectOld: cp(1, metav1.ConditionTrue), ObjectNew: cp(1, metav1.ConditionTrue), + }), "no spec/Ready change must not fire") + assert.True(t, p.Update(event.UpdateEvent{ + ObjectOld: cp(1, metav1.ConditionTrue), ObjectNew: cp(2, metav1.ConditionTrue), + }), "a spec (generation) change must fire") + assert.True(t, p.Create(event.CreateEvent{})) + assert.True(t, p.Delete(event.DeleteEvent{})) +} + const scValidKubeConfig = `apiVersion: v1 kind: Config clusters: @@ -61,79 +114,102 @@ func scScheme(t *testing.T) *runtime.Scheme { return s } -func gitTargetWithKubeConfig(secretName, key string) *configbutleraiv1alpha3.GitTarget { - t := &configbutleraiv1alpha3.GitTarget{ - ObjectMeta: metav1.ObjectMeta{Name: "gt", Namespace: "team-a"}, +func TestCheckSourceAuthorization(t *testing.T) { + provider := func(policy *configbutleraiv1alpha3.AllowedNamespaces) *configbutleraiv1alpha3.ClusterProvider { + return &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "prod-eu-1"}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{AllowedNamespaces: policy}, + } } - if secretName != "" { - t.Spec.KubeConfig = &meta.KubeConfigReference{ - SecretRef: &meta.SecretKeyReference{Name: secretName, Key: key}, + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "team-a", Labels: map[string]string{"tier": "trusted"}}} + target := func(providerName string) *configbutleraiv1alpha3.GitTarget { + return &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "gt", Namespace: "team-a"}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + ClusterProviderRef: &configbutleraiv1alpha3.ClusterProviderReference{Name: providerName}, + }, } } - return t -} -func TestValidateKubeConfig_AllScenarios(t *testing.T) { tests := []struct { - name string - secretName string - key string - secretData map[string][]byte - safety kubeconfig.SafetyPolicy - wantOK bool - wantReason string + name string + objects []client.Object + providerRef string + wantAuthorized bool + wantReason string }{ - {name: "omitted is valid (local cluster)", secretName: "", wantOK: true}, - {name: "missing Secret", secretName: "absent", wantOK: false, wantReason: kubeconfig.ReasonSecretNotFound}, { - name: "missing key", secretName: "kc", key: "value", - secretData: map[string][]byte{"elsewhere": []byte(scValidKubeConfig)}, - wantOK: false, wantReason: kubeconfig.ReasonKeyNotFound, + name: "provider not found -> hard gate (NotReady, no mirroring)", + objects: []client.Object{ns}, + providerRef: "absent", + wantAuthorized: false, + wantReason: GitTargetReasonClusterProviderNotFound, + }, + { + name: "provider allows the namespace by name", + objects: []client.Object{ + provider(&configbutleraiv1alpha3.AllowedNamespaces{Names: []string{"team-a"}}), ns, + }, + providerRef: "prod-eu-1", wantAuthorized: true, }, { - name: "unparseable", secretName: "kc", - secretData: map[string][]byte{"value": []byte("not a kubeconfig")}, - wantOK: false, wantReason: kubeconfig.ReasonInvalid, + name: "provider does not allow the namespace -> refused", + objects: []client.Object{ + provider(&configbutleraiv1alpha3.AllowedNamespaces{Names: []string{"team-b"}}), ns, + }, + providerRef: "prod-eu-1", wantAuthorized: false, wantReason: GitTargetReasonNamespaceNotAuthorized, }, { - name: "exec rejected", secretName: "kc", - secretData: map[string][]byte{"value": []byte(scExecKubeConfig)}, - wantOK: false, wantReason: kubeconfig.ReasonExecNotAllowed, + name: "provider allows the namespace by label selector", + objects: []client.Object{ + provider(&configbutleraiv1alpha3.AllowedNamespaces{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "trusted"}}, + }), ns, + }, + providerRef: "prod-eu-1", wantAuthorized: true, }, { - name: "insecure TLS rejected", secretName: "kc", - secretData: map[string][]byte{"value": []byte(scInsecureKubeConfig)}, - wantOK: false, wantReason: kubeconfig.ReasonInsecureTLSNotAllowed, + // A selector the API accepted but that cannot compile must FAIL CLOSED. Treating an + // unevaluatable policy as "allow" would hand a namespace access it was never granted. + name: "invalid allowedNamespaces selector -> refused, not allowed", + objects: []client.Object{ + provider(&configbutleraiv1alpha3.AllowedNamespaces{ + Selector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "tier", Operator: "BogusOperator", Values: []string{"trusted"}}, + }, + }, + }), ns, + }, + providerRef: "prod-eu-1", wantAuthorized: false, wantReason: GitTargetReasonNamespaceNotAuthorized, }, { - name: "valid via value.yaml fallback", secretName: "kc", - secretData: map[string][]byte{"value.yaml": []byte(scValidKubeConfig)}, - wantOK: true, + // A missing Namespace object is not an error: the policy is still evaluated, just with + // no labels. A name-based allow still works; a selector-only policy then denies. + name: "namespace object absent -> evaluated with no labels, name allow still holds", + objects: []client.Object{ + provider(&configbutleraiv1alpha3.AllowedNamespaces{Names: []string{"team-a"}}), + }, + providerRef: "prod-eu-1", wantAuthorized: true, }, { - name: "exec allowed when opted in", secretName: "kc", - secretData: map[string][]byte{"value": []byte(scExecKubeConfig)}, - safety: kubeconfig.SafetyPolicy{AllowExec: true}, - wantOK: true, + name: "namespace object absent -> selector-only policy denies (no labels to match)", + objects: []client.Object{ + provider(&configbutleraiv1alpha3.AllowedNamespaces{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "trusted"}}, + }), + }, + providerRef: "prod-eu-1", wantAuthorized: false, wantReason: GitTargetReasonNamespaceNotAuthorized, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - builder := fake.NewClientBuilder().WithScheme(scScheme(t)) - if tc.secretData != nil { - builder = builder.WithObjects(&corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "kc"}, - Data: tc.secretData, - }) - } - r := &GitTargetReconciler{Client: builder.Build(), KubeConfigSafety: tc.safety} - ok, reason, msg, err := r.validateKubeConfig( - context.Background(), - gitTargetWithKubeConfig(tc.secretName, tc.key), - ) + cl := fake.NewClientBuilder().WithScheme(scScheme(t)).WithObjects(tc.objects...).Build() + r := &GitTargetReconciler{Client: cl} + ok, reason, msg, err := r.checkSourceAuthorization(context.Background(), target(tc.providerRef)) require.NoError(t, err) - assert.Equal(t, tc.wantOK, ok) - if !tc.wantOK { + assert.Equal(t, tc.wantAuthorized, ok) + if !tc.wantAuthorized { assert.Equal(t, tc.wantReason, reason) assert.NotEmpty(t, msg) } @@ -141,6 +217,237 @@ func TestValidateKubeConfig_AllScenarios(t *testing.T) { } } +// TestCheckSourceAuthorization_ReadErrorsRequeue pins the fail-closed contract for TRANSIENT +// failures. A read error is not a verdict: the gate must return an error so the reconcile requeues, +// rather than returning authorized=false (which would look like a policy denial and flap the +// GitTarget's status) or authorized=true (which would open a watch on an unevaluated policy). +func TestCheckSourceAuthorization_ReadErrorsRequeue(t *testing.T) { + target := &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "gt", Namespace: "team-a"}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + ClusterProviderRef: &configbutleraiv1alpha3.ClusterProviderReference{Name: "prod-eu-1"}, + }, + } + provider := &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "prod-eu-1"}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{ + AllowedNamespaces: &configbutleraiv1alpha3.AllowedNamespaces{Names: []string{"team-a"}}, + }, + } + + tests := []struct { + name string + failOn func(client.Object) bool + wantErr string + }{ + { + name: "ClusterProvider read fails", + failOn: func(o client.Object) bool { _, ok := o.(*configbutleraiv1alpha3.ClusterProvider); return ok }, + wantErr: "read ClusterProvider", + }, + { + name: "Namespace read fails", + failOn: func(o client.Object) bool { _, ok := o.(*corev1.Namespace); return ok }, + wantErr: "read namespace", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cl := fake.NewClientBuilder(). + WithScheme(scScheme(t)). + WithObjects(provider). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func( + ctx context.Context, + c client.WithWatch, + key client.ObjectKey, + obj client.Object, + opts ...client.GetOption, + ) error { + if tc.failOn(obj) { + return errors.New("api server unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }). + Build() + r := &GitTargetReconciler{Client: cl} + + ok, reason, _, err := r.checkSourceAuthorization(context.Background(), target) + require.Error(t, err, "a transient read failure must requeue, not decide the policy") + assert.Contains(t, err.Error(), tc.wantErr) + assert.False(t, ok) + assert.Empty(t, reason, "an error carries no verdict reason") + }) + } +} + +// TestConditionStatusFromString maps the watch layer's stringly-typed state onto the API type. +// Anything unrecognised must land on Unknown rather than being read as a False (which would +// downgrade a GitTarget on a state the data plane never actually reported). +func TestConditionStatusFromString(t *testing.T) { + tests := []struct { + state string + want metav1.ConditionStatus + }{ + {"True", metav1.ConditionTrue}, + {"False", metav1.ConditionFalse}, + {"Unknown", metav1.ConditionUnknown}, + {"", metav1.ConditionUnknown}, + {"true", metav1.ConditionUnknown}, + {"garbage", metav1.ConditionUnknown}, + } + for _, tc := range tests { + t.Run("state="+tc.state, func(t *testing.T) { + assert.Equal(t, tc.want, conditionStatusFromString(tc.state)) + }) + } +} + +// TestDescribeKubeConfigKey checks the "key not found" hint names the value→value.yaml fallback +// when the spec omitted a key, so the message tells the user what was actually tried. +func TestDescribeKubeConfigKey(t *testing.T) { + assert.Equal(t, "value or value.yaml", describeKubeConfigKey("")) + assert.Equal(t, "kubeconfig", describeKubeConfigKey("kubeconfig")) +} + +// TestReconcile_UnauthorizedNamespaceStartsNoWatch pins the whole security property in one place, +// through the REAL Reconcile rather than checkSourceAuthorization in isolation: namespace +// authorization is enforced at reconcile only (there is no admission webhook for it), so this path +// is the entire boundary. A GitTarget whose namespace its ClusterProvider does not admit must end +// up Validated=False/NamespaceNotAuthorized AND must never reach DeclareForGitTarget — no watch is +// started, so nothing is ever routed to a branch worker and nothing is written to Git. +func TestReconcile_UnauthorizedNamespaceStartsNoWatch(t *testing.T) { + const ns, providerName = "team-a", "prod-eu-1" + + // The provider admits team-b only; the GitTarget lives in team-a. + provider := &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: providerName}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{ + AllowedNamespaces: &configbutleraiv1alpha3.AllowedNamespaces{Names: []string{"team-b"}}, + }, + } + gitProvider := &configbutleraiv1alpha3.GitProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "gp", Namespace: ns}, + Spec: configbutleraiv1alpha3.GitProviderSpec{AllowedBranches: []string{"main"}}, + } + target := &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "gt", Namespace: ns, UID: "gt-uid"}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + ProviderRef: configbutleraiv1alpha3.GitProviderReference{Name: "gp"}, + Branch: "main", + Path: "apps", + ClusterProviderRef: &configbutleraiv1alpha3.ClusterProviderReference{Name: providerName}, + }, + } + + cl := fake.NewClientBuilder().WithScheme(scScheme(t)). + WithObjects(provider, gitProvider, target, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}). + WithStatusSubresource(&configbutleraiv1alpha3.GitTarget{}). + Build() + + watchManager := &watch.Manager{Client: cl, Log: logr.Discard()} + r := &GitTargetReconciler{ + Client: cl, + EventRouter: &watch.EventRouter{WatchManager: watchManager}, + } + + _, err := r.Reconcile(context.Background(), + ctrl.Request{NamespacedName: k8stypes.NamespacedName{Name: "gt", Namespace: ns}}) + require.NoError(t, err) + + var got configbutleraiv1alpha3.GitTarget + require.NoError(t, cl.Get(context.Background(), + k8stypes.NamespacedName{Name: "gt", Namespace: ns}, &got)) + + validated := apimeta.FindStatusCondition(got.Status.Conditions, GitTargetConditionValidated) + require.NotNil(t, validated, "Validated condition must be set") + assert.Equal(t, metav1.ConditionFalse, validated.Status) + assert.Equal(t, GitTargetReasonNamespaceNotAuthorized, validated.Reason) + assert.Contains(t, validated.Message, "not permitted to reference") + + // The data plane is reported blocked, not merely un-ready... + streams := apimeta.FindStatusCondition(got.Status.Conditions, GitTargetConditionStreamsRunning) + require.NotNil(t, streams) + assert.Equal(t, metav1.ConditionUnknown, streams.Status) + + // ...and, the point of the test, no declaration ever reached the watch manager. The Validated + // gate returns before DeclareForGitTarget, so the capture-on-Declare map stays empty for it. + gitDest := types.NewResourceReference("gt", ns).WithUID("gt-uid") + _, declared := watchManager.DeclaredSourceCluster(gitDest) + assert.False(t, declared, "a refused GitTarget must never declare watches against its source cluster") + + // Positive control, through the real Declare path: the same manager DOES record a declaration, + // so the assertion above cannot pass vacuously. DeclareForGitTarget captures the source cluster + // before it opens any watch, so it records even though opening watches fails here (no discovery + // client is wired) — which is exactly the capture a refused GitTarget must not produce. + other := types.NewResourceReference("authorized", ns).WithUID("other-uid") + _ = watchManager.DeclareForGitTarget(context.Background(), other, providerName) + id, declaredOther := watchManager.DeclaredSourceCluster(other) + require.True(t, declaredOther, "the positive control must declare, or the assertion above proves nothing") + assert.Equal(t, providerName, id) +} + +func TestClusterProviderReadiness_AllScenarios(t *testing.T) { + provider := func(conds []metav1.Condition) *configbutleraiv1alpha3.ClusterProvider { + return &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "prod-eu-1"}, + Status: configbutleraiv1alpha3.ClusterProviderStatus{Conditions: conds}, + } + } + ready := metav1.Condition{Type: ConditionTypeReady, Status: metav1.ConditionTrue, Reason: "OK"} + notReady := metav1.Condition{ + Type: ConditionTypeReady, + Status: metav1.ConditionFalse, + Reason: "Bad", + Message: "nope", + } + + unknownReady := metav1.Condition{ + Type: ConditionTypeReady, + Status: metav1.ConditionUnknown, + Reason: "Checking", + Message: "validating", + } + + tests := []struct { + name string + cp *configbutleraiv1alpha3.ClusterProvider + want metav1.ConditionStatus + }{ + {"ready", provider([]metav1.Condition{ready}), metav1.ConditionTrue}, + {"not ready -> False (downgrades)", provider([]metav1.Condition{notReady}), metav1.ConditionFalse}, + {"no condition -> Unknown (does not downgrade)", provider(nil), metav1.ConditionUnknown}, + {"absent provider -> Unknown", nil, metav1.ConditionUnknown}, + // Only an EXPLICIT Ready=False downgrades. A provider reporting Ready=Unknown is mid-flight, + // not broken, so it must not hold its GitTargets down. + { + "explicit Ready=Unknown -> Unknown (does not downgrade)", + provider([]metav1.Condition{unknownReady}), + metav1.ConditionUnknown, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + builder := fake.NewClientBuilder().WithScheme(scScheme(t)) + if tc.cp != nil { + builder = builder.WithObjects(tc.cp) + } + r := &GitTargetReconciler{Client: builder.Build()} + target := &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "gt", Namespace: "team-a"}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + ClusterProviderRef: &configbutleraiv1alpha3.ClusterProviderReference{Name: "prod-eu-1"}, + }, + } + status, _, msg := r.clusterProviderReadiness(context.Background(), target) + assert.Equal(t, tc.want, status) + assert.NotEmpty(t, msg) + }) + } +} + func TestGitProviderReadiness_AllScenarios(t *testing.T) { provider := func(conds []metav1.Condition) *configbutleraiv1alpha3.GitProvider { return &configbutleraiv1alpha3.GitProvider{ diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index d5d39140..c30431e4 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -90,6 +91,13 @@ var _ = BeforeSuite(func() { }).SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred()) + err = (&ClusterProviderReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + OperatorNamespace: "default", + }).SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + err = (&GitTargetReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), @@ -126,6 +134,17 @@ var _ = BeforeSuite(func() { // Note: Old git.Worker has been replaced by WorkerManager + BranchWorker architecture // Webhook tests are handled separately in webhook package + // Ship the reserved "default" ClusterProvider the way a real install (chart / config SUT) does, + // so a GitTarget that references it (the schema default) passes the reconcile-time hard gate + // that now REQUIRES the referenced ClusterProvider to exist. Its empty selector admits every + // namespace, matching the chart default. + Expect(k8sClient.Create(ctx, &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: configbutleraiv1alpha3.DefaultClusterProviderName}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{ + AllowedNamespaces: &configbutleraiv1alpha3.AllowedNamespaces{Selector: &metav1.LabelSelector{}}, + }, + })).To(Succeed()) + go func() { defer GinkgoRecover() err = mgr.Start(ctx) diff --git a/internal/controller/watchrule_controller_test.go b/internal/controller/watchrule_controller_test.go index a09d0d25..c46c848f 100644 --- a/internal/controller/watchrule_controller_test.go +++ b/internal/controller/watchrule_controller_test.go @@ -232,15 +232,15 @@ var _ = Describe("WatchRule Controller", func() { // until the GitTarget settles on its deterministic Ready=False/Progressing state // (no streams run in envtest) so the WatchRule reconcile observes a stable dependency. gitTarget := &configbutleraiv1alpha3.GitTarget{} - ready := eventuallyConditionStatus( + eventuallyConditionStatusReason( ctx, types.NamespacedName{Name: "local-target", Namespace: "default"}, gitTarget, func() []metav1.Condition { return gitTarget.Status.Conditions }, ConditionTypeReady, metav1.ConditionFalse, + ReasonProgressing, ) - Expect(ready.Reason).To(Equal(ReasonProgressing)) By("Reconciling the WatchRule directly") result, err := reconciler.Reconcile(ctx, reconcile.Request{ diff --git a/internal/git/pending_writes.go b/internal/git/pending_writes.go index c0f2f89c..e3a9371c 100644 --- a/internal/git/pending_writes.go +++ b/internal/git/pending_writes.go @@ -168,7 +168,7 @@ func (w *BranchWorker) resolveTargetMetadata( BootstrapOptions: buildBootstrapOptions(encryptionConfig), EncryptionConfig: encryptionConfig, Placement: resolvePlacementPolicy(target.Spec.Placement), - SourceClusterID: target.SourceClusterID(), + SourceCluster: target.SourceCluster(), }, nil } diff --git a/internal/git/plan_flush.go b/internal/git/plan_flush.go index 498441a1..04c9818f 100644 --- a/internal/git/plan_flush.go +++ b/internal/git/plan_flush.go @@ -57,8 +57,8 @@ func (w *BranchWorker) mapperForCluster(clusterID string) typeset.Lookup { // first non-empty id wins, and an all-empty set is the local cluster. func clusterIDForEvents(events []Event) string { for _, ev := range events { - if ev.SourceClusterID != "" { - return ev.SourceClusterID + if ev.SourceCluster != "" { + return ev.SourceCluster } } return "" diff --git a/internal/git/resync_flush.go b/internal/git/resync_flush.go index b272f090..59a42244 100644 --- a/internal/git/resync_flush.go +++ b/internal/git/resync_flush.go @@ -230,7 +230,7 @@ func (w *BranchWorker) executeResyncPendingWrite( target := pendingWrite.Target() base := sanitizePath(target.Path) - if err := w.refuseUnsafeWorktree(ctx, worktree, base, target.SourceClusterID); err != nil { + if err := w.refuseUnsafeWorktree(ctx, worktree, base, target.SourceCluster); err != nil { return 0, err } @@ -248,7 +248,7 @@ func (w *BranchWorker) executeResyncPendingWrite( } stats, anyChanges, err := w.applyResyncToWorktree( - ctx, worktree, base, target.SourceClusterID, pendingWrite.Desired, pendingWrite.ScopeGVR, target.Placement, + ctx, worktree, base, target.SourceCluster, pendingWrite.Desired, pendingWrite.ScopeGVR, target.Placement, ) if err != nil { return 0, err diff --git a/internal/git/types.go b/internal/git/types.go index d380944d..4a23676f 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -158,11 +158,12 @@ type ResolvedTargetMetadata struct { // from spec.placement. Nil when the GitTarget declares none, in which case new // resources are placed by sibling inference and then the canonical path. Placement *manifestanalyzer.PlacementPolicy - // SourceClusterID is the id of the cluster the GitTarget mirrors from — - // (api/v1alpha3).GitTarget.SourceClusterID(), "" for the local cluster. The resync - // mark-and-sweep resolves this subtree's documents' GVK->GVR against that cluster's - // registry, so a folder mirroring a remote is swept against the right cluster's mapping. - SourceClusterID string + // SourceCluster is the NAME of the source cluster the GitTarget mirrors from — + // (api/v1alpha3).GitTarget.SourceCluster(), the referenced ClusterProvider's name + // ("default" for the in-cluster provider). The resync mark-and-sweep resolves this subtree's + // documents' GVK->GVR against that cluster's registry, so a folder mirroring a remote is swept + // against the right cluster's mapping. + SourceCluster string } // PendingWrite is the unit retained until a push succeeds. @@ -341,11 +342,12 @@ type Event struct { // GitTargetNamespace is the namespace of the target owning this event. GitTargetNamespace string - // SourceClusterID is the id of the cluster this object was watched on — - // (api/v1alpha3).GitTarget.SourceClusterID(), "" for the local cluster. The writer - // resolves this document's GVK->GVR against that cluster's type registry, so a folder - // mirroring a remote is never indexed against the local cluster's mapping. - SourceClusterID string + // SourceCluster is the NAME of the source cluster this object was watched on — + // (api/v1alpha3).GitTarget.SourceCluster(), the referenced ClusterProvider's name + // ("default" for the in-cluster provider). The writer resolves this document's GVK->GVR + // against that cluster's type registry, so a folder mirroring a remote is never indexed + // against the local cluster's mapping. + SourceCluster string // BootstrapOptions controls path-scoped bootstrap file staging for this event. BootstrapOptions pathBootstrapOptions diff --git a/internal/queue/attribution_index.go b/internal/queue/attribution_index.go index f11e6bf6..85137222 100644 --- a/internal/queue/attribution_index.go +++ b/internal/queue/attribution_index.go @@ -26,7 +26,7 @@ import ( // they expire on their own — nothing deletes them. After it elapses a miss is simply // "absent": the v3 schema keeps no tombstone, so an aged-out fact is indistinguishable // from one that never arrived. Configurable via --author-attribution-ttl. -const DefaultAttributionFactTTL = 15 * time.Minute +const DefaultAttributionFactTTL = 10 * time.Minute // DefaultKeyPrefix is the root namespace every Redis key (cursors, facts, and command // author records alike) carries when --redis-key-prefix is not set. It is also the value @@ -35,8 +35,15 @@ const DefaultKeyPrefix = "gitops-reverser" const ( // attributionKeySuffix namespaces audit-sourced resource author facts under the - // top-level author domain, e.g. "gitops-reverser:author:v1:audit::...". + // top-level author domain, e.g. + // "gitops-reverser:author:v1:audit:cluster:::...". attributionKeySuffix = ":author:v1:audit:" + // clusterKeyInfix carries the SOURCE CLUSTER dimension (a ClusterProvider name, "default" + // for the in-cluster provider) so a fact from cluster A never joins a watch event from + // cluster B — the rv-only hatch especially, since RV is not globally unique. It sits right + // after attributionKeySuffix so a provider's facts share a single glob prefix + // "…:author:v1:audit:cluster::*", which the delete-on-provider purge scans. + clusterKeyInfix = "cluster:" // factObjectInfix groups every fact for one object under one prefix, so a SCAN of // ":object::*" shows the whole history-in-flight for that object. factObjectInfix = ":object:" @@ -119,7 +126,7 @@ type AttributionIndex struct { // It is a no-op for events without an objectRef, a resolvable name, or a user — those // can never name an author. The caller (the audit handler) has already rejected reads, // failures, dry-runs, and non-ResponseComplete stages. -func (a *AttributionIndex) RecordFact(ctx context.Context, event auditv1.Event) error { +func (a *AttributionIndex) RecordFact(ctx context.Context, providerName string, event auditv1.Event) error { if event.ObjectRef == nil { return nil } @@ -132,7 +139,7 @@ func (a *AttributionIndex) RecordFact(ctx context.Context, event auditv1.Event) // A deletecollection is name-less, so it can never name a single object. When the // API server returns the deleted set, expand it into one fact per object instead. if strings.EqualFold(event.Verb, "deletecollection") { - return a.RecordDeleteCollectionFacts(ctx, event) + return a.RecordDeleteCollectionFacts(ctx, providerName, event) } op, _ := auditutil.VerbToOperation(event.Verb) @@ -172,7 +179,7 @@ func (a *AttributionIndex) RecordFact(ctx context.Context, event auditv1.Event) return fmt.Errorf("marshal attribution fact: %w", err) } - wrote, err := a.writeFactKeys(ctx, gr, uid, rv, raw) + wrote, err := a.writeFactKeys(ctx, providerName, gr, uid, rv, raw) if err != nil { return err } @@ -188,15 +195,15 @@ func (a *AttributionIndex) RecordFact(ctx context.Context, event auditv1.Event) // known, or the type-scoped rv-only escape hatch when the fact has an RV but no UID. // The exact key is written once per (uid, rv) and never contended, so there is no // conflict marking. It reports whether any key was written. -func (a *AttributionIndex) writeFactKeys(ctx context.Context, gr, uid, rv string, raw []byte) (bool, error) { +func (a *AttributionIndex) writeFactKeys(ctx context.Context, cluster, gr, uid, rv string, raw []byte) (bool, error) { switch { case uid != "": if rv != "" { - if err := a.setFact(ctx, a.factKeyExact(gr, uid, rv), raw); err != nil { + if err := a.setFact(ctx, a.factKeyExact(cluster, gr, uid, rv), raw); err != nil { return false, fmt.Errorf("store exact attribution fact: %w", err) } } - if err := a.setFact(ctx, a.factKeyLast(gr, uid), raw); err != nil { + if err := a.setFact(ctx, a.factKeyLast(cluster, gr, uid), raw); err != nil { return false, fmt.Errorf("store last attribution fact: %w", err) } return true, nil @@ -204,7 +211,7 @@ func (a *AttributionIndex) writeFactKeys(ctx context.Context, gr, uid, rv string // The §5 escape hatch: a UID-bearing fact's rv-only key would be dead (the watch // side always carries a UID and resolves via object::… first), so it is // written only when there is no UID. - if err := a.setFact(ctx, a.factKeyRV(gr, rv), raw); err != nil { + if err := a.setFact(ctx, a.factKeyRV(cluster, gr, rv), raw); err != nil { return false, fmt.Errorf("store rv-only attribution fact: %w", err) } return true, nil @@ -225,7 +232,11 @@ func (a *AttributionIndex) writeFactKeys(ctx context.Context, gr, uid, rv string // deletionTimestamp already removes the file, so the actor who ran the collection delete // is credited with that removal even while Kubernetes finalization is still in flight. // See docs/spec/deletecollection-attribution-expander.md. -func (a *AttributionIndex) RecordDeleteCollectionFacts(ctx context.Context, event auditv1.Event) error { +func (a *AttributionIndex) RecordDeleteCollectionFacts( + ctx context.Context, + providerName string, + event auditv1.Event, +) error { if !strings.EqualFold(event.Verb, "deletecollection") || event.ObjectRef == nil || event.ObjectRef.Resource == "" { return nil } @@ -249,7 +260,14 @@ func (a *AttributionIndex) RecordDeleteCollectionFacts(ctx context.Context, even if !event.StageTimestamp.IsZero() { base.StageTimestamp = event.StageTimestamp.UTC().Format(time.RFC3339Nano) } - return a.storeDeleteCollectionFacts(ctx, event.ObjectRef.APIGroup, event.ObjectRef.Resource, items, base) + return a.storeDeleteCollectionFacts( + ctx, + providerName, + event.ObjectRef.APIGroup, + event.ObjectRef.Resource, + items, + base, + ) } // storeDeleteCollectionFacts writes one :last fact per joinable item, carrying the @@ -257,6 +275,7 @@ func (a *AttributionIndex) RecordDeleteCollectionFacts(ctx context.Context, even // one item was written. func (a *AttributionIndex) storeDeleteCollectionFacts( ctx context.Context, + cluster string, group, resource string, items []deleteCollectionItem, base AuthorFact, @@ -276,7 +295,7 @@ func (a *AttributionIndex) storeDeleteCollectionFacts( if err != nil { return fmt.Errorf("marshal deletecollection fact: %w", err) } - if err := a.setFact(ctx, a.factKeyLast(gr, string(item.UID)), raw); err != nil { + if err := a.setFact(ctx, a.factKeyLast(cluster, gr, string(item.UID)), raw); err != nil { return fmt.Errorf("store deletecollection fact %q: %w", item.Name, err) } expanded = true @@ -332,12 +351,13 @@ func deleteCollectionItems(obj *runtime.Unknown) []deleteCollectionItem { // policy: see LookupAuthorResolution. func (a *AttributionIndex) LookupAuthor( ctx context.Context, + providerName string, gvr schema.GroupVersionResource, uid types.UID, rv string, exactCapable bool, ) (AuthorFact, bool) { - resolution := a.LookupAuthorResolution(ctx, gvr, uid, rv, exactCapable) + resolution := a.LookupAuthorResolution(ctx, providerName, gvr, uid, rv, exactCapable) return resolution.Fact, resolution.Result != AttributionAbsent } @@ -354,6 +374,7 @@ func (a *AttributionIndex) LookupAuthor( // A miss returns AttributionAbsent; there is no tombstone and so no expired outcome. func (a *AttributionIndex) LookupAuthorResolution( ctx context.Context, + providerName string, gvr schema.GroupVersionResource, uid types.UID, rv string, @@ -361,17 +382,17 @@ func (a *AttributionIndex) LookupAuthorResolution( ) AuthorResolution { gr := groupResourceKey(gvr.Group, gvr.Resource) if uid != "" && rv != "" { - if res, ok := a.matchFactKey(ctx, a.factKeyExact(gr, string(uid), rv), false); ok { + if res, ok := a.matchFactKey(ctx, a.factKeyExact(providerName, gr, string(uid), rv), false); ok { return res } } if !exactCapable && uid != "" { - if res, ok := a.matchFactKey(ctx, a.factKeyLast(gr, string(uid)), true); ok { + if res, ok := a.matchFactKey(ctx, a.factKeyLast(providerName, gr, string(uid)), true); ok { return res } } if rv != "" { - if res, ok := a.matchFactKey(ctx, a.factKeyRV(gr, rv), true); ok { + if res, ok := a.matchFactKey(ctx, a.factKeyRV(providerName, gr, rv), true); ok { return res } } @@ -409,29 +430,37 @@ func attributionResultForFact(fact AuthorFact, weak bool) AttributionResult { return AttributionExactUser } -// factKeyBase is the per-type prefix shared by every fact key, e.g. -// "gitops-reverser:author:v1:audit:apps/deployments". -func (a *AttributionIndex) factKeyBase(gr string) string { - return resolveKeyPrefix(a.keyPrefix) + attributionKeySuffix + gr +// clusterFactPrefix is the per-cluster glob prefix under which every fact for one source cluster +// lives, e.g. "gitops-reverser:author:v1:audit:cluster:prod-eu-1:". The delete-on-provider purge +// SCANs "*". +func (a *AttributionIndex) clusterFactPrefix(cluster string) string { + return resolveKeyPrefix(a.keyPrefix) + attributionKeySuffix + clusterKeyInfix + escapeKeyField(cluster) + ":" +} + +// factKeyBase is the per-(cluster,type) prefix shared by every fact key, e.g. +// "gitops-reverser:author:v1:audit:cluster:default:apps/deployments". +func (a *AttributionIndex) factKeyBase(cluster, gr string) string { + return a.clusterFactPrefix(cluster) + gr } // factKeyExact is the immutable per-write fact key, e.g. -// "gitops-reverser:author:v1:audit:apps/deployments:object::101". -func (a *AttributionIndex) factKeyExact(gr, uid, rv string) string { - return a.factKeyBase(gr) + factObjectInfix + escapeKeyField(uid) + ":" + escapeKeyField(rv) +// "gitops-reverser:author:v1:audit:cluster:default:apps/deployments:object::101". +func (a *AttributionIndex) factKeyExact(cluster, gr, uid, rv string) string { + return a.factKeyBase(cluster, gr) + factObjectInfix + escapeKeyField(uid) + ":" + escapeKeyField(rv) } // factKeyLast is the latest-writer-wins pointer for an object, e.g. -// "gitops-reverser:author:v1:audit:apps/deployments:object::last". -func (a *AttributionIndex) factKeyLast(gr, uid string) string { - return a.factKeyBase(gr) + factObjectInfix + escapeKeyField(uid) + ":" + factLastLeaf +// "gitops-reverser:author:v1:audit:cluster:default:apps/deployments:object::last". +func (a *AttributionIndex) factKeyLast(cluster, gr, uid string) string { + return a.factKeyBase(cluster, gr) + factObjectInfix + escapeKeyField(uid) + ":" + factLastLeaf } -// factKeyRV is the type-scoped rv-only escape hatch, e.g. -// "gitops-reverser:author:v1:audit:apps/deployments:rv:101". RV is opaque per the -// Kubernetes API contract and not globally unique, so this key always includes the type. -func (a *AttributionIndex) factKeyRV(gr, rv string) string { - return a.factKeyBase(gr) + factRVInfix + escapeKeyField(rv) +// factKeyRV is the (cluster, type)-scoped rv-only escape hatch, e.g. +// "gitops-reverser:author:v1:audit:cluster:default:apps/deployments:rv:101". RV is opaque per the +// Kubernetes API contract and not globally unique — not even within a cluster's type, and +// certainly not across clusters — so this key always includes both the cluster and the type. +func (a *AttributionIndex) factKeyRV(cluster, gr, rv string) string { + return a.factKeyBase(cluster, gr) + factRVInfix + escapeKeyField(rv) } // setFact writes one fact value under its key with the bounded fact TTL. No sibling @@ -440,6 +469,38 @@ func (a *AttributionIndex) setFact(ctx context.Context, key string, raw []byte) return a.client.Set(ctx, key, raw, a.factTTL).Err() } +// PurgeClusterFacts deletes every attribution fact keyed to one source cluster (a ClusterProvider +// name). It is run by the ClusterProvider delete finalizer so a recreated name — a cluster torn +// down and stood up again under the same provider name — starts with a clean keyspace and can +// never join a stale predecessor's facts. It SCANs the per-cluster glob prefix and deletes in +// batches (facts also self-expire on the TTL, so this is the belt to that suspenders). It returns +// the number of keys removed. +func (a *AttributionIndex) PurgeClusterFacts(ctx context.Context, providerName string) (int, error) { + pattern := a.clusterFactPrefix(providerName) + "*" + var cursor uint64 + var removed int + for { + keys, next, err := a.client.Scan(ctx, cursor, pattern, attributionFactScanBatchSize).Result() + if err != nil { + return removed, fmt.Errorf("scan cluster facts %q: %w", pattern, err) + } + if len(keys) > 0 { + if err := a.client.Del(ctx, keys...).Err(); err != nil { + return removed, fmt.Errorf("delete cluster facts %q: %w", pattern, err) + } + removed += len(keys) + } + cursor = next + if cursor == 0 { + break + } + } + if removed > 0 { + a.recordFactEvent(ctx, "cluster_purged") + } + return removed, nil +} + func (a *AttributionIndex) recordFactEvent(ctx context.Context, op string) { if telemetry.AttributionFactEventsTotal == nil { return diff --git a/internal/queue/attribution_index_deletecollection_test.go b/internal/queue/attribution_index_deletecollection_test.go index ec54d51e..f6c73d6b 100644 --- a/internal/queue/attribution_index_deletecollection_test.go +++ b/internal/queue/attribution_index_deletecollection_test.go @@ -60,14 +60,14 @@ func deleteCollectionEvent(username string, items ...dcItem) auditv1.Event { // proving the join is by UID (the body item carried no RV at all) via the :last pointer. // A collection removal is a known RV-mismatch event, so it is not exact-capable. func resolveDC(ctx context.Context, idx *AttributionIndex, _, uid string) AuthorResolution { - return idx.LookupAuthorResolution(ctx, coreConfigmapsGVR(), k8stypes.UID(uid), "9999", false) + return idx.LookupAuthorResolution(ctx, "default", coreConfigmapsGVR(), k8stypes.UID(uid), "9999", false) } func TestRecordDeleteCollectionFacts_ExpandsListToPerObjectFacts(t *testing.T) { idx := newTestAttributionIndex(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, deleteCollectionEvent("alice", + require.NoError(t, idx.RecordFact(ctx, "default", deleteCollectionEvent("alice", dcItem{namespace: "team-a", name: "a", uid: "uid-a"}, dcItem{namespace: "team-a", name: "b", uid: "uid-b"}, dcItem{namespace: "team-a", name: "c", uid: "uid-c"}, @@ -87,7 +87,7 @@ func TestRecordDeleteCollectionFacts_FinalizerItemAttributed(t *testing.T) { idx := newTestAttributionIndex(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, deleteCollectionEvent("alice", + require.NoError(t, idx.RecordFact(ctx, "default", deleteCollectionEvent("alice", dcItem{namespace: "team-a", name: "plain", uid: "uid-plain"}, dcItem{namespace: "team-a", name: "stuck", uid: "uid-stuck", terminating: true}, ))) @@ -108,15 +108,15 @@ func TestRecordDeleteCollectionFacts_HollowBodyWritesNothing(t *testing.T) { statusEvent := deleteCollectionEvent("alice") statusEvent.ResponseObject = &runtime.Unknown{Raw: []byte(`{"kind":"Status","status":"Success"}`)} - require.NoError(t, idx.RecordFact(ctx, statusEvent)) + require.NoError(t, idx.RecordFact(ctx, "default", statusEvent)) absentEvent := deleteCollectionEvent("alice") absentEvent.ResponseObject = nil - require.NoError(t, idx.RecordFact(ctx, absentEvent)) + require.NoError(t, idx.RecordFact(ctx, "default", absentEvent)) badEvent := deleteCollectionEvent("alice") badEvent.ResponseObject = &runtime.Unknown{Raw: []byte(`{not json`)} - require.NoError(t, idx.RecordFact(ctx, badEvent)) + require.NoError(t, idx.RecordFact(ctx, "default", badEvent)) require.Equal(t, AttributionAbsent, resolveDC(ctx, idx, "anything", "uid-x").Result) } @@ -128,7 +128,7 @@ func TestRecordDeleteCollectionFacts_PartialListOnlyWritesPresent(t *testing.T) idx := newTestAttributionIndex(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, deleteCollectionEvent("alice", + require.NoError(t, idx.RecordFact(ctx, "default", deleteCollectionEvent("alice", dcItem{namespace: "team-a", name: "a", uid: "uid-a"}, dcItem{namespace: "team-a", name: "b", uid: "uid-b"}, ))) @@ -145,7 +145,7 @@ func TestRecordDeleteCollectionFacts_SkipsItemsMissingUIDOrName(t *testing.T) { idx := newTestAttributionIndex(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, deleteCollectionEvent("alice", + require.NoError(t, idx.RecordFact(ctx, "default", deleteCollectionEvent("alice", dcItem{namespace: "team-a", name: "good", uid: "uid-good"}, dcItem{namespace: "team-a", name: "", uid: "uid-noname"}, dcItem{namespace: "team-a", name: "nouid", uid: ""}, @@ -161,7 +161,7 @@ func TestRecordDeleteCollectionFacts_ServiceAccountActor(t *testing.T) { ctx := context.Background() const sa = "system:serviceaccount:flux-system:kustomize-controller" - require.NoError(t, idx.RecordFact(ctx, deleteCollectionEvent(sa, + require.NoError(t, idx.RecordFact(ctx, "default", deleteCollectionEvent(sa, dcItem{namespace: "team-a", name: "a", uid: "uid-a"}, ))) @@ -176,7 +176,10 @@ func TestRecordDeleteCollectionFacts_NonDeleteCollectionVerbIsNoOp(t *testing.T) idx := newTestAttributionIndex(t) ctx := context.Background() - require.NoError(t, idx.RecordDeleteCollectionFacts(ctx, mutationEvent("delete", "uid-1", "101", "alice"))) + require.NoError( + t, + idx.RecordDeleteCollectionFacts(ctx, "default", mutationEvent("delete", "uid-1", "101", "alice")), + ) require.Equal(t, AttributionAbsent, - idx.LookupAuthorResolution(ctx, appsDeploymentGVR(), "uid-1", "101", true).Result) + idx.LookupAuthorResolution(ctx, "default", appsDeploymentGVR(), "uid-1", "101", true).Result) } diff --git a/internal/queue/attribution_index_test.go b/internal/queue/attribution_index_test.go index 254ccbc1..95b2966a 100644 --- a/internal/queue/attribution_index_test.go +++ b/internal/queue/attribution_index_test.go @@ -78,9 +78,9 @@ func TestAttributionIndex_RecordAndLookupExact(t *testing.T) { idx := newTestAttributionIndex(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, mutationEvent("update", "uid-1", "101", "alice"))) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "uid-1", "101", "alice"))) - fact, ok := idx.LookupAuthor(ctx, appsDeploymentGVR(), "uid-1", "101", true) + fact, ok := idx.LookupAuthor(ctx, "default", appsDeploymentGVR(), "uid-1", "101", true) require.True(t, ok) require.Equal(t, "alice", fact.Author) require.Equal(t, "101", fact.ResourceVersion) @@ -95,11 +95,11 @@ func TestAttributionIndex_LookupByUIDWhenRVDiffers(t *testing.T) { idx := newTestAttributionIndex(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, mutationEvent("delete", "uid-1", "101", "alice"))) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("delete", "uid-1", "101", "alice"))) // Watch DELETE lands at a later RV; the uid-latest :last pointer still resolves the // author (a delete is not exact-capable, so it may consult :last). - fact, ok := idx.LookupAuthor(ctx, appsDeploymentGVR(), "uid-1", "999", false) + fact, ok := idx.LookupAuthor(ctx, "default", appsDeploymentGVR(), "uid-1", "999", false) require.True(t, ok) require.Equal(t, "alice", fact.Author) } @@ -108,9 +108,9 @@ func TestAttributionIndex_LookupResolutionWeakWhenExactMisses(t *testing.T) { idx := newTestAttributionIndex(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, mutationEvent("delete", "uid-1", "101", "alice"))) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("delete", "uid-1", "101", "alice"))) - resolution := idx.LookupAuthorResolution(ctx, appsDeploymentGVR(), "uid-1", "999", false) + resolution := idx.LookupAuthorResolution(ctx, "default", appsDeploymentGVR(), "uid-1", "999", false) require.Equal(t, AttributionWeak, resolution.Result) require.Equal(t, "alice", resolution.Fact.Author) } @@ -120,15 +120,15 @@ func TestAttributionIndex_ExactCapableDoesNotFallThroughToLast(t *testing.T) { ctx := context.Background() // alice's write seeds both the exact key (uid-1:101) and the :last pointer. - require.NoError(t, idx.RecordFact(ctx, mutationEvent("update", "uid-1", "101", "alice"))) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "uid-1", "101", "alice"))) // An exact-capable event at a different RV whose exact key is absent must NOT borrow // the :last author — it is absent and ships as committer. - res := idx.LookupAuthorResolution(ctx, appsDeploymentGVR(), "uid-1", "202", true) + res := idx.LookupAuthorResolution(ctx, "default", appsDeploymentGVR(), "uid-1", "202", true) require.Equal(t, AttributionAbsent, res.Result) // The same miss for a known RV-mismatch event DOES consult :last. - weak := idx.LookupAuthorResolution(ctx, appsDeploymentGVR(), "uid-1", "202", false) + weak := idx.LookupAuthorResolution(ctx, "default", appsDeploymentGVR(), "uid-1", "202", false) require.Equal(t, AttributionWeak, weak.Result) require.Equal(t, "alice", weak.Fact.Author) } @@ -138,19 +138,19 @@ func TestAttributionIndex_BurstKeepsEachWritePrecise(t *testing.T) { ctx := context.Background() // Two authors write the same object in a burst at distinct RVs. - require.NoError(t, idx.RecordFact(ctx, mutationEvent("update", "uid-1", "1", "alice"))) - require.NoError(t, idx.RecordFact(ctx, mutationEvent("update", "uid-1", "2", "bob"))) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "uid-1", "1", "alice"))) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "uid-1", "2", "bob"))) // Each watch event hits its own immutable exact key → both precise, no conflict. - f1, ok := idx.LookupAuthor(ctx, appsDeploymentGVR(), "uid-1", "1", true) + f1, ok := idx.LookupAuthor(ctx, "default", appsDeploymentGVR(), "uid-1", "1", true) require.True(t, ok) require.Equal(t, "alice", f1.Author) - f2, ok := idx.LookupAuthor(ctx, appsDeploymentGVR(), "uid-1", "2", true) + f2, ok := idx.LookupAuthor(ctx, "default", appsDeploymentGVR(), "uid-1", "2", true) require.True(t, ok) require.Equal(t, "bob", f2.Author) // :last is last-writer-wins (bob), consulted only by an RV-mismatch event. - fl, ok := idx.LookupAuthor(ctx, appsDeploymentGVR(), "uid-1", "999", false) + fl, ok := idx.LookupAuthor(ctx, "default", appsDeploymentGVR(), "uid-1", "999", false) require.True(t, ok) require.Equal(t, "bob", fl.Author) } @@ -159,15 +159,15 @@ func TestAttributionIndex_NoUIDFactWritesRVKeyOnly(t *testing.T) { idx, mr := newTestAttributionIndexWithRedis(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, mutationEvent("update", "", "202", "alice"))) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "", "202", "alice"))) // The §5 escape hatch: a no-UID fact writes the type-scoped rv-only key and no // object keys. - require.True(t, mr.Exists(idx.factKeyRV("apps/deployments", "202"))) - require.False(t, mr.Exists(idx.factKeyLast("apps/deployments", ""))) + require.True(t, mr.Exists(idx.factKeyRV("default", "apps/deployments", "202"))) + require.False(t, mr.Exists(idx.factKeyLast("default", "apps/deployments", ""))) // An exact-capable watch event (which carries a UID) joins it via the rv-only fallback. - res := idx.LookupAuthorResolution(ctx, appsDeploymentGVR(), "uid-live", "202", true) + res := idx.LookupAuthorResolution(ctx, "default", appsDeploymentGVR(), "uid-live", "202", true) require.Equal(t, AttributionWeak, res.Result) require.Equal(t, "alice", res.Fact.Author) } @@ -176,11 +176,11 @@ func TestAttributionIndex_UIDFactWritesNoDeadRVKey(t *testing.T) { idx, mr := newTestAttributionIndexWithRedis(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, mutationEvent("update", "uid-1", "303", "alice"))) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "uid-1", "303", "alice"))) - require.True(t, mr.Exists(idx.factKeyExact("apps/deployments", "uid-1", "303"))) - require.True(t, mr.Exists(idx.factKeyLast("apps/deployments", "uid-1"))) - require.False(t, mr.Exists(idx.factKeyRV("apps/deployments", "303")), + require.True(t, mr.Exists(idx.factKeyExact("default", "apps/deployments", "uid-1", "303"))) + require.True(t, mr.Exists(idx.factKeyLast("default", "apps/deployments", "uid-1"))) + require.False(t, mr.Exists(idx.factKeyRV("default", "apps/deployments", "303")), "a UID-bearing fact's rv-only key would be dead, so it is not written") } @@ -188,10 +188,10 @@ func TestAttributionIndex_ServiceAccountFlagged(t *testing.T) { idx := newTestAttributionIndex(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, mutationEvent("update", "uid-1", "303", + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "uid-1", "303", "system:serviceaccount:flux-system:kustomize-controller"))) - fact, ok := idx.LookupAuthor(ctx, appsDeploymentGVR(), "uid-1", "303", true) + fact, ok := idx.LookupAuthor(ctx, "default", appsDeploymentGVR(), "uid-1", "303", true) require.True(t, ok) require.True(t, fact.IsServiceAccount) } @@ -200,15 +200,15 @@ func TestAttributionIndex_NoUserIsNoOp(t *testing.T) { idx := newTestAttributionIndex(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, mutationEvent("update", "uid-1", "101", ""))) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "uid-1", "101", ""))) - _, ok := idx.LookupAuthor(ctx, appsDeploymentGVR(), "uid-1", "101", true) + _, ok := idx.LookupAuthor(ctx, "default", appsDeploymentGVR(), "uid-1", "101", true) require.False(t, ok) } func TestAttributionIndex_LookupMiss(t *testing.T) { idx := newTestAttributionIndex(t) - _, ok := idx.LookupAuthor(context.Background(), appsDeploymentGVR(), "uid-x", "1", true) + _, ok := idx.LookupAuthor(context.Background(), "default", appsDeploymentGVR(), "uid-x", "1", true) require.False(t, ok) } @@ -217,12 +217,12 @@ func TestAttributionIndex_AgedOutFactIsAbsent(t *testing.T) { idx := store.AttributionIndex(time.Minute) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, mutationEvent("update", "uid-1", "101", "alice"))) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "uid-1", "101", "alice"))) mr.FastForward(time.Minute + time.Second) // No tombstone: an aged-out fact is indistinguishable from one that never arrived. - resolution := idx.LookupAuthorResolution(ctx, appsDeploymentGVR(), "uid-1", "101", true) + resolution := idx.LookupAuthorResolution(ctx, "default", appsDeploymentGVR(), "uid-1", "101", true) require.Equal(t, AttributionAbsent, resolution.Result) } @@ -232,8 +232,8 @@ func TestAttributionIndex_FactLifecycleMetrics(t *testing.T) { idx := newTestAttributionIndex(t) ctx := context.Background() - require.NoError(t, idx.RecordFact(ctx, mutationEvent("update", "uid-1", "101", "alice"))) - _ = idx.LookupAuthorResolution(ctx, appsDeploymentGVR(), "uid-1", "101", true) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "uid-1", "101", "alice"))) + _ = idx.LookupAuthorResolution(ctx, "default", appsDeploymentGVR(), "uid-1", "101", true) written, ok := telemetry.CollectInt64Sum(reader, "gitopsreverser_attribution_fact_events_total", map[string]string{"op": "written"}) @@ -254,17 +254,20 @@ func TestAttributionIndex_RecordFactNoOpCases(t *testing.T) { ctx := context.Background() // No objectRef → nothing to key on. - require.NoError(t, idx.RecordFact(ctx, auditv1.Event{Verb: "create", User: authnv1.UserInfo{Username: "a"}})) + require.NoError( + t, + idx.RecordFact(ctx, "default", auditv1.Event{Verb: "create", User: authnv1.UserInfo{Username: "a"}}), + ) // Empty resource → cannot build a key. - require.NoError(t, idx.RecordFact(ctx, auditv1.Event{ + require.NoError(t, idx.RecordFact(ctx, "default", auditv1.Event{ Verb: "create", User: authnv1.UserInfo{Username: "a"}, ObjectRef: &auditv1.ObjectReference{APIGroup: "apps", Name: "web"}, })) // No resolvable name → no author can be attributed to an object. - require.NoError(t, idx.RecordFact(ctx, auditv1.Event{ + require.NoError(t, idx.RecordFact(ctx, "default", auditv1.Event{ Verb: "create", User: authnv1.UserInfo{Username: "a"}, ObjectRef: &auditv1.ObjectReference{APIGroup: "apps", Resource: "deployments"}, @@ -327,11 +330,14 @@ func TestAttributionIndex_FactTTLConfigurable(t *testing.T) { store, mr := newTestRedisStoreWithRedis(t) idx := store.AttributionIndex(5 * time.Minute) - require.NoError(t, idx.RecordFact(context.Background(), mutationEvent("update", "uid-1", "101", "alice"))) + require.NoError( + t, + idx.RecordFact(context.Background(), "default", mutationEvent("update", "uid-1", "101", "alice")), + ) for _, key := range []string{ - idx.factKeyExact("apps/deployments", "uid-1", "101"), - idx.factKeyLast("apps/deployments", "uid-1"), + idx.factKeyExact("default", "apps/deployments", "uid-1", "101"), + idx.factKeyLast("default", "apps/deployments", "uid-1"), } { require.Equal(t, 5*time.Minute, mr.TTL(key), "fact key %q", key) } @@ -341,9 +347,12 @@ func TestAttributionIndex_FactTTLDefaultsWhenUnset(t *testing.T) { store, mr := newTestRedisStoreWithRedis(t) idx := store.AttributionIndex(0) - require.NoError(t, idx.RecordFact(context.Background(), mutationEvent("update", "uid-1", "101", "alice"))) + require.NoError( + t, + idx.RecordFact(context.Background(), "default", mutationEvent("update", "uid-1", "101", "alice")), + ) - require.Equal(t, DefaultAttributionFactTTL, mr.TTL(idx.factKeyExact("apps/deployments", "uid-1", "101"))) + require.Equal(t, DefaultAttributionFactTTL, mr.TTL(idx.factKeyExact("default", "apps/deployments", "uid-1", "101"))) } func TestEscapeKeyField(t *testing.T) { @@ -367,19 +376,247 @@ func TestGroupResourceKey(t *testing.T) { require.Equal(t, "rbac.authorization.k8s.io/roles", groupResourceKey("rbac.authorization.k8s.io", "roles")) } +// rawObject wraps a JSON body the way the audit pipeline delivers request/response bodies. +func rawObject(body string) *runtime.Unknown { + return &runtime.Unknown{Raw: []byte(body)} +} + +// deploymentBody renders a minimal Deployment whose metadata.resourceVersion is rv. +func deploymentBody(rv string) string { + return fmt.Sprintf(`{"apiVersion":"apps/v1","kind":"Deployment",`+ + `"metadata":{"name":"web","namespace":"team-a","uid":"uid-1","resourceVersion":%q}}`, rv) +} + +// TestResourceVersionFromEvent_Precedence pins the RV precedence the join depends on. The RV is +// half of the fact key, so reading the wrong one files the fact under an object version that will +// never be looked up — the write silently ships the committer instead of its real author. Only the +// POST-write RV identifies the version a mutation produced: that lives in responseObject. +// requestObject carries the PRE-write RV and must never be consulted, and objectRef.resourceVersion +// is usually the empty precondition RV on writes, so it is a last resort only. +func TestResourceVersionFromEvent_Precedence(t *testing.T) { + cases := []struct { + name string + mutate func(*auditv1.Event) + wantRV string + wantWhy string + }{ + { + name: "response object wins over a different objectRef RV", + mutate: func(e *auditv1.Event) { + e.ResponseObject = rawObject(deploymentBody("202")) + e.ObjectRef.ResourceVersion = "101" + }, + wantRV: "202", + wantWhy: "the post-write RV in the response body is authoritative", + }, + { + name: "request object is ignored even when the response object has none", + mutate: func(e *auditv1.Event) { + e.RequestObject = rawObject(deploymentBody("101")) + e.ResponseObject = nil + e.ObjectRef.ResourceVersion = "" + }, + wantRV: "", + wantWhy: "requestObject holds the pre-write RV and is never a source", + }, + { + name: "request object never outranks the response object", + mutate: func(e *auditv1.Event) { + e.RequestObject = rawObject(deploymentBody("101")) + e.ResponseObject = rawObject(deploymentBody("202")) + }, + wantRV: "202", + wantWhy: "responseObject is consulted first and short-circuits", + }, + { + name: "objectRef is the fallback when the response object is absent", + mutate: func(e *auditv1.Event) { + e.ResponseObject = nil + e.ObjectRef.ResourceVersion = "101" + }, + wantRV: "101", + wantWhy: "objectRef is the last resort, not the first choice", + }, + { + name: "objectRef is the fallback when the response body carries no RV", + mutate: func(e *auditv1.Event) { + e.ResponseObject = rawObject(`{"metadata":{"name":"web"}}`) + e.ObjectRef.ResourceVersion = "101" + }, + wantRV: "101", + wantWhy: "a shallow body yields nothing, so the fallback still applies", + }, + { + name: "objectRef is the fallback when the response body is malformed", + mutate: func(e *auditv1.Event) { + e.ResponseObject = rawObject(`{"metadata":`) + e.ObjectRef.ResourceVersion = "101" + }, + wantRV: "101", + wantWhy: "an unparseable body must not poison the fallback", + }, + { + name: "empty precondition RV on objectRef yields nothing", + mutate: func(e *auditv1.Event) { + e.ResponseObject = nil + e.ObjectRef.ResourceVersion = "" + }, + wantRV: "", + wantWhy: "writes usually leave objectRef.resourceVersion empty", + }, + { + name: "nil objectRef and nil response object yield nothing", + mutate: func(e *auditv1.Event) { + e.ResponseObject = nil + e.ObjectRef = nil + }, + wantRV: "", + wantWhy: "collection verbs and deletes legitimately have no RV", + }, + { + name: "nil objectRef does not fall through to the request object", + mutate: func(e *auditv1.Event) { + e.ResponseObject = nil + e.ObjectRef = nil + e.RequestObject = rawObject(deploymentBody("101")) + }, + wantRV: "", + wantWhy: "requestObject stays ignored on every path", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + event := mutationEvent("update", "uid-1", "202", "alice") + c.mutate(&event) + require.Equal(t, c.wantRV, resourceVersionFromEvent(event), c.wantWhy) + }) + } +} + +// TestRVFromRawObject_Cases covers the body shapes the audit stream actually delivers. Every +// non-answer must be "" rather than a partial or panicking read: a truncated or bodyless audit +// event has to degrade into "no RV recorded", not into a bogus RV that keys a fact nobody finds. +func TestRVFromRawObject_Cases(t *testing.T) { + cases := []struct { + name string + obj *runtime.Unknown + want string + }{ + {name: "nil object", obj: nil, want: ""}, + {name: "nil raw bytes", obj: &runtime.Unknown{}, want: ""}, + {name: "zero-length raw bytes", obj: &runtime.Unknown{Raw: []byte{}}, want: ""}, + {name: "malformed json", obj: rawObject(`{"metadata":{"resourceVersion":`), want: ""}, + {name: "non-object json", obj: rawObject(`"a string"`), want: ""}, + {name: "empty json object", obj: rawObject(`{}`), want: ""}, + {name: "object without metadata", obj: rawObject(`{"kind":"Deployment"}`), want: ""}, + {name: "metadata without resourceVersion", obj: rawObject(`{"metadata":{"name":"web"}}`), want: ""}, + {name: "explicitly empty resourceVersion", obj: rawObject(`{"metadata":{"resourceVersion":""}}`), want: ""}, + {name: "well-formed body", obj: rawObject(deploymentBody("101")), want: "101"}, + { + name: "resourceVersion alongside unknown fields", + obj: rawObject(`{"spec":{"replicas":3},"metadata":{"name":"web","resourceVersion":"99"}}`), + want: "99", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + require.Equal(t, c.want, rvFromRawObject(c.obj)) + }) + } +} + +// TestAttributionIndex_CrossClusterIsolation is the multi-cluster centerpiece: two clusters +// record the SAME object identity (uid, rv) with different authors, and each cluster's read joins +// ONLY its own fact — never the other's. A third cluster that recorded nothing misses (ships +// committer) instead of borrowing a neighbor's author. +func TestAttributionIndex_CrossClusterIsolation(t *testing.T) { + idx := newTestAttributionIndex(t) + ctx := context.Background() + + require.NoError(t, idx.RecordFact(ctx, "prod-eu-1", mutationEvent("update", "uid-1", "101", "alice"))) + require.NoError(t, idx.RecordFact(ctx, "prod-us-1", mutationEvent("update", "uid-1", "101", "bob"))) + + a, ok := idx.LookupAuthor(ctx, "prod-eu-1", appsDeploymentGVR(), "uid-1", "101", true) + require.True(t, ok) + require.Equal(t, "alice", a.Author) + + b, ok := idx.LookupAuthor(ctx, "prod-us-1", appsDeploymentGVR(), "uid-1", "101", true) + require.True(t, ok) + require.Equal(t, "bob", b.Author) + + _, ok = idx.LookupAuthor(ctx, "default", appsDeploymentGVR(), "uid-1", "101", true) + require.False(t, ok, "a cluster with no fact for this identity must miss, not cross-join") +} + +// TestAttributionIndex_RVOnlyHatchIsClusterScoped proves the correctness fix: the no-UID rv-only +// hatch is keyed by cluster, so the same RV in two clusters resolves to each cluster's own author +// (RV is not globally unique). +func TestAttributionIndex_RVOnlyHatchIsClusterScoped(t *testing.T) { + idx := newTestAttributionIndex(t) + ctx := context.Background() + + require.NoError(t, idx.RecordFact(ctx, "prod-eu-1", mutationEvent("update", "", "202", "alice"))) + require.NoError(t, idx.RecordFact(ctx, "prod-us-1", mutationEvent("update", "", "202", "bob"))) + + eu := idx.LookupAuthorResolution(ctx, "prod-eu-1", appsDeploymentGVR(), "uid-live", "202", true) + require.Equal(t, AttributionWeak, eu.Result) + require.Equal(t, "alice", eu.Fact.Author) + + us := idx.LookupAuthorResolution(ctx, "prod-us-1", appsDeploymentGVR(), "uid-live", "202", true) + require.Equal(t, "bob", us.Fact.Author, "same RV in another cluster must not leak across") +} + +// TestAttributionIndex_PurgeClusterFacts checks the delete-on-provider purge removes exactly one +// cluster's facts and leaves the others intact. +func TestAttributionIndex_PurgeClusterFacts(t *testing.T) { + idx := newTestAttributionIndex(t) + ctx := context.Background() + + require.NoError(t, idx.RecordFact(ctx, "prod-eu-1", mutationEvent("update", "uid-1", "101", "alice"))) + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "uid-2", "1", "bob"))) + + removed, err := idx.PurgeClusterFacts(ctx, "prod-eu-1") + require.NoError(t, err) + require.Positive(t, removed) + + _, ok := idx.LookupAuthor(ctx, "prod-eu-1", appsDeploymentGVR(), "uid-1", "101", true) + require.False(t, ok, "purged cluster's facts are gone") + other, ok := idx.LookupAuthor(ctx, "default", appsDeploymentGVR(), "uid-2", "1", true) + require.True(t, ok, "another cluster's facts are untouched") + require.Equal(t, "bob", other.Author) +} + +// TestAttributionIndex_SingleProviderMatchesBareInstall proves a single-(default)-provider install +// round-trips correctly: what RecordFact writes under "default" is exactly what a "default" read +// joins, so a bare single-cluster install behaves as before the cluster dimension existed. +func TestAttributionIndex_SingleProviderMatchesBareInstall(t *testing.T) { + idx := newTestAttributionIndex(t) + ctx := context.Background() + require.NoError(t, idx.RecordFact(ctx, "default", mutationEvent("update", "uid-1", "101", "alice"))) + fact, ok := idx.LookupAuthor(ctx, "default", appsDeploymentGVR(), "uid-1", "101", true) + require.True(t, ok) + require.Equal(t, "alice", fact.Author) +} + func TestAttributionIndex_FactKeyReadableFormat(t *testing.T) { idx := newTestAttributionIndex(t) - require.Equal(t, "gitops-reverser:author:v1:audit:apps/deployments:object:uid-1:101", - idx.factKeyExact("apps/deployments", "uid-1", "101")) - require.Equal(t, "gitops-reverser:author:v1:audit:apps/deployments:object:uid-1:last", - idx.factKeyLast("apps/deployments", "uid-1")) - require.Equal(t, "gitops-reverser:author:v1:audit:apps/deployments:rv:101", - idx.factKeyRV("apps/deployments", "101")) + require.Equal(t, "gitops-reverser:author:v1:audit:cluster:default:apps/deployments:object:uid-1:101", + idx.factKeyExact("default", "apps/deployments", "uid-1", "101")) + require.Equal(t, "gitops-reverser:author:v1:audit:cluster:default:apps/deployments:object:uid-1:last", + idx.factKeyLast("default", "apps/deployments", "uid-1")) + require.Equal(t, "gitops-reverser:author:v1:audit:cluster:default:apps/deployments:rv:101", + idx.factKeyRV("default", "apps/deployments", "101")) + + // A remote provider keys under its own name, so its facts never collide with the local ones. + require.Equal(t, "gitops-reverser:author:v1:audit:cluster:prod-eu-1:apps/deployments:object:uid-1:101", + idx.factKeyExact("prod-eu-1", "apps/deployments", "uid-1", "101")) // The core group drops the group segment. - require.Equal(t, "gitops-reverser:author:v1:audit:configmaps:object:uid-2:last", - idx.factKeyLast("configmaps", "uid-2")) + require.Equal(t, "gitops-reverser:author:v1:audit:cluster:default:configmaps:object:uid-2:last", + idx.factKeyLast("default", "configmaps", "uid-2")) } func TestRedisStore_WatchCursorKeyReadableFormat(t *testing.T) { diff --git a/internal/queue/key_prefix_test.go b/internal/queue/key_prefix_test.go index 6294ee95..e1c72e2a 100644 --- a/internal/queue/key_prefix_test.go +++ b/internal/queue/key_prefix_test.go @@ -87,12 +87,12 @@ func TestRedisStore_KeyPrefixReachesEveryKeyFamily(t *testing.T) { store.watchCursorKey("gtuid-3", gvr, "team-a")) idx := store.AttributionIndex(0) - require.Equal(t, "cell-a:tenant-7:author:v1:audit:apps/deployments:object:uid-1:101", - idx.factKeyExact("apps/deployments", "uid-1", "101")) - require.Equal(t, "cell-a:tenant-7:author:v1:audit:apps/deployments:object:uid-1:last", - idx.factKeyLast("apps/deployments", "uid-1")) - require.Equal(t, "cell-a:tenant-7:author:v1:audit:apps/deployments:rv:101", - idx.factKeyRV("apps/deployments", "101")) + require.Equal(t, "cell-a:tenant-7:author:v1:audit:cluster:default:apps/deployments:object:uid-1:101", + idx.factKeyExact("default", "apps/deployments", "uid-1", "101")) + require.Equal(t, "cell-a:tenant-7:author:v1:audit:cluster:default:apps/deployments:object:uid-1:last", + idx.factKeyLast("default", "apps/deployments", "uid-1")) + require.Equal(t, "cell-a:tenant-7:author:v1:audit:cluster:default:apps/deployments:rv:101", + idx.factKeyRV("default", "apps/deployments", "101")) require.Equal(t, "cell-a:tenant-7:author:v1:command:cr-uid", store.CommandAuthorStore().key("cr-uid")) } @@ -119,7 +119,8 @@ func TestRedisStore_ZeroValueStoreStillWritesPrefixedKeys(t *testing.T) { require.Equal(t, "gitops-reverser:watch:v1:target:gtuid-3:configmaps:cluster:last-rv", store.watchCursorKey("gtuid-3", coreConfigmapsGVR(), "")) require.Equal(t, "gitops-reverser:author:v1:command:cr-uid", store.CommandAuthorStore().key("cr-uid")) - require.Equal(t, "gitops-reverser:author:v1:audit:", store.AttributionIndex(0).factKeyBase("")) + require.Equal(t, "gitops-reverser:author:v1:audit:cluster:default:", + store.AttributionIndex(0).clusterFactPrefix("default")) } // Two reversers sharing one Redis/Valkey and one logical database must not read each @@ -152,13 +153,14 @@ func TestRedisStore_DistinctPrefixesIsolateCursors(t *testing.T) { require.Equal(t, "111", rv, "tenant-b's write must not clobber tenant-a's cursor") } -// The attribution telemetry gauge SCANs ":author:v1:audit:*". A prefix that -// contained a glob metacharacter would make it count the wrong keyspace; validation -// rejects those, so the pattern is always a literal prefix plus one trailing star. +// The attribution telemetry gauge SCANs ":author:v1:audit:*" and the per-provider purge +// SCANs ":author:v1:audit:cluster::*". A prefix that contained a glob metacharacter +// would make either count/delete the wrong keyspace; validation rejects those, so the pattern is +// always a literal prefix plus one trailing star. func TestAttributionIndex_ScanPatternIsPrefixed(t *testing.T) { t.Parallel() store := newPrefixedRedisStore(t, "tenant-a") idx := store.AttributionIndex(0) - require.Equal(t, "tenant-a:author:v1:audit:", idx.factKeyBase("")) + require.Equal(t, "tenant-a:author:v1:audit:cluster:prod-eu-1:", idx.clusterFactPrefix("prod-eu-1")) } diff --git a/internal/watch/author_resolver.go b/internal/watch/author_resolver.go index 26100c31..1f8c000f 100644 --- a/internal/watch/author_resolver.go +++ b/internal/watch/author_resolver.go @@ -36,6 +36,7 @@ type AttributionLookup interface { // (also consult the last-writer-wins /last pointer). LookupAuthorResolution( ctx context.Context, + providerName string, gvr schema.GroupVersionResource, uid k8stypes.UID, rv string, @@ -72,6 +73,7 @@ type AuthorResolver interface { // ADDED/MODIFIED events (true) from known RV-mismatch removals (false). ResolveAuthor( ctx context.Context, + providerName string, gvr schema.GroupVersionResource, uid k8stypes.UID, rv string, @@ -99,6 +101,7 @@ func NewAuthorResolver( func (r *attributionResolver) ResolveAuthor( ctx context.Context, + providerName string, gvr schema.GroupVersionResource, uid k8stypes.UID, rv string, @@ -111,7 +114,7 @@ func (r *attributionResolver) ResolveAuthor( } deadline := time.Now().Add(r.grace) for { - resolution := r.lookup.LookupAuthorResolution(ctx, gvr, uid, rv, exactCapable) + resolution := r.lookup.LookupAuthorResolution(ctx, providerName, gvr, uid, rv, exactCapable) if resolution.Result != queue.AttributionAbsent { ui, ok, result := r.userInfoForResolution(resolution) recordAttributionResolution(ctx, gvr, result, time.Since(start)) diff --git a/internal/watch/author_resolver_test.go b/internal/watch/author_resolver_test.go index 8abbe66f..33e1f780 100644 --- a/internal/watch/author_resolver_test.go +++ b/internal/watch/author_resolver_test.go @@ -24,13 +24,15 @@ type fakeLookup struct { hitAfter int calls int lastExactCapable bool + lastProvider string } func (f *fakeLookup) LookupAuthorResolution( - _ context.Context, _ schema.GroupVersionResource, _ k8stypes.UID, _ string, exactCapable bool, + _ context.Context, providerName string, _ schema.GroupVersionResource, _ k8stypes.UID, _ string, exactCapable bool, ) queue.AuthorResolution { f.calls++ f.lastExactCapable = exactCapable + f.lastProvider = providerName if f.calls >= f.hitAfter { return f.resolution } @@ -49,7 +51,7 @@ func TestAuthorResolver_HumanHit(t *testing.T) { } r := NewAuthorResolver(lookup, DefaultAttributionGraceWindow, logr.Discard()) - ui, ok := r.ResolveAuthor(context.Background(), resolverGVR, "uid-1", "101", true) + ui, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) require.True(t, ok) assert.Equal(t, "alice", ui.Username) assert.Equal(t, "a@x.io", ui.Email) @@ -73,7 +75,7 @@ func TestAuthorResolver_ServiceAccountIsNamed(t *testing.T) { } r := NewAuthorResolver(lookup, DefaultAttributionGraceWindow, logr.Discard()) - ui, ok := r.ResolveAuthor(context.Background(), resolverGVR, "uid-1", "101", true) + ui, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) require.True(t, ok, "a matched service account is named, not collapsed to the committer") assert.Equal(t, sa, ui.Username) @@ -94,7 +96,7 @@ func TestAuthorResolver_MissExpiresToCommitter(t *testing.T) { // A zero grace does a single lookup and, on a miss, ships as committer (ok=false). // There is no longer a miss-marker write-back. - _, ok := r.ResolveAuthor(context.Background(), resolverGVR, "uid-1", "101", true) + _, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) assert.False(t, ok) assert.Equal(t, 1, lookup.calls) } @@ -109,7 +111,7 @@ func TestAuthorResolver_DeleteEventIsNotExactCapable(t *testing.T) { } r := NewAuthorResolver(lookup, DefaultAttributionGraceWindow, logr.Discard()) - _, ok := r.ResolveAuthor(context.Background(), resolverGVR, "uid-1", "999", false) + _, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "999", false) require.True(t, ok) assert.False(t, lookup.lastExactCapable, "a removal event may consult the /last pointer") } @@ -124,7 +126,7 @@ func TestAuthorResolver_WaitsThroughGraceWindowForLateFact(t *testing.T) { } r := NewAuthorResolver(lookup, 2*time.Second, logr.Discard()) - ui, ok := r.ResolveAuthor(context.Background(), resolverGVR, "uid-1", "101", true) + ui, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) require.True(t, ok) assert.Equal(t, "bob", ui.Username) assert.GreaterOrEqual(t, lookup.calls, 3) @@ -132,6 +134,6 @@ func TestAuthorResolver_WaitsThroughGraceWindowForLateFact(t *testing.T) { func TestAuthorResolver_NilLookupIsCommitter(t *testing.T) { r := NewAuthorResolver(nil, DefaultAttributionGraceWindow, logr.Discard()) - _, ok := r.ResolveAuthor(context.Background(), resolverGVR, "uid-1", "101", true) + _, ok := r.ResolveAuthor(context.Background(), "prod-eu-1", resolverGVR, "uid-1", "101", true) assert.False(t, ok) } diff --git a/internal/watch/cluster_context.go b/internal/watch/cluster_context.go index fd811063..e5762ca9 100644 --- a/internal/watch/cluster_context.go +++ b/internal/watch/cluster_context.go @@ -21,36 +21,63 @@ import ( "github.com/ConfigButler/gitops-reverser/internal/typeset" ) -// LocalClusterID identifies the cluster the operator runs in — the config plane, which is -// also the watched cluster in a single-cluster install. It is the zero value on purpose: -// every code path that does not know about source clusters lands on it. It matches -// (api/v1alpha3).GitTarget.SourceClusterID()'s "" return for an omitted spec.kubeConfig. -const LocalClusterID = "" - -// SourceClusterResolver turns a GitTarget's source-cluster id into a rest.Config by reading -// the kubeconfig Secret it names from the config plane. It is an interface so the watch -// manager grows no Kubernetes client of its own for this, and so tests can stand up a -// remote cluster without a Secret. The concrete implementation lives in -// source_cluster_resolver.go. +// configPlaneClusterID identifies the CONFIG PLANE context — the cluster the operator itself runs +// in, where its own CRs live. It owns the operator's own duties (the CRD/APIService trigger +// informers, the singleton catalog metrics, the injected test clients) and is always present. +// +// It is deliberately the empty string, which no ClusterProvider name can ever be (name is required, +// MinLength=1), so the config plane can never collide with — or be mistaken for — a source cluster. +// NO PROVIDER NAME IS SPECIAL: what makes a source cluster in-cluster is an absent +// spec.kubeConfig, resolved per provider, never the name "default". +const configPlaneClusterID = "" + +// inClusterConfigVersion is the version token the resolver returns for a provider that omits +// kubeConfig. It is constant because there is no Secret to rotate: the in-cluster config is fixed +// for the process, so a credential refresh can never see it "change". +const inClusterConfigVersion = "in-cluster" + +// SourceClusterResolver turns a source-cluster NAME (a ClusterProvider's name) into a rest.Config +// by looking up the ClusterProvider and reading the kubeconfig Secret it names from the operator +// namespace. It is an interface so the watch manager grows no Kubernetes client of its own for +// this, and so tests can stand up a remote cluster without a Secret. The concrete implementation +// lives in source_cluster_resolver.go. type SourceClusterResolver interface { - // ResolveSourceCluster returns the rest.Config for a cluster id, and an opaque version - // token that changes when the underlying kubeconfig changes (the Secret's - // resourceVersion). An unknown or unreadable id is an error: mirroring the wrong - // cluster into a folder is worse than mirroring none. - ResolveSourceCluster(ctx context.Context, clusterID string) (cfg *rest.Config, version string, err error) + // ResolveSourceCluster returns the rest.Config for a ClusterProvider name, and an opaque + // version token that changes when the resolved config changes (the provider generation and + // the kubeconfig Secret's resourceVersion). An unknown or unreadable name is an error: + // mirroring the wrong cluster into a folder is worse than mirroring none. + // + // A NIL config with a nil error means the provider omits spec.kubeConfig and therefore names + // the operator's OWN cluster — the in-cluster answer, available to every provider name. This + // is the ONLY authority on whether a source cluster is in-cluster; nothing keys that off the + // provider's name. + ResolveSourceCluster(ctx context.Context, providerName string) (cfg *rest.Config, version string, err error) } // clusterContext holds everything that used to be a Manager-wide singleton and is in fact a // property of ONE cluster: its API surface catalog, the followability registry derived from // it, and the clients that reach it. // -// A single-cluster install has exactly one, keyed by LocalClusterID, and behaves exactly as -// before — the trigger informers, the catalog refresh, and the type registry all land on it. -// A GitTarget that names a source cluster (spec.kubeConfig) gets its own, created on first -// use and torn down when the last such GitTarget is gone. +// There are two kinds. The CONFIG PLANE context (configPlaneClusterID) is the operator's own +// cluster: it owns the API-surface trigger informers, the singleton catalog metrics, and the +// injected test clients, and it is never torn down. A SOURCE context is one per ClusterProvider a +// GitTarget mirrors from, created on first use and torn down when the last such GitTarget is gone; +// whether it talks to the operator's own cluster or a remote is decided by RESOLVING its provider +// (spec.kubeConfig absent ⇒ in-cluster), never by its name. type clusterContext struct { id string + // configPlane marks the operator's own context (id configPlaneClusterID). It is set at + // construction and never changes. + configPlane bool + + // inCluster records that this SOURCE cluster resolved to the operator's own cluster — its + // ClusterProvider omits spec.kubeConfig. It is RESOLVED, not inferred from the id: it is + // false until the first successful resolution, so an unreached provider is treated as remote + // (fail-closed — a cluster we have not resolved never silently borrows in-cluster + // credentials). Guarded by clientsMu alongside the config fields below. + inCluster bool + // catalog and registry are the "Scan -> Registry" pipeline for this cluster. A CRD // installed only on the remote is followable only there — and, more importantly, a type // served only locally never resolves for a remote target. @@ -129,13 +156,25 @@ func newClusterContext(id string) *clusterContext { } } -// isLocal reports whether this context is the cluster the operator runs in. -func (c *clusterContext) isLocal() bool { return c.id == LocalClusterID } +// isLocal reports whether this context talks to the cluster the operator runs in — either because +// it IS the config plane, or because its ClusterProvider resolved without a kubeConfig. It is never +// a test on the provider's name. +func (c *clusterContext) isLocal() bool { + if c.configPlane { + return true + } + c.clientsMu.Lock() + defer c.clientsMu.Unlock() + return c.inCluster +} + +// isLocalLocked is isLocal for callers already holding clientsMu. +func (c *clusterContext) isLocalLocked() bool { return c.configPlane || c.inCluster } -// describeCluster renders a cluster id for logs. The local cluster has no name of its own. +// describeCluster renders a cluster id for logs. The config plane has no provider name of its own. func describeCluster(id string) string { - if id == LocalClusterID { - return "local" + if id == configPlaneClusterID { + return "config-plane" } return id } @@ -154,8 +193,9 @@ func (m *Manager) cluster(id string) *clusterContext { return cc } cc := newClusterContext(id) - if id == LocalClusterID { - m.seedLocalClusterLocked(cc) + if id == configPlaneClusterID { + cc.configPlane = true + m.seedConfigPlaneLocked(cc) } else { m.Log.Info("source cluster registered", "clusterID", id) } @@ -164,10 +204,10 @@ func (m *Manager) cluster(id string) *clusterContext { return cc } -// seedLocalClusterLocked wires the local context's catalog to m.resourceCatalog (a test may +// seedConfigPlaneLocked wires the config-plane context's catalog to m.resourceCatalog (a test may // have injected one; otherwise the two are aliased so apiResourceCatalog() and the context see // the same object) and marks it reachable — the operator runs in it. Must hold clustersMu. -func (m *Manager) seedLocalClusterLocked(cc *clusterContext) { +func (m *Manager) seedConfigPlaneLocked(cc *clusterContext) { if m.resourceCatalog != nil { cc.catalog = m.resourceCatalog } else { @@ -176,9 +216,10 @@ func (m *Manager) seedLocalClusterLocked(cc *clusterContext) { cc.reachable = sourceClusterReachability{state: reachTrue, reason: reasonLocalCluster} } -// localCluster is the config plane, and the watched cluster of every GitTarget that does -// not name a source. -func (m *Manager) localCluster() *clusterContext { return m.cluster(LocalClusterID) } +// configPlaneCluster is the operator's OWN cluster: where its CRs live, what the API-surface +// trigger informers watch, and what the singleton catalog metrics describe. It is not a source +// cluster and has no ClusterProvider — a GitTarget mirrors from a provider, never from this. +func (m *Manager) configPlaneCluster() *clusterContext { return m.cluster(configPlaneClusterID) } // registryForGitTarget returns the followability registry of the cluster a GitTarget mirrors // from — its OWN cluster's surface, never a union. A single-cluster GitTarget resolves against @@ -201,13 +242,13 @@ func (m *Manager) ClusterTypeLookup(clusterID string) typeset.Lookup { return m.cluster(clusterID).registry } -// activeClusterIDs is every cluster some GitTarget currently mirrors from, plus the local -// one. The local cluster is always active: the operator's own CRs live there, and a -// rule-less install still refreshes its catalog. The remote ids come from the Declare-time -// capture (gitTargetClusters), not from the rules — spec.kubeConfig is immutable and a -// GitTarget property, so there is no rules-disagree window to reconcile. +// activeClusterIDs is every source cluster some GitTarget currently mirrors from, plus the config +// plane. The config plane is always active: the operator's own CRs live there and its catalog arms +// the API-surface trigger informers, so a rule-less install still refreshes it. The source ids come +// from the Declare-time capture (gitTargetClusters), not from the rules — spec.clusterProviderRef +// is immutable and a GitTarget property, so there is no rules-disagree window to reconcile. func (m *Manager) activeClusterIDs() []string { - seen := map[string]struct{}{LocalClusterID: {}} + seen := map[string]struct{}{configPlaneClusterID: {}} m.gitTargetClustersMu.Lock() for _, id := range m.gitTargetClusters { seen[id] = struct{}{} @@ -223,7 +264,7 @@ func (m *Manager) activeClusterIDs() []string { // rememberGitTargetCluster captures the source cluster a GitTarget mirrors from, keyed by // GitTarget — the same capture-on-Declare pattern as rememberGitTargetUID. Because -// spec.kubeConfig is immutable, this is learned once and never changes for a given +// spec.clusterProviderRef is immutable, this is learned once and never changes for a given // GitTarget, so there is no per-rule propagation and no cross-rule disagreement window. func (m *Manager) rememberGitTargetCluster(gitDest types.ResourceReference, clusterID string) { m.gitTargetClustersMu.Lock() @@ -234,13 +275,29 @@ func (m *Manager) rememberGitTargetCluster(gitDest types.ResourceReference, clus m.gitTargetClusters[gitDest.Key()] = clusterID } +// DeclaredSourceCluster reports the source cluster captured for a GitTarget at Declare time and +// whether that GitTarget has declared at all. It is the observable form of the capture-on-Declare +// contract: a GitTarget the controller's Validated gate refused never reaches DeclareForGitTarget, +// so it never appears here. That makes "an unauthorized namespace starts no watch" assertable from +// outside this package — unlike clusterIDForGitTarget, which deliberately hides the +// not-yet-declared case behind the local-cluster default. +func (m *Manager) DeclaredSourceCluster(gitDest types.ResourceReference) (string, bool) { + m.gitTargetClustersMu.Lock() + defer m.gitTargetClustersMu.Unlock() + id, ok := m.gitTargetClusters[gitDest.Key()] + return id, ok +} + // clusterIDForGitTarget resolves the source cluster of a GitTarget from the Declare-time // capture, defaulting to the local cluster for a GitTarget that has not declared yet (a // status read racing the first Declare) or that names no source cluster. func (m *Manager) clusterIDForGitTarget(gitDest types.ResourceReference) string { m.gitTargetClustersMu.Lock() defer m.gitTargetClustersMu.Unlock() - return m.gitTargetClusters[gitDest.Key()] + if id := m.gitTargetClusters[gitDest.Key()]; id != "" { + return id + } + return configPlaneClusterID } // forgetGitTargetCluster drops a deleted GitTarget's captured cluster and tears down that @@ -262,7 +319,7 @@ func (m *Manager) forgetGitTargetCluster(gitDest types.ResourceReference) { } m.gitTargetClustersMu.Unlock() - if !had || clusterID == LocalClusterID || stillReferenced { + if !had || clusterID == configPlaneClusterID || stillReferenced { return } m.teardownCluster(clusterID) @@ -270,9 +327,9 @@ func (m *Manager) forgetGitTargetCluster(gitDest types.ResourceReference) { // teardownCluster drops a source cluster's context once no GitTarget references it: its // clients and catalog/registry are released so a deleted remote GitTarget leaks nothing. The -// local cluster is never torn down. +// config plane is never torn down — it is the operator's own cluster, not a source. func (m *Manager) teardownCluster(clusterID string) { - if clusterID == LocalClusterID { + if clusterID == configPlaneClusterID { return } m.clustersMu.Lock() @@ -286,17 +343,24 @@ func (m *Manager) teardownCluster(clusterID string) { } // recordClusterReachability updates a source cluster's SourceClusterReachable state from the -// outcome of a discovery attempt. The local cluster is always reachable — the operator runs in -// it — so it is never touched here. A remote's failure class (unreachable / auth / access -// denied) is derived by classifySourceClusterReachFailure. +// outcome of a discovery attempt. Only the CONFIG PLANE is skipped — it is the operator's own +// cluster, reachable by definition and not a source. A source cluster that resolved in-cluster is +// still recorded from its real attempt (it reports reasonLocalCluster on success), because "this +// provider omits kubeConfig" is a resolved fact, not an assumption made from its name. A failure +// class (unreachable / auth / access denied) is derived by classifySourceClusterReachFailure. func (m *Manager) recordClusterReachability(cc *clusterContext, err error) { - if cc.isLocal() { + if cc.configPlane { return } + inCluster := cc.isLocal() m.clustersMu.Lock() defer m.clustersMu.Unlock() if err == nil { - cc.reachable = sourceClusterReachability{state: reachTrue, reason: reasonSourceClusterReachable} + reason := reasonSourceClusterReachable + if inCluster { + reason = reasonLocalCluster + } + cc.reachable = sourceClusterReachability{state: reachTrue, reason: reason} return } cc.reachable = classifySourceClusterReachFailure(err) @@ -359,7 +423,7 @@ func (m *Manager) SourceClusterReachable(clusterID string) SourceClusterReachabl // clusterReachability returns a snapshot of a cluster's SourceClusterReachable state for // projection onto its GitTargets. An unknown id is reported Unknown. func (m *Manager) clusterReachability(clusterID string) sourceClusterReachability { - if clusterID == LocalClusterID { + if clusterID == configPlaneClusterID { return sourceClusterReachability{state: reachTrue, reason: reasonLocalCluster} } m.clustersMu.Lock() @@ -380,36 +444,64 @@ func (m *Manager) clusterRESTConfigLocked(ctx context.Context, cc *clusterContex if cc.restConfig != nil { return cc.restConfig, nil } - if cc.isLocal() { - cfg, err := ctrl.GetConfig() + if cc.configPlane { + cfg, err := inClusterRESTConfig() if err != nil { - return nil, fmt.Errorf("no REST config for the local cluster: %w", err) + return nil, err } cc.restConfig = cfg + cc.configVersion = inClusterConfigVersion return cfg, nil } - cfg, version, err := m.resolveRemoteConfig(ctx, cc) + cfg, version, inCluster, err := m.resolveSourceConfig(ctx, cc) if err != nil { return nil, err } cc.restConfig = cfg cc.configVersion = version + cc.inCluster = inCluster return cfg, nil } -// resolveRemoteConfig reads a remote cluster's kubeconfig Secret from the config plane. -func (m *Manager) resolveRemoteConfig(ctx context.Context, cc *clusterContext) (*rest.Config, string, error) { +// inClusterRESTConfig is the operator's own cluster config, used by the config plane and by any +// ClusterProvider that omits spec.kubeConfig. +func inClusterRESTConfig() (*rest.Config, error) { + cfg, err := ctrl.GetConfig() + if err != nil { + return nil, fmt.Errorf("no REST config for the operator's own cluster: %w", err) + } + return cfg, nil +} + +// resolveSourceConfig resolves a SOURCE cluster's config from its ClusterProvider, and reports +// whether that provider turned out to be in-cluster. This is the ONLY place in-cluster-ness is +// decided for a source cluster: a provider that omits spec.kubeConfig resolves to the operator's +// own config whatever it is named, and one that sets it is remote even if it is named "default". +// +// It returns the verdict rather than storing it, because one caller resolves OUTSIDE clientsMu +// (the credential refresh) and must not race the field. The returns are, in order: the config, its +// opaque version token, whether the provider resolved in-cluster, and the error. +func (m *Manager) resolveSourceConfig( + ctx context.Context, + cc *clusterContext, +) (*rest.Config, string, bool, error) { if m.SourceClusters == nil { - return nil, "", fmt.Errorf("cannot reach source cluster %q: no source-cluster resolver configured", cc.id) + return nil, "", false, fmt.Errorf( + "cannot reach source cluster %q: no source-cluster resolver configured", cc.id) } - cfg, version, err := m.SourceClusters.ResolveSourceCluster(ctx, cc.id) + resolved, version, err := m.SourceClusters.ResolveSourceCluster(ctx, cc.id) if err != nil { - return nil, "", fmt.Errorf("resolve source cluster %q: %w", cc.id, err) + return nil, "", false, fmt.Errorf("resolve source cluster %q: %w", cc.id, err) } - if cfg == nil { - return nil, "", fmt.Errorf("resolve source cluster %q: nil REST config", cc.id) + if resolved == nil { + // The provider omits kubeConfig: it names the operator's own cluster. + local, localErr := inClusterRESTConfig() + if localErr != nil { + return nil, "", false, localErr + } + return local, inClusterConfigVersion, true, nil } - return cfg, version, nil + return resolved, version, false, nil } // refreshClusterCredentials re-reads a remote cluster's kubeconfig Secret on the catalog-refresh @@ -425,7 +517,7 @@ func (m *Manager) refreshClusterCredentials(ctx context.Context, cc *clusterCont if cc.isLocal() { return } - cfg, version, err := m.resolveRemoteConfig(ctx, cc) + cfg, version, inCluster, err := m.resolveSourceConfig(ctx, cc) if err != nil { if isDefinitiveCredentialFailure(err) && m.dropClusterClients(cc) { m.invalidateClusterWatches(cc.id) @@ -434,6 +526,7 @@ func (m *Manager) refreshClusterCredentials(ctx context.Context, cc *clusterCont } cc.clientsMu.Lock() + cc.inCluster = inCluster if cc.restConfig != nil && version == cc.configVersion { cc.clientsMu.Unlock() return @@ -562,7 +655,9 @@ func (m *Manager) clusterDiscovery(ctx context.Context, clusterID string) (apiRe return nil, err } discoCfg := cfg - if !cc.isLocal() { + // isLocalLocked, not isLocal: clientsMu is held here, and clusterRESTConfigLocked has just + // resolved cc.inCluster, so this reads the freshly-decided verdict. + if !cc.isLocalLocked() { // Discovery uses the legacy, non-context ServerGroupsAndResources(), and a remote config // deliberately carries no request-level timeout (its watches must stay open). Bound the // finite discovery call with a request timeout on a COPY, so a remote that accepts the @@ -591,9 +686,9 @@ func (m *Manager) orderedClusters() []*clusterContext { if snapshot := m.clusterOrder.Load(); snapshot != nil { return *snapshot } - // No cluster has been created yet: force the local one into existence, which publishes + // No cluster has been created yet: force the config plane into existence, which publishes // the first snapshot. - m.localCluster() + m.configPlaneCluster() if snapshot := m.clusterOrder.Load(); snapshot != nil { return *snapshot } @@ -619,7 +714,14 @@ func (m *Manager) publishClusterOrderLocked() { for id := range m.clusters { ids = append(ids, id) } - sort.Strings(ids) // LocalClusterID is "" and sorts first. + // Sort deterministically with the config plane first, then source clusters by name, so a + // status read sees a stable order. + sort.Slice(ids, func(i, j int) bool { + if (ids[i] == configPlaneClusterID) != (ids[j] == configPlaneClusterID) { + return ids[i] == configPlaneClusterID + } + return ids[i] < ids[j] + }) out := make([]*clusterContext, 0, len(ids)) for _, id := range ids { out = append(out, m.clusters[id]) diff --git a/internal/watch/cluster_context_test.go b/internal/watch/cluster_context_test.go index 64c53e56..948a05cc 100644 --- a/internal/watch/cluster_context_test.go +++ b/internal/watch/cluster_context_test.go @@ -19,38 +19,83 @@ func gd(name string) types.ResourceReference { return types.NewResourceReference(name, "team-a") } -func TestLocalCluster_SeededAndReachable(t *testing.T) { +func TestConfigPlaneCluster_SeededAndReachable(t *testing.T) { m := &Manager{Log: logr.Discard()} - local := m.localCluster() - assert.True(t, local.isLocal()) - assert.Same(t, local, m.cluster(LocalClusterID), "the local id is stable") - assert.Same(t, local.catalog, m.apiResourceCatalog(), "apiResourceCatalog() is the local catalog") + cp := m.configPlaneCluster() + assert.True(t, cp.isLocal()) + assert.True(t, cp.configPlane, "the config plane is not a source cluster") + assert.Same(t, cp, m.cluster(configPlaneClusterID), "the config-plane id is stable") + assert.Same(t, cp.catalog, m.apiResourceCatalog(), "apiResourceCatalog() is the config-plane catalog") - reach := m.clusterReachability(LocalClusterID) + reach := m.clusterReachability(configPlaneClusterID) assert.Equal(t, reachTrue, reach.state) assert.Equal(t, reasonLocalCluster, reach.reason) } +// TestSourceClusterInClusterIsResolvedNotNamed pins the model: a source cluster is in-cluster +// because its ClusterProvider omits kubeConfig, never because of what it is called. A provider +// named "default" is an ordinary source context and starts out un-resolved (fail-closed). +func TestSourceClusterInClusterIsResolvedNotNamed(t *testing.T) { + m := &Manager{Log: logr.Discard()} + + named := m.cluster("default") + assert.False(t, named.configPlane, "no provider name is the config plane") + assert.False(t, named.isLocal(), + "un-resolved source clusters are treated as remote until their provider says otherwise") + assert.NotSame(t, m.configPlaneCluster(), named, "\"default\" is a source, distinct from the config plane") +} + func TestActiveClusterIDs_FromDeclareCapture(t *testing.T) { m := &Manager{Log: logr.Discard()} - assert.Equal(t, []string{LocalClusterID}, m.activeClusterIDs(), "only local before any declare") + assert.Equal(t, []string{configPlaneClusterID}, m.activeClusterIDs(), + "only the config plane before any declare") - m.rememberGitTargetCluster(gd("a"), "team-a/kc/value") - m.rememberGitTargetCluster(gd("b"), "team-a/kc/value") // same remote, shared - m.rememberGitTargetCluster(gd("c"), "team-b/kc2/value") + m.rememberGitTargetCluster(gd("a"), "prod-eu-1") + m.rememberGitTargetCluster(gd("b"), "prod-eu-1") // same provider, shared + m.rememberGitTargetCluster(gd("c"), "prod-us-1") assert.ElementsMatch(t, - []string{LocalClusterID, "team-a/kc/value", "team-b/kc2/value"}, + []string{configPlaneClusterID, "prod-eu-1", "prod-us-1"}, m.activeClusterIDs(), - "active ids are the deduped Declare-captured remotes plus local") + "active ids are the deduped Declare-captured providers plus the config plane") - assert.Equal(t, "team-a/kc/value", m.clusterIDForGitTarget(gd("a"))) - assert.Equal(t, LocalClusterID, m.clusterIDForGitTarget(gd("never-declared"))) + assert.Equal(t, "prod-eu-1", m.clusterIDForGitTarget(gd("a"))) + assert.Equal(t, configPlaneClusterID, m.clusterIDForGitTarget(gd("never-declared"))) +} + +// TestDeclaredSourceCluster_ReportsCaptureAndDeclaration pins the difference between +// DeclaredSourceCluster and clusterIDForGitTarget: the latter defaults a never-declared GitTarget +// to the config plane, which is indistinguishable from one that declared it. Only the ok flag makes +// "a GitTarget the Validated gate refused starts no watch" assertable from outside this package, +// so a refused (never-declared) target must report ok=false even for the config-plane id. +func TestDeclaredSourceCluster_ReportsCaptureAndDeclaration(t *testing.T) { + m := &Manager{Log: logr.Discard()} + + id, ok := m.DeclaredSourceCluster(gd("refused")) + assert.False(t, ok, "a GitTarget that never reached Declare has no captured cluster") + assert.Empty(t, id) + + m.rememberGitTargetCluster(gd("remote"), "prod-eu-1") + m.rememberGitTargetCluster(gd("local"), configPlaneClusterID) + + id, ok = m.DeclaredSourceCluster(gd("remote")) + assert.True(t, ok) + assert.Equal(t, "prod-eu-1", id, "the cluster captured at Declare time is reported verbatim") + + id, ok = m.DeclaredSourceCluster(gd("local")) + assert.True(t, ok, "declaring the config plane is still a declaration, not an absence") + assert.Equal(t, configPlaneClusterID, id) + + // Forgetting a deleted GitTarget returns it to the never-declared state, so a torn-down + // target cannot be mistaken for one still mirroring the config plane. + m.forgetGitTargetCluster(gd("remote")) + _, ok = m.DeclaredSourceCluster(gd("remote")) + assert.False(t, ok) } func TestRefcountedTeardown(t *testing.T) { m := &Manager{Log: logr.Discard()} - const remote = "team-a/kc/value" + const remote = "prod-eu-1" // Two GitTargets mirror the same remote; create its context. m.rememberGitTargetCluster(gd("a"), remote) @@ -66,30 +111,30 @@ func TestRefcountedTeardown(t *testing.T) { m.forgetGitTargetCluster(gd("b")) assert.Nil(t, m.clusterContextByID(remote), "last referencing GitTarget gone -> torn down") - // The local cluster is never torn down by a forget, even when the forgotten GitTarget + // The config plane is never torn down by a forget, even when the forgotten GitTarget // mapped to it. - require.NotNil(t, m.localCluster()) - m.rememberGitTargetCluster(gd("c"), LocalClusterID) + require.NotNil(t, m.configPlaneCluster()) + m.rememberGitTargetCluster(gd("c"), configPlaneClusterID) m.forgetGitTargetCluster(gd("c")) - assert.NotNil(t, m.clusterContextByID(LocalClusterID)) + assert.NotNil(t, m.clusterContextByID(configPlaneClusterID)) } func TestRegistryForGitTarget_PerCluster(t *testing.T) { m := &Manager{Log: logr.Discard()} - m.rememberGitTargetCluster(gd("remote"), "team-a/kc/value") + m.rememberGitTargetCluster(gd("remote"), "prod-eu-1") localReg := m.registryForGitTarget(gd("local-target")) remoteReg := m.registryForGitTarget(gd("remote")) - assert.Same(t, m.localCluster().registry, localReg) + assert.Same(t, m.configPlaneCluster().registry, localReg) assert.NotSame(t, localReg, remoteReg, "a remote GitTarget resolves against its own registry, not local") } func TestClusterTypeLookup(t *testing.T) { m := &Manager{Log: logr.Discard()} - // Local lookup is the local registry. - assert.Same(t, m.localCluster().registry, m.ClusterTypeLookup(LocalClusterID)) + // The config-plane lookup is its registry. + assert.Same(t, m.configPlaneCluster().registry, m.ClusterTypeLookup(configPlaneClusterID)) // An unknown remote yields a (fresh, unready) registry that fails closed, never nil. - lk := m.ClusterTypeLookup("team-a/kc/value") + lk := m.ClusterTypeLookup("prod-eu-1") require.NotNil(t, lk) assert.False(t, lk.Ready(), "an unobserved remote registry is not ready — the writer falls closed") } diff --git a/internal/watch/cluster_model_test.go b/internal/watch/cluster_model_test.go new file mode 100644 index 00000000..4f5d2715 --- /dev/null +++ b/internal/watch/cluster_model_test.go @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "errors" + "testing" + + meta "github.com/fluxcd/pkg/apis/meta" + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/rest" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// stubResolver is a SourceClusterResolver whose answer is fixed per provider name, so the +// in-cluster verdict can be driven without a cluster or a Secret. +type stubResolver struct { + cfg *rest.Config + version string + err error +} + +func (s stubResolver) ResolveSourceCluster(context.Context, string) (*rest.Config, string, error) { + return s.cfg, s.version, s.err +} + +// TestResolveSourceConfig_InClusterIsResolvedFromTheProvider is the core of the cluster model: a +// provider that omits spec.kubeConfig resolves to the operator's own cluster, and one that sets it +// is remote — for ANY provider name. Nothing keys this off the name "default". +func TestResolveSourceConfig_InClusterIsResolvedFromTheProvider(t *testing.T) { + remote := &rest.Config{Host: "https://192.0.2.1:6443"} + + tests := []struct { + name string + providerName string + resolver SourceClusterResolver + wantInCluster bool + }{ + { + name: "a provider with a kubeConfig is remote", + providerName: configv1alpha3.DefaultClusterProviderName, + resolver: stubResolver{cfg: remote, version: "v1"}, + wantInCluster: false, + }, + { + name: "even when it is named default", + providerName: "default", + resolver: stubResolver{cfg: remote, version: "v1"}, + wantInCluster: false, + }, + { + name: "a provider without a kubeConfig is in-cluster", + providerName: "prod-eu-1", + resolver: stubResolver{cfg: nil, version: inClusterConfigVersion}, + wantInCluster: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := &Manager{Log: logr.Discard(), SourceClusters: tt.resolver} + cc := m.cluster(tt.providerName) + + cfg, version, inCluster, err := m.resolveSourceConfig(context.Background(), cc) + if tt.wantInCluster { + // There is no cluster under a unit test, so the in-cluster branch surfaces + // ctrl.GetConfig's failure. Either way it must have taken the in-cluster path + // rather than treating the provider as remote. + if err != nil { + assert.Contains(t, err.Error(), "operator's own cluster") + return + } + assert.True(t, inCluster) + assert.Equal(t, inClusterConfigVersion, version) + return + } + require.NoError(t, err) + assert.False(t, inCluster, "a provider with a kubeConfig is never in-cluster") + assert.Equal(t, remote, cfg) + assert.Equal(t, "v1", version) + }) + } +} + +// TestResolveSourceConfig_Failures covers the two ways resolution refuses: no resolver wired, and +// the provider being unreadable. Both must be errors, never a silent in-cluster fallback — that +// would mirror the wrong cluster into a folder. +func TestResolveSourceConfig_Failures(t *testing.T) { + t.Run("no resolver configured", func(t *testing.T) { + m := &Manager{Log: logr.Discard()} + _, _, inCluster, err := m.resolveSourceConfig(context.Background(), m.cluster("prod-eu-1")) + require.Error(t, err) + assert.Contains(t, err.Error(), "no source-cluster resolver configured") + assert.False(t, inCluster) + }) + + t.Run("resolver error is propagated, not defaulted to in-cluster", func(t *testing.T) { + m := &Manager{Log: logr.Discard(), SourceClusters: stubResolver{err: errors.New("boom")}} + _, _, inCluster, err := m.resolveSourceConfig(context.Background(), m.cluster("prod-eu-1")) + require.Error(t, err) + assert.Contains(t, err.Error(), "resolve source cluster") + assert.False(t, inCluster) + }) +} + +// TestClusterContext_IsLocal covers both spellings of the in-cluster test, including the locked +// variant used from inside clientsMu (calling the unlocked one there would self-deadlock). +func TestClusterContext_IsLocal(t *testing.T) { + m := &Manager{Log: logr.Discard()} + + cp := m.configPlaneCluster() + assert.True(t, cp.isLocal(), "the config plane is always the operator's own cluster") + assert.True(t, cp.isLocalLocked()) + + src := m.cluster("prod-eu-1") + assert.False(t, src.isLocal(), "an unresolved source is treated as remote (fail-closed)") + + src.clientsMu.Lock() + src.inCluster = true + assert.True(t, src.isLocalLocked(), "a resolved in-cluster source reads true under the lock") + src.clientsMu.Unlock() + assert.True(t, src.isLocal()) +} + +// TestDescribeCluster keeps the config plane legible in logs without giving it a provider name. +func TestDescribeCluster(t *testing.T) { + assert.Equal(t, "config-plane", describeCluster(configPlaneClusterID)) + assert.Equal(t, "prod-eu-1", describeCluster("prod-eu-1")) +} + +// TestSourceClusterReachable_Projection pins the GitTarget-facing status contract: the tri-state a +// source cluster's reachability projects onto SourceClusterReachable. +func TestSourceClusterReachable_Projection(t *testing.T) { + tests := []struct { + name string + reach sourceClusterReachability + wantState string + wantReason string + wantMessage string + }{ + { + name: "reachable", + reach: sourceClusterReachability{state: reachTrue, reason: reasonSourceClusterReachable}, + wantState: "True", + wantReason: reasonSourceClusterReachable, + }, + { + name: "reachable with no reason falls back to a concrete one", + reach: sourceClusterReachability{state: reachTrue}, + wantState: "True", + wantReason: reasonSourceClusterReachable, + }, + { + name: "unreachable keeps its classified reason", + reach: sourceClusterReachability{ + state: reachFalse, + reason: reasonSourceClusterAuthFailed, + message: "401", + }, + wantState: "False", + wantReason: reasonSourceClusterAuthFailed, + wantMessage: "401", + }, + { + name: "before the first discovery attempt", + reach: sourceClusterReachability{state: reachUnknown}, + wantState: "Unknown", + wantReason: reasonAwaitingDiscovery, + wantMessage: "source cluster not yet reached; awaiting first discovery", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := &Manager{Log: logr.Discard()} + cc := m.cluster("prod-eu-1") + m.clustersMu.Lock() + cc.reachable = tt.reach + m.clustersMu.Unlock() + + got := m.SourceClusterReachable("prod-eu-1") + assert.Equal(t, tt.wantState, got.State) + assert.Equal(t, tt.wantReason, got.Reason) + assert.Equal(t, tt.wantMessage, got.Message) + }) + } +} + +// TestSourceClusterReachable_ConfigPlaneAlwaysReachable — the operator runs in it, so it never +// depends on a discovery attempt. +func TestSourceClusterReachable_ConfigPlaneAlwaysReachable(t *testing.T) { + m := &Manager{Log: logr.Discard()} + got := m.SourceClusterReachable(configPlaneClusterID) + assert.Equal(t, "True", got.State) + assert.Equal(t, reasonLocalCluster, got.Reason) +} + +// TestRecordClusterReachability_InClusterSourceReportsLocalCluster: a source cluster that resolved +// in-cluster still records a real verdict (unlike the config plane, which is skipped), and reports +// the LocalCluster reason so its GitTargets say why they are reachable. +func TestRecordClusterReachability_InClusterSourceReportsLocalCluster(t *testing.T) { + m := &Manager{Log: logr.Discard()} + + cc := m.cluster("prod-eu-1") + cc.clientsMu.Lock() + cc.inCluster = true + cc.clientsMu.Unlock() + + m.recordClusterReachability(cc, nil) + assert.Equal(t, reasonLocalCluster, m.SourceClusterReachable("prod-eu-1").Reason) + + remote := m.cluster("prod-us-1") + m.recordClusterReachability(remote, nil) + assert.Equal(t, reasonSourceClusterReachable, m.SourceClusterReachable("prod-us-1").Reason) + + // The config plane is skipped entirely: it is not a source and has no discovery verdict. + cp := m.configPlaneCluster() + m.recordClusterReachability(cp, errors.New("ignored")) + assert.Equal(t, "True", m.SourceClusterReachable(configPlaneClusterID).State) +} + +// TestClusterProviderIsInCluster mirrors the API-side predicate the resolver implements, so the +// two cannot drift: an absent kubeConfig is what makes a provider local, not its name. +func TestClusterProviderIsInCluster(t *testing.T) { + local := &configv1alpha3.ClusterProvider{ObjectMeta: metav1.ObjectMeta{Name: "prod-eu-1"}} + assert.True(t, local.IsInCluster(), "no kubeConfig means the operator's own cluster") + + named := &configv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: configv1alpha3.DefaultClusterProviderName}, + Spec: configv1alpha3.ClusterProviderSpec{KubeConfig: remoteKubeConfigRef()}, + } + assert.False(t, named.IsInCluster(), `"default" with a kubeConfig is a remote cluster`) +} + +// remoteKubeConfigRef is a minimal kubeConfig reference marking a provider as remote. +func remoteKubeConfigRef() *meta.KubeConfigReference { + return &meta.KubeConfigReference{SecretRef: &meta.SecretKeyReference{Name: "kc"}} +} + +// TestOrderedClusters_ConfigPlaneFirstThenByName pins the published snapshot's ordering. The git +// writer's cluster-scoped GVK lookup reads this slice per document, so a stable order keeps status +// reads and lookups deterministic rather than map-iteration random. +func TestOrderedClusters_ConfigPlaneFirstThenByName(t *testing.T) { + m := &Manager{Log: logr.Discard()} + // Create out of order on purpose. + m.cluster("prod-us-1") + m.cluster("alpha") + m.configPlaneCluster() + m.cluster("prod-eu-1") + + ids := make([]string, 0, 4) + for _, cc := range m.orderedClusters() { + ids = append(ids, cc.id) + } + assert.Equal(t, []string{configPlaneClusterID, "alpha", "prod-eu-1", "prod-us-1"}, ids) +} + +// TestOrderedClusters_ForcesConfigPlaneIntoExistence: a Manager nobody has touched still publishes +// a snapshot, so the writer's lookup never sees nil. +func TestOrderedClusters_ForcesConfigPlaneIntoExistence(t *testing.T) { + m := &Manager{Log: logr.Discard()} + ordered := m.orderedClusters() + require.Len(t, ordered, 1) + assert.True(t, ordered[0].configPlane) +} + +// TestClusterDynamicClient_UsesInjectedClientForConfigPlane covers the seam unit tests rely on: an +// injected dynamic client serves the operator's own cluster without any REST config at all. +func TestClusterDynamicClient_UsesInjectedClientForConfigPlane(t *testing.T) { + injected := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + m := &Manager{Log: logr.Discard(), dynamicClient: injected} + + got, err := m.clusterDynamicClient(context.Background(), configPlaneClusterID) + require.NoError(t, err) + assert.Same(t, injected, got, "the config plane uses the injected client, not a built one") +} + +// TestTeardownCluster_NeverDropsTheConfigPlane — the operator's own context outlives every +// GitTarget; only source clusters are refcounted away. +func TestTeardownCluster_NeverDropsTheConfigPlane(t *testing.T) { + m := &Manager{Log: logr.Discard()} + m.configPlaneCluster() + m.cluster("prod-eu-1") + + m.teardownCluster(configPlaneClusterID) + assert.NotNil(t, m.clusterContextByID(configPlaneClusterID), "the config plane is never torn down") + + m.teardownCluster("prod-eu-1") + assert.Nil(t, m.clusterContextByID("prod-eu-1"), "a source cluster is torn down") + + // Tearing down an id that is not live is a no-op, not a panic. + m.teardownCluster("never-created") +} diff --git a/internal/watch/manager.go b/internal/watch/manager.go index b293320f..7d4f6aad 100644 --- a/internal/watch/manager.go +++ b/internal/watch/manager.go @@ -102,14 +102,17 @@ type Manager struct { // against what git already holds. See routeLiveTargetWatchEvent. liveContentDedup sync.Map - // SourceClusters resolves a GitTarget's source-cluster id into a rest.Config by reading - // the kubeconfig Secret it names from the config plane. Nil is the single-cluster - // install: every GitTarget mirrors the cluster the operator runs in. + // SourceClusters resolves a GitTarget's source cluster — a ClusterProvider NAME — into a + // rest.Config, reading the kubeconfig Secret the provider names from the config plane. It is + // required for any GitTarget to mirror, single-cluster installs included: a source cluster is + // always a ClusterProvider, and only this resolver can say whether that provider is in-cluster + // (kubeConfig omitted) or remote. Nil leaves every source cluster unresolvable; only the config + // plane, which needs no provider, still works. SourceClusters SourceClusterResolver - // clusters holds one clusterContext per distinct source cluster — its API catalog, type - // registry, and clients. LocalClusterID is the cluster the operator runs in, and the only - // one a single-cluster install ever creates. See cluster_context.go. + // clusters holds one clusterContext per distinct cluster — its API catalog, type registry, + // and clients. configPlaneClusterID is the operator's own cluster (always present, never a + // source); every other key is a ClusterProvider name. See cluster_context.go. clustersMu sync.Mutex clusters map[string]*clusterContext // clusterOrder is the published, ordered snapshot of clusters (local first). The git @@ -118,7 +121,7 @@ type Manager struct { // reconcile loop. clusterOrder atomic.Pointer[[]*clusterContext] // gitTargetClusters maps a GitTarget key to the source-cluster id it mirrors from, - // captured on Declare (the gitTargetUIDs pattern). Because spec.kubeConfig is immutable + // captured on Declare (the gitTargetUIDs pattern). Because spec.clusterProviderRef is immutable // this is learned once and never changes — no per-rule propagation, no disagreement // window. Guarded by gitTargetClustersMu. gitTargetClustersMu sync.Mutex diff --git a/internal/watch/manager_catalog.go b/internal/watch/manager_catalog.go index 3c7bf165..ba99e490 100644 --- a/internal/watch/manager_catalog.go +++ b/internal/watch/manager_catalog.go @@ -289,14 +289,14 @@ func recordCatalogStats(ctx context.Context, stats CatalogStats) { // the back-compatible accessor every source-cluster-unaware caller uses; per-cluster callers // read cc.catalog directly. func (m *Manager) apiResourceCatalog() *APIResourceCatalog { - return m.localCluster().catalog + return m.configPlaneCluster().catalog } // typeRegistryInstance returns the LOCAL cluster's followability registry, so a zero-value // Manager (used widely in tests) needs no explicit setup. Per-cluster callers read // cc.registry directly. func (m *Manager) typeRegistryInstance() *typeset.Registry { - return m.localCluster().registry + return m.configPlaneCluster().registry } // refreshClusterTypeRegistry publishes one cluster's catalog scan to its typeset registry, @@ -349,20 +349,20 @@ func (m *Manager) logTypeRefusals(cc *clusterContext, reg *typeset.Registry) { // writer's manager-wide mapper wiring (cmd/main.go); the cluster-scoped writer lookup uses // ClusterTypeLookup instead (see gvr.go / Step 4). func (m *Manager) TypeRegistry() *typeset.Registry { - return m.localCluster().registry + return m.configPlaneCluster().registry } // FollowableTypeRecords returns the LOCAL cluster's currently-followable type records (verdict // followable or retained), sorted by identity. It is the inventory the status and visibility // surfaces read; it never recomputes followability. func (m *Manager) FollowableTypeRecords() []typeset.TypeRecord { - return m.localCluster().registry.Followable() + return m.configPlaneCluster().registry.Followable() } // TypeRecords returns every known type record — followable, retained, and refused — for the // LOCAL cluster, for inventory and "why is this type not picked up?" views. func (m *Manager) TypeRecords() []typeset.TypeRecord { - return m.localCluster().registry.All() + return m.configPlaneCluster().registry.All() } // ruleResourceSelector is one rule's (apiGroups, apiVersions, resources, scope) tuple, diff --git a/internal/watch/materialization.go b/internal/watch/materialization.go index 7cc5a915..e4142332 100644 --- a/internal/watch/materialization.go +++ b/internal/watch/materialization.go @@ -9,10 +9,11 @@ import ( ) // DeclareForGitTarget ensures the GitTarget's watch-first data plane is running against the -// source cluster it mirrors from. clusterID is (api/v1alpha3).GitTarget.SourceClusterID() — "" -// for the cluster the operator runs in. It is captured here, the same capture-on-Declare -// pattern as the UID: because spec.kubeConfig is immutable it is learned once and never -// changes, so there is no per-rule propagation and no cross-rule disagreement window. +// source cluster it mirrors from. clusterID is (api/v1alpha3).GitTarget.SourceCluster() — the +// referenced ClusterProvider's name, "default" for the cluster the operator runs in. It is +// captured here, the same capture-on-Declare pattern as the UID: because spec.clusterProviderRef +// is immutable it is learned once and never changes, so there is no per-rule propagation and no +// cross-rule disagreement window. func (m *Manager) DeclareForGitTarget( ctx context.Context, gitDest types.ResourceReference, diff --git a/internal/watch/source_cluster_resolver.go b/internal/watch/source_cluster_resolver.go index 97bbfc76..fd1f5487 100644 --- a/internal/watch/source_cluster_resolver.go +++ b/internal/watch/source_cluster_resolver.go @@ -6,13 +6,13 @@ import ( "context" "fmt" "net" - "strings" "time" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/kubeconfig" ) @@ -29,27 +29,36 @@ const sourceClusterDialTimeout = 15 * time.Second // to a remote is kept healthy rather than silently half-open. const sourceClusterKeepAlive = 30 * time.Second -// secretSourceClusterResolver resolves a source-cluster id — "//", as -// (api/v1alpha3).GitTarget.SourceClusterID() renders it — into a rest.Config, by reading that -// Secret from the CONFIG PLANE: the cluster the operator runs in. +// secretSourceClusterResolver resolves a source-cluster NAME — a ClusterProvider's name, as +// (api/v1alpha3).GitTarget.SourceCluster() carries it — into a rest.Config by looking up the +// cluster-scoped ClusterProvider and reading its kubeConfig Secret from the OPERATOR NAMESPACE +// (the config plane): the cluster the operator runs in. // -// This is the whole point of the split. The credential for a cluster never has to live on -// that cluster, and the watched cluster holds nothing but the watched resources — no Secret, -// no configbutler.ai CRDs at all. +// This is the whole point of the split. The credential for a cluster never has to live on that +// cluster, and the watched cluster holds nothing but the watched resources — no Secret, no +// configbutler.ai CRDs at all. The resolver is only ever asked to resolve a REMOTE provider; +// the in-cluster "default" provider is served by the manager's own in-cluster config and never +// reaches here. type secretSourceClusterResolver struct { - // client reads Secrets from the config plane. Secret reads bypass the controller-runtime - // cache (the manager client sets Cache.DisableFor on Secrets), so a rotated kubeconfig is - // seen without a Secret informer — the same reasoning as the SOPS age-key reads. + // client reads the ClusterProvider (cluster-scoped) and its kubeConfig Secret. Secret reads + // bypass the controller-runtime cache (the manager client sets Cache.DisableFor on Secrets), + // so a rotated kubeconfig is seen without a Secret informer — the same reasoning as the SOPS + // age-key reads. client client.Client + // operatorNamespace is where a ClusterProvider's kubeConfig Secret is pinned. A cluster-scoped + // provider has no namespace of its own, so the credential for a cluster is always read from + // here — never from the source cluster. + operatorNamespace string + // safety is the operator's exec / insecure-TLS opt-in. Both default off: an // operator-supplied kubeconfig is attacker-adjacent input, so we REJECT rather than // silently strip (diverging from Flux) unless a flag opts in. safety kubeconfig.SafetyPolicy - // qps and burst bound the rate at which the operator talks to a source cluster. A remote - // cluster is reached over a network the local one is not, so it gets client-side - // throttling the in-cluster config does not carry by default. + // qps and burst are the GLOBAL defaults (--source-cluster-qps/-burst) bounding the rate at + // which the operator talks to a source cluster. A ClusterProvider may override them per + // cluster via spec.qps/spec.burst. qps float32 burst int } @@ -57,68 +66,65 @@ type secretSourceClusterResolver struct { // NewSecretSourceClusterResolver builds the production source-cluster resolver. func NewSecretSourceClusterResolver( c client.Client, + operatorNamespace string, safety kubeconfig.SafetyPolicy, qps float32, burst int, ) SourceClusterResolver { - return &secretSourceClusterResolver{client: c, safety: safety, qps: qps, burst: burst} -} - -// sourceClusterRef is a source-cluster id parsed back into the Secret it names. -type sourceClusterRef struct { - Namespace string - Name string - Key string -} - -// sourceClusterIDSegments is the number of "/"-separated parts in a source-cluster id. -const sourceClusterIDSegments = 3 - -// parseSourceClusterID splits the id GitTarget.SourceClusterID() produces. It is a private -// encoding, never a user input: a GitTarget's namespace cannot contain "/" and neither can a -// Secret name, so the first two segments are unambiguous and the rest is the data key. The KEY -// segment MAY be empty — an omitted spec key is its own identity, and the resolver then falls -// back value→value.yaml. -func parseSourceClusterID(id string) (sourceClusterRef, error) { - parts := strings.SplitN(id, "/", sourceClusterIDSegments) - if len(parts) != sourceClusterIDSegments || parts[0] == "" || parts[1] == "" { - return sourceClusterRef{}, fmt.Errorf("malformed source cluster id %q, want //", id) + return &secretSourceClusterResolver{ + client: c, + operatorNamespace: operatorNamespace, + safety: safety, + qps: qps, + burst: burst, } - return sourceClusterRef{Namespace: parts[0], Name: parts[1], Key: parts[2]}, nil } func (r *secretSourceClusterResolver) ResolveSourceCluster( ctx context.Context, - clusterID string, + providerName string, ) (*rest.Config, string, error) { - ref, err := parseSourceClusterID(clusterID) - if err != nil { - return nil, "", err + var provider configv1alpha3.ClusterProvider + if err := r.client.Get(ctx, client.ObjectKey{Name: providerName}, &provider); err != nil { + return nil, "", fmt.Errorf("read ClusterProvider %q: %w", providerName, err) + } + if provider.Spec.KubeConfig == nil { + // No kubeConfig means the operator's OWN cluster — legal for every provider name, so this + // is the in-cluster answer (nil config), never an error. The name is irrelevant: what makes + // a provider local is the absent kubeConfig, not being called "default". + return nil, inClusterConfigVersion, nil } + if provider.Spec.KubeConfig.SecretRef == nil { + return nil, "", &kubeconfig.RejectionError{ + Reason: kubeconfig.ReasonInvalid, + Message: fmt.Sprintf("ClusterProvider %q sets kubeConfig without a secretRef", providerName), + } + } + ref := provider.Spec.KubeConfig.SecretRef var secret corev1.Secret - if err := r.client.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, &secret); err != nil { - return nil, "", fmt.Errorf("read kubeconfig Secret %s/%s: %w", ref.Namespace, ref.Name, err) + secretKey := client.ObjectKey{Namespace: r.operatorNamespace, Name: ref.Name} + if err := r.client.Get(ctx, secretKey, &secret); err != nil { + return nil, "", fmt.Errorf("read kubeconfig Secret %s for ClusterProvider %q: %w", secretKey, providerName, err) } raw, usedKey, ok := kubeconfig.ResolveKey(secret.Data, ref.Key) if !ok { return nil, "", &kubeconfig.RejectionError{ Reason: kubeconfig.ReasonKeyNotFound, - Message: fmt.Sprintf("kubeconfig Secret %s/%s has no kubeconfig under key %q", - ref.Namespace, ref.Name, describeKey(ref.Key)), + Message: fmt.Sprintf("kubeconfig Secret %s has no kubeconfig under key %q", + secretKey, describeKey(ref.Key)), } } - // Parse and REJECT unsafe kubeconfigs before building the config — a legible failure that - // the controller's Validated gate reports with the same typed reason. Never dials. + // Parse and REJECT unsafe kubeconfigs before building the config — a legible failure that the + // ClusterProvider's Validated gate reports with the same typed reason. Never dials. cfg, err := kubeconfig.BuildRESTConfig(raw, r.safety) if err != nil { - return nil, "", fmt.Errorf("kubeconfig Secret %s/%s key %q: %w", - ref.Namespace, ref.Name, usedKey, err) + return nil, "", fmt.Errorf("kubeconfig Secret %s key %q: %w", secretKey, usedKey, err) } - if r.qps > 0 { - cfg.QPS = r.qps - cfg.Burst = r.burst + if qps, burst := r.throttleFor(&provider); qps > 0 { + cfg.QPS = qps + cfg.Burst = burst } // Bound CONNECTION SETUP so an unreachable remote surfaces as SourceClusterReachable=False // promptly — but do NOT set rest.Config.Timeout, which applies to the full HTTP request and @@ -128,10 +134,26 @@ func (r *secretSourceClusterResolver) ResolveSourceCluster( // timeout; list callers pass a context deadline). cfg.Dial = (&net.Dialer{Timeout: sourceClusterDialTimeout, KeepAlive: sourceClusterKeepAlive}).DialContext - // The Secret's resourceVersion is the version token: it changes on every rotation, and on - // nothing else. The kubeconfig bytes themselves are dropped here — only the built - // rest.Config survives the call. - return cfg, secret.ResourceVersion, nil + // The version token changes exactly when the resolved config could change: on a kubeconfig + // Secret rotation (its resourceVersion) OR on a provider spec change that alters qps/burst + // (its generation). A change in either re-resolves and rebuilds the clients. + version := fmt.Sprintf("%d/%s", provider.Generation, secret.ResourceVersion) + return cfg, version, nil +} + +// throttleFor returns the effective client QPS/burst for a provider: its per-provider override +// when set, else the operator-wide default. A zero global default leaves the rest.Config defaults +// untouched (the same behavior as before per-provider overrides existed). +func (r *secretSourceClusterResolver) throttleFor(provider *configv1alpha3.ClusterProvider) (float32, int) { + qps := r.qps + burst := r.burst + if provider.Spec.QPS != nil { + qps = float32(*provider.Spec.QPS) + } + if provider.Spec.Burst != nil { + burst = int(*provider.Spec.Burst) + } + return qps, burst } // describeKey renders the resolved-key hint for a "key not found" message: an omitted spec key diff --git a/internal/watch/source_cluster_resolver_test.go b/internal/watch/source_cluster_resolver_test.go index fe40c033..ec88c0ae 100644 --- a/internal/watch/source_cluster_resolver_test.go +++ b/internal/watch/source_cluster_resolver_test.go @@ -6,15 +6,21 @@ import ( "context" "testing" + meta "github.com/fluxcd/pkg/apis/meta" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/kubeconfig" ) +const resolverOperatorNS = "gitops-reverser-system" + const resolverKubeConfig = `apiVersion: v1 kind: Config clusters: @@ -50,87 +56,127 @@ users: interactiveMode: Never ` -func TestParseSourceClusterID(t *testing.T) { - ref, err := parseSourceClusterID("team-a/kc/value") - require.NoError(t, err) - assert.Equal(t, sourceClusterRef{Namespace: "team-a", Name: "kc", Key: "value"}, ref) - - // An empty key segment is valid — an omitted spec key is its own identity. - ref, err = parseSourceClusterID("team-a/kc/") - require.NoError(t, err) - assert.Equal(t, sourceClusterRef{Namespace: "team-a", Name: "kc", Key: ""}, ref) - - for _, bad := range []string{"", "onlyone", "ns/name", "/name/key", "ns//key"} { - _, err := parseSourceClusterID(bad) - assert.Error(t, err, "malformed id %q must error", bad) - } +func resolverScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + require.NoError(t, configv1alpha3.AddToScheme(s)) + require.NoError(t, corev1.AddToScheme(s)) + return s } -func newResolver(t *testing.T, secret *corev1.Secret, safety kubeconfig.SafetyPolicy) SourceClusterResolver { - t.Helper() - builder := fake.NewClientBuilder() - if secret != nil { - builder = builder.WithObjects(secret) +// clusterProvider builds the remote ClusterProvider "prod-eu-1" whose kubeconfig Secret is "kc" +// under the given data key. +func clusterProvider(key string) *configv1alpha3.ClusterProvider { + return &configv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "prod-eu-1"}, + Spec: configv1alpha3.ClusterProviderSpec{ + KubeConfig: &meta.KubeConfigReference{SecretRef: &meta.SecretKeyReference{Name: "kc", Key: key}}, + }, } - return NewSecretSourceClusterResolver(builder.Build(), safety, 20, 30) } func kubeconfigSecret(key, body string) *corev1.Secret { return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "kc"}, + ObjectMeta: metav1.ObjectMeta{Namespace: resolverOperatorNS, Name: "kc"}, Data: map[string][]byte{key: []byte(body)}, } } +func newResolver(t *testing.T, safety kubeconfig.SafetyPolicy, objs ...client.Object) SourceClusterResolver { + t.Helper() + cl := fake.NewClientBuilder().WithScheme(resolverScheme(t)).WithObjects(objs...).Build() + return NewSecretSourceClusterResolver(cl, resolverOperatorNS, safety, 20, 30) +} + func TestResolveSourceCluster_ValidAppliesThrottleAndVersion(t *testing.T) { - secret := kubeconfigSecret("value", resolverKubeConfig) - r := newResolver(t, secret, kubeconfig.SafetyPolicy{}) + r := newResolver(t, kubeconfig.SafetyPolicy{}, + clusterProvider("value"), kubeconfigSecret("value", resolverKubeConfig)) - cfg, version, err := r.ResolveSourceCluster(context.Background(), "team-a/kc/value") + cfg, version, err := r.ResolveSourceCluster(context.Background(), "prod-eu-1") require.NoError(t, err) require.NotNil(t, cfg) assert.Equal(t, "https://192.0.2.1:6443", cfg.Host) - assert.InDelta(t, 20.0, cfg.QPS, 0.001, "source-cluster QPS applied") - assert.Equal(t, 30, cfg.Burst, "source-cluster burst applied") - assert.NotEmpty(t, version, "the Secret resourceVersion is the version token") + assert.InDelta(t, 20.0, cfg.QPS, 0.001, "global source-cluster QPS applied") + assert.Equal(t, 30, cfg.Burst, "global source-cluster burst applied") + assert.NotEmpty(t, version, "the provider generation + Secret resourceVersion form the version token") +} + +func TestResolveSourceCluster_PerProviderThrottleOverride(t *testing.T) { + qps := int32(5) + burst := int32(7) + provider := clusterProvider("value") + provider.Spec.QPS = &qps + provider.Spec.Burst = &burst + r := newResolver(t, kubeconfig.SafetyPolicy{}, provider, kubeconfigSecret("value", resolverKubeConfig)) + + cfg, _, err := r.ResolveSourceCluster(context.Background(), "prod-eu-1") + require.NoError(t, err) + assert.InDelta(t, 5.0, cfg.QPS, 0.001, "per-provider QPS overrides the global default") + assert.Equal(t, 7, cfg.Burst, "per-provider burst overrides the global default") } func TestResolveSourceCluster_KeyFallbackValueYaml(t *testing.T) { - // Secret stores under value.yaml (Flux Kustomization shape); id has an empty key segment. - secret := kubeconfigSecret("value.yaml", resolverKubeConfig) - r := newResolver(t, secret, kubeconfig.SafetyPolicy{}) + // Secret stores under value.yaml (Flux Kustomization shape); the provider's secretRef.key is empty. + r := newResolver(t, kubeconfig.SafetyPolicy{}, + clusterProvider(""), kubeconfigSecret("value.yaml", resolverKubeConfig)) - cfg, _, err := r.ResolveSourceCluster(context.Background(), "team-a/kc/") + cfg, _, err := r.ResolveSourceCluster(context.Background(), "prod-eu-1") require.NoError(t, err) assert.Equal(t, "https://192.0.2.1:6443", cfg.Host) } -func TestResolveSourceCluster_MissingSecretAndKey(t *testing.T) { - r := newResolver(t, nil, kubeconfig.SafetyPolicy{}) - _, _, err := r.ResolveSourceCluster(context.Background(), "team-a/absent/value") +func TestResolveSourceCluster_MissingProviderSecretAndKey(t *testing.T) { + // Absent ClusterProvider -> error. + r := newResolver(t, kubeconfig.SafetyPolicy{}) + _, _, err := r.ResolveSourceCluster(context.Background(), "absent") + require.Error(t, err, "an absent ClusterProvider is an error, not a nil config") + + // Provider present, kubeconfig Secret absent -> error. + r = newResolver(t, kubeconfig.SafetyPolicy{}, clusterProvider("value")) + _, _, err = r.ResolveSourceCluster(context.Background(), "prod-eu-1") require.Error(t, err, "an absent Secret is an error, not a nil config") // Secret present but no kubeconfig under the resolved key -> typed KeyNotFound. - secret := kubeconfigSecret("elsewhere", resolverKubeConfig) - r = newResolver(t, secret, kubeconfig.SafetyPolicy{}) - _, _, err = r.ResolveSourceCluster(context.Background(), "team-a/kc/value") + r = newResolver(t, kubeconfig.SafetyPolicy{}, + clusterProvider("value"), kubeconfigSecret("elsewhere", resolverKubeConfig)) + _, _, err = r.ResolveSourceCluster(context.Background(), "prod-eu-1") require.Error(t, err) rej, ok := kubeconfig.AsRejection(err) require.True(t, ok) assert.Equal(t, kubeconfig.ReasonKeyNotFound, rej.Reason) } +// TestDescribeKey covers the hint a "key not found" error carries. An omitted secretRef.key is +// not "no key": the resolver tried both fallbacks, and the message must say so or the human is +// told to look for a key they never wrote. +func TestDescribeKey(t *testing.T) { + tests := []struct { + name string + specKey string + want string + }{ + {"omitted key names both fallbacks", "", "value or value.yaml"}, + {"explicit key is reported verbatim", "kubeconfig", "kubeconfig"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, describeKey(tc.specKey)) + }) + } +} + func TestResolveSourceCluster_RejectsUnsafe(t *testing.T) { - secret := kubeconfigSecret("value", resolverExecKubeConfig) - r := newResolver(t, secret, kubeconfig.SafetyPolicy{}) - _, _, err := r.ResolveSourceCluster(context.Background(), "team-a/kc/value") + r := newResolver(t, kubeconfig.SafetyPolicy{}, + clusterProvider("value"), kubeconfigSecret("value", resolverExecKubeConfig)) + _, _, err := r.ResolveSourceCluster(context.Background(), "prod-eu-1") require.Error(t, err) rej, ok := kubeconfig.AsRejection(err) require.True(t, ok) assert.Equal(t, kubeconfig.ReasonExecNotAllowed, rej.Reason) // Opting in (a deliberate trust decision) lets it through. - r = newResolver(t, secret, kubeconfig.SafetyPolicy{AllowExec: true}) - _, _, err = r.ResolveSourceCluster(context.Background(), "team-a/kc/value") + r = newResolver(t, kubeconfig.SafetyPolicy{AllowExec: true}, + clusterProvider("value"), kubeconfigSecret("value", resolverExecKubeConfig)) + _, _, err = r.ResolveSourceCluster(context.Background(), "prod-eu-1") require.NoError(t, err) } diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index 010c17aa..01183d49 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -697,7 +697,7 @@ func (m *Manager) routeLiveTargetWatchEvent( event := targetWatchGitEvent(key.GVR, u, op) // Carry the source cluster so the git writer resolves this document's GVK->GVR // against the cluster it was watched on, never a union of all clusters. - event.SourceClusterID = m.clusterIDForGitTarget(gitDest) + event.SourceCluster = m.clusterIDForGitTarget(gitDest) // Drop a no-op UPDATE before it reaches the worker: a /status-only change // sanitizes to identical git content but ships unattributed (its /status audit // is dropped), so routing it would split an open commit window on the author @@ -741,8 +741,12 @@ func (m *Manager) attachAuthor( // author fact's post-write RV, so it may consult the /last pointer; a create/update is // exact-capable and must not fall through to /last. exactCapable := event.Operation != string(configv1alpha3.OperationDelete) + // event.SourceCluster (stamped just above, before this call) is the SOURCE CLUSTER — the + // ClusterProvider name — this event was watched on. It keys the author read against exactly + // the facts the audit handler recorded for that cluster, so a fact from cluster A can never + // name the author of an object watched on cluster B. if userInfo, ok := m.AuthorResolver.ResolveAuthor( - ctx, gvr, u.GetUID(), u.GetResourceVersion(), exactCapable, + ctx, event.SourceCluster, gvr, u.GetUID(), u.GetResourceVersion(), exactCapable, ); ok { event.UserInfo = userInfo } diff --git a/internal/watch/target_watch_test.go b/internal/watch/target_watch_test.go index ce4550cd..c705ae91 100644 --- a/internal/watch/target_watch_test.go +++ b/internal/watch/target_watch_test.go @@ -504,7 +504,7 @@ func TestOpenTargetWatch_UsesConfiguredHook(t *testing.T) { w, err := manager.openTargetWatch( context.Background(), - LocalClusterID, + configPlaneClusterID, configmapsGVR, "apps", metav1.ListOptions{ResourceVersion: "42"}, diff --git a/internal/webhook/audit_handler.go b/internal/webhook/audit_handler.go index cb8f8b65..e24b428e 100644 --- a/internal/webhook/audit_handler.go +++ b/internal/webhook/audit_handler.go @@ -40,14 +40,26 @@ type auditHandlerFirsts struct { request sync.Once factRecorded sync.Once impersonatedEvent sync.Once + unroutableEvent sync.Once } // AuditFactRecorder stores the minimal author-attribution fact for one accepted, -// mutating audit event. It is the only thing the audit webhook does now: watch -// carries the object body, so audit is a pure attribution lookup table. A nil -// recorder means configured-author mode — the handler is not wired at all. +// mutating audit event under a SOURCE CLUSTER (a ClusterProvider name), so a fact from +// one cluster never joins a watch event from another. It is the only thing the audit +// webhook does now: watch carries the object body, so audit is a pure attribution lookup +// table. A nil recorder means configured-author mode — the handler is not wired at all. type AuditFactRecorder interface { - RecordFact(ctx context.Context, event auditv1.Event) error + RecordFact(ctx context.Context, providerName string, event auditv1.Event) error +} + +// AuditProviderResolver reports whether a named source cluster (a ClusterProvider) exists, so an +// /audit-webhook/ route is accepted only for a configured cluster. It is the gate behind the +// connection's mTLS: the audit server already requires a CA-signed client cert +// (RequireAndVerifyClientCert), so an unauthenticated apiserver never reaches here; this then +// refuses a route for a provider that does not exist, rather than accumulating orphan facts. Every +// name is gated the same way, "default" included. A nil resolver means no route can be served. +type AuditProviderResolver interface { + ProviderExists(ctx context.Context, name string) (bool, error) } // AuditHandlerConfig contains configuration for the audit handler. @@ -58,6 +70,16 @@ type AuditHandlerConfig struct { // A write failure returns an audit-request error so the API server retries // delivery; mirrored-resource author attribution depends on these facts. FactRecorder AuditFactRecorder + // ProviderResolver gates every /audit-webhook/ route on the ClusterProvider existing. + // Nil means named routes are all 404. + ProviderResolver AuditProviderResolver + // ClusterAnnotationKey enables the bare /audit-webhook endpoint for a SHARED audit stream that + // carries several logical clusters: the owning ClusterProvider is read PER EVENT from this + // audit-event annotation, so one batch may fan out to several source clusters. Empty (the + // default) means the bare endpoint is NOT enabled and every producer must post to a named + // /audit-webhook/. Setting it requires a ProviderResolver — an annotation naming an + // unknown provider must be rejected, never guessed. + ClusterAnnotationKey string } // AuditHandler receives kube-apiserver audit events on /audit-webhook and records @@ -75,6 +97,11 @@ func NewAuditHandler(config AuditHandlerConfig) (*AuditHandler, error) { if config.MaxRequestBodyBytes <= 0 { config.MaxRequestBodyBytes = DefaultAuditMaxRequestBodyBytes } + // Annotation routing must be able to reject an unknown provider, so it cannot run without a + // resolver. Fail at startup rather than 400-ing every bare request at runtime. + if config.ClusterAnnotationKey != "" && config.ProviderResolver == nil { + return nil, errors.New("cluster annotation routing requires a ProviderResolver") + } scheme := runtime.NewScheme() if err := audit.AddToScheme(scheme); err != nil { @@ -115,15 +142,80 @@ func (h *AuditHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + route, ok := h.resolveRoute(ctx, w, r) + if !ok { + return + } + start := time.Now() - result, eventCount := h.serveEventListRequest(ctx, w, r, log) + result, eventCount := h.serveEventListRequest(ctx, route, w, r, log) h.recordEventListRequest(ctx, result, eventCount, time.Since(start)) } +// resolveRoute maps an already-syntactically-valid path to the route its events belong to. It +// writes the rejection itself and reports ok=false when the request cannot be served at all — as +// opposed to a per-event rejection, which still returns 200 for the rest of the batch. +func (h *AuditHandler) resolveRoute(ctx context.Context, w http.ResponseWriter, r *http.Request) (auditRoute, bool) { + providerName, named := providerRouteForPath(r.URL.Path) + if !named { + // The bare endpoint never guesses a source cluster. It exists only to demultiplex a SHARED + // stream by annotation, so without a configured key there is nothing it could mean and the + // producer is simply misconfigured: reject the whole request rather than silently dropping + // every event in it. + if h.config.ClusterAnnotationKey == "" { + http.Error(w, "the bare /audit-webhook endpoint is not enabled; post to "+ + "/audit-webhook/", http.StatusBadRequest) + return auditRoute{}, false + } + return auditRoute{annotationKey: h.config.ClusterAnnotationKey}, true + } + + // The connection is already CA-authenticated (the audit server requires a client cert), so gate + // on the named ClusterProvider existing rather than accepting facts for an unknown cluster. + // Every name is gated, "default" included — it is an ordinary provider. + if h.config.ProviderResolver == nil { + http.NotFound(w, r) + return auditRoute{}, false + } + exists, err := h.config.ProviderResolver.ProviderExists(ctx, providerName) + if err != nil { + http.Error(w, "resolve source cluster", http.StatusServiceUnavailable) + return auditRoute{}, false + } + if !exists { + http.NotFound(w, r) + return auditRoute{}, false + } + return auditRoute{provider: providerName}, true +} + +// auditRoute is how one accepted request maps its events to SOURCE CLUSTERS. Exactly one field is +// set: a named /audit-webhook/ route fixes `provider` for the whole batch, while the bare +// /audit-webhook route carries `annotationKey` and resolves a provider PER EVENT, so a single batch +// from a shared stream may fan out to several source clusters. +type auditRoute struct { + provider string + annotationKey string +} + +// providerRouteForPath maps an accepted audit route to the SOURCE CLUSTER its facts belong to and +// whether it is a NAMED route. Audit routes are named: /audit-webhook/ carries the provider +// (named=true), including /audit-webhook/default. The bare /audit-webhook names no provider — +// it is the shared, annotation-routed endpoint, so it returns ("", false) and the caller resolves +// each event separately. validateAuditWebhookPath has already accepted the path syntactically. +func providerRouteForPath(path string) (string, bool) { + segment := strings.TrimPrefix(path, "/audit-webhook/") + if segment == path || segment == "" { + return "", false + } + return segment, true +} + // serveEventListRequest decodes and processes one EventList request, returning the // bounded outcome and the number of decoded event items for the ingress metrics. func (h *AuditHandler) serveEventListRequest( ctx context.Context, + route auditRoute, w http.ResponseWriter, r *http.Request, log logr.Logger, @@ -148,7 +240,7 @@ func (h *AuditHandler) serveEventListRequest( reqLog.Info("Received first audit request", "eventCount", eventCount) }) - if err := h.processEvents(ctx, eventListV1.Items); err != nil { + if err := h.processEvents(ctx, route, eventListV1.Items); err != nil { reqLog.Error(err, "Failed to process audit events") http.Error(w, err.Error(), http.StatusInternalServerError) return outcomeProcessError, eventCount @@ -208,21 +300,23 @@ func (h *AuditHandler) decodeEventList(r *http.Request) (*auditv1.EventList, err return &eventListV1, nil } -// processEvents processes a list of audit events. -func (h *AuditHandler) processEvents(ctx context.Context, events []auditv1.Event) error { +// processEvents processes a list of audit events for one route. On the annotation-routed bare +// endpoint the events in one list may belong to different source clusters, so each is resolved +// independently and an unroutable event only drops itself. +func (h *AuditHandler) processEvents(ctx context.Context, route auditRoute, events []auditv1.Event) error { for i := range events { - if err := h.processEvent(ctx, events[i]); err != nil { + if err := h.processEvent(ctx, route, events[i]); err != nil { return err } } return nil } -// processEvent applies the intrinsic accept gate and records the attribution fact -// for an accepted, mutating event. A rejected event is recorded with its terminal -// outcome and dropped; only a fact-store failure returns an error (the API server -// then retries delivery). -func (h *AuditHandler) processEvent(ctx context.Context, event auditv1.Event) error { +// processEvent applies the intrinsic accept gate, resolves the event's source cluster, and records +// the attribution fact for an accepted, mutating event. A rejected event is recorded with its +// terminal outcome and dropped; only a fact-store or provider-lookup failure returns an error (the +// API server then retries delivery). +func (h *AuditHandler) processEvent(ctx context.Context, route auditRoute, event auditv1.Event) error { log := logf.Log.WithName("audit-handler") h.logAuditEventReceived(event) @@ -239,8 +333,16 @@ func (h *AuditHandler) processEvent(ctx context.Context, event auditv1.Event) er return nil } + providerName, routed, err := h.resolveEventProvider(ctx, route, &event) + if err != nil { + return err + } + if !routed { + return nil + } + if h.config.FactRecorder != nil { - if err := h.config.FactRecorder.RecordFact(ctx, event); err != nil { + if err := h.config.FactRecorder.RecordFact(ctx, providerName, event); err != nil { outcome.Record(ctx, &event, outcome.WriteError) return fmt.Errorf("record attribution fact %q: %w", event.AuditID, err) } @@ -256,6 +358,71 @@ func (h *AuditHandler) processEvent(ctx context.Context, event auditv1.Event) er return nil } +// resolveEventProvider returns the ClusterProvider that owns one accepted event, and whether it +// routed at all. On a named route that is the route's provider, unconditionally. On the shared, +// annotation-routed bare endpoint it is read from the event's own annotations and existence-checked: +// an event carrying no annotation, or naming a ClusterProvider that does not exist, is REJECTED — +// it produces no fact and is never credited to a fallback provider, because a wrong source cluster +// would let a user from one logical cluster author a matching object in another. The rejection is +// per EVENT so correctly-annotated events in the same batch still land. A resolver failure is +// transient rather than a verdict, so it is returned as an error and the API server retries the +// whole batch. +func (h *AuditHandler) resolveEventProvider( + ctx context.Context, + route auditRoute, + event *auditv1.Event, +) (string, bool, error) { + if route.annotationKey == "" { + return route.provider, true, nil + } + + name := event.Annotations[route.annotationKey] + if name == "" { + h.rejectUnroutableEvent(ctx, event, outcome.MissingClusterAnnotation, route.annotationKey, name) + return "", false, nil + } + + exists, err := h.config.ProviderResolver.ProviderExists(ctx, name) + if err != nil { + return "", false, fmt.Errorf("resolve source cluster %q: %w", name, err) + } + if !exists { + h.rejectUnroutableEvent(ctx, event, outcome.UnknownClusterProvider, route.annotationKey, name) + return "", false, nil + } + return name, true, nil +} + +// rejectUnroutableEvent counts and logs one event the shared endpoint could not route. Counting it +// on the ordinary per-event outcome counter is the point: a producer that is not stamping the +// annotation shows up as a rising drop rate rather than as silence. The first rejection is logged +// at Info so the misconfiguration is visible without reading metrics; the rest stay at V(1) so a +// steady stream of them cannot flood the log. +func (h *AuditHandler) rejectUnroutableEvent( + ctx context.Context, + event *auditv1.Event, + reason outcome.Outcome, + annotationKey string, + sourceCluster string, +) { + outcome.Record(ctx, event, reason) + + log := logf.Log.WithName("audit-handler") + fields := []any{ + "reason", string(reason), + "annotationKey", annotationKey, + "sourceCluster", sourceCluster, + "auditID", event.AuditID, + "gvr", extractGVR(event), + } + h.firsts.unroutableEvent.Do(func() { + log.Info("Rejected an unroutable audit event on the shared /audit-webhook endpoint; it names no "+ + "existing ClusterProvider and is never credited to a fallback. Stamp the annotation, or point "+ + "this producer at /audit-webhook/", fields...) + }) + log.V(1).Info("Rejected unroutable audit event", fields...) +} + // logAuditEventReceived emits the structured "audit event received" log (and the // first-impersonation banner). The per-event count is recorded once as the event's // outcome on gitopsreverser_audit_events_total, so there is no separate counter here. @@ -389,21 +556,25 @@ func effectiveAuditUsername(event auditv1.Event) string { return event.User.Username } -// validateAuditWebhookPath accepts only the canonical /audit-webhook path. The -// aggregated-API body proxy and its /audit-webhook-additional endpoint were removed -// with the watch-first rewrite — watch carries the body, so there is no body to join. +// validateAuditWebhookPath accepts the bare shared /audit-webhook and a single-segment +// /audit-webhook/. It is a purely SYNTACTIC check; whether the named provider +// actually exists, and whether the bare endpoint is enabled at all, is enforced in ServeHTTP. It +// rejects a trailing slash and any extra path segment. func validateAuditWebhookPath(path string) error { - switch path { - case "/audit-webhook": + if path == "/audit-webhook" { return nil - case "/audit-webhook/": + } + if !strings.HasPrefix(path, "/audit-webhook/") { + return errors.New("invalid path; expected /audit-webhook or /audit-webhook/") + } + segment := strings.TrimPrefix(path, "/audit-webhook/") + if segment == "" { return errors.New("audit webhook path must not include a trailing slash") - default: - if strings.HasPrefix(path, "/audit-webhook/") { - return errors.New("audit webhook path must not include a cluster ID or extra path segment") - } - return errors.New("invalid path; expected /audit-webhook") } + if strings.Contains(segment, "/") { + return errors.New("audit webhook path must name exactly one ClusterProvider: /audit-webhook/") + } + return nil } // gvrParts splits an audit event's objectRef into bounded group/version/resource diff --git a/internal/webhook/audit_handler_test.go b/internal/webhook/audit_handler_test.go index b79ddd45..352467b8 100644 --- a/internal/webhook/audit_handler_test.go +++ b/internal/webhook/audit_handler_test.go @@ -37,21 +37,41 @@ func TestMain(m *testing.M) { // fakeFactRecorder is an in-memory AuditFactRecorder. It appends every accepted // event and can be told to fail with an injectable error. type fakeFactRecorder struct { - mu sync.Mutex - err error - events []auditv1.Event + mu sync.Mutex + err error + events []auditv1.Event + providers []string } -func (r *fakeFactRecorder) RecordFact(_ context.Context, event auditv1.Event) error { +func (r *fakeFactRecorder) RecordFact(_ context.Context, providerName string, event auditv1.Event) error { r.mu.Lock() defer r.mu.Unlock() if r.err != nil { return r.err } r.events = append(r.events, event) + r.providers = append(r.providers, providerName) return nil } +// lastProvider returns the provider name threaded into the most recent RecordFact call. +func (r *fakeFactRecorder) lastProvider() string { + r.mu.Lock() + defer r.mu.Unlock() + if len(r.providers) == 0 { + return "" + } + return r.providers[len(r.providers)-1] +} + +// recordedProviders returns the provider name threaded into each RecordFact call, in order — the +// fan-out a single annotation-routed batch produced. +func (r *fakeFactRecorder) recordedProviders() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.providers...) +} + func (r *fakeFactRecorder) auditIDs() []string { r.mu.Lock() defer r.mu.Unlock() @@ -88,6 +108,22 @@ func eventListFixtureBody(t *testing.T, path string) string { return string(body) } +// defaultRoute is the named audit route for a ClusterProvider called "default". Audit routes are +// NAMED, and the bare /audit-webhook is the shared, annotation-routed endpoint that is off unless +// ClusterAnnotationKey is set — so every test about event classification (rather than routing) +// posts here, exactly as a single-cluster apiserver would. +const defaultRoute = "/audit-webhook/default" + +// routedConfig fills in the ProviderResolver that defaultRoute is existence-gated on, so a +// classification test can keep stating only what it is actually about. A test that cares about the +// gate supplies its own resolver, which is left untouched. +func routedConfig(config AuditHandlerConfig) AuditHandlerConfig { + if config.ProviderResolver == nil { + config.ProviderResolver = fakeProviderResolver{existing: map[string]bool{"default": true}} + } + return config +} + // serveBody runs one POST request through the handler and returns the recorder. func serveBody(t *testing.T, handler *AuditHandler, method, path, body string) *httptest.ResponseRecorder { t.Helper() @@ -106,20 +142,235 @@ const acceptedCreateEvent = `{"kind":"Event","level":"RequestResponse","auditID" `"responseStatus":{"code":200},` + `"responseObject":{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"cm","namespace":"default"}}}` +// TestAuditHandler_NamedDefaultRouteThreadsItsProvider checks that /audit-webhook/default is an +// ordinary named route: it records its facts under the "default" ClusterProvider name. +func TestAuditHandler_NamedDefaultRouteThreadsItsProvider(t *testing.T) { + recorder := &fakeFactRecorder{} + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) + require.NoError(t, err) + + body := `{"kind":"EventList","apiVersion":"audit.k8s.io/v1","items":[` + acceptedCreateEvent + `]}` + w := serveBody(t, handler, http.MethodPost, defaultRoute, body) + require.Equal(t, http.StatusOK, w.Code) + require.Equal(t, []string{"create-1"}, recorder.auditIDs()) + assert.Equal(t, "default", recorder.lastProvider(), "the named default route keys facts by its own name") +} + +// fakeProviderResolver answers ProviderExists from a fixed set, or with an injectable error. +type fakeProviderResolver struct { + existing map[string]bool + err error +} + +func (f fakeProviderResolver) ProviderExists(_ context.Context, name string) (bool, error) { + if f.err != nil { + return false, f.err + } + return f.existing[name], nil +} + +// TestAuditHandler_NamedRouting checks the /audit-webhook/ gate: an existing provider is +// served and its facts keyed by name; a missing provider is 404 (for "default" too); a resolver +// error is 503; and the bare endpoint is not a fallback for any of them. +func TestAuditHandler_NamedRouting(t *testing.T) { + body := eventListBody(acceptedCreateEvent) + + t.Run("existing provider records under its name", func(t *testing.T) { + recorder := &fakeFactRecorder{} + handler, err := NewAuditHandler(AuditHandlerConfig{ + FactRecorder: recorder, + ProviderResolver: fakeProviderResolver{existing: map[string]bool{"prod-eu-1": true}}, + }) + require.NoError(t, err) + w := serveBody(t, handler, http.MethodPost, "/audit-webhook/prod-eu-1", body) + require.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "prod-eu-1", recorder.lastProvider()) + }) + + t.Run("missing provider is 404, records nothing", func(t *testing.T) { + recorder := &fakeFactRecorder{} + handler, err := NewAuditHandler(AuditHandlerConfig{ + FactRecorder: recorder, + ProviderResolver: fakeProviderResolver{existing: map[string]bool{}}, + }) + require.NoError(t, err) + w := serveBody(t, handler, http.MethodPost, "/audit-webhook/gone", body) + assert.Equal(t, http.StatusNotFound, w.Code) + assert.Zero(t, recorder.len()) + }) + + t.Run("resolver error is 503", func(t *testing.T) { + handler, err := NewAuditHandler(AuditHandlerConfig{ + FactRecorder: &fakeFactRecorder{}, + ProviderResolver: fakeProviderResolver{err: assert.AnError}, + }) + require.NoError(t, err) + w := serveBody(t, handler, http.MethodPost, "/audit-webhook/prod-eu-1", body) + assert.Equal(t, http.StatusServiceUnavailable, w.Code) + }) + + t.Run("default is existence-gated like any other name", func(t *testing.T) { + recorder := &fakeFactRecorder{} + handler, err := NewAuditHandler(AuditHandlerConfig{ + FactRecorder: recorder, + ProviderResolver: fakeProviderResolver{existing: map[string]bool{}}, // default absent + }) + require.NoError(t, err) + w := serveBody(t, handler, http.MethodPost, defaultRoute, body) + assert.Equal(t, http.StatusNotFound, w.Code, "default has no privileged route") + assert.Zero(t, recorder.len()) + }) + + t.Run("bare endpoint is 400 while no annotation key is configured", func(t *testing.T) { + recorder := &fakeFactRecorder{} + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) + require.NoError(t, err) + w := serveBody(t, handler, http.MethodPost, "/audit-webhook", body) + assert.Equal(t, http.StatusBadRequest, w.Code, + "the bare endpoint resolves no provider of its own, so a producer posting to it is misconfigured") + assert.Zero(t, recorder.len()) + }) +} + +// clusterAnnotation is the annotation key a shared audit stream stamps its source cluster into. +const clusterAnnotation = "example.io/source-cluster" + +// annotatedEvent builds an otherwise-acceptable create event carrying an audit-event annotation +// map. A nil map is an event a producer did not stamp at all. +func annotatedEvent(auditID string, annotations map[string]string) string { + encoded, err := json.Marshal(annotations) + if err != nil { + panic(err) + } + return `{"kind":"Event","level":"RequestResponse","auditID":"` + auditID + `",` + + `"stage":"ResponseComplete","verb":"create","user":{"username":"test-user"},` + + `"requestURI":"/api/v1/namespaces/default/configmaps",` + + `"annotations":` + string(encoded) + `,` + + `"objectRef":{"resource":"configmaps","namespace":"default","name":"cm","apiVersion":"v1"},` + + `"responseStatus":{"code":200},` + + `"responseObject":{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"cm","namespace":"default"}}}` +} + +// TestAuditHandler_AnnotationRouting pins the shared-stream contract: with an annotation key +// configured the bare endpoint resolves the ClusterProvider PER EVENT, so one batch fans out to +// several source clusters, and an event that names none — or names one that does not exist — is +// rejected by itself, never credited to a fallback, while the rest of the batch still lands. +func TestAuditHandler_AnnotationRouting(t *testing.T) { + newHandler := func(t *testing.T, recorder *fakeFactRecorder, resolver AuditProviderResolver) *AuditHandler { + t.Helper() + handler, err := NewAuditHandler(AuditHandlerConfig{ + FactRecorder: recorder, + ProviderResolver: resolver, + ClusterAnnotationKey: clusterAnnotation, + }) + require.NoError(t, err) + return handler + } + known := fakeProviderResolver{existing: map[string]bool{"prod-eu-1": true, "prod-us-1": true}} + + t.Run("one batch fans out to several source clusters", func(t *testing.T) { + recorder := &fakeFactRecorder{} + handler := newHandler(t, recorder, known) + + w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody( + annotatedEvent("eu", map[string]string{clusterAnnotation: "prod-eu-1"}), + annotatedEvent("us", map[string]string{clusterAnnotation: "prod-us-1"}), + )) + require.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, []string{"eu", "us"}, recorder.auditIDs()) + assert.Equal(t, []string{"prod-eu-1", "prod-us-1"}, recorder.recordedProviders()) + }) + + t.Run("an unstamped or unknown event is rejected without failing the batch", func(t *testing.T) { + for _, tt := range []struct { + name string + annotations map[string]string + }{ + {"no annotation at all", nil}, + {"the key present but empty", map[string]string{clusterAnnotation: ""}}, + {"a different key", map[string]string{"other.io/cluster": "prod-eu-1"}}, + {"an unknown provider", map[string]string{clusterAnnotation: "never-created"}}, + } { + t.Run(tt.name, func(t *testing.T) { + recorder := &fakeFactRecorder{} + handler := newHandler(t, recorder, known) + + w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody( + annotatedEvent("rejected", tt.annotations), + annotatedEvent("kept", map[string]string{clusterAnnotation: "prod-eu-1"}), + )) + require.Equal(t, http.StatusOK, w.Code, + "a heterogeneous stream must not be retried wholesale for one bad event") + assert.Equal(t, []string{"kept"}, recorder.auditIDs(), "the rejected event produces no fact") + assert.Equal(t, []string{"prod-eu-1"}, recorder.recordedProviders(), + "and is never credited to a fallback provider") + }) + } + }) + + t.Run("a resolver failure retries the whole batch", func(t *testing.T) { + recorder := &fakeFactRecorder{} + handler := newHandler(t, recorder, fakeProviderResolver{err: assert.AnError}) + + w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody( + annotatedEvent("eu", map[string]string{clusterAnnotation: "prod-eu-1"}), + )) + assert.Equal(t, http.StatusInternalServerError, w.Code, + "a lookup failure is transient, not a verdict that the provider is absent") + assert.Zero(t, recorder.len()) + }) + + t.Run("named routes ignore the annotation", func(t *testing.T) { + recorder := &fakeFactRecorder{} + handler := newHandler(t, recorder, known) + + w := serveBody(t, handler, http.MethodPost, "/audit-webhook/prod-us-1", eventListBody( + annotatedEvent("eu", map[string]string{clusterAnnotation: "prod-eu-1"}), + )) + require.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, []string{"prod-us-1"}, recorder.recordedProviders(), + "a named route fixes the source cluster for its whole batch") + }) +} + +// TestNewAuditHandler_AnnotationRoutingRequiresResolver pins the startup guard: annotation routing +// must be able to reject an unknown provider, so it cannot be configured without a resolver. +func TestNewAuditHandler_AnnotationRoutingRequiresResolver(t *testing.T) { + _, err := NewAuditHandler(AuditHandlerConfig{ClusterAnnotationKey: clusterAnnotation}) + require.Error(t, err) + assert.Contains(t, err.Error(), "ProviderResolver") +} + +// TestProviderRouteForPath covers the path -> (provider, named) mapping directly. The bare path +// names no provider at all: it is the shared, annotation-routed endpoint. +func TestProviderRouteForPath(t *testing.T) { + name, named := providerRouteForPath("/audit-webhook") + assert.Empty(t, name, "the bare path resolves no provider by itself") + assert.False(t, named) + + name, named = providerRouteForPath("/audit-webhook/prod-eu-1") + assert.Equal(t, "prod-eu-1", name) + assert.True(t, named, "a segment names a provider") + + name, named = providerRouteForPath(defaultRoute) + assert.Equal(t, "default", name, "default is an ordinary named route") + assert.True(t, named) +} + func TestNewAuditHandler_DefaultsMaxBody(t *testing.T) { - handler, err := NewAuditHandler(AuditHandlerConfig{}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{})) require.NoError(t, err) assert.Equal(t, DefaultAuditMaxRequestBodyBytes, handler.config.MaxRequestBodyBytes) - handler, err = NewAuditHandler(AuditHandlerConfig{MaxRequestBodyBytes: 4096}) + handler, err = NewAuditHandler(routedConfig(AuditHandlerConfig{MaxRequestBodyBytes: 4096})) require.NoError(t, err) assert.Equal(t, int64(4096), handler.config.MaxRequestBodyBytes) } // TestAuditHandler_MethodAndPathValidation pins the HTTP-method and path gates: -// only POST to the canonical /audit-webhook is accepted; the removed +// only POST to a named /audit-webhook/ is accepted; the removed // /audit-webhook-additional endpoint, trailing slashes, and extra segments are -// all 400. +// all 400, and so is the bare endpoint while annotation routing is off. func TestAuditHandler_MethodAndPathValidation(t *testing.T) { tests := []struct { name string @@ -127,19 +378,24 @@ func TestAuditHandler_MethodAndPathValidation(t *testing.T) { path string wantStatus int }{ - {"valid POST", http.MethodPost, "/audit-webhook", http.StatusOK}, - {"GET rejected", http.MethodGet, "/audit-webhook", http.StatusMethodNotAllowed}, - {"PUT rejected", http.MethodPut, "/audit-webhook", http.StatusMethodNotAllowed}, - {"DELETE rejected", http.MethodDelete, "/audit-webhook", http.StatusMethodNotAllowed}, + {"valid POST to a named route", http.MethodPost, defaultRoute, http.StatusOK}, + {"GET rejected", http.MethodGet, defaultRoute, http.StatusMethodNotAllowed}, + {"PUT rejected", http.MethodPut, defaultRoute, http.StatusMethodNotAllowed}, + {"DELETE rejected", http.MethodDelete, defaultRoute, http.StatusMethodNotAllowed}, {"trailing slash rejected", http.MethodPost, "/audit-webhook/", http.StatusBadRequest}, {"removed additional endpoint rejected", http.MethodPost, "/audit-webhook-additional", http.StatusBadRequest}, - {"extra segment rejected", http.MethodPost, "/audit-webhook/extra", http.StatusBadRequest}, + {"two segments rejected", http.MethodPost, "/audit-webhook/a/b", http.StatusBadRequest}, {"unrelated path rejected", http.MethodPost, "/wrong", http.StatusBadRequest}, + // The bare endpoint only means something with an annotation key configured; this handler has + // none, so a producer posting there is misconfigured rather than routed to a default. + {"bare endpoint rejected without an annotation key", http.MethodPost, "/audit-webhook", http.StatusBadRequest}, + // A name with no ClusterProvider behind it is 404 — the gate applies to every name. + {"unknown named route is 404", http.MethodPost, "/audit-webhook/prod-eu-1", http.StatusNotFound}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - handler, err := NewAuditHandler(AuditHandlerConfig{}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{})) require.NoError(t, err) w := serveBody(t, handler, tt.method, tt.path, eventListBody(acceptedCreateEvent)) @@ -157,8 +413,8 @@ func TestValidateAuditWebhookPath(t *testing.T) { {"canonical", "/audit-webhook", false}, {"trailing slash", "/audit-webhook/", true}, {"removed additional endpoint", "/audit-webhook-additional", true}, - {"extra segment", "/audit-webhook/extra", true}, - {"cluster id segment", "/audit-webhook/cluster-a", true}, + {"named provider segment is valid", "/audit-webhook/prod-eu-1", false}, + {"two segments", "/audit-webhook/a/b", true}, {"unrelated", "/healthz", true}, {"root", "/", true}, } @@ -188,10 +444,10 @@ func TestAuditHandler_DecodeErrors(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", tt.body) + w := serveBody(t, handler, http.MethodPost, defaultRoute, tt.body) assert.Equal(t, http.StatusBadRequest, w.Code) assert.Zero(t, recorder.len(), "a decode failure records no facts") }) @@ -200,10 +456,10 @@ func TestAuditHandler_DecodeErrors(t *testing.T) { func TestAuditHandler_RejectsOversizedBody(t *testing.T) { recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{MaxRequestBodyBytes: 32, FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{MaxRequestBodyBytes: 32, FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody(acceptedCreateEvent)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(acceptedCreateEvent)) assert.Equal(t, http.StatusBadRequest, w.Code) assert.Contains(t, w.Body.String(), "request body too large") assert.Zero(t, recorder.len()) @@ -211,10 +467,10 @@ func TestAuditHandler_RejectsOversizedBody(t *testing.T) { func TestAuditHandler_EmptyEventListRecordsNothing(t *testing.T) { recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody()) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody()) assert.Equal(t, http.StatusOK, w.Code) assert.Zero(t, recorder.len(), "an empty event list records no facts") } @@ -223,10 +479,10 @@ func TestAuditHandler_EmptyEventListRecordsNothing(t *testing.T) { // mutating event reaches the FactRecorder and the request returns 200. func TestAuditHandler_AcceptedEventRecordsFact(t *testing.T) { recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody(acceptedCreateEvent)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(acceptedCreateEvent)) assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, []string{"create-1"}, recorder.auditIDs()) } @@ -234,10 +490,10 @@ func TestAuditHandler_AcceptedEventRecordsFact(t *testing.T) { // TestAuditHandler_NilRecorderAcceptsWithoutRecording confirms configured-author // mode: a nil FactRecorder records nothing yet still returns 200. func TestAuditHandler_NilRecorderAcceptsWithoutRecording(t *testing.T) { - handler, err := NewAuditHandler(AuditHandlerConfig{}) // FactRecorder nil + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{})) // FactRecorder nil require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody(acceptedCreateEvent)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(acceptedCreateEvent)) assert.Equal(t, http.StatusOK, w.Code) } @@ -245,7 +501,7 @@ func TestAuditHandler_NilRecorderAcceptsWithoutRecording(t *testing.T) { // the whole list, recording each accepted event in order. func TestAuditHandler_RecordsEveryAcceptedEventInBatch(t *testing.T) { recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) second := `{"kind":"Event","auditID":"update-1","stage":"ResponseComplete","verb":"update",` + @@ -254,7 +510,7 @@ func TestAuditHandler_RecordsEveryAcceptedEventInBatch(t *testing.T) { `"namespace":"prod","name":"web"},"responseStatus":{"code":200},` + `"responseObject":{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"name":"web","resourceVersion":"7"}}}` - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody(acceptedCreateEvent, second)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(acceptedCreateEvent, second)) assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, []string{"create-1", "update-1"}, recorder.auditIDs()) } @@ -263,10 +519,10 @@ func TestAuditHandler_RecordsEveryAcceptedEventInBatch(t *testing.T) { // fact-store failure surfaces as 500 so the API server redelivers. func TestAuditHandler_RecordFactErrorFailsRequest(t *testing.T) { recorder := &fakeFactRecorder{err: errors.New("fact store down")} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody(acceptedCreateEvent)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(acceptedCreateEvent)) assert.Equal(t, http.StatusInternalServerError, w.Code) assert.Contains(t, w.Body.String(), "fact store down") } @@ -352,10 +608,10 @@ func TestAuditHandler_RejectedEventsAreDropped(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody(tt.event)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(tt.event)) assert.Equal(t, http.StatusOK, w.Code, tt.why) assert.Zero(t, recorder.len(), tt.why) }) @@ -398,10 +654,10 @@ func TestAuditHandler_AcceptedEdgeCases(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody(tt.event)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(tt.event)) assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, []string{tt.auditID}, recorder.auditIDs()) }) @@ -413,14 +669,14 @@ func TestAuditHandler_AcceptedEdgeCases(t *testing.T) { // the whole request is 500. func TestAuditHandler_BatchStopsOnFirstRecordError(t *testing.T) { recorder := &fakeFactRecorder{err: errors.New("fact store down")} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) second := `{"kind":"Event","auditID":"update-1","stage":"ResponseComplete","verb":"update",` + `"objectRef":{"resource":"configmaps","apiVersion":"v1"},` + `"responseObject":{"metadata":{"resourceVersion":"5"}}}` - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody(acceptedCreateEvent, second)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(acceptedCreateEvent, second)) assert.Equal(t, http.StatusInternalServerError, w.Code) assert.Zero(t, recorder.len(), "no facts persist when the recorder errors") } @@ -434,10 +690,10 @@ func TestAuditHandler_ForwardsRealScaleSubresourceRecording(t *testing.T) { require.NoError(t, err, "the captured scale recording must be readable") recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody(string(recording))) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(string(recording))) require.Equal(t, http.StatusOK, w.Code) require.Equal(t, 1, recorder.len(), "the real deployments/scale recording must be recorded") @@ -462,10 +718,10 @@ func TestAuditHandler_FixtureDryRunAndUnchangedRVDropped(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListFixtureBody(t, tt.fixture)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListFixtureBody(t, tt.fixture)) assert.Equal(t, http.StatusOK, w.Code) assert.Zero(t, recorder.len(), "filtered events must not be recorded") }) @@ -496,10 +752,10 @@ func TestAuditHandler_FixturePersistedAndCreateRecorded(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListFixtureBody(t, tt.fixture)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListFixtureBody(t, tt.fixture)) assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, []string{tt.wantID}, recorder.auditIDs()) }) diff --git a/internal/webhook/audit_identity_test.go b/internal/webhook/audit_identity_test.go new file mode 100644 index 00000000..cc65bd50 --- /dev/null +++ b/internal/webhook/audit_identity_test.go @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: Apache-2.0 + +package webhook + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + authnv1 "k8s.io/api/authentication/v1" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" +) + +// TestEffectiveAuditUsername_PrefersImpersonatedUser pins whose name reaches Git. When a request is +// impersonated the ACTOR is the impersonated identity — that is who changed the cluster — while the +// authenticating account is only the one permitted to impersonate. Crediting the authenticating +// user would attribute every `kubectl --as` change to the operator or a CI robot. +func TestEffectiveAuditUsername_PrefersImpersonatedUser(t *testing.T) { + tests := []struct { + name string + event auditv1.Event + want string + }{ + { + name: "plain request uses the authenticated user", + event: auditv1.Event{User: authnv1.UserInfo{Username: "alice"}}, + want: "alice", + }, + { + name: "impersonated request uses the impersonated user", + event: auditv1.Event{ + User: authnv1.UserInfo{Username: "admin"}, + ImpersonatedUser: &authnv1.UserInfo{Username: "alice"}, + }, + want: "alice", + }, + { + name: "an empty impersonated username falls back to the authenticated user", + event: auditv1.Event{ + User: authnv1.UserInfo{Username: "admin"}, + ImpersonatedUser: &authnv1.UserInfo{}, + }, + want: "admin", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, effectiveAuditUsername(tt.event)) + }) + } +} + +// TestGVRParts_BoundedLabels keeps the metric label set small: an objectRef with no usable identity +// must collapse to fixed strings rather than emit unbounded cardinality. +func TestGVRParts_BoundedLabels(t *testing.T) { + tests := []struct { + name string + event *auditv1.Event + group, version, resource string + }{ + { + name: "no objectRef", + event: &auditv1.Event{}, + group: "unknown", version: "unknown", resource: "unknown", + }, + { + name: "core group", + event: &auditv1.Event{ObjectRef: &auditv1.ObjectReference{APIVersion: "v1", Resource: "configmaps"}}, + group: "", version: "v1", resource: "configmaps", + }, + { + name: "grouped resource", + event: &auditv1.Event{ObjectRef: &auditv1.ObjectReference{APIVersion: "apps/v1", Resource: "deployments"}}, + group: "apps", version: "v1", resource: "deployments", + }, + { + name: "missing resource collapses to unknown", + event: &auditv1.Event{ObjectRef: &auditv1.ObjectReference{APIVersion: "v1"}}, + group: "", version: "v1", resource: "unknown", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g, v, r := gvrParts(tt.event) + assert.Equal(t, tt.group, g) + assert.Equal(t, tt.version, v) + assert.Equal(t, tt.resource, r) + }) + } +} + +// TestExtractGVR_CoreGroupRendersLeadingSlash covers the log-field rendering for both group shapes. +func TestExtractGVR_CoreGroupRendersLeadingSlash(t *testing.T) { + core := &auditv1.Event{ObjectRef: &auditv1.ObjectReference{APIVersion: "v1", Resource: "configmaps"}} + assert.Equal(t, "/v1/configmaps", extractGVR(core)) + + grouped := &auditv1.Event{ObjectRef: &auditv1.ObjectReference{APIVersion: "apps/v1", Resource: "deployments"}} + assert.Equal(t, "apps/v1/deployments", extractGVR(grouped)) +} + +// TestAuditHandler_ImpersonatedEventIsRecordedUnderTheImpersonatedUser drives the whole ingress path +// for an impersonated mutation, so the identity rule above is proven end-to-end rather than only at +// the helper. +func TestAuditHandler_ImpersonatedEventIsRecordedUnderTheImpersonatedUser(t *testing.T) { + recorder := &fakeFactRecorder{} + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) + require.NoError(t, err) + + const impersonated = `{"kind":"Event","level":"RequestResponse","auditID":"imp-1",` + + `"stage":"ResponseComplete","verb":"create","user":{"username":"admin"},` + + `"impersonatedUser":{"username":"alice"},` + + `"requestURI":"/api/v1/namespaces/default/configmaps",` + + `"objectRef":{"resource":"configmaps","namespace":"default","name":"cm","apiVersion":"v1"},` + + `"responseStatus":{"code":200},` + + `"responseObject":{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"cm"}}}` + + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(impersonated)) + require.Equal(t, http.StatusOK, w.Code) + require.Equal(t, 1, recorder.len()) + assert.Equal(t, "alice", effectiveAuditUsername(recorder.events[0]), + "the impersonated actor is the author, not the account permitted to impersonate") +} diff --git a/internal/webhook/audit_metrics_test.go b/internal/webhook/audit_metrics_test.go index be302e9f..5e7ba52e 100644 --- a/internal/webhook/audit_metrics_test.go +++ b/internal/webhook/audit_metrics_test.go @@ -91,10 +91,10 @@ func TestServeHTTP_EventListIngressMetrics(t *testing.T) { if tt.recorderErr { recorder.err = errAuditTest } - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", tt.body) + w := serveBody(t, handler, http.MethodPost, defaultRoute, tt.body) assert.Equal(t, tt.wantStatus, w.Code) match := map[string]string{"outcome": tt.wantOutcome} @@ -125,10 +125,10 @@ func TestServeHTTP_AcceptedEventQueuedOutcome(t *testing.T) { require.NoError(t, err) recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody(acceptedCreateEvent)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(acceptedCreateEvent)) require.Equal(t, http.StatusOK, w.Code) queued, ok := telemetry.CollectInt64Sum(reader, auditEventsMetric, map[string]string{ @@ -139,6 +139,47 @@ func TestServeHTTP_AcceptedEventQueuedOutcome(t *testing.T) { assert.Equal(t, int64(1), queued) } +// TestServeHTTP_UnroutableEventOutcomes confirms an event the shared endpoint could not route is +// counted on audit_events_total under its own outcome, so a producer that stops stamping the +// annotation shows up as a rising drop rate instead of as silence. +func TestServeHTTP_UnroutableEventOutcomes(t *testing.T) { + tests := []struct { + name string + annotations map[string]string + wantOutcome string + }{ + {"unstamped event", nil, "missing_cluster_annotation"}, + {"unknown provider", map[string]string{clusterAnnotation: "never-created"}, "unknown_cluster_provider"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + recorder := &fakeFactRecorder{} + handler, err := NewAuditHandler(AuditHandlerConfig{ + FactRecorder: recorder, + ProviderResolver: fakeProviderResolver{existing: map[string]bool{"prod-eu-1": true}}, + ClusterAnnotationKey: clusterAnnotation, + }) + require.NoError(t, err) + + w := serveBody(t, handler, http.MethodPost, "/audit-webhook", + eventListBody(annotatedEvent("unroutable", tt.annotations))) + require.Equal(t, http.StatusOK, w.Code) + assert.Zero(t, recorder.len()) + + dropped, ok := telemetry.CollectInt64Sum(reader, auditEventsMetric, map[string]string{ + "outcome": tt.wantOutcome, "category": "dropped", + "resource": "configmaps", "verb": "create", + }) + require.True(t, ok, "expected a %s outcome sample", tt.wantOutcome) + assert.Equal(t, int64(1), dropped) + }) + } +} + // TestServeHTTP_NonScaleSubresourceDropped confirms a non-/scale subresource // (pods/exec) is dropped before recording and recorded on audit_events_total as // the non_scale_subresource outcome (resource="pods"), so a pods/exec flood is @@ -148,10 +189,10 @@ func TestServeHTTP_NonScaleSubresourceDropped(t *testing.T) { require.NoError(t, err) recorder := &fakeFactRecorder{} - handler, err := NewAuditHandler(AuditHandlerConfig{FactRecorder: recorder}) + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{FactRecorder: recorder})) require.NoError(t, err) - w := serveBody(t, handler, http.MethodPost, "/audit-webhook", eventListBody(subresourceExecEvent)) + w := serveBody(t, handler, http.MethodPost, defaultRoute, eventListBody(subresourceExecEvent)) require.Equal(t, http.StatusOK, w.Code) assert.Zero(t, recorder.len(), "pods/exec must not be recorded") diff --git a/internal/webhook/fuzz_test.go b/internal/webhook/fuzz_test.go index 4cd653f0..588c0e94 100644 --- a/internal/webhook/fuzz_test.go +++ b/internal/webhook/fuzz_test.go @@ -36,7 +36,9 @@ func FuzzDecodeEventList(f *testing.F) { f.Add([]byte(s)) } - handler, err := NewAuditHandler(AuditHandlerConfig{MaxRequestBodyBytes: 1 << 20}) + // A served route, not the bare endpoint: a bare request is rejected before the body is read, so + // fuzzing it would never reach the decoder this target exists to exercise. + handler, err := NewAuditHandler(routedConfig(AuditHandlerConfig{MaxRequestBodyBytes: 1 << 20})) if err != nil { f.Fatalf("NewAuditHandler: %v", err) } @@ -44,7 +46,7 @@ func FuzzDecodeEventList(f *testing.F) { f.Fuzz(func(t *testing.T, body []byte) { // The full ingress path must never panic on an untrusted body and must // always answer with a syntactically valid HTTP status code. - req := httptest.NewRequest(http.MethodPost, "/audit-webhook", bytes.NewReader(body)) + req := httptest.NewRequest(http.MethodPost, defaultRoute, bytes.NewReader(body)) w := httptest.NewRecorder() handler.ServeHTTP(w, req) if w.Code < 100 || w.Code > 599 { diff --git a/test/e2e/Taskfile.yml b/test/e2e/Taskfile.yml index 6b2b463a..b0323c98 100644 --- a/test/e2e/Taskfile.yml +++ b/test/e2e/Taskfile.yml @@ -149,7 +149,7 @@ vars: ARGOCD_DIR: '{{.ARGOCD_DIR | default "test/e2e/setup/argocd"}}' ARGOCD_PORT: '{{.ARGOCD_PORT | default "18080"}}' ARGOCD_WAIT_TIMEOUT: '{{.ARGOCD_WAIT_TIMEOUT | default "300s"}}' - # kcp — installed ONLY for the source-cluster corner (docs/design/config-plane-split.md), + # kcp — installed ONLY for the source-cluster corner (docs/finished/config-plane-split.md), # the same way Argo CD hangs off the bi-directional corner. kcp-operator arrives as a Flux # HelmRelease (test/e2e/setup/kcp/base), like every other e2e dependency; the source-cluster # specs mirror kcp *workspaces* (cheap logical clusters) as remote source clusters. Not a @@ -999,7 +999,7 @@ tasks: _kcp-ready: # kcp for the source-cluster corner. NOT a dependency of prepare-e2e — only # test-e2e-source-cluster pulls it in, so the other CI legs never install kcp. - # See docs/design/config-plane-split.md. Modeled on _argocd-installed. + # See docs/finished/config-plane-split.md. Modeled on _argocd-installed. # # _cluster-ready + _flux-setup-ready, because the kcp-operator HelmRelease needs Flux # and its dependsOn: cert-manager (both arrive via the Flux setup). @@ -1229,10 +1229,38 @@ tasks: mv "${tmp_manifest}" "{{.CS}}/{{.NAMESPACE}}/helm/install.yaml" fi + # _crds-installed applies the project CRDs and waits for them to be Established BEFORE any + # kustomize/flat-file install applies a custom resource of them. The config-dir bundle (and, + # once regenerated, dist/install.yaml) ships the reserved `default` ClusterProvider CR in the + # SAME `kubectl apply` as its CRD; kubectl builds its RESTMapper up front, so a CR whose CRD + # is created in that same apply fails with `no matches for kind "ClusterProvider"`. Applying + # the CRDs first — and waiting for Established — closes that race. Re-runs whenever the + # generated CRDs change (sources). The Helm chart ships CRDs in charts/gitops-reverser/crds/ + # (helm's own first phase), so _install-helm deliberately does NOT depend on this: the + # helm/quickstart legs must keep exercising helm's own CRD install. + _crds-installed: + deps: + - _cluster-ready + - manifests + sources: + - config/crd/bases/*.yaml + generates: + - '{{.CS}}/crds.installed' + cmds: + - mkdir -p "{{.CS}}" + - '{{.KUSTOMIZE}} build config/crd | {{.KUBECTL}} --context "{{.CTX}}" apply -f -' + - | + {{.KUBECTL}} --context "{{.CTX}}" wait --for=condition=Established --timeout=60s \ + crd/clusterproviders.configbutler.ai crd/clusterwatchrules.configbutler.ai \ + crd/commitrequests.configbutler.ai crd/gitproviders.configbutler.ai \ + crd/gittargets.configbutler.ai crd/watchrules.configbutler.ai + - touch "{{.CS}}/crds.installed" + _install-config-dir: deps: - _services-ready - manifests + - _crds-installed - _install-cleanup sources: - '{{.CS}}/services.ready' @@ -1250,12 +1278,17 @@ tasks: _install-plain-manifests-file: method: timestamp + # Deliberately does NOT depend on _crds-installed: install-plain-manifests.sh applies + # dist/crds.yaml itself (then dist/install.yaml), so this leg exercises the real two-step + # user flow end to end. Pre-installing the CRDs from config/crd here would mask whether the + # shipped dist/crds.yaml actually works. deps: - _services-ready - dist-install - _install-cleanup sources: - '{{.CS}}/services.ready' + - dist/crds.yaml - dist/install.yaml generates: - '{{.CS}}/{{.NAMESPACE}}/plain-manifests-file/install.yaml' diff --git a/test/e2e/cluster/audit/webhook-config.yaml b/test/e2e/cluster/audit/webhook-config.yaml index ab628ff3..e06e0998 100644 --- a/test/e2e/cluster/audit/webhook-config.yaml +++ b/test/e2e/cluster/audit/webhook-config.yaml @@ -9,7 +9,10 @@ preferences: {} clusters: - name: audit-webhook cluster: - server: https://127.0.0.1:30444/audit-webhook + # Audit routes are NAMED: this cluster's source identity is the "default" ClusterProvider the + # install creates. The bare /audit-webhook is the shared, annotation-routed endpoint and is + # rejected with 400 unless --author-attribution-cluster-annotation-key is set. + server: https://127.0.0.1:30444/audit-webhook/default insecure-skip-tls-verify: true contexts: - name: webhook diff --git a/test/e2e/quickstart_framework_e2e_test.go b/test/e2e/quickstart_framework_e2e_test.go index 6076d27d..e0bf895d 100644 --- a/test/e2e/quickstart_framework_e2e_test.go +++ b/test/e2e/quickstart_framework_e2e_test.go @@ -137,6 +137,11 @@ func (r *quickstartFrameworkRun) helmInstallReadmeQuickstart() { chart = "charts/gitops-reverser" } + // ONE apply, exactly as the README's step 4 does it. This works because no webhook this chart + // installs is failurePolicy: Fail — the starter GitTarget is gated by nothing, so it is admitted + // while the Deployment it rolls is still coming up. Namespace authorization is enforced at + // reconcile instead (checkSourceAuthorization, inside the Validated gate), which does not need + // the manager to be reachable at admission time. args := []string{ "--kube-context", ctx, "upgrade", "--install", release, chart, diff --git a/test/e2e/source_cluster_e2e_test.go b/test/e2e/source_cluster_e2e_test.go index f05b7ab2..c3fb7aa9 100644 --- a/test/e2e/source_cluster_e2e_test.go +++ b/test/e2e/source_cluster_e2e_test.go @@ -13,18 +13,22 @@ import ( . "github.com/onsi/gomega" ) -// This file is the source-cluster corner for the config-plane split -// (docs/design/config-plane-split.md): a GitTarget may name the cluster it mirrors FROM via -// spec.kubeConfig (Flux's meta.KubeConfigReference). +// This file is the source-cluster corner for multi-cluster author attribution +// (docs/finished/multi-cluster-author-attribution.md): a GitTarget names the cluster it mirrors +// FROM by referencing a cluster-scoped ClusterProvider (spec.clusterProviderRef). The +// ClusterProvider is the home for that cluster's kubeconfig credential (spec.kubeConfig, a Flux +// meta.KubeConfigReference resolved from the operator namespace), namespace-access authorization, +// and connectivity status. // // Two kinds of spec live here: -// - Input-validation / reachability specs (Scenarios 1-3) need no remote cluster: they -// assert the controller's Validated / SourceClusterReachable projection from bad, valid- -// but-unroutable, and omitted kubeconfigs. -// - Remote-mirror specs (Scenarios 4, 8) mirror real REMOTE clusters — kcp WORKSPACES, -// cheap logical clusters installed by Flux (test/e2e/setup/kcp, see kcp_workspace_test.go). -// Scenario 8 is the centerpiece: three workspaces holding the SAME namespace + resource -// with different content, proving state is keyed by source cluster, not (namespace, GVR). +// - Input-validation specs (Scenario 1) assert the ClusterProvider reconciler's Validated +// verdict on bad kubeconfigs, without a remote cluster and without a dial. +// - Reachability specs (Scenarios 2-3) assert the GitTarget's SourceClusterReachable projection +// for valid-but-unroutable and default (local) providers. +// - Remote-mirror specs (Scenarios 4, 8) mirror real REMOTE clusters — kcp WORKSPACES, cheap +// logical clusters installed by Flux (test/e2e/setup/kcp, see kcp_workspace_test.go). +// Scenario 8 is the centerpiece: three workspaces holding the SAME namespace + resource with +// different content, proving state is keyed by source cluster, not (namespace, GVR). // // The whole suite is gated by skipUnlessSourceClusterEnabled() (env E2E_ENABLE_SOURCE_CLUSTER), // and the kcp specs additionally Skip when kcp is not installed — so a default `task test-e2e` @@ -36,6 +40,9 @@ const ( // server, guaranteed not to route, so a kubeconfig pointing at it parses cleanly // (Validated=True) yet can never be dialed (SourceClusterReachable=False). unreachableAPIServer = "https://192.0.2.1:6443" + // sourceClusterOperatorNS is the namespace a ClusterProvider's kubeConfig Secret is pinned to + // (the operator's own namespace). A cluster-scoped provider has no namespace of its own. + sourceClusterOperatorNS = defaultE2ENamespace ) func sourceClusterEnabled() bool { @@ -139,8 +146,9 @@ users: ` } -// writeKubeConfigSecret applies a Secret holding a kubeconfig under the given key. -func writeKubeConfigSecret(ns, name, key, kubeconfig string) { +// writeKubeConfigSecret applies a Secret holding a kubeconfig under the given key, in the OPERATOR +// namespace — a ClusterProvider's secretRef is resolved from there, never from the source cluster. +func writeKubeConfigSecret(name, key, kubeconfig string) { GinkgoHelper() f, err := os.CreateTemp("", "e2e-kubeconfig-*.yaml") Expect(err).NotTo(HaveOccurred()) @@ -149,26 +157,47 @@ func writeKubeConfigSecret(ns, name, key, kubeconfig string) { Expect(err).NotTo(HaveOccurred()) Expect(f.Close()).To(Succeed()) - manifest, err := kubectlRunInNamespace(ns, "create", "secret", "generic", name, + manifest, err := kubectlRunInNamespace(sourceClusterOperatorNS, "create", "secret", "generic", name, "--from-file="+key+"="+f.Name(), "--dry-run=client", "-o", "yaml") Expect(err).NotTo(HaveOccurred(), "failed to render kubeconfig Secret manifest") - _, err = kubectlRunWithStdin(ns, manifest, "apply", "-f", "-") + _, err = kubectlRunWithStdin(sourceClusterOperatorNS, manifest, "apply", "-f", "-") Expect(err).NotTo(HaveOccurred(), "failed to apply kubeconfig Secret") } -// applyGitTargetWithKubeConfig applies a GitTarget whose spec.kubeConfig.secretRef names a -// kubeconfig Secret. It returns the kubectl error so a spec can assert the apply succeeded (the -// spec is well-formed; the kubeconfig, not the CR, is what a validation case makes bad). -// -//nolint:unparam // provider is kept an explicit argument for readability; the corner uses one. -func applyGitTargetWithKubeConfig(ns, name, provider, path, secretName, key string) (string, error) { +// applyClusterProvider applies a cluster-scoped ClusterProvider whose kubeConfig.secretRef names a +// kubeconfig Secret in the operator namespace, allowing the given namespace to reference it. It +// returns the kubectl error so a spec can assert the apply succeeded (the CR is well-formed; the +// kubeconfig, not the CR, is what a validation case makes bad). +func applyClusterProvider(name, secretName, key, allowedNS string) (string, error) { keyLine := "" if key != "" { - // key: must be a sibling of name: (6-space indent, under secretRef:), not nested - // under it — an 8-space indent produces "mapping values are not allowed here". keyLine = "\n key: " + key } manifest := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: ClusterProvider +metadata: + name: %s +spec: + kubeConfig: + secretRef: + name: %s%s + allowedNamespaces: + names: [%s] +`, name, secretName, keyLine, allowedNS) + return kubectlRunWithStdin("", manifest, "apply", "-f", "-") +} + +// deleteClusterProvider removes a cluster-scoped ClusterProvider (not covered by namespace cleanup). +func deleteClusterProvider(name string) { + _, _ = kubectlRun("delete", "clusterprovider", name, "--ignore-not-found", "--wait=false") +} + +// applyGitTargetWithClusterProvider applies a GitTarget whose spec.clusterProviderRef names a +// ClusterProvider (the source cluster it mirrors from). +// +//nolint:unparam // gitProvider is kept an explicit argument for readability; the corner uses one. +func applyGitTargetWithClusterProvider(ns, name, gitProvider, path, clusterProvider string) (string, error) { + manifest := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 kind: GitTarget metadata: name: %s @@ -179,10 +208,9 @@ spec: name: %s branch: main path: %s - kubeConfig: - secretRef: - name: %s%s -`, name, ns, provider, path, secretName, keyLine) + clusterProviderRef: + name: %s +`, name, ns, gitProvider, path, clusterProvider) return kubectlRunWithStdin(ns, manifest, "apply", "-f", "-") } @@ -199,7 +227,7 @@ func findFileByBasename(root, basename string) string { return hit } -var _ = Describe("Manager source cluster / config-plane split", Label("source-cluster"), Ordered, func() { +var _ = Describe("Manager source cluster / ClusterProvider attribution", Label("source-cluster"), Ordered, func() { const providerName = "sc-provider" var ( @@ -237,11 +265,11 @@ var _ = Describe("Manager source cluster / config-plane split", Label("source-cl SetDefaultEventuallyTimeout(60 * time.Second) SetDefaultEventuallyPollingInterval(2 * time.Second) - // Scenario 1 — input validation is legible, and never dials. + // Scenario 1 — the ClusterProvider reconciler's input validation is legible, and never dials. inputCases := []struct { name string reason string - setup func(ns string) (secretName, key string) + setup func(cpName string) (secretName, key string) }{ { name: "a missing Secret", @@ -251,74 +279,78 @@ var _ = Describe("Manager source cluster / config-plane split", Label("source-cl { name: "a missing key", reason: "KubeConfigKeyNotFound", - setup: func(ns string) (string, string) { - kubeconfig := rawKubeConfigWithServer(unreachableAPIServer) - writeKubeConfigSecret(ns, "sc-wrongkey", "somewhere-else", kubeconfig) - return "sc-wrongkey", "value" + setup: func(cp string) (string, string) { + writeKubeConfigSecret(cp+"-kc", "somewhere-else", rawKubeConfigWithServer(unreachableAPIServer)) + return cp + "-kc", "value" }, }, { name: "an unparseable kubeconfig", reason: "KubeConfigInvalid", - setup: func(ns string) (string, string) { - writeKubeConfigSecret(ns, "sc-garbage", "value", "this is not a kubeconfig") - return "sc-garbage", "" + setup: func(cp string) (string, string) { + writeKubeConfigSecret(cp+"-kc", "value", "this is not a kubeconfig") + return cp + "-kc", "" }, }, { name: "an exec auth provider", reason: "KubeConfigExecNotAllowed", - setup: func(ns string) (string, string) { - writeKubeConfigSecret(ns, "sc-exec", "value", execKubeConfig()) - return "sc-exec", "" + setup: func(cp string) (string, string) { + writeKubeConfigSecret(cp+"-kc", "value", execKubeConfig()) + return cp + "-kc", "" }, }, { name: "insecure TLS", reason: "KubeConfigInsecureTLSNotAllowed", - setup: func(ns string) (string, string) { - writeKubeConfigSecret(ns, "sc-insecure", "value", insecureKubeConfig()) - return "sc-insecure", "" + setup: func(cp string) (string, string) { + writeKubeConfigSecret(cp+"-kc", "value", insecureKubeConfig()) + return cp + "-kc", "" }, }, { name: "a file-path credential", reason: "KubeConfigFileReferenceNotAllowed", - setup: func(ns string) (string, string) { - writeKubeConfigSecret(ns, "sc-filepath", "value", fileReferenceKubeConfig()) - return "sc-filepath", "" + setup: func(cp string) (string, string) { + writeKubeConfigSecret(cp+"-kc", "value", fileReferenceKubeConfig()) + return cp + "-kc", "" }, }, } for _, tc := range inputCases { - It("fails Validated (no dial) for "+tc.name+" with reason "+tc.reason, func() { - secretName, key := tc.setup(testNs) - target := "sc-input-" + strings.ToLower(tc.reason) - path := "clusters/input/" + tc.reason - // The GitTarget spec is well-formed (the kubeconfig is bad, not the CR), so the apply - // itself must succeed — the controller then reports the typed reason on Validated. A - // discarded apply error would hide a real rejection as a 90s "condition not found". - _, err := applyGitTargetWithKubeConfig(testNs, target, providerName, path, secretName, key) + It("fails ClusterProvider Validated (no dial) for "+tc.name+" with reason "+tc.reason, func() { + cpName := "sc-input-" + strings.ToLower(tc.reason) + secretName, key := tc.setup(cpName) + DeferCleanup(func() { deleteClusterProvider(cpName) }) + // The ClusterProvider CR is well-formed (the kubeconfig is bad, not the CR), so the + // apply itself must succeed — the controller then reports the typed reason on Validated. + _, err := applyClusterProvider(cpName, secretName, key, testNs) Expect(err).NotTo(HaveOccurred()) - verifyResourceCondition("gittarget", target, testNs, "Validated", "False", tc.reason, "") + verifyResourceCondition("clusterprovider", cpName, "", "Validated", "False", tc.reason, "") }) } - // Scenario 2 — a valid kubeconfig that cannot be dialed: Validated=True, reachability=False. + // Scenario 2 — a valid kubeconfig that cannot be dialed: the ClusterProvider is Validated=True, + // and the GitTarget that references it projects SourceClusterReachable=False. It("separates Validated (inputs) from SourceClusterReachable (runtime)", func() { - writeKubeConfigSecret(testNs, "sc-unreachable", "value", rawKubeConfigWithServer(unreachableAPIServer)) - target := "sc-unreachable-target" - _, err := applyGitTargetWithKubeConfig( - testNs, target, providerName, "clusters/unreachable", "sc-unreachable", "") + const cpName = "sc-unreachable" + writeKubeConfigSecret(cpName+"-kc", "value", rawKubeConfigWithServer(unreachableAPIServer)) + DeferCleanup(func() { deleteClusterProvider(cpName) }) + _, err := applyClusterProvider(cpName, cpName+"-kc", "", testNs) Expect(err).NotTo(HaveOccurred()) + verifyResourceCondition("clusterprovider", cpName, "", "Validated", "True", "Validated", "") + target := "sc-unreachable-target" + _, err = applyGitTargetWithClusterProvider(testNs, target, providerName, "clusters/unreachable", cpName) + Expect(err).NotTo(HaveOccurred()) verifyResourceCondition("gittarget", target, testNs, "Validated", "True", "OK", "") verifyResourceCondition("gittarget", target, testNs, "SourceClusterReachable", "False", "SourceClusterUnreachable", "", "150s") }) - // Scenario 3 — omitted kubeConfig is unchanged local behavior. - It("treats an omitted kubeConfig as the local cluster", func() { + // Scenario 3 — the default (in-cluster) provider: an omitted clusterProviderRef defaults to + // {name: default} and mirrors the operator's own cluster. + It("treats an omitted clusterProviderRef as the default (local) cluster", func() { target := "sc-local-target" manifest := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 kind: GitTarget @@ -335,10 +367,24 @@ spec: "SourceClusterReachable", "True", "LocalCluster", "") }) - // Scenario 4 — a REAL remote cluster: mirror a ConfigMap out of one kcp workspace. Proves - // the whole remote path end to end (resolver -> per-cluster clientContext -> per-cluster - // discovery -> per-cluster watch -> target-scoped writer) against an actually-remote API, - // not the self-referencing in-cluster server the scaffold used to fake it with. + // Scenario 3b — the hard gate: a GitTarget that references a ClusterProvider which does NOT + // exist is held NotReady (Validated=False, ClusterProviderNotFound) and never mirrors. This is + // the regression for "clusterProvider.createDefault: false" — with no default provider a + // GitTarget referencing it (or any missing provider) does not fall back to an implicit local + // identity, and the operator never creates the object to rescue it. + It("holds a GitTarget NotReady when its ClusterProvider does not exist (no bypass)", func() { + target := "sc-missing-cp-target" + _, err := applyGitTargetWithClusterProvider( + testNs, target, providerName, "clusters/missing", "sc-does-not-exist") + Expect(err).NotTo(HaveOccurred()) + verifyResourceCondition("gittarget", target, testNs, + "Validated", "False", "ClusterProviderNotFound", "") + }) + + // Scenario 4 — a REAL remote cluster: mirror a ConfigMap out of one kcp workspace. Proves the + // whole remote path end to end (ClusterProvider resolver -> per-cluster clientContext -> + // per-cluster discovery -> per-cluster watch -> target-scoped writer) against an actually-remote + // API, not the self-referencing in-cluster server the scaffold used to fake it with. It("mirrors a ConfigMap from a kcp workspace", func() { if kcp == nil { Skip("kcp is not installed; run this corner via `task test-e2e-source-cluster`") @@ -346,6 +392,7 @@ spec: const ws = "sc-mirror" hash := kcp.createWorkspace(ws) DeferCleanup(func() { kcp.cleanupWorkspaceTarget(testNs, ws) }) + DeferCleanup(func() { deleteClusterProvider(ws) }) // A WatchRule watches its OWN namespace name on the source cluster, so the ConfigMap must // live under that namespace (testNs) IN the workspace — a separate cluster, so creating a @@ -356,9 +403,11 @@ spec: "--from-literal=hello=from-kcp") Expect(err).NotTo(HaveOccurred(), "create ConfigMap in the workspace") - writeKubeConfigSecret(testNs, ws+"-kubeconfig", "value", kcp.operatorKubeConfig(hash)) + writeKubeConfigSecret(ws+"-kubeconfig", "value", kcp.operatorKubeConfig(hash)) + _, err = applyClusterProvider(ws, ws+"-kubeconfig", "", testNs) + Expect(err).NotTo(HaveOccurred()) target := ws + "-target" - _, err = applyGitTargetWithKubeConfig(testNs, target, providerName, "clusters/kcp", ws+"-kubeconfig", "") + _, err = applyGitTargetWithClusterProvider(testNs, target, providerName, "clusters/kcp", ws) Expect(err).NotTo(HaveOccurred()) verifyResourceCondition("gittarget", target, testNs, "SourceClusterReachable", "True", "", "", "180s") @@ -384,12 +433,13 @@ spec: }).WithTimeout(180 * time.Second).Should(Succeed()) }) - // Scenario 8 — the centerpiece: source-cluster identity is load-bearing. Three workspaces - // each hold the SAME namespace + the SAME resource name (demo/ConfigMap "shared") with - // DIFFERENT content, mirrored by three GitTargets into three folders. If the operator keyed - // state by (namespace, GVR) alone — a union / first-wins lookup — the three identical - // identities would collapse into one; that they land as three distinct files, each carrying - // its own workspace's value, is the proof that everything is keyed by SOURCE CLUSTER. + // Scenario 8 — the centerpiece: source-cluster identity is load-bearing. Three workspaces each + // hold the SAME namespace + the SAME resource name (demo/ConfigMap "shared") with DIFFERENT + // content, mirrored by three GitTargets (each naming its own ClusterProvider) into three + // folders. If the operator keyed state by (namespace, GVR) alone — a union / first-wins lookup — + // the three identical identities would collapse into one; that they land as three distinct + // files, each carrying its own workspace's value, is the proof that everything is keyed by + // SOURCE CLUSTER (the ClusterProvider name). It("mirrors identical resources from three workspaces as distinct GitOps state", func() { if kcp == nil { Skip("kcp is not installed; run this corner via `task test-e2e-source-cluster`") @@ -408,6 +458,7 @@ spec: c := cases[i] hash := kcp.createWorkspace(c.ws) DeferCleanup(func() { kcp.cleanupWorkspaceTarget(testNs, c.ws) }) + DeferCleanup(func() { deleteClusterProvider(c.ws) }) // The SAME namespace name (testNs) and the SAME resource name (shared) in every // workspace; only the value differs. A WatchRule watches its own namespace name on the @@ -418,9 +469,11 @@ spec: "--from-literal=which="+c.value) Expect(err).NotTo(HaveOccurred(), "create ConfigMap shared in %s", c.ws) - writeKubeConfigSecret(testNs, c.ws+"-kubeconfig", "value", kcp.operatorKubeConfig(hash)) + writeKubeConfigSecret(c.ws+"-kubeconfig", "value", kcp.operatorKubeConfig(hash)) + _, err = applyClusterProvider(c.ws, c.ws+"-kubeconfig", "", testNs) + Expect(err).NotTo(HaveOccurred()) target := c.ws + "-target" - _, err = applyGitTargetWithKubeConfig(testNs, target, providerName, c.folder, c.ws+"-kubeconfig", "") + _, err = applyGitTargetWithClusterProvider(testNs, target, providerName, c.folder, c.ws) Expect(err).NotTo(HaveOccurred()) verifyResourceCondition("gittarget", target, testNs, "SourceClusterReachable", "True", "", "", "180s")