From ba79b54f625c24364de9f2490098f0497d55fe4d Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Wed, 22 Jul 2026 15:03:37 -0700 Subject: [PATCH] [PATCH 05/08] build(hub-afd): hub-afd-controller-manager binary + Helm chart + Dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part 05 of an 8-patch split of #373. Introduces the sibling binary that hosts the AFD reconcilers, along with its container image build, Helm chart, and Makefile targets. - cmd/hub-afd-controller-manager/main.go — entry point for the AFD binary. Runs under its own Workload-Identity federated subject so ATM-only tenants do not inherit AFD write permissions (SFI-NS253, docs/first-party/001 §7). - cmd/hub-net-controller-manager/main.go — no longer registers AFD reconcilers (they moved to the new binary). - charts/hub-afd-controller-manager/** — new Helm chart sibling to hub-net-controller-manager (Deployment, RBAC, PDB, values). - docker/hub-afd-controller-manager.Dockerfile — image build recipe. - Makefile — adds hub-afd-controller-manager targets (build, image, push) mirroring the hub-net targets. The AFD reconcilers themselves land in PATCH 06/07/08 and are registered against the manager set up here. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a8fabc68-e113-49ef-b82d-d99a8502da11 --- Makefile | 39 ++- charts/hub-afd-controller-manager/.helmignore | 23 ++ charts/hub-afd-controller-manager/Chart.yaml | 22 ++ charts/hub-afd-controller-manager/README.md | 86 +++++++ .../templates/_helpers.tpl | 51 ++++ .../templates/deployment.yaml | 74 ++++++ .../templates/poddisruptionbudget.yaml | 23 ++ .../templates/rbac.yaml | 119 +++++++++ .../templates/serviceaccount.yaml | 27 ++ charts/hub-afd-controller-manager/values.yaml | 86 +++++++ cmd/hub-afd-controller-manager/main.go | 237 ++++++++++++++++++ cmd/hub-net-controller-manager/main.go | 9 + docker/hub-afd-controller-manager.Dockerfile | 40 +++ 13 files changed, 834 insertions(+), 2 deletions(-) create mode 100644 charts/hub-afd-controller-manager/.helmignore create mode 100644 charts/hub-afd-controller-manager/Chart.yaml create mode 100644 charts/hub-afd-controller-manager/README.md create mode 100644 charts/hub-afd-controller-manager/templates/_helpers.tpl create mode 100644 charts/hub-afd-controller-manager/templates/deployment.yaml create mode 100644 charts/hub-afd-controller-manager/templates/poddisruptionbudget.yaml create mode 100644 charts/hub-afd-controller-manager/templates/rbac.yaml create mode 100644 charts/hub-afd-controller-manager/templates/serviceaccount.yaml create mode 100644 charts/hub-afd-controller-manager/values.yaml create mode 100644 cmd/hub-afd-controller-manager/main.go create mode 100644 docker/hub-afd-controller-manager.Dockerfile diff --git a/Makefile b/Makefile index d8bc01b6..7d4bdd41 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,18 @@ ifndef TAG TAG ?= $(shell git rev-parse --short=7 HEAD) endif HUB_NET_CONTROLLER_MANAGER_IMAGE_VERSION ?= $(TAG) +HUB_AFD_CONTROLLER_MANAGER_IMAGE_VERSION ?= $(TAG) MEMBER_NET_CONTROLLER_MANAGER_IMAGE_VERSION ?= $(TAG) MCS_CONTROLLER_MANAGER_IMAGE_VERSION ?= $(TAG) NET_CRD_INSTALLER_IMAGE_VERSION ?= $(TAG) HUB_NET_CONTROLLER_MANAGER_IMAGE_NAME ?= hub-net-controller-manager +# hub-afd-controller-manager is a sibling of hub-net-controller-manager. +# The image name is intentionally distinct: images are baked into distinct +# Deployments (Proposal 001 §7 identity split — see +# cmd/hub-afd-controller-manager/main.go), and mirroring the naming makes +# CVE-scan tooling and release automation treat both binaries symmetrically. +HUB_AFD_CONTROLLER_MANAGER_IMAGE_NAME ?= hub-afd-controller-manager MEMBER_NET_CONTROLLER_MANAGER_IMAGE_NAME ?= member-net-controller-manager MCS_CONTROLLER_MANAGER_IMAGE_NAME ?= mcs-controller-manager NET_CRD_INSTALLER_IMAGE_NAME ?= net-crd-installer @@ -183,6 +190,7 @@ generate: $(CONTROLLER_GEN) .PHONY: build build: generate fmt vet ## Build binaries. go build -o bin/hub-net-controller-manager cmd/hub-net-controller-manager/main.go + go build -o bin/hub-afd-controller-manager cmd/hub-afd-controller-manager/main.go go build -o bin/member-net-controller-manager cmd/member-net-controller-manager/main.go go build -o bin/mcs-controller-manager cmd/mcs-controller-manager/main.go @@ -190,6 +198,16 @@ build: generate fmt vet ## Build binaries. run-hub-net-controller-manager: manifests generate fmt vet ## Run a controllers from your host. go run ./cmd/hub-net-controller-manager/main.go +# The AFD controller-manager is a separate binary from hub-net-controller-manager +# because Proposal 001 §7 (SFI-NS253) requires the AFD Workload-Identity subject +# to be distinct from the ATM subject. See cmd/hub-afd-controller-manager/main.go +# for the full rationale. This target expects AZURE_TENANT_ID / AZURE_CLIENT_ID / +# AZURE_FEDERATED_TOKEN_FILE / AZURE_SUBSCRIPTION_ID to be set in the environment +# (the binary loads them via pkg/common/azurefrontdoor.LoadConfigFromEnv). +.PHONY: run-hub-afd-controller-manager +run-hub-afd-controller-manager: manifests generate fmt vet ## Run the AFD controller from your host. + go run ./cmd/hub-afd-controller-manager/main.go + .PHONY: run-member-net-controller-manager run-member-net-controller-manager: manifests generate fmt vet ## Run a controllers from your host. go run ./cmd/member-net-controller-manager/main.go @@ -213,11 +231,11 @@ tidy: .PHONY: image image: - $(MAKE) OUTPUT_TYPE="type=docker" docker-build-hub-net-controller-manager docker-build-member-net-controller-manager docker-build-mcs-controller-manager docker-build-net-crd-installer + $(MAKE) OUTPUT_TYPE="type=docker" docker-build-hub-net-controller-manager docker-build-hub-afd-controller-manager docker-build-member-net-controller-manager docker-build-mcs-controller-manager docker-build-net-crd-installer .PHONY: push push: - $(MAKE) OUTPUT_TYPE="type=registry" docker-build-hub-net-controller-manager docker-build-member-net-controller-manager docker-build-mcs-controller-manager docker-build-net-crd-installer + $(MAKE) OUTPUT_TYPE="type=registry" docker-build-hub-net-controller-manager docker-build-hub-afd-controller-manager docker-build-member-net-controller-manager docker-build-mcs-controller-manager docker-build-net-crd-installer # By default, docker buildx create will pull image moby/buildkit:buildx-stable-1 and hit the too many requests error. .PHONY: docker-buildx-builder @@ -253,6 +271,23 @@ docker-build-hub-net-controller-manager: docker-buildx-builder tidy --build-arg GOARCH=$(TARGET_ARCH) \ --build-arg GOOS=$(TARGET_OS) . +# docker-build-hub-afd-controller-manager mirrors the hub-net target above. +# The two binaries share a base image, distroless runtime, and non-root UID +# so the same CVE / supply-chain policy applies uniformly. Kept as a separate +# target (rather than a matrix over an image list) so operators can iterate +# on just the AFD image during POC without rebuilding the ATM one. +.PHONY: docker-build-hub-afd-controller-manager +docker-build-hub-afd-controller-manager: docker-buildx-builder tidy + docker buildx build \ + --file docker/$(HUB_AFD_CONTROLLER_MANAGER_IMAGE_NAME).Dockerfile \ + --output=$(OUTPUT_TYPE) \ + --platform=$(TARGET_OS)/$(TARGET_ARCH) \ + --pull \ + --tag $(REGISTRY)/$(HUB_AFD_CONTROLLER_MANAGER_IMAGE_NAME):$(HUB_AFD_CONTROLLER_MANAGER_IMAGE_VERSION) \ + --progress=$(BUILDKIT_PROGRESS_TYPE) \ + --build-arg GOARCH=$(TARGET_ARCH) \ + --build-arg GOOS=$(TARGET_OS) . + .PHONY: docker-build-member-net-controller-manager docker-build-member-net-controller-manager: docker-buildx-builder tidy docker buildx build \ diff --git a/charts/hub-afd-controller-manager/.helmignore b/charts/hub-afd-controller-manager/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/charts/hub-afd-controller-manager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/hub-afd-controller-manager/Chart.yaml b/charts/hub-afd-controller-manager/Chart.yaml new file mode 100644 index 00000000..c9aaa8d7 --- /dev/null +++ b/charts/hub-afd-controller-manager/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +name: hub-afd-controller-manager +description: | + Helm chart for the fleet-networking Azure Front Door controller-manager on the hub cluster. + This chart is a sibling of the hub-net-controller-manager chart. The two are packaged + separately (rather than as one chart with a feature toggle) because Proposal 001 §7 + (SFI-NS253) requires the AFD Workload-Identity federated subject to be distinct from + the ATM subject, which is only enforceable at the Pod boundary. See + docs/first-party/001-afd-global-load-balancing.md §7 and + docs/first-party/003-pre-implementation-checklist.md §2.4. + +# Application chart, not a library — this ships templates that render Kubernetes objects. +type: application + +# This is the chart version. Bump on any chart or template change. +# Follows Semantic Versioning (https://semver.org/). +version: 0.1.0 + +# The application version tracks the AFD controller binary version. It intentionally +# matches hub-net-controller-manager's appVersion so release automation can promote +# both binaries together. +appVersion: "v0.1.0" diff --git a/charts/hub-afd-controller-manager/README.md b/charts/hub-afd-controller-manager/README.md new file mode 100644 index 00000000..a7bb0bf5 --- /dev/null +++ b/charts/hub-afd-controller-manager/README.md @@ -0,0 +1,86 @@ +# hub-afd-controller-manager + +A Helm chart for the fleet-networking **Azure Front Door** controller-manager +on the hub cluster. This chart is a **sibling** of `hub-net-controller-manager` +and must be installed alongside it, not in place of it. + +## Why a separate chart? + +Proposal 001 §7 (SFI-NS253) requires the AFD controller to run under a +Workload-Identity federated subject that is **distinct** from the +ATM/MCS controller's subject, so ATM-only tenants do not inherit AFD +write permissions on the shared Azure subscription. A Kubernetes pod +projects exactly one WI token, so the identity split is only +enforceable at the Pod boundary — one binary per Deployment, one +ServiceAccount per binary, one AAD federated identity per +ServiceAccount. See: + +- `docs/first-party/001-afd-global-load-balancing.md` §7 +- `docs/first-party/003-pre-implementation-checklist.md` §2.4 +- `cmd/hub-afd-controller-manager/main.go` (package comment) + +## Prerequisites + +1. **CRDs installed.** This chart does **not** install CRDs. The + `net-crd-installer` job attached to the `hub-net-controller-manager` + chart (or an equivalent) must have applied the following CRDs first: + - `frontdoorprofiles.networking.fleet.azure.com` + - `frontdoorcustomdomains.networking.fleet.azure.com` + + The controller startup fails fast if either is missing (see + `cmd/hub-afd-controller-manager/main.go`). + +2. **Workload Identity webhook.** The AKS cluster hosting the hub must + have the `azure-workload-identity` mutating webhook enabled (on AKS, + enable the `WorkloadIdentity` feature; on OSS clusters, install the + webhook chart from `Azure/azure-workload-identity`). + +3. **Federated AAD identity provisioned.** Before `helm install`, an + operator must: + - Create an AAD app registration (or User-Assigned MI) for AFD. + - Federate it to the hub cluster's OIDC issuer with subject + `system:serviceaccount::-hub-afd-controller-manager-sa`. + - Grant the identity `Contributor` (or the equivalent least-privilege + `Front Door Domain Contributor` + `Front Door Endpoint Contributor`) + scoped to the resource groups referenced by `FrontDoorProfile.Spec.ResourceGroup`. + + > **SFI invariant.** This AAD identity **must not** be the same as + > the identity backing the `hub-net-controller-manager` chart. The + > chart cannot cross-check this — enforcement lives in operator + > tooling (Terraform / `az` CLI). + +## Required values + +| Value | Description | +| --- | --- | +| `azure.tenantId` | AAD tenant hosting the federated identity. | +| `azure.clientId` | Client ID of the federated AAD app / MI. **Distinct from ATM.** | +| `azure.subscriptionID` | Subscription that owns the AFD profiles. | + +`helm install` fails loudly if any of these is empty. + +## Coexistence with hub-net-controller-manager + +Both charts can (and typically will) be installed into the same +namespace on the same hub. They are designed for peaceful coexistence: + +- **Distinct leader-election lease names** (`afd.hub.networking.fleet.azure.com` + vs `2bf2b407.hub.networking.fleet.azure.com`) — no lease contention. +- **Distinct container ports** (metrics `:8082` vs `:8080`, probe `:8083` + vs `:8081`). +- **Distinct ServiceAccounts** with distinct WI annotations — no + identity crossover. +- **Non-overlapping RBAC** — the AFD ClusterRole grants zero verbs on + ATM/MCS CRDs, and vice versa. + +## Uninstall + +```bash +helm uninstall -n +``` + +Uninstall leaves the CRDs in place (they are shared with the +hub-net-controller-manager install). Front Door profiles that are still +present when the controller pod is removed will keep their finalizers +until either the controller is re-installed or the finalizers are +manually cleared — see `pkg/controllers/hub/frontdoorprofile/controller.go`. diff --git a/charts/hub-afd-controller-manager/templates/_helpers.tpl b/charts/hub-afd-controller-manager/templates/_helpers.tpl new file mode 100644 index 00000000..bc30165e --- /dev/null +++ b/charts/hub-afd-controller-manager/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "hub-afd-controller-manager.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "hub-afd-controller-manager.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "hub-afd-controller-manager.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "hub-afd-controller-manager.labels" -}} +helm.sh/chart: {{ include "hub-afd-controller-manager.chart" . }} +{{ include "hub-afd-controller-manager.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hub-afd-controller-manager.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hub-afd-controller-manager.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/charts/hub-afd-controller-manager/templates/deployment.yaml b/charts/hub-afd-controller-manager/templates/deployment.yaml new file mode 100644 index 00000000..68c4455f --- /dev/null +++ b/charts/hub-afd-controller-manager/templates/deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hub-afd-controller-manager.fullname" . }} + namespace: {{ .Values.fleetSystemNamespace }} + labels: + {{- include "hub-afd-controller-manager.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "hub-afd-controller-manager.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "hub-afd-controller-manager.selectorLabels" . | nindent 8 }} + # Pod-level label required by the azure-workload-identity webhook — + # without it, no federated token is projected even if the + # ServiceAccount is annotated. Belt-and-braces with the + # ServiceAccount label; both are required in current AKS versions. + azure.workload.identity/use: "true" + spec: + serviceAccountName: {{ include "hub-afd-controller-manager.fullname" . }}-sa + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --leader-election-namespace={{ .Values.leaderElectionNamespace }} + - --v={{ .Values.logVerbosity }} + - --add_dir_header + env: + # AZURE_TENANT_ID, AZURE_CLIENT_ID, and AZURE_FEDERATED_TOKEN_FILE + # are auto-injected by the azure-workload-identity mutating webhook + # from the ServiceAccount annotations. AZURE_SUBSCRIPTION_ID is + # NOT part of Workload Identity and must be supplied explicitly. + - name: AZURE_SUBSCRIPTION_ID + value: {{ required "azure.subscriptionID is required" .Values.azure.subscriptionID | quote }} + ports: + # Distinct from hub-net-controller-manager (:8080 / :8081) so + # both binaries can run on the same node without collision when + # hostNetwork is enabled or ports are exposed on a Service. + - name: metrics + containerPort: 8082 + protocol: TCP + - name: healthz + containerPort: 8083 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + readinessProbe: + httpGet: + path: /readyz + port: healthz + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/hub-afd-controller-manager/templates/poddisruptionbudget.yaml b/charts/hub-afd-controller-manager/templates/poddisruptionbudget.yaml new file mode 100644 index 00000000..9a151623 --- /dev/null +++ b/charts/hub-afd-controller-manager/templates/poddisruptionbudget.yaml @@ -0,0 +1,23 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{/* +PodDisruptionBudget so voluntary disruptions (upgrades, node drains, +AKS Automatic NAP-driven scaling) cannot take both replicas down at +once. With replicaCount=2 and minAvailable=1 the deployment tolerates +one voluntary eviction per drain cycle. + +Disable via podDisruptionBudget.enabled=false on single-replica installs +(otherwise the PDB blocks all voluntary evictions and drains hang). +*/}} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "hub-afd-controller-manager.fullname" . }}-pdb + namespace: {{ .Values.fleetSystemNamespace }} + labels: + {{- include "hub-afd-controller-manager.labels" . | nindent 4 }} +spec: + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + selector: + matchLabels: + {{- include "hub-afd-controller-manager.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/hub-afd-controller-manager/templates/rbac.yaml b/charts/hub-afd-controller-manager/templates/rbac.yaml new file mode 100644 index 00000000..b2759814 --- /dev/null +++ b/charts/hub-afd-controller-manager/templates/rbac.yaml @@ -0,0 +1,119 @@ +{{/* +Least-privilege ClusterRole for the hub-afd-controller-manager. + +Scope is deliberately narrow: only the AFD-owned CRDs, plus the minimum +Kubernetes primitives every controller-runtime manager needs +(leases for leader election, events for the Recorder). Notably ABSENT: + * No verbs on ATM CRDs (trafficmanagerprofiles / trafficmanagerbackends). + The AFD binary does not import their packages; granting perms would + weaken the SFI-NS253 §7 identity split at the K8s RBAC layer too. + * No verbs on MCS CRDs (serviceimports, endpointsliceexports, etc.). + * No cluster.kubernetes-fleet.io/memberclusters — the AFD controller + does not enumerate member clusters (backend selection lands with + FrontDoorBackend in Phase 4, at which point this ClusterRole + is extended, not the ATM binary's). + * No apiextensions.k8s.io/customresourcedefinitions — CRD install is + handled by the shared net-crd-installer job attached to the hub-net + chart, not by this chart. + +The verbs mirror the +kubebuilder:rbac markers on the FrontDoorProfile +and FrontDoorCustomDomain reconcilers exactly, so config/rbac/role.yaml +regenerations from controller-gen match this template semantically. +*/}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hub-afd-controller-manager.fullname" . }}-role + labels: + {{- include "hub-afd-controller-manager.labels" . | nindent 4 }} +rules: +# Leader election: coordination.k8s.io leases in the leader-election namespace. +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - update +# Event recorder: needed by both reconcilers (they call +# mgr.GetEventRecorderFor(...) and emit Normal/Warning events). +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +# FrontDoorProfile CRUD. +- apiGroups: + - networking.fleet.azure.com + resources: + - frontdoorprofiles + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - networking.fleet.azure.com + resources: + - frontdoorprofiles/status + verbs: + - get + - update + - patch +- apiGroups: + - networking.fleet.azure.com + resources: + - frontdoorprofiles/finalizers + verbs: + - get + - update +# FrontDoorCustomDomain CRUD. +- apiGroups: + - networking.fleet.azure.com + resources: + - frontdoorcustomdomains + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - networking.fleet.azure.com + resources: + - frontdoorcustomdomains/status + verbs: + - get + - update + - patch +- apiGroups: + - networking.fleet.azure.com + resources: + - frontdoorcustomdomains/finalizers + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hub-afd-controller-manager.fullname" . }}-role-binding + labels: + {{- include "hub-afd-controller-manager.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hub-afd-controller-manager.fullname" . }}-role +subjects: + - kind: ServiceAccount + name: {{ include "hub-afd-controller-manager.fullname" . }}-sa + namespace: {{ .Values.fleetSystemNamespace }} diff --git a/charts/hub-afd-controller-manager/templates/serviceaccount.yaml b/charts/hub-afd-controller-manager/templates/serviceaccount.yaml new file mode 100644 index 00000000..b67d23bb --- /dev/null +++ b/charts/hub-afd-controller-manager/templates/serviceaccount.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hub-afd-controller-manager.fullname" . }}-sa + namespace: {{ .Values.fleetSystemNamespace }} + labels: + {{- include "hub-afd-controller-manager.labels" . | nindent 4 }} + # The azure-workload-identity mutating webhook uses this label to + # decide whether to project a federated token into pods that reference + # this ServiceAccount. Without it, no token is projected and the + # controller falls back to whatever chain azidentity picks — which + # would defeat the identity-split invariant. + azure.workload.identity/use: "true" + annotations: + # These annotations are what actually bind THIS ServiceAccount to a + # specific AAD identity. The workload-identity webhook reads them + # and materializes AZURE_CLIENT_ID / AZURE_TENANT_ID / + # AZURE_FEDERATED_TOKEN_FILE into the pod's env at admission time. + # + # SFI-NS253 §7 invariant: the client-id here MUST be distinct from + # the hub-net-controller-manager ServiceAccount's client-id. The + # chart cannot enforce it (values are opaque strings); enforcement + # lives in the operator's Terraform / az CLI federated-credential + # setup. Chart install fails loudly if these values are empty + # (see values.yaml comments). + azure.workload.identity/client-id: {{ required "azure.clientId is required for Workload Identity" .Values.azure.clientId | quote }} + azure.workload.identity/tenant-id: {{ required "azure.tenantId is required for Workload Identity" .Values.azure.tenantId | quote }} diff --git a/charts/hub-afd-controller-manager/values.yaml b/charts/hub-afd-controller-manager/values.yaml new file mode 100644 index 00000000..d1231db4 --- /dev/null +++ b/charts/hub-afd-controller-manager/values.yaml @@ -0,0 +1,86 @@ +# Default values for hub-afd-controller-manager. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Two replicas by default so leader-election survives a single-pod eviction +# (e.g. node scaling on AKS Automatic). Higher availability than the 1-replica +# hub-net default because the AFD controller manages external customer-facing +# infra (Front Door profiles) — a stuck reconciler is more user-visible. +replicaCount: 2 + +image: + repository: ghcr.io/azure/fleet-networking/hub-afd-controller-manager + pullPolicy: Always + # Overrides the image tag whose default is the chart appVersion. + tag: "v0.1.0" + +logVerbosity: 2 + +leaderElectionNamespace: fleet-system +fleetSystemNamespace: fleet-system + +# -------------------------------------------------------------------------- +# Azure Workload Identity configuration. +# +# The AFD controller authenticates to ARM exclusively via Workload Identity +# (see pkg/common/azurefrontdoor/client.go). The three sub-fields below are +# REQUIRED at install time — the chart intentionally does not provide +# fallback default credentials, so a misconfigured install fails at deploy +# time rather than surfacing as opaque 401s at reconcile time. +# +# SFI-NS253 §7 invariant: the AAD app-registration identified by +# `azure.clientId` here MUST NOT be the same identity used by the +# hub-net-controller-manager chart. If it is, the whole point of the +# chart split is defeated. This is enforced operationally (Terraform / az +# CLI federated-credential setup) — the chart cannot cross-check it. +# -------------------------------------------------------------------------- +azure: + # tenantId is the AAD tenant that hosts the federated app registration + # for this controller. Required. + tenantId: "" + # clientId is the AAD app (or User-Assigned MI) client ID that the + # controller's ServiceAccount is federated to. Required. MUST be + # distinct from the ATM controller's identity (SFI-NS253 §7). + clientId: "" + # subscriptionID is the subscription that owns the AFD profiles this + # controller reconciles. Required. Note the ID casing: the underlying + # env var read by pkg/common/azurefrontdoor is AZURE_SUBSCRIPTION_ID. + subscriptionID: "" + +# Resource requests/limits. Sized for a single AFD tenant workload; adjust +# upward if reconciling many FrontDoorProfile CRs simultaneously (rare — +# most fleets have <10 profiles). +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +podAnnotations: {} + +# Two-replica default (see replicaCount rationale) benefits from spreading +# pods across nodes. Override with node-affinity if the cluster is +# single-zone or has fewer nodes than replicas. +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: hub-afd-controller-manager + +nodeSelector: {} + +tolerations: [] + +# PodDisruptionBudget prevents both replicas from being evicted simultaneously +# during voluntary disruptions (upgrades, node drains, NAP scaling on AKS +# Automatic). Set enabled=false on single-replica installs to avoid blocking +# node drains — a single pod with minAvailable=1 is unevictable. +podDisruptionBudget: + enabled: true + minAvailable: 1 diff --git a/cmd/hub-afd-controller-manager/main.go b/cmd/hub-afd-controller-manager/main.go new file mode 100644 index 00000000..35fa73a8 --- /dev/null +++ b/cmd/hub-afd-controller-manager/main.go @@ -0,0 +1,237 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +// Binary hub-afd-controller-manager runs the Azure Front Door reconcilers +// (FrontDoorProfile, FrontDoorCustomDomain) against a Fleet hub cluster. +// +// This binary is INTENTIONALLY SEPARATE from cmd/hub-net-controller-manager +// (the ATM/MCS binary). Proposal 001 §7 (SFI-NS253) requires the AFD +// controller to run under its own Workload-Identity federated subject so +// that ATM-only tenants do not inherit AFD write permissions on the shared +// Azure subscription. A Kubernetes pod projects exactly ONE Workload-Identity +// token, so the identity split is only enforceable at the Pod boundary — +// i.e. by having two independent binaries in two independent Deployments +// backed by two independent ServiceAccounts. See: +// - docs/first-party/001-afd-global-load-balancing.md §7 +// - docs/first-party/003-pre-implementation-checklist.md §2.4 +// +// Consequently this binary: +// - Loads only the AFD CRDs (FrontDoorProfile, FrontDoorCustomDomain); +// the ATM CRDs are intentionally not registered here. +// - Loads its credential from environment-projected Workload Identity +// (AZURE_TENANT_ID / AZURE_CLIENT_ID / AZURE_FEDERATED_TOKEN_FILE / +// AZURE_SUBSCRIPTION_ID) — NOT from the shared azure.json cloud config +// used by the ATM binary. +// - Uses a distinct leader-election ID, metrics port, and probe port so +// that a single hub can safely run both binaries side by side without +// lease collision or port collision. +// +// The chart wiring (charts/hub-afd-controller-manager) lands in a follow-up +// commit; this commit only introduces the binary and removes the AFD block +// from cmd/hub-net-controller-manager. Until that chart lands, this binary +// is not deployable via helm — that is intentional: the POC installation +// path is being closed BEFORE the new one opens so the shared-subject +// deployment topology cannot regress silently. +package main + +import ( + "flag" + "os" + "time" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/rand" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/discovery" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "go.goms.io/fleet/pkg/utils" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + "go.goms.io/fleet-networking/pkg/common/azurefrontdoor" + "go.goms.io/fleet-networking/pkg/controllers/hub/frontdoorbackend" + "go.goms.io/fleet-networking/pkg/controllers/hub/frontdoorcustomdomain" + "go.goms.io/fleet-networking/pkg/controllers/hub/frontdoorprofile" +) + +var ( + scheme = runtime.NewScheme() + + // Distinct defaults from cmd/hub-net-controller-manager so both binaries + // can be co-scheduled on the same node/pod network without port collision. + // If you change the defaults here, keep charts/hub-afd-controller-manager + // (containerPort + probe port + Service ports) in sync. + metricsAddr = flag.String("metrics-bind-address", ":8082", "The address the metric endpoint binds to.") + probeAddr = flag.String("health-probe-bind-address", ":8083", "The address the probe endpoint binds to.") + + enableLeaderElection = flag.Bool("leader-elect", true, + "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + leaderElectionNamespace = flag.String("leader-election-namespace", "fleet-system", "The namespace in which the leader election resource will be created.") +) + +// frontDoorFeatureRequiredGVKs is the AFD-specific CRD set this binary +// depends on. Startup fails fast if any is missing so operators discover +// misconfiguration at deployment time, not at first reconcile. Silently +// starting an idle pod would be strictly worse: healthz would report +// green while nothing is being reconciled. +var frontDoorFeatureRequiredGVKs = []schema.GroupVersionKind{ + fleetnetv1alpha1.GroupVersion.WithKind(fleetnetv1alpha1.FrontDoorProfileKind), + fleetnetv1alpha1.GroupVersion.WithKind(fleetnetv1alpha1.FrontDoorCustomDomainKind), +} + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + // Only the v1alpha1 group is registered: FrontDoorProfile and + // FrontDoorCustomDomain both live there. Deliberately no v1beta1 / + // clusterv1beta1 registration — this binary must not reconcile any + // ATM / MemberCluster types even if their CRDs happen to be installed + // on the same cluster. + utilruntime.Must(fleetnetv1alpha1.AddToScheme(scheme)) + klog.InitFlags(nil) +} + +func main() { + flag.Parse() + rand.Seed(time.Now().UnixNano()) + + handleExit := func() { klog.Flush() } + exitWithError := func() { + handleExit() + os.Exit(1) + } + defer handleExit() + + flag.VisitAll(func(f *flag.Flag) { + klog.InfoS("flag:", "name", f.Name, "value", f.Value) + }) + + ctrl.SetLogger(zap.New(zap.UseDevMode(true))) + + hubConfig := ctrl.GetConfigOrDie() + mgr, err := ctrl.NewManager(hubConfig, ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{ + BindAddress: *metricsAddr, + }, + // The webhook server is initialized even though no webhooks are + // currently registered, so a future AFD-specific validating webhook + // (e.g. FrontDoorBackend AFD/ATM coexistence guard from Proposal 001 + // §3.5) can be added without a chart/binary co-change. + WebhookServer: webhook.NewServer(webhook.Options{ + Port: 9443, + }), + HealthProbeBindAddress: *probeAddr, + LeaderElection: *enableLeaderElection, + LeaderElectionNamespace: *leaderElectionNamespace, + // Distinct lease name from the ATM binary + // ("2bf2b407.hub.networking.fleet.azure.com"). Both binaries can be + // deployed simultaneously into the same namespace without either + // stealing the other's lease. + LeaderElectionID: "afd.hub.networking.fleet.azure.com", + }) + if err != nil { + klog.ErrorS(err, "Unable to start manager") + exitWithError() + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + klog.ErrorS(err, "Unable to set up health check") + exitWithError() + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + klog.ErrorS(err, "Unable to set up ready check") + exitWithError() + } + + ctx := ctrl.SetupSignalHandler() + + // CRD gating: this binary is single-purpose, so a missing AFD CRD is a + // hard failure. The corresponding block in hub-net-controller-manager + // was gated by --enable-frontdoor-feature; here the feature is + // unconditionally on and there is no flag to disable it. + discoverClient := discovery.NewDiscoveryClientForConfigOrDie(hubConfig) + klog.V(1).InfoS("Checking required Front Door CRDs") + for _, gvk := range frontDoorFeatureRequiredGVKs { + if err = utils.CheckCRDInstalled(discoverClient, gvk); err != nil { + klog.ErrorS(err, "Unable to find the required Front Door CRD", "GVK", gvk) + exitWithError() + } + } + + // Load AFD-scoped Workload Identity from the environment. The projected + // token here MUST come from a ServiceAccount federated to an AAD app + // that is DISTINCT from the ATM ServiceAccount's federated subject — + // that is the SFI-NS253 §7 invariant this binary exists to enforce. + // Enforcement itself lives outside this Go code (in the chart's + // ServiceAccount annotations + the operator's federated-credential + // setup); the effect is: this pod is authorized only for AFD ARM ops. + klog.V(1).InfoS("Loading Workload Identity config and creating AFD clients") + afdConfig, err := azurefrontdoor.LoadConfigFromEnv() + if err != nil { + klog.ErrorS(err, "Unable to load AFD Workload Identity config from environment") + exitWithError() + } + afdCred, err := azurefrontdoor.NewCredential(afdConfig) + if err != nil { + klog.ErrorS(err, "Unable to create AFD Workload Identity credential") + exitWithError() + } + afdClients, err := azurefrontdoor.NewClients(afdCred, afdConfig.SubscriptionID, azurefrontdoor.DefaultARMClientOptions()) + if err != nil { + klog.ErrorS(err, "Unable to create AFD clients") + exitWithError() + } + + klog.V(1).InfoS("Start to setup FrontDoorProfile controller") + if err := (&frontdoorprofile.Reconciler{ + Client: mgr.GetClient(), + ProfilesClient: afdClients.Profiles, + EndpointsClient: afdClients.AFDEndpoints, + WAFPoliciesClient: afdClients.WAFPolicies, + SecurityPoliciesClient: afdClients.SecurityPolicies, + SubscriptionID: afdConfig.SubscriptionID, + Recorder: mgr.GetEventRecorderFor(frontdoorprofile.ControllerName), + }).SetupWithManager(mgr); err != nil { + klog.ErrorS(err, "Unable to create FrontDoorProfile controller") + exitWithError() + } + + klog.V(1).InfoS("Start to setup FrontDoorCustomDomain controller") + if err := (&frontdoorcustomdomain.Reconciler{ + Client: mgr.GetClient(), + CustomDomainsClient: afdClients.CustomDomains, + Recorder: mgr.GetEventRecorderFor(frontdoorcustomdomain.ControllerName), + }).SetupWithManager(mgr); err != nil { + klog.ErrorS(err, "Unable to create FrontDoorCustomDomain controller") + exitWithError() + } + + klog.V(1).InfoS("Start to setup FrontDoorBackend controller") + if err := (&frontdoorbackend.Reconciler{ + Client: mgr.GetClient(), + OriginGroupsClient: afdClients.OriginGroups, + OriginsClient: afdClients.Origins, + Recorder: mgr.GetEventRecorderFor(frontdoorbackend.ControllerName), + }).SetupWithManager(mgr); err != nil { + klog.ErrorS(err, "Unable to create FrontDoorBackend controller") + exitWithError() + } + + klog.V(1).InfoS("Starting hub-afd-controller-manager") + if err := mgr.Start(ctx); err != nil { + klog.ErrorS(err, "Problem running manager") + exitWithError() + } +} diff --git a/cmd/hub-net-controller-manager/main.go b/cmd/hub-net-controller-manager/main.go index 487900a3..d883351f 100644 --- a/cmd/hub-net-controller-manager/main.go +++ b/cmd/hub-net-controller-manager/main.go @@ -68,6 +68,15 @@ var ( enableTrafficManagerFeature = flag.Bool("enable-traffic-manager-feature", true, "If set, the traffic manager feature will be enabled.") + // NOTE: the Azure Front Door feature used to be gated here behind + // --enable-frontdoor-feature. It has been moved to a dedicated binary + // (cmd/hub-afd-controller-manager) to satisfy the SFI-NS253 §7 + // identity-split requirement — the AFD controllers must run under a + // distinct Workload-Identity federated subject from the ATM controllers, + // and a Kubernetes pod projects exactly one WI token. See: + // - docs/first-party/001-afd-global-load-balancing.md §7 + // - docs/first-party/003-pre-implementation-checklist.md §2.4 + cloudConfigFile = flag.String("cloud-config", "/etc/kubernetes/provider/azure.json", "The path to the cloud config file which will be used to access the Azure resource.") ) diff --git a/docker/hub-afd-controller-manager.Dockerfile b/docker/hub-afd-controller-manager.Dockerfile new file mode 100644 index 00000000..d0575320 --- /dev/null +++ b/docker/hub-afd-controller-manager.Dockerfile @@ -0,0 +1,40 @@ +# Build the hub-afd-controller-manager binary. +# +# This binary is a sibling of hub-net-controller-manager. The two run as +# separate Deployments so their Workload-Identity federated subjects can +# be distinct — required by Proposal 001 §7 (SFI-NS253). See +# cmd/hub-afd-controller-manager/main.go for the full rationale. The +# Dockerfile is intentionally kept in sync with hub-net-controller-manager.Dockerfile +# (same base images, same distroless runtime, same non-root UID) so that +# CVE-scan and supply-chain policies apply uniformly across the fleet +# controller-manager binaries. +FROM mcr.microsoft.com/oss/go/microsoft/golang:1.25.12 AS builder + +ARG GOOS=linux +ARG GOARCH=amd64 + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# Cache the downloaded dependency modules across different builds to expedite the progress. +# This also helps reduce downloading related reliability issues in our build environment. +RUN --mount=type=cache,target=/go/pkg/mod go mod download + +# Copy the go source +COPY cmd/hub-afd-controller-manager/main.go main.go +COPY api/ api/ +COPY pkg/ pkg/ + +# Build with CGO enabled for internal usage +RUN echo "Building images with GOOS=$GOOS GOARCH=$GOARCH" +RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GO111MODULE=on go build -o hub-afd-controller-manager main.go + +# Use Azure Linux distroless base image to package hub-afd-controller-manager binary +# Refer to https://mcr.microsoft.com/en-us/artifact/mar/azurelinux/distroless/base/about for more details +FROM mcr.microsoft.com/azurelinux/distroless/base:3.0 +WORKDIR / +COPY --from=builder /workspace/hub-afd-controller-manager . +USER 65532:65532 + +ENTRYPOINT ["/hub-afd-controller-manager"]