From 589f904b1c5b422570a269f3ec6ad99b8994ba69 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 08:35:32 +0000 Subject: [PATCH 01/38] feat: add CoderProvisioner CRD types --- api/v1alpha1/coderprovisioner_types.go | 104 +++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 api/v1alpha1/coderprovisioner_types.go diff --git a/api/v1alpha1/coderprovisioner_types.go b/api/v1alpha1/coderprovisioner_types.go new file mode 100644 index 00000000..2f008b26 --- /dev/null +++ b/api/v1alpha1/coderprovisioner_types.go @@ -0,0 +1,104 @@ +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // CoderProvisionerPhasePending indicates the provisioner deployment is not ready. + CoderProvisionerPhasePending = "Pending" + // CoderProvisionerPhaseReady indicates at least one provisioner pod is ready. + CoderProvisionerPhaseReady = "Ready" + + // DefaultProvisionerKeySecretKey is the default data key for provisioner key secrets. + DefaultProvisionerKeySecretKey = "key" + + // ProvisionerKeyCleanupFinalizer is applied to ensure coderd key cleanup on deletion. + ProvisionerKeyCleanupFinalizer = "coder.com/provisioner-key-cleanup" +) + +// CoderProvisionerBootstrapSpec configures credentials for provisioner key management. +type CoderProvisionerBootstrapSpec struct { + // CredentialsSecretRef points to a Secret containing a Coder session token + // with permission to manage provisioner keys. + CredentialsSecretRef SecretKeySelector `json:"credentialsSecretRef"` +} + +// CoderProvisionerKeySpec configures provisioner key naming and storage. +type CoderProvisionerKeySpec struct { + // Name is the provisioner key name in coderd. Defaults to the CR name. + Name string `json:"name,omitempty"` + // SecretName is the Kubernetes Secret to store the key. Defaults to "{crName}-provisioner-key". + SecretName string `json:"secretName,omitempty"` + // SecretKey is the data key in the Secret. Defaults to "key". + SecretKey string `json:"secretKey,omitempty"` +} + +// CoderProvisionerSpec defines the desired state of a CoderProvisioner. +type CoderProvisionerSpec struct { + // ControlPlaneRef identifies which CoderControlPlane instance to join. + ControlPlaneRef corev1.LocalObjectReference `json:"controlPlaneRef"` + // OrganizationName is the Coder organization. Defaults to "default". + OrganizationName string `json:"organizationName,omitempty"` + // Bootstrap configures credentials for provisioner key management. + Bootstrap CoderProvisionerBootstrapSpec `json:"bootstrap"` + // Key configures provisioner key naming and secret storage. + Key CoderProvisionerKeySpec `json:"key,omitempty"` + // Replicas is the desired number of provisioner pods. + Replicas *int32 `json:"replicas,omitempty"` + // Tags are attached to the provisioner key for job routing. + Tags map[string]string `json:"tags,omitempty"` + // Image is the container image. Defaults to the control plane image. + Image string `json:"image,omitempty"` + // ExtraArgs are appended after "provisionerd start". + ExtraArgs []string `json:"extraArgs,omitempty"` + // ExtraEnv are injected into the provisioner container. + ExtraEnv []corev1.EnvVar `json:"extraEnv,omitempty"` + // Resources for the provisioner container. + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + // ImagePullSecrets are used by the pod to pull private images. + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + // TerminationGracePeriodSeconds for the provisioner pods. + TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"` +} + +// CoderProvisionerStatus defines the observed state of a CoderProvisioner. +type CoderProvisionerStatus struct { + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + ReadyReplicas int32 `json:"readyReplicas,omitempty"` + Phase string `json:"phase,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + OrganizationID string `json:"organizationID,omitempty"` + ProvisionerKeyID string `json:"provisionerKeyID,omitempty"` + ProvisionerKeyName string `json:"provisionerKeyName,omitempty"` + SecretRef *SecretKeySelector `json:"secretRef,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced +// +kubebuilder:subresource:status + +// CoderProvisioner is the schema for Coder external provisioner daemon resources. +type CoderProvisioner struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CoderProvisionerSpec `json:"spec,omitempty"` + Status CoderProvisionerStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// CoderProvisionerList contains a list of CoderProvisioner objects. +type CoderProvisionerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CoderProvisioner `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CoderProvisioner{}, &CoderProvisionerList{}) +} From 4a58ddf737e45d915f0f2d292121063fdfcf8a8d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 08:38:18 +0000 Subject: [PATCH 02/38] feat: add coderbootstrap provisioner key management --- internal/coderbootstrap/client.go | 4 +- internal/coderbootstrap/provisionerkeys.go | 198 +++++++++++++ .../coderbootstrap/provisionerkeys_test.go | 279 ++++++++++++++++++ 3 files changed, 480 insertions(+), 1 deletion(-) create mode 100644 internal/coderbootstrap/provisionerkeys.go create mode 100644 internal/coderbootstrap/provisionerkeys_test.go diff --git a/internal/coderbootstrap/client.go b/internal/coderbootstrap/client.go index 536f0d75..6e0b2a0c 100644 --- a/internal/coderbootstrap/client.go +++ b/internal/coderbootstrap/client.go @@ -35,9 +35,11 @@ type RegisterWorkspaceProxyResponse struct { // Client provides optional bootstrap operations against the Coder API. type Client interface { EnsureWorkspaceProxy(context.Context, RegisterWorkspaceProxyRequest) (RegisterWorkspaceProxyResponse, error) + EnsureProvisionerKey(context.Context, EnsureProvisionerKeyRequest) (EnsureProvisionerKeyResponse, error) + DeleteProvisionerKey(ctx context.Context, coderURL, sessionToken, orgName, keyName string) error } -// SDKClient uses codersdk to register workspace proxies. +// SDKClient uses codersdk to perform bootstrap operations. type SDKClient struct{} // NewSDKClient returns a bootstrap client backed by codersdk. diff --git a/internal/coderbootstrap/provisionerkeys.go b/internal/coderbootstrap/provisionerkeys.go new file mode 100644 index 00000000..cb3eead8 --- /dev/null +++ b/internal/coderbootstrap/provisionerkeys.go @@ -0,0 +1,198 @@ +package coderbootstrap + +import ( + "context" + "errors" + "net/http" + "net/url" + + "github.com/google/uuid" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/codersdk" +) + +// EnsureProvisionerKeyRequest describes how to create or look up a provisioner key in Coder. +type EnsureProvisionerKeyRequest struct { + CoderURL string + SessionToken string + OrganizationName string + KeyName string + Tags map[string]string +} + +// EnsureProvisionerKeyResponse contains provisioner key metadata. +type EnsureProvisionerKeyResponse struct { + OrganizationID uuid.UUID + KeyID uuid.UUID + KeyName string + // Key is the plaintext provisioner key. It is only non-empty when a key is created. + Key string +} + +// EnsureProvisionerKey creates a provisioner key if it does not already exist, +// otherwise it returns the existing key metadata. +func (c *SDKClient) EnsureProvisionerKey(ctx context.Context, req EnsureProvisionerKeyRequest) (EnsureProvisionerKeyResponse, error) { + if err := validateProvisionerKeyInputs(req.CoderURL, req.SessionToken, req.KeyName); err != nil { + return EnsureProvisionerKeyResponse{}, err + } + + client, err := newAuthenticatedClient(req.CoderURL, req.SessionToken) + if err != nil { + return EnsureProvisionerKeyResponse{}, err + } + + organizationName := req.OrganizationName + if organizationName == "" { + organizationName = codersdk.DefaultOrganization + } + + organization, err := resolveOrganizationByName(ctx, client, organizationName) + if err != nil { + return EnsureProvisionerKeyResponse{}, err + } + + existing, err := findOrganizationProvisionerKey(ctx, client, organization.ID, req.KeyName) + if err != nil { + return EnsureProvisionerKeyResponse{}, err + } + if existing != nil { + if existing.ID == uuid.Nil { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("assertion failed: provisioner key %q returned an empty ID", req.KeyName) + } + return EnsureProvisionerKeyResponse{ + OrganizationID: organization.ID, + KeyID: existing.ID, + KeyName: existing.Name, + }, nil + } + + created, err := client.CreateProvisionerKey(ctx, organization.ID, codersdk.CreateProvisionerKeyRequest{ + Name: req.KeyName, + Tags: req.Tags, + }) + if err != nil { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("create provisioner key %q: %w", req.KeyName, err) + } + if created.Key == "" { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("assertion failed: created provisioner key %q returned an empty key", req.KeyName) + } + + createdMetadata, err := findOrganizationProvisionerKey(ctx, client, organization.ID, req.KeyName) + if err != nil { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("query created provisioner key %q: %w", req.KeyName, err) + } + if createdMetadata == nil { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("assertion failed: created provisioner key %q was not returned by list", req.KeyName) + } + if createdMetadata.ID == uuid.Nil { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("assertion failed: created provisioner key %q returned an empty ID", req.KeyName) + } + + return EnsureProvisionerKeyResponse{ + OrganizationID: organization.ID, + KeyID: createdMetadata.ID, + KeyName: createdMetadata.Name, + Key: created.Key, + }, nil +} + +// DeleteProvisionerKey deletes a provisioner key by name. +// A missing key is treated as success for idempotency. +func (c *SDKClient) DeleteProvisionerKey(ctx context.Context, coderURL, sessionToken, orgName, keyName string) error { + if err := validateProvisionerKeyInputs(coderURL, sessionToken, keyName); err != nil { + return err + } + + client, err := newAuthenticatedClient(coderURL, sessionToken) + if err != nil { + return err + } + + organizationName := orgName + if organizationName == "" { + organizationName = codersdk.DefaultOrganization + } + + organization, err := resolveOrganizationByName(ctx, client, organizationName) + if err != nil { + return err + } + + err = client.DeleteProvisionerKey(ctx, organization.ID, keyName) + if err == nil { + return nil + } + + var apiErr *codersdk.Error + if errors.As(err, &apiErr) && apiErr.StatusCode() == http.StatusNotFound { + return nil + } + + return xerrors.Errorf("delete provisioner key %q: %w", keyName, err) +} + +func validateProvisionerKeyInputs(coderURL, sessionToken, keyName string) error { + if coderURL == "" { + return xerrors.New("coder URL is required") + } + if sessionToken == "" { + return xerrors.New("session token is required") + } + if keyName == "" { + return xerrors.New("provisioner key name is required") + } + + return nil +} + +func newAuthenticatedClient(coderURL, sessionToken string) (*codersdk.Client, error) { + coderAPIURL, err := url.Parse(coderURL) + if err != nil { + return nil, xerrors.Errorf("parse coder URL: %w", err) + } + + client := codersdk.New(coderAPIURL) + if client == nil { + return nil, xerrors.New("assertion failed: codersdk client is nil after successful construction") + } + client.SetSessionToken(sessionToken) + if client.HTTPClient == nil { + client.HTTPClient = &http.Client{} + } + client.HTTPClient.Timeout = coderSDKRequestTimeout + + return client, nil +} + +func resolveOrganizationByName(ctx context.Context, client *codersdk.Client, organizationName string) (codersdk.Organization, error) { + organization, err := client.OrganizationByName(ctx, organizationName) + if err != nil { + return codersdk.Organization{}, xerrors.Errorf("query organization %q: %w", organizationName, err) + } + if organization.ID == uuid.Nil { + return codersdk.Organization{}, xerrors.Errorf("assertion failed: organization %q returned an empty ID", organizationName) + } + + return organization, nil +} + +func findOrganizationProvisionerKey(ctx context.Context, client *codersdk.Client, organizationID uuid.UUID, keyName string) (*codersdk.ProvisionerKey, error) { + keys, err := client.ListProvisionerKeys(ctx, organizationID) + if err != nil { + return nil, xerrors.Errorf("list provisioner keys for organization %q: %w", organizationID, err) + } + + var match *codersdk.ProvisionerKey + for i := range keys { + if keys[i].Name != keyName { + continue + } + if match != nil { + return nil, xerrors.Errorf("assertion failed: found multiple provisioner keys named %q in organization %q", keyName, organizationID) + } + match = &keys[i] + } + + return match, nil +} diff --git a/internal/coderbootstrap/provisionerkeys_test.go b/internal/coderbootstrap/provisionerkeys_test.go new file mode 100644 index 00000000..a1032aaf --- /dev/null +++ b/internal/coderbootstrap/provisionerkeys_test.go @@ -0,0 +1,279 @@ +package coderbootstrap_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder-k8s/internal/coderbootstrap" +) + +func TestEnsureProvisionerKey_Create(t *testing.T) { + t.Parallel() + + const keyName = "provisioner-key" + orgID := uuid.New() + keyID := uuid.New() + now := time.Now().UTC().Format(time.RFC3339) + + createCalls := 0 + listCalls := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/default": + writeJSONResponse(t, w, http.StatusOK, map[string]any{ + "id": orgID.String(), + "name": "default", + "created_at": now, + "updated_at": now, + }) + return + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys": + listCalls++ + if createCalls == 0 { + writeJSONResponse(t, w, http.StatusOK, []any{}) + return + } + writeJSONResponse(t, w, http.StatusOK, []map[string]any{{ + "id": keyID.String(), + "name": keyName, + "organization": orgID.String(), + "created_at": now, + "tags": map[string]string{ + "cluster": "dev", + }, + }}) + return + case r.Method == http.MethodPost && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys": + createCalls++ + var payload struct { + Name string `json:"name"` + Tags map[string]string `json:"tags"` + } + err := json.NewDecoder(r.Body).Decode(&payload) + require.NoError(t, err) + require.Equal(t, keyName, payload.Name) + require.Equal(t, map[string]string{"cluster": "dev"}, payload.Tags) + + writeJSONResponse(t, w, http.StatusCreated, map[string]any{ + "key": "plaintext-provisioner-key", + }) + return + default: + writeJSONResponse(t, w, http.StatusNotFound, map[string]any{ + "message": "unexpected route", + }) + return + } + })) + defer server.Close() + + client := coderbootstrap.NewSDKClient() + resp, err := client.EnsureProvisionerKey(context.Background(), coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: server.URL, + SessionToken: "session-token", + KeyName: keyName, + Tags: map[string]string{ + "cluster": "dev", + }, + }) + require.NoError(t, err) + require.Equal(t, 1, createCalls) + require.Equal(t, 2, listCalls) + require.Equal(t, orgID, resp.OrganizationID) + require.Equal(t, keyID, resp.KeyID) + require.Equal(t, keyName, resp.KeyName) + require.Equal(t, "plaintext-provisioner-key", resp.Key) +} + +func TestEnsureProvisionerKey_Exists(t *testing.T) { + t.Parallel() + + const ( + orgName = "engineering" + keyName = "existing-key" + ) + orgID := uuid.New() + keyID := uuid.New() + now := time.Now().UTC().Format(time.RFC3339) + createCalled := false + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/"+orgName: + writeJSONResponse(t, w, http.StatusOK, map[string]any{ + "id": orgID.String(), + "name": orgName, + "created_at": now, + "updated_at": now, + }) + return + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys": + writeJSONResponse(t, w, http.StatusOK, []map[string]any{{ + "id": keyID.String(), + "name": keyName, + "organization": orgID.String(), + "created_at": now, + "tags": map[string]string{"cluster": "prod"}, + }}) + return + case r.Method == http.MethodPost && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys": + createCalled = true + writeJSONResponse(t, w, http.StatusCreated, map[string]any{"key": "should-not-be-created"}) + return + default: + writeJSONResponse(t, w, http.StatusNotFound, map[string]any{"message": "unexpected route"}) + return + } + })) + defer server.Close() + + client := coderbootstrap.NewSDKClient() + resp, err := client.EnsureProvisionerKey(context.Background(), coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: server.URL, + SessionToken: "session-token", + OrganizationName: orgName, + KeyName: keyName, + }) + require.NoError(t, err) + require.False(t, createCalled) + require.Equal(t, orgID, resp.OrganizationID) + require.Equal(t, keyID, resp.KeyID) + require.Equal(t, keyName, resp.KeyName) + require.Empty(t, resp.Key) +} + +func TestEnsureProvisionerKey_ValidationErrors(t *testing.T) { + t.Parallel() + + client := coderbootstrap.NewSDKClient() + tests := []struct { + name string + request coderbootstrap.EnsureProvisionerKeyRequest + errSubstr string + }{ + { + name: "missing coder URL", + request: coderbootstrap.EnsureProvisionerKeyRequest{ + SessionToken: "session-token", + KeyName: "key", + }, + errSubstr: "coder URL is required", + }, + { + name: "missing session token", + request: coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: "https://coder.example.com", + KeyName: "key", + }, + errSubstr: "session token is required", + }, + { + name: "missing key name", + request: coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: "https://coder.example.com", + SessionToken: "session-token", + }, + errSubstr: "provisioner key name is required", + }, + } + + for _, testCase := range tests { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + _, err := client.EnsureProvisionerKey(context.Background(), testCase.request) + require.Error(t, err) + require.Contains(t, err.Error(), testCase.errSubstr) + }) + } +} + +func TestDeleteProvisionerKey_Success(t *testing.T) { + t.Parallel() + + const keyName = "delete-me" + orgID := uuid.New() + now := time.Now().UTC().Format(time.RFC3339) + deleteCalls := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/default": + writeJSONResponse(t, w, http.StatusOK, map[string]any{ + "id": orgID.String(), + "name": "default", + "created_at": now, + "updated_at": now, + }) + return + case r.Method == http.MethodDelete && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys/"+keyName: + deleteCalls++ + w.WriteHeader(http.StatusNoContent) + return + default: + writeJSONResponse(t, w, http.StatusNotFound, map[string]any{"message": "unexpected route"}) + return + } + })) + defer server.Close() + + client := coderbootstrap.NewSDKClient() + err := client.DeleteProvisionerKey(context.Background(), server.URL, "session-token", "", keyName) + require.NoError(t, err) + require.Equal(t, 1, deleteCalls) +} + +func TestDeleteProvisionerKey_NotFound(t *testing.T) { + t.Parallel() + + const keyName = "already-deleted" + orgID := uuid.New() + now := time.Now().UTC().Format(time.RFC3339) + deleteCalls := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/default": + writeJSONResponse(t, w, http.StatusOK, map[string]any{ + "id": orgID.String(), + "name": "default", + "created_at": now, + "updated_at": now, + }) + return + case r.Method == http.MethodDelete && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys/"+keyName: + deleteCalls++ + writeJSONResponse(t, w, http.StatusNotFound, map[string]any{ + "message": "provisioner key not found", + }) + return + default: + writeJSONResponse(t, w, http.StatusNotFound, map[string]any{"message": "unexpected route"}) + return + } + })) + defer server.Close() + + client := coderbootstrap.NewSDKClient() + err := client.DeleteProvisionerKey(context.Background(), server.URL, "session-token", "", keyName) + require.NoError(t, err) + require.Equal(t, 1, deleteCalls) +} + +func writeJSONResponse(t *testing.T, w http.ResponseWriter, statusCode int, payload any) { + t.Helper() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + err := json.NewEncoder(w).Encode(payload) + require.NoError(t, err) +} From e8d68806433f89b052ba23ebc40329c666a4bb81 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 08:41:18 +0000 Subject: [PATCH 03/38] chore: generate deepcopy and CRD manifest for CoderProvisioner --- api/v1alpha1/zz_generated.deepcopy.go | 176 +++++++ .../bases/coder.com_coderprovisioners.yaml | 451 ++++++++++++++++++ 2 files changed, 627 insertions(+) create mode 100644 config/crd/bases/coder.com_coderprovisioners.yaml diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9c6c1e9f..f231cf81 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -140,6 +140,182 @@ func (in *CoderControlPlaneStatus) DeepCopy() *CoderControlPlaneStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisioner) DeepCopyInto(out *CoderProvisioner) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisioner. +func (in *CoderProvisioner) DeepCopy() *CoderProvisioner { + if in == nil { + return nil + } + out := new(CoderProvisioner) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CoderProvisioner) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisionerBootstrapSpec) DeepCopyInto(out *CoderProvisionerBootstrapSpec) { + *out = *in + out.CredentialsSecretRef = in.CredentialsSecretRef + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisionerBootstrapSpec. +func (in *CoderProvisionerBootstrapSpec) DeepCopy() *CoderProvisionerBootstrapSpec { + if in == nil { + return nil + } + out := new(CoderProvisionerBootstrapSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisionerKeySpec) DeepCopyInto(out *CoderProvisionerKeySpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisionerKeySpec. +func (in *CoderProvisionerKeySpec) DeepCopy() *CoderProvisionerKeySpec { + if in == nil { + return nil + } + out := new(CoderProvisionerKeySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisionerList) DeepCopyInto(out *CoderProvisionerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CoderProvisioner, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisionerList. +func (in *CoderProvisionerList) DeepCopy() *CoderProvisionerList { + if in == nil { + return nil + } + out := new(CoderProvisionerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CoderProvisionerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisionerSpec) DeepCopyInto(out *CoderProvisionerSpec) { + *out = *in + out.ControlPlaneRef = in.ControlPlaneRef + out.Bootstrap = in.Bootstrap + out.Key = in.Key + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ExtraArgs != nil { + in, out := &in.ExtraArgs, &out.ExtraArgs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExtraEnv != nil { + in, out := &in.ExtraEnv, &out.ExtraEnv + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Resources.DeepCopyInto(&out.Resources) + if in.ImagePullSecrets != nil { + in, out := &in.ImagePullSecrets, &out.ImagePullSecrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } + if in.TerminationGracePeriodSeconds != nil { + in, out := &in.TerminationGracePeriodSeconds, &out.TerminationGracePeriodSeconds + *out = new(int64) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisionerSpec. +func (in *CoderProvisionerSpec) DeepCopy() *CoderProvisionerSpec { + if in == nil { + return nil + } + out := new(CoderProvisionerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisionerStatus) DeepCopyInto(out *CoderProvisionerStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(SecretKeySelector) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisionerStatus. +func (in *CoderProvisionerStatus) DeepCopy() *CoderProvisionerStatus { + if in == nil { + return nil + } + out := new(CoderProvisionerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorAccessSpec) DeepCopyInto(out *OperatorAccessSpec) { *out = *in diff --git a/config/crd/bases/coder.com_coderprovisioners.yaml b/config/crd/bases/coder.com_coderprovisioners.yaml new file mode 100644 index 00000000..37144a19 --- /dev/null +++ b/config/crd/bases/coder.com_coderprovisioners.yaml @@ -0,0 +1,451 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: coderprovisioners.coder.com +spec: + group: coder.com + names: + kind: CoderProvisioner + listKind: CoderProvisionerList + plural: coderprovisioners + singular: coderprovisioner + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: CoderProvisioner is the schema for Coder external provisioner + daemon resources. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CoderProvisionerSpec defines the desired state of a CoderProvisioner. + properties: + bootstrap: + description: Bootstrap configures credentials for provisioner key + management. + properties: + credentialsSecretRef: + description: |- + CredentialsSecretRef points to a Secret containing a Coder session token + with permission to manage provisioner keys. + properties: + key: + description: Key is the key inside the Secret data map. + type: string + name: + description: Name is the Kubernetes Secret name. + type: string + required: + - name + type: object + required: + - credentialsSecretRef + type: object + controlPlaneRef: + description: ControlPlaneRef identifies which CoderControlPlane instance + to join. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + extraArgs: + description: ExtraArgs are appended after "provisionerd start". + items: + type: string + type: array + extraEnv: + description: ExtraEnv are injected into the provisioner container. + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + description: Image is the container image. Defaults to the control + plane image. + type: string + imagePullSecrets: + description: ImagePullSecrets are used by the pod to pull private + images. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + key: + description: Key configures provisioner key naming and secret storage. + properties: + name: + description: Name is the provisioner key name in coderd. Defaults + to the CR name. + type: string + secretKey: + description: SecretKey is the data key in the Secret. Defaults + to "key". + type: string + secretName: + description: SecretName is the Kubernetes Secret to store the + key. Defaults to "{crName}-provisioner-key". + type: string + type: object + organizationName: + description: OrganizationName is the Coder organization. Defaults + to "default". + type: string + replicas: + description: Replicas is the desired number of provisioner pods. + format: int32 + type: integer + resources: + description: Resources for the provisioner container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tags: + additionalProperties: + type: string + description: Tags are attached to the provisioner key for job routing. + type: object + terminationGracePeriodSeconds: + description: TerminationGracePeriodSeconds for the provisioner pods. + format: int64 + type: integer + required: + - bootstrap + - controlPlaneRef + type: object + status: + description: CoderProvisionerStatus defines the observed state of a CoderProvisioner. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + observedGeneration: + format: int64 + type: integer + organizationID: + type: string + phase: + type: string + provisionerKeyID: + type: string + provisionerKeyName: + type: string + readyReplicas: + format: int32 + type: integer + secretRef: + description: SecretKeySelector identifies a key in a Secret. + properties: + key: + description: Key is the key inside the Secret data map. + type: string + name: + description: Name is the Kubernetes Secret name. + type: string + required: + - name + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} From 713e70d8c3dfbb1f44e4cb4c8fa935d1b2bdab9a Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 08:49:18 +0000 Subject: [PATCH 04/38] feat: add coder provisioner reconciler --- internal/app/controllerapp/controllerapp.go | 9 + .../controller/coderprovisioner_controller.go | 660 ++++++++++++++++++ 2 files changed, 669 insertions(+) create mode 100644 internal/controller/coderprovisioner_controller.go diff --git a/internal/app/controllerapp/controllerapp.go b/internal/app/controllerapp/controllerapp.go index c99d0dcf..65388326 100644 --- a/internal/app/controllerapp/controllerapp.go +++ b/internal/app/controllerapp/controllerapp.go @@ -103,6 +103,15 @@ func Run(ctx context.Context) error { return fmt.Errorf("unable to create workspace proxy controller: %w", err) } + provisionerReconciler := &controller.CoderProvisionerReconciler{ + Client: client, + Scheme: managerScheme, + BootstrapClient: coderbootstrap.NewSDKClient(), + } + if err := provisionerReconciler.SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create provisioner controller: %w", err) + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { return fmt.Errorf("unable to set up health check: %w", err) } diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go new file mode 100644 index 00000000..a01e52cd --- /dev/null +++ b/internal/controller/coderprovisioner_controller.go @@ -0,0 +1,660 @@ +// Package controller contains Kubernetes controllers for coder-k8s resources. +package controller + +import ( + "context" + "fmt" + "hash/fnv" + "maps" + + "github.com/google/uuid" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" + "github.com/coder/coder-k8s/internal/coderbootstrap" +) + +const ( + defaultProvisionerReplicas = int32(1) + defaultProvisionerTerminationGracePeriodSeconds = int64(600) + defaultProvisionerOrganizationName = "default" + provisionerNamePrefix = "provisioner-" + provisionerServiceAccountSuffix = "-provisioner" + provisionerKeyChecksumAnnotation = "checksum/provisioner-key" +) + +// CoderProvisionerReconciler reconciles a CoderProvisioner object. +type CoderProvisionerReconciler struct { + client.Client + Scheme *runtime.Scheme + BootstrapClient coderbootstrap.Client +} + +// +kubebuilder:rbac:groups=coder.com,resources=coderprovisioners,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=coder.com,resources=coderprovisioners/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=coder.com,resources=coderprovisioners/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=secrets;serviceaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete + +// Reconcile converges the desired CoderProvisioner spec into Deployment, RBAC, and Secret resources. +func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + if r.Client == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: reconciler client must not be nil") + } + if r.Scheme == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: reconciler scheme must not be nil") + } + if r.BootstrapClient == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: reconciler bootstrap client must not be nil") + } + + provisioner := &coderv1alpha1.CoderProvisioner{} + if err := r.Get(ctx, req.NamespacedName, provisioner); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, fmt.Errorf("get coderprovisioner %s: %w", req.NamespacedName, err) + } + + if provisioner.Name != req.Name || provisioner.Namespace != req.Namespace { + return ctrl.Result{}, fmt.Errorf("assertion failed: fetched object %s/%s does not match request %s/%s", + provisioner.Namespace, provisioner.Name, req.Namespace, req.Name) + } + + if !provisioner.DeletionTimestamp.IsZero() { + return r.reconcileDeletion(ctx, provisioner) + } + + finalizerAdded, err := r.ensureCleanupFinalizer(ctx, provisioner) + if err != nil { + return ctrl.Result{}, err + } + if finalizerAdded { + return ctrl.Result{}, nil + } + + controlPlane, err := r.fetchControlPlane(ctx, provisioner) + if err != nil { + return ctrl.Result{}, err + } + + organizationName := provisionerOrganizationName(provisioner.Spec.OrganizationName) + keyName, keySecretName, keySecretKey := provisionerKeyConfig(provisioner) + + sessionToken, err := r.readBootstrapSessionToken(ctx, provisioner) + if err != nil { + return ctrl.Result{}, err + } + + secretNamespacedName := types.NamespacedName{Name: keySecretName, Namespace: provisioner.Namespace} + existingSecret := &corev1.Secret{} + secretExists := true + if err := r.Get(ctx, secretNamespacedName, existingSecret); err != nil { + if apierrors.IsNotFound(err) { + secretExists = false + } else { + return ctrl.Result{}, fmt.Errorf("get provisioner key secret %s: %w", secretNamespacedName, err) + } + } + + organizationID := provisioner.Status.OrganizationID + provisionerKeyID := provisioner.Status.ProvisionerKeyID + provisionerKeyName := provisioner.Status.ProvisionerKeyName + if provisionerKeyName == "" { + provisionerKeyName = keyName + } + + keyMaterial := "" + if !secretExists { + response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if ensureErr != nil { + return ctrl.Result{}, fmt.Errorf("ensure provisioner key %q: %w", keyName, ensureErr) + } + if response.OrganizationID != uuid.Nil { + organizationID = response.OrganizationID.String() + } + if response.KeyID != uuid.Nil { + provisionerKeyID = response.KeyID.String() + } + if response.KeyName != "" { + provisionerKeyName = response.KeyName + } + keyMaterial = response.Key + if keyMaterial == "" { + return ctrl.Result{}, fmt.Errorf("provisioner key %q exists in coderd but no key material is available to create secret %q", keyName, keySecretName) + } + } + + provisionerKeySecret, err := r.ensureProvisionerKeySecret(ctx, provisioner, keySecretName, keySecretKey, keyMaterial) + if err != nil { + return ctrl.Result{}, err + } + + secretValue, ok := provisionerKeySecret.Data[keySecretKey] + if !ok { + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key secret %q is missing key %q after reconciliation", keySecretName, keySecretKey) + } + if len(secretValue) == 0 { + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key secret %q key %q is empty after reconciliation", keySecretName, keySecretKey) + } + secretChecksum := hashProvisionerSecret(secretValue) + + serviceAccountName := provisionerServiceAccountName(provisioner.Name) + if _, err := r.reconcileServiceAccount(ctx, provisioner, serviceAccountName); err != nil { + return ctrl.Result{}, err + } + + roleName := provisionerResourceName(provisioner.Name) + role, err := r.reconcileRole(ctx, provisioner, roleName) + if err != nil { + return ctrl.Result{}, err + } + if _, err := r.reconcileRoleBinding(ctx, provisioner, roleName, role.Name, serviceAccountName); err != nil { + return ctrl.Result{}, err + } + + image := provisioner.Spec.Image + if image == "" { + image = controlPlane.Spec.Image + } + if image == "" { + image = defaultCoderImage + } + + secretRef := &coderv1alpha1.SecretKeySelector{Name: keySecretName, Key: keySecretKey} + deployment, err := r.reconcileDeployment(ctx, provisioner, image, controlPlane.Status.URL, organizationName, secretRef, serviceAccountName, secretChecksum) + if err != nil { + return ctrl.Result{}, err + } + + if err := r.reconcileStatus(ctx, provisioner, deployment, secretRef, organizationID, provisionerKeyID, provisionerKeyName); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func (r *CoderProvisionerReconciler) reconcileDeletion(ctx context.Context, provisioner *coderv1alpha1.CoderProvisioner) (ctrl.Result, error) { + if !controllerutil.ContainsFinalizer(provisioner, coderv1alpha1.ProvisionerKeyCleanupFinalizer) { + return ctrl.Result{}, nil + } + + controlPlane, err := r.fetchControlPlane(ctx, provisioner) + if err != nil { + return ctrl.Result{}, err + } + + sessionToken, err := r.readBootstrapSessionToken(ctx, provisioner) + if err != nil { + return ctrl.Result{}, err + } + + organizationName := provisionerOrganizationName(provisioner.Spec.OrganizationName) + keyName, _, _ := provisionerKeyConfig(provisioner) + if err := r.BootstrapClient.DeleteProvisionerKey( + ctx, + controlPlane.Status.URL, + sessionToken, + organizationName, + keyName, + ); err != nil { + return ctrl.Result{}, fmt.Errorf("delete provisioner key %q: %w", keyName, err) + } + + controllerutil.RemoveFinalizer(provisioner, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + if err := r.Update(ctx, provisioner); err != nil { + return ctrl.Result{}, fmt.Errorf("remove finalizer from coderprovisioner %s/%s: %w", provisioner.Namespace, provisioner.Name, err) + } + + return ctrl.Result{}, nil +} + +func (r *CoderProvisionerReconciler) ensureCleanupFinalizer(ctx context.Context, provisioner *coderv1alpha1.CoderProvisioner) (bool, error) { + if controllerutil.ContainsFinalizer(provisioner, coderv1alpha1.ProvisionerKeyCleanupFinalizer) { + return false, nil + } + + controllerutil.AddFinalizer(provisioner, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + if err := r.Update(ctx, provisioner); err != nil { + return false, fmt.Errorf("add finalizer to coderprovisioner %s/%s: %w", provisioner.Namespace, provisioner.Name, err) + } + + return true, nil +} + +func (r *CoderProvisionerReconciler) fetchControlPlane(ctx context.Context, provisioner *coderv1alpha1.CoderProvisioner) (*coderv1alpha1.CoderControlPlane, error) { + controlPlaneName := provisioner.Spec.ControlPlaneRef.Name + if controlPlaneName == "" { + return nil, fmt.Errorf("coderprovisioner %s/%s spec.controlPlaneRef.name is required", provisioner.Namespace, provisioner.Name) + } + + controlPlane := &coderv1alpha1.CoderControlPlane{} + namespacedName := types.NamespacedName{Name: controlPlaneName, Namespace: provisioner.Namespace} + if err := r.Get(ctx, namespacedName, controlPlane); err != nil { + return nil, fmt.Errorf("get referenced codercontrolplane %s for coderprovisioner %s/%s: %w", namespacedName, provisioner.Namespace, provisioner.Name, err) + } + + if controlPlane.Name != controlPlaneName || controlPlane.Namespace != provisioner.Namespace { + return nil, fmt.Errorf("assertion failed: fetched control plane %s/%s does not match expected %s/%s", + controlPlane.Namespace, controlPlane.Name, provisioner.Namespace, controlPlaneName) + } + if controlPlane.Status.URL == "" { + return nil, fmt.Errorf("codercontrolplane %s/%s status.url is empty", controlPlane.Namespace, controlPlane.Name) + } + + return controlPlane, nil +} + +func (r *CoderProvisionerReconciler) readBootstrapSessionToken(ctx context.Context, provisioner *coderv1alpha1.CoderProvisioner) (string, error) { + credentialsRef := provisioner.Spec.Bootstrap.CredentialsSecretRef + if credentialsRef.Name == "" { + return "", fmt.Errorf("coderprovisioner %s/%s spec.bootstrap.credentialsSecretRef.name is required", provisioner.Namespace, provisioner.Name) + } + + credentialsKey := credentialsRef.Key + if credentialsKey == "" { + credentialsKey = coderv1alpha1.DefaultTokenSecretKey + } + + token, err := r.readSecretValue(ctx, provisioner.Namespace, credentialsRef.Name, credentialsKey) + if err != nil { + return "", fmt.Errorf("read bootstrap credentials secret %q/%q key %q: %w", provisioner.Namespace, credentialsRef.Name, credentialsKey, err) + } + + return token, nil +} + +func (r *CoderProvisionerReconciler) ensureProvisionerKeySecret( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + secretName string, + secretKey string, + keyMaterial string, +) (*corev1.Secret, error) { + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: provisioner.Namespace}} + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, secret, func() error { + labels := provisionerLabels(provisioner.Name) + secret.Labels = maps.Clone(labels) + secret.Type = corev1.SecretTypeOpaque + if secret.Data == nil { + secret.Data = make(map[string][]byte) + } + if keyMaterial != "" { + secret.Data[secretKey] = []byte(keyMaterial) + } + if len(secret.Data[secretKey]) == 0 { + return fmt.Errorf("provisioner key secret %q key %q is empty", secretName, secretKey) + } + if err := controllerutil.SetControllerReference(provisioner, secret, r.Scheme); err != nil { + return fmt.Errorf("set controller reference: %w", err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("reconcile provisioner key secret %q: %w", secretName, err) + } + + if err := r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: provisioner.Namespace}, secret); err != nil { + return nil, fmt.Errorf("get reconciled provisioner key secret %q: %w", secretName, err) + } + + return secret, nil +} + +func (r *CoderProvisionerReconciler) reconcileServiceAccount( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + serviceAccountName string, +) (*corev1.ServiceAccount, error) { + serviceAccount := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: serviceAccountName, Namespace: provisioner.Namespace}} + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, serviceAccount, func() error { + labels := provisionerLabels(provisioner.Name) + serviceAccount.Labels = maps.Clone(labels) + if err := controllerutil.SetControllerReference(provisioner, serviceAccount, r.Scheme); err != nil { + return fmt.Errorf("set controller reference: %w", err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("reconcile serviceaccount %q: %w", serviceAccountName, err) + } + + if err := r.Get(ctx, types.NamespacedName{Name: serviceAccount.Name, Namespace: serviceAccount.Namespace}, serviceAccount); err != nil { + return nil, fmt.Errorf("get reconciled serviceaccount %q: %w", serviceAccountName, err) + } + + return serviceAccount, nil +} + +func (r *CoderProvisionerReconciler) reconcileRole( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + roleName string, +) (*rbacv1.Role, error) { + role := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: roleName, Namespace: provisioner.Namespace}} + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, role, func() error { + labels := provisionerLabels(provisioner.Name) + role.Labels = maps.Clone(labels) + role.Rules = []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"pods", "persistentvolumeclaims"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }} + if err := controllerutil.SetControllerReference(provisioner, role, r.Scheme); err != nil { + return fmt.Errorf("set controller reference: %w", err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("reconcile role %q: %w", roleName, err) + } + + if err := r.Get(ctx, types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, role); err != nil { + return nil, fmt.Errorf("get reconciled role %q: %w", roleName, err) + } + + return role, nil +} + +func (r *CoderProvisionerReconciler) reconcileRoleBinding( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + roleBindingName string, + roleName string, + serviceAccountName string, +) (*rbacv1.RoleBinding, error) { + roleBinding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: roleBindingName, Namespace: provisioner.Namespace}} + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, roleBinding, func() error { + labels := provisionerLabels(provisioner.Name) + roleBinding.Labels = maps.Clone(labels) + roleBinding.RoleRef = rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: roleName, + } + roleBinding.Subjects = []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: serviceAccountName, + Namespace: provisioner.Namespace, + }} + if err := controllerutil.SetControllerReference(provisioner, roleBinding, r.Scheme); err != nil { + return fmt.Errorf("set controller reference: %w", err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("reconcile rolebinding %q: %w", roleBindingName, err) + } + + if err := r.Get(ctx, types.NamespacedName{Name: roleBinding.Name, Namespace: roleBinding.Namespace}, roleBinding); err != nil { + return nil, fmt.Errorf("get reconciled rolebinding %q: %w", roleBindingName, err) + } + + return roleBinding, nil +} + +func (r *CoderProvisionerReconciler) reconcileDeployment( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + image string, + coderURL string, + organizationName string, + secretRef *coderv1alpha1.SecretKeySelector, + serviceAccountName string, + secretChecksum string, +) (*appsv1.Deployment, error) { + deploymentName := provisionerResourceName(provisioner.Name) + deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: provisioner.Namespace}} + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + labels := provisionerLabels(provisioner.Name) + deployment.Labels = maps.Clone(labels) + + if err := controllerutil.SetControllerReference(provisioner, deployment, r.Scheme); err != nil { + return fmt.Errorf("set controller reference: %w", err) + } + + replicas := defaultProvisionerReplicas + if provisioner.Spec.Replicas != nil { + replicas = *provisioner.Spec.Replicas + } + terminationGracePeriodSeconds := defaultProvisionerTerminationGracePeriodSeconds + if provisioner.Spec.TerminationGracePeriodSeconds != nil { + terminationGracePeriodSeconds = *provisioner.Spec.TerminationGracePeriodSeconds + } + + args := []string{"provisionerd", "start"} + args = append(args, provisioner.Spec.ExtraArgs...) + + env := []corev1.EnvVar{ + {Name: "CODER_URL", Value: coderURL}, + { + Name: "CODER_PROVISIONER_DAEMON_KEY", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: secretRef.Name}, + Key: secretRef.Key, + }}, + }, + } + if organizationName != "" && organizationName != defaultProvisionerOrganizationName { + env = append(env, corev1.EnvVar{Name: "CODER_ORGANIZATION", Value: organizationName}) + } + env = append(env, provisioner.Spec.ExtraEnv...) + + deployment.Spec.Replicas = &replicas + deployment.Spec.Selector = &metav1.LabelSelector{MatchLabels: maps.Clone(labels)} + deployment.Spec.Template = corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: maps.Clone(labels), + Annotations: map[string]string{ + provisionerKeyChecksumAnnotation: secretChecksum, + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccountName, + ImagePullSecrets: provisioner.Spec.ImagePullSecrets, + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + Containers: []corev1.Container{{ + Name: "provisioner", + Image: image, + Args: args, + Env: env, + Resources: provisioner.Spec.Resources, + }}, + }, + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("reconcile provisioner deployment: %w", err) + } + + if err := r.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment); err != nil { + return nil, fmt.Errorf("get reconciled deployment %q: %w", deployment.Name, err) + } + + return deployment, nil +} + +func (r *CoderProvisionerReconciler) reconcileStatus( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + deployment *appsv1.Deployment, + secretRef *coderv1alpha1.SecretKeySelector, + organizationID string, + provisionerKeyID string, + provisionerKeyName string, +) error { + phase := coderv1alpha1.CoderProvisionerPhasePending + if deployment.Status.ReadyReplicas > 0 { + phase = coderv1alpha1.CoderProvisionerPhaseReady + } + + nextStatus := coderv1alpha1.CoderProvisionerStatus{ + ObservedGeneration: provisioner.Generation, + ReadyReplicas: deployment.Status.ReadyReplicas, + Phase: phase, + OrganizationID: organizationID, + ProvisionerKeyID: provisionerKeyID, + ProvisionerKeyName: provisionerKeyName, + SecretRef: &coderv1alpha1.SecretKeySelector{ + Name: secretRef.Name, + Key: secretRef.Key, + }, + } + if equality.Semantic.DeepEqual(provisioner.Status, nextStatus) { + return nil + } + + provisioner.Status = nextStatus + if err := r.Status().Update(ctx, provisioner); err != nil { + return fmt.Errorf("update coderprovisioner status: %w", err) + } + + return nil +} + +func (r *CoderProvisionerReconciler) readSecretValue(ctx context.Context, namespace, name, key string) (string, error) { + secret := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, secret); err != nil { + return "", err + } + + value, ok := secret.Data[key] + if !ok { + return "", fmt.Errorf("secret %q does not contain key %q", name, key) + } + if len(value) == 0 { + return "", fmt.Errorf("secret %q key %q is empty", name, key) + } + + return string(value), nil +} + +// SetupWithManager wires the reconciler into controller-runtime. +func (r *CoderProvisionerReconciler) SetupWithManager(mgr ctrl.Manager) error { + if mgr == nil { + return fmt.Errorf("assertion failed: manager must not be nil") + } + if r.Client == nil { + return fmt.Errorf("assertion failed: reconciler client must not be nil") + } + if r.Scheme == nil { + return fmt.Errorf("assertion failed: reconciler scheme must not be nil") + } + if r.BootstrapClient == nil { + return fmt.Errorf("assertion failed: reconciler bootstrap client must not be nil") + } + + return ctrl.NewControllerManagedBy(mgr). + For(&coderv1alpha1.CoderProvisioner{}). + Owns(&appsv1.Deployment{}). + Owns(&corev1.Secret{}). + Owns(&corev1.ServiceAccount{}). + Named("coderprovisioner"). + Complete(r) +} + +func provisionerResourceName(name string) string { + candidate := provisionerNamePrefix + name + if len(candidate) <= 63 { + return candidate + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 63 - len(provisionerNamePrefix) - len(suffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s%s-%s", provisionerNamePrefix, name[:available], suffix) +} + +func provisionerLabels(name string) map[string]string { + return map[string]string{ + "app.kubernetes.io/name": "coder-provisioner", + "app.kubernetes.io/instance": provisionerInstanceLabelValue(name), + "app.kubernetes.io/managed-by": "coder-k8s", + } +} + +func provisionerInstanceLabelValue(name string) string { + if len(name) <= 63 { + return name + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 63 - len(suffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s-%s", name[:available], suffix) +} + +func provisionerServiceAccountName(name string) string { + return fmt.Sprintf("%s%s", name, provisionerServiceAccountSuffix) +} + +func provisionerOrganizationName(name string) string { + if name == "" { + return defaultProvisionerOrganizationName + } + + return name +} + +func provisionerKeyConfig(provisioner *coderv1alpha1.CoderProvisioner) (string, string, string) { + keyName := provisioner.Spec.Key.Name + if keyName == "" { + keyName = provisioner.Name + } + + secretName := provisioner.Spec.Key.SecretName + if secretName == "" { + secretName = fmt.Sprintf("%s-provisioner-key", provisioner.Name) + } + + secretKey := provisioner.Spec.Key.SecretKey + if secretKey == "" { + secretKey = coderv1alpha1.DefaultProvisionerKeySecretKey + } + + return keyName, secretName, secretKey +} + +func hashProvisionerSecret(secretValue []byte) string { + hasher := fnv.New32a() + _, _ = hasher.Write(secretValue) + return fmt.Sprintf("%08x", hasher.Sum32()) +} From 08b086c77d146bcb523793ca6fdcc6ebcaca75ee Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 08:59:57 +0000 Subject: [PATCH 05/38] test: add coder provisioner controller envtests --- .../coderprovisioner_controller_test.go | 509 ++++++++++++++++++ .../workspaceproxy_controller_test.go | 17 + 2 files changed, 526 insertions(+) create mode 100644 internal/controller/coderprovisioner_controller_test.go diff --git a/internal/controller/coderprovisioner_controller_test.go b/internal/controller/coderprovisioner_controller_test.go new file mode 100644 index 00000000..40c0f5ec --- /dev/null +++ b/internal/controller/coderprovisioner_controller_test.go @@ -0,0 +1,509 @@ +package controller_test + +import ( + "context" + "fmt" + "hash/fnv" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" + "github.com/coder/coder-k8s/internal/coderbootstrap" + "github.com/coder/coder-k8s/internal/controller" +) + +func createTestNamespace(t *testing.T, ctx context.Context, prefix string) string { + t.Helper() + + namespaceName := fmt.Sprintf("%s-%s", prefix, strings.ToLower(uuid.NewString()[:8])) + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}} + require.NoError(t, k8sClient.Create(ctx, ns)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), ns) + }) + + return namespaceName +} + +// createTestControlPlane creates a test CoderControlPlane and optionally sets status.url. +func createTestControlPlane(t *testing.T, ctx context.Context, namespace, name, url string) *coderv1alpha1.CoderControlPlane { + t.Helper() + + controlPlane := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + Image: "coder-control-plane:test", + }, + } + require.NoError(t, k8sClient.Create(ctx, controlPlane)) + if url != "" { + controlPlane.Status.URL = url + require.NoError(t, k8sClient.Status().Update(ctx, controlPlane)) + } + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), controlPlane) + }) + + return controlPlane +} + +// createBootstrapSecret creates the bootstrap credentials secret used by provisioner reconciliation. +func createBootstrapSecret(t *testing.T, ctx context.Context, namespace, name, key, value string) *corev1.Secret { + t.Helper() + + if key == "" { + key = coderv1alpha1.DefaultTokenSecretKey + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + key: []byte(value), + }, + } + require.NoError(t, k8sClient.Create(ctx, secret)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), secret) + }) + + return secret +} + +func expectedProvisionerResourceName(name string) string { + const prefix = "provisioner-" + candidate := prefix + name + if len(candidate) <= 63 { + return candidate + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 63 - len(prefix) - len(suffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s%s-%s", prefix, name[:available], suffix) +} + +func expectedProvisionerServiceAccountName(name string) string { + return fmt.Sprintf("%s-provisioner", name) +} + +func reconcileProvisioner(t *testing.T, ctx context.Context, reconciler *controller.CoderProvisionerReconciler, namespacedName types.NamespacedName) { + t.Helper() + + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) +} + +func requireOwnerReference(t *testing.T, owner metav1.Object, child metav1.Object) { + t.Helper() + + ownerReferences := child.GetOwnerReferences() + require.NotEmpty(t, ownerReferences) + + for _, ownerReference := range ownerReferences { + if ownerReference.Name == owner.GetName() && ownerReference.UID == owner.GetUID() { + return + } + } + + require.Failf(t, "missing owner reference", "expected %s/%s to own %s/%s", owner.GetNamespace(), owner.GetName(), child.GetNamespace(), child.GetName()) +} + +func TestCoderProvisionerReconciler_BasicCreate(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(t, ctx, "coderprov-basic") + controlPlane := createTestControlPlane(t, ctx, namespace, "controlplane-basic", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(t, ctx, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + organizationID := uuid.New() + provisionerKeyID := uuid.New() + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponse: coderbootstrap.EnsureProvisionerKeyResponse{ + OrganizationID: organizationID, + KeyID: provisionerKeyID, + KeyName: "provisioner-key-name", + Key: "provisioner-key-material", + }, + } + reconciler := &controller.CoderProvisionerReconciler{ + Client: k8sClient, + Scheme: scheme, + BootstrapClient: bootstrapClient, + } + + replicas := int32(2) + terminationGracePeriodSeconds := int64(120) + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provisioner-basic", + Namespace: namespace, + }, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + OrganizationName: "acme", + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{ + Name: bootstrapSecret.Name, + Key: coderv1alpha1.DefaultTokenSecretKey, + }, + }, + Key: coderv1alpha1.CoderProvisionerKeySpec{ + Name: "provisioner-key-name", + SecretName: "provisioner-basic-key", + SecretKey: "daemon-key", + }, + Replicas: &replicas, + Tags: map[string]string{"region": "test"}, + Image: "provisioner-image:test", + ExtraArgs: []string{"--test-mode=true"}, + ExtraEnv: []corev1.EnvVar{{Name: "EXTRA_ENV", Value: "extra-value"}}, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "regcred"}}, + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + reconcileProvisioner(t, ctx, reconciler, namespacedName) + reconcileProvisioner(t, ctx, reconciler, namespacedName) + + reconciledProvisioner := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, reconciledProvisioner)) + require.Contains(t, reconciledProvisioner.Finalizers, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + + require.Equal(t, 1, bootstrapClient.provisionerKeyCalls) + require.Equal(t, 0, bootstrapClient.deleteKeyCalls) + + keySecret := &corev1.Secret{} + keySecretName := types.NamespacedName{Name: provisioner.Spec.Key.SecretName, Namespace: provisioner.Namespace} + require.NoError(t, k8sClient.Get(ctx, keySecretName, keySecret)) + require.Equal(t, "provisioner-key-material", string(keySecret.Data[provisioner.Spec.Key.SecretKey])) + requireOwnerReference(t, reconciledProvisioner, keySecret) + + serviceAccount := &corev1.ServiceAccount{} + saNamespacedName := types.NamespacedName{Name: expectedProvisionerServiceAccountName(provisioner.Name), Namespace: provisioner.Namespace} + require.NoError(t, k8sClient.Get(ctx, saNamespacedName, serviceAccount)) + requireOwnerReference(t, reconciledProvisioner, serviceAccount) + + roleName := expectedProvisionerResourceName(provisioner.Name) + role := &rbacv1.Role{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: roleName, Namespace: provisioner.Namespace}, role)) + requireOwnerReference(t, reconciledProvisioner, role) + require.Len(t, role.Rules, 1) + require.ElementsMatch(t, []string{""}, role.Rules[0].APIGroups) + require.ElementsMatch(t, []string{"pods", "persistentvolumeclaims"}, role.Rules[0].Resources) + require.ElementsMatch(t, []string{"get", "list", "watch", "create", "update", "patch", "delete"}, role.Rules[0].Verbs) + + roleBinding := &rbacv1.RoleBinding{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: roleName, Namespace: provisioner.Namespace}, roleBinding)) + requireOwnerReference(t, reconciledProvisioner, roleBinding) + require.Equal(t, rbacv1.GroupName, roleBinding.RoleRef.APIGroup) + require.Equal(t, "Role", roleBinding.RoleRef.Kind) + require.Equal(t, role.Name, roleBinding.RoleRef.Name) + require.Len(t, roleBinding.Subjects, 1) + require.Equal(t, rbacv1.ServiceAccountKind, roleBinding.Subjects[0].Kind) + require.Equal(t, serviceAccount.Name, roleBinding.Subjects[0].Name) + require.Equal(t, provisioner.Namespace, roleBinding.Subjects[0].Namespace) + + deployment := &appsv1.Deployment{} + deploymentName := types.NamespacedName{Name: roleName, Namespace: provisioner.Namespace} + require.NoError(t, k8sClient.Get(ctx, deploymentName, deployment)) + requireOwnerReference(t, reconciledProvisioner, deployment) + + require.NotNil(t, deployment.Spec.Replicas) + require.Equal(t, replicas, *deployment.Spec.Replicas) + require.Equal(t, expectedProvisionerServiceAccountName(provisioner.Name), deployment.Spec.Template.Spec.ServiceAccountName) + require.Equal(t, []corev1.LocalObjectReference{{Name: "regcred"}}, deployment.Spec.Template.Spec.ImagePullSecrets) + require.NotNil(t, deployment.Spec.Template.Spec.TerminationGracePeriodSeconds) + require.Equal(t, terminationGracePeriodSeconds, *deployment.Spec.Template.Spec.TerminationGracePeriodSeconds) + require.NotEmpty(t, deployment.Spec.Template.Annotations["checksum/provisioner-key"]) + + require.Len(t, deployment.Spec.Template.Spec.Containers, 1) + container := deployment.Spec.Template.Spec.Containers[0] + require.Equal(t, "provisioner", container.Name) + require.Equal(t, "provisioner-image:test", container.Image) + require.Equal(t, []string{"provisionerd", "start", "--test-mode=true"}, container.Args) + + envByName := make(map[string]corev1.EnvVar, len(container.Env)) + for _, envVar := range container.Env { + envByName[envVar.Name] = envVar + } + require.Equal(t, "https://coder.example.com", envByName["CODER_URL"].Value) + require.Equal(t, "acme", envByName["CODER_ORGANIZATION"].Value) + require.Equal(t, "extra-value", envByName["EXTRA_ENV"].Value) + keyEnv, ok := envByName["CODER_PROVISIONER_DAEMON_KEY"] + require.True(t, ok) + require.NotNil(t, keyEnv.ValueFrom) + require.NotNil(t, keyEnv.ValueFrom.SecretKeyRef) + require.Equal(t, provisioner.Spec.Key.SecretName, keyEnv.ValueFrom.SecretKeyRef.Name) + require.Equal(t, provisioner.Spec.Key.SecretKey, keyEnv.ValueFrom.SecretKeyRef.Key) + + require.Equal(t, reconciledProvisioner.Generation, reconciledProvisioner.Status.ObservedGeneration) + require.Equal(t, int32(0), reconciledProvisioner.Status.ReadyReplicas) + require.Equal(t, coderv1alpha1.CoderProvisionerPhasePending, reconciledProvisioner.Status.Phase) + require.Equal(t, organizationID.String(), reconciledProvisioner.Status.OrganizationID) + require.Equal(t, provisionerKeyID.String(), reconciledProvisioner.Status.ProvisionerKeyID) + require.Equal(t, "provisioner-key-name", reconciledProvisioner.Status.ProvisionerKeyName) + require.NotNil(t, reconciledProvisioner.Status.SecretRef) + require.Equal(t, provisioner.Spec.Key.SecretName, reconciledProvisioner.Status.SecretRef.Name) + require.Equal(t, provisioner.Spec.Key.SecretKey, reconciledProvisioner.Status.SecretRef.Key) +} + +func TestCoderProvisionerReconciler_ExistingSecret(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(t, ctx, "coderprov-existing") + controlPlane := createTestControlPlane(t, ctx, namespace, "controlplane-existing", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(t, ctx, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisionerName := "provisioner-existing" + secretName := fmt.Sprintf("%s-provisioner-key", provisionerName) + existingSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: namespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + coderv1alpha1.DefaultProvisionerKeySecretKey: []byte("existing-key-material"), + }, + } + require.NoError(t, k8sClient.Create(ctx, existingSecret)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), existingSecret) + }) + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: provisionerName, Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Image: "provisioner-image:test", + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponse: coderbootstrap.EnsureProvisionerKeyResponse{ + OrganizationID: uuid.New(), + KeyID: uuid.New(), + KeyName: provisionerName, + Key: "new-key-material", + }, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + reconcileProvisioner(t, ctx, reconciler, namespacedName) + reconcileProvisioner(t, ctx, reconciler, namespacedName) + + require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) + require.Equal(t, 0, bootstrapClient.deleteKeyCalls) + + reconciledSecret := &corev1.Secret{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: secretName, Namespace: namespace}, reconciledSecret)) + require.Equal(t, "existing-key-material", string(reconciledSecret.Data[coderv1alpha1.DefaultProvisionerKeySecretKey])) + + deployment := &appsv1.Deployment{} + resourceName := expectedProvisionerResourceName(provisioner.Name) + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: resourceName, Namespace: namespace}, deployment)) + require.Len(t, deployment.Spec.Template.Spec.Containers, 1) + + envByName := make(map[string]corev1.EnvVar, len(deployment.Spec.Template.Spec.Containers[0].Env)) + for _, envVar := range deployment.Spec.Template.Spec.Containers[0].Env { + envByName[envVar.Name] = envVar + } + keyEnv, ok := envByName["CODER_PROVISIONER_DAEMON_KEY"] + require.True(t, ok) + require.NotNil(t, keyEnv.ValueFrom) + require.NotNil(t, keyEnv.ValueFrom.SecretKeyRef) + require.Equal(t, secretName, keyEnv.ValueFrom.SecretKeyRef.Name) + require.Equal(t, coderv1alpha1.DefaultProvisionerKeySecretKey, keyEnv.ValueFrom.SecretKeyRef.Key) + + reconciledProvisioner := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, reconciledProvisioner)) + require.Equal(t, provisioner.Name, reconciledProvisioner.Status.ProvisionerKeyName) + require.NotNil(t, reconciledProvisioner.Status.SecretRef) + require.Equal(t, secretName, reconciledProvisioner.Status.SecretRef.Name) + require.Equal(t, coderv1alpha1.DefaultProvisionerKeySecretKey, reconciledProvisioner.Status.SecretRef.Key) +} + +func TestCoderProvisionerReconciler_Deletion(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(t, ctx, "coderprov-delete") + controlPlane := createTestControlPlane(t, ctx, namespace, "controlplane-delete", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(t, ctx, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-delete", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Image: "provisioner-image:test", + Key: coderv1alpha1.CoderProvisionerKeySpec{ + Name: "cleanup-key", + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponse: coderbootstrap.EnsureProvisionerKeyResponse{Key: "provisioner-key-material"}, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + reconcileProvisioner(t, ctx, reconciler, namespacedName) + reconcileProvisioner(t, ctx, reconciler, namespacedName) + require.Equal(t, 1, bootstrapClient.provisionerKeyCalls) + + latest := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, latest)) + require.Contains(t, latest.Finalizers, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + + require.NoError(t, k8sClient.Delete(ctx, latest)) + markedForDeletion := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, markedForDeletion)) + require.False(t, markedForDeletion.DeletionTimestamp.IsZero()) + + reconcileProvisioner(t, ctx, reconciler, namespacedName) + require.Equal(t, 1, bootstrapClient.deleteKeyCalls) + + require.Eventually(t, func() bool { + reconciled := &coderv1alpha1.CoderProvisioner{} + err := k8sClient.Get(ctx, namespacedName, reconciled) + if apierrors.IsNotFound(err) { + return true + } + if err != nil { + t.Logf("get reconciled provisioner: %v", err) + return false + } + + return !controllerutil.ContainsFinalizer(reconciled, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + }, 5*time.Second, 100*time.Millisecond) +} + +func TestCoderProvisionerReconciler_NotFound(t *testing.T) { + t.Parallel() + + reconciler := &controller.CoderProvisionerReconciler{ + Client: k8sClient, + Scheme: scheme, + BootstrapClient: &fakeBootstrapClient{}, + } + + result, err := reconciler.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "does-not-exist", Namespace: "default"}, + }) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) +} + +func TestCoderProvisionerReconciler_NilChecks(t *testing.T) { + t.Parallel() + + request := ctrl.Request{NamespacedName: types.NamespacedName{Name: "test", Namespace: "default"}} + + t.Run("nil client", func(t *testing.T) { + t.Parallel() + + reconciler := &controller.CoderProvisionerReconciler{ + Client: nil, + Scheme: scheme, + BootstrapClient: &fakeBootstrapClient{}, + } + + _, err := reconciler.Reconcile(context.Background(), request) + require.ErrorContains(t, err, "assertion failed: reconciler client must not be nil") + }) + + t.Run("nil scheme", func(t *testing.T) { + t.Parallel() + + reconciler := &controller.CoderProvisionerReconciler{ + Client: k8sClient, + Scheme: nil, + BootstrapClient: &fakeBootstrapClient{}, + } + + _, err := reconciler.Reconcile(context.Background(), request) + require.ErrorContains(t, err, "assertion failed: reconciler scheme must not be nil") + }) + + t.Run("nil bootstrap client", func(t *testing.T) { + t.Parallel() + + reconciler := &controller.CoderProvisionerReconciler{ + Client: k8sClient, + Scheme: scheme, + BootstrapClient: nil, + } + + _, err := reconciler.Reconcile(context.Background(), request) + require.ErrorContains(t, err, "assertion failed: reconciler bootstrap client must not be nil") + }) +} + +func TestCoderProvisionerReconciler_ControlPlaneNotReady(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(t, ctx, "coderprov-cpnotready") + controlPlane := createTestControlPlane(t, ctx, namespace, "controlplane-notready", "") + bootstrapSecret := createBootstrapSecret(t, ctx, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-notready", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{} + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(t, ctx, reconciler, namespacedName) + + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.ErrorContains(t, err, fmt.Sprintf("codercontrolplane %s/%s status.url is empty", controlPlane.Namespace, controlPlane.Name)) + require.Equal(t, ctrl.Result{}, result) + require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) +} diff --git a/internal/controller/workspaceproxy_controller_test.go b/internal/controller/workspaceproxy_controller_test.go index fcaf9596..7a0a7797 100644 --- a/internal/controller/workspaceproxy_controller_test.go +++ b/internal/controller/workspaceproxy_controller_test.go @@ -22,6 +22,13 @@ type fakeBootstrapClient struct { response coderbootstrap.RegisterWorkspaceProxyResponse err error calls int + + // Provisioner key support (for interface compliance). + provisionerKeyResponse coderbootstrap.EnsureProvisionerKeyResponse + provisionerKeyErr error + provisionerKeyCalls int + deleteKeyErr error + deleteKeyCalls int } func (f *fakeBootstrapClient) EnsureWorkspaceProxy(_ context.Context, _ coderbootstrap.RegisterWorkspaceProxyRequest) (coderbootstrap.RegisterWorkspaceProxyResponse, error) { @@ -29,6 +36,16 @@ func (f *fakeBootstrapClient) EnsureWorkspaceProxy(_ context.Context, _ coderboo return f.response, f.err } +func (f *fakeBootstrapClient) EnsureProvisionerKey(_ context.Context, _ coderbootstrap.EnsureProvisionerKeyRequest) (coderbootstrap.EnsureProvisionerKeyResponse, error) { + f.provisionerKeyCalls++ + return f.provisionerKeyResponse, f.provisionerKeyErr +} + +func (f *fakeBootstrapClient) DeleteProvisionerKey(_ context.Context, _, _, _, _ string) error { + f.deleteKeyCalls++ + return f.deleteKeyErr +} + func workspaceProxyResourceName(name string) string { const prefix = "wsproxy-" candidate := prefix + name From d7b06d5b6e1eefa47f69dcb1c68e180b72603f67 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 09:03:29 +0000 Subject: [PATCH 06/38] chore: add CoderProvisioner sample manifest and API reference docs --- .../coder_v1alpha1_coderprovisioner.yaml | 15 +++++++ docs/reference/api/coderprovisioner.md | 45 +++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 61 insertions(+) create mode 100644 config/samples/coder_v1alpha1_coderprovisioner.yaml create mode 100644 docs/reference/api/coderprovisioner.md diff --git a/config/samples/coder_v1alpha1_coderprovisioner.yaml b/config/samples/coder_v1alpha1_coderprovisioner.yaml new file mode 100644 index 00000000..307406c1 --- /dev/null +++ b/config/samples/coder_v1alpha1_coderprovisioner.yaml @@ -0,0 +1,15 @@ +apiVersion: coder.com/v1alpha1 +kind: CoderProvisioner +metadata: + name: coderprovisioner-sample + namespace: default +spec: + controlPlaneRef: + name: codercontrolplane-sample + bootstrap: + credentialsSecretRef: + name: coder-bootstrap-token + key: token + tags: + scope: organization + replicas: 1 diff --git a/docs/reference/api/coderprovisioner.md b/docs/reference/api/coderprovisioner.md new file mode 100644 index 00000000..9e2be183 --- /dev/null +++ b/docs/reference/api/coderprovisioner.md @@ -0,0 +1,45 @@ + + +# `CoderProvisioner` + +## API identity + +- Group/version: `coder.com/v1alpha1` +- Kind: `CoderProvisioner` +- Resource: `coderprovisioners` +- Scope: namespaced + +## Spec + +| Field | Type | Description | +| --- | --- | --- | +| `spec.controlPlaneRef` | `k8s.io/api/core/v1.LocalObjectReference` | ControlPlaneRef identifies which CoderControlPlane instance to join. | +| `spec.organizationName` | `string` | OrganizationName is the Coder organization. Defaults to "default". | +| `spec.bootstrap` | `github.com/coder/coder-k8s/api/v1alpha1.CoderProvisionerBootstrapSpec` | Bootstrap configures credentials for provisioner key management. | +| `spec.key` | `github.com/coder/coder-k8s/api/v1alpha1.CoderProvisionerKeySpec` | Key configures provisioner key naming and secret storage. | +| `spec.replicas` | `int32` | Replicas is the desired number of provisioner pods. | +| `spec.tags` | `map[string]string` | Tags are attached to the provisioner key for job routing. | +| `spec.image` | `string` | Image is the container image. Defaults to the control plane image. | +| `spec.extraArgs` | `[]string` | ExtraArgs are appended after "provisionerd start". | +| `spec.extraEnv` | `[]k8s.io/api/core/v1.EnvVar` | ExtraEnv are injected into the provisioner container. | +| `spec.resources` | `k8s.io/api/core/v1.ResourceRequirements` | Resources for the provisioner container. | +| `spec.imagePullSecrets` | `[]k8s.io/api/core/v1.LocalObjectReference` | ImagePullSecrets are used by the pod to pull private images. | +| `spec.terminationGracePeriodSeconds` | `int64` | TerminationGracePeriodSeconds for the provisioner pods. | + +## Status + +| Field | Type | Description | +| --- | --- | --- | +| `status.observedGeneration` | `int64` | | +| `status.readyReplicas` | `int32` | | +| `status.phase` | `string` | | +| `status.conditions` | `[]metav1.Condition` | | +| `status.organizationID` | `string` | | +| `status.provisionerKeyID` | `string` | | +| `status.provisionerKeyName` | `string` | | +| `status.secretRef` | `github.com/coder/coder-k8s/api/v1alpha1.SecretKeySelector` | | + +## Source + +- Go type: `api/v1alpha1/coderprovisioner_types.go` +- Generated CRD: `config/crd/bases/coder.com_coderprovisioners.yaml` diff --git a/mkdocs.yml b/mkdocs.yml index 47d437d5..b457ed5f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,6 +44,7 @@ nav: - API: # BEGIN GENERATED API NAV - CoderControlPlane: reference/api/codercontrolplane.md + - CoderProvisioner: reference/api/coderprovisioner.md - WorkspaceProxy: reference/api/workspaceproxy.md - CoderTemplate: reference/api/codertemplate.md - CoderWorkspace: reference/api/coderworkspace.md From e6aeadbfc5cc96ff9b921b501ac35032e1da9859 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 09:05:59 +0000 Subject: [PATCH 07/38] chore: fix lint issues in provisioner types and tests --- api/v1alpha1/coderprovisioner_types.go | 12 ++--- .../coderprovisioner_controller_test.go | 50 +++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/api/v1alpha1/coderprovisioner_types.go b/api/v1alpha1/coderprovisioner_types.go index 2f008b26..da79b098 100644 --- a/api/v1alpha1/coderprovisioner_types.go +++ b/api/v1alpha1/coderprovisioner_types.go @@ -65,13 +65,13 @@ type CoderProvisionerSpec struct { // CoderProvisionerStatus defines the observed state of a CoderProvisioner. type CoderProvisionerStatus struct { - ObservedGeneration int64 `json:"observedGeneration,omitempty"` - ReadyReplicas int32 `json:"readyReplicas,omitempty"` - Phase string `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + ReadyReplicas int32 `json:"readyReplicas,omitempty"` + Phase string `json:"phase,omitempty"` Conditions []metav1.Condition `json:"conditions,omitempty"` - OrganizationID string `json:"organizationID,omitempty"` - ProvisionerKeyID string `json:"provisionerKeyID,omitempty"` - ProvisionerKeyName string `json:"provisionerKeyName,omitempty"` + OrganizationID string `json:"organizationID,omitempty"` + ProvisionerKeyID string `json:"provisionerKeyID,omitempty"` + ProvisionerKeyName string `json:"provisionerKeyName,omitempty"` SecretRef *SecretKeySelector `json:"secretRef,omitempty"` } diff --git a/internal/controller/coderprovisioner_controller_test.go b/internal/controller/coderprovisioner_controller_test.go index 40c0f5ec..b1eebe94 100644 --- a/internal/controller/coderprovisioner_controller_test.go +++ b/internal/controller/coderprovisioner_controller_test.go @@ -24,7 +24,7 @@ import ( "github.com/coder/coder-k8s/internal/controller" ) -func createTestNamespace(t *testing.T, ctx context.Context, prefix string) string { +func createTestNamespace(ctx context.Context, t *testing.T, prefix string) string { t.Helper() namespaceName := fmt.Sprintf("%s-%s", prefix, strings.ToLower(uuid.NewString()[:8])) @@ -38,7 +38,7 @@ func createTestNamespace(t *testing.T, ctx context.Context, prefix string) strin } // createTestControlPlane creates a test CoderControlPlane and optionally sets status.url. -func createTestControlPlane(t *testing.T, ctx context.Context, namespace, name, url string) *coderv1alpha1.CoderControlPlane { +func createTestControlPlane(ctx context.Context, t *testing.T, namespace, name, url string) *coderv1alpha1.CoderControlPlane { t.Helper() controlPlane := &coderv1alpha1.CoderControlPlane{ @@ -60,7 +60,7 @@ func createTestControlPlane(t *testing.T, ctx context.Context, namespace, name, } // createBootstrapSecret creates the bootstrap credentials secret used by provisioner reconciliation. -func createBootstrapSecret(t *testing.T, ctx context.Context, namespace, name, key, value string) *corev1.Secret { +func createBootstrapSecret(ctx context.Context, t *testing.T, namespace, name, key, value string) *corev1.Secret { t.Helper() if key == "" { @@ -103,7 +103,7 @@ func expectedProvisionerServiceAccountName(name string) string { return fmt.Sprintf("%s-provisioner", name) } -func reconcileProvisioner(t *testing.T, ctx context.Context, reconciler *controller.CoderProvisionerReconciler, namespacedName types.NamespacedName) { +func reconcileProvisioner(ctx context.Context, t *testing.T, reconciler *controller.CoderProvisionerReconciler, namespacedName types.NamespacedName) { t.Helper() result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) @@ -111,7 +111,7 @@ func reconcileProvisioner(t *testing.T, ctx context.Context, reconciler *control require.Equal(t, ctrl.Result{}, result) } -func requireOwnerReference(t *testing.T, owner metav1.Object, child metav1.Object) { +func requireOwnerReference(t *testing.T, owner, child metav1.Object) { t.Helper() ownerReferences := child.GetOwnerReferences() @@ -130,9 +130,9 @@ func TestCoderProvisionerReconciler_BasicCreate(t *testing.T) { t.Parallel() ctx := context.Background() - namespace := createTestNamespace(t, ctx, "coderprov-basic") - controlPlane := createTestControlPlane(t, ctx, namespace, "controlplane-basic", "https://coder.example.com") - bootstrapSecret := createBootstrapSecret(t, ctx, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + namespace := createTestNamespace(ctx, t, "coderprov-basic") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-basic", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") organizationID := uuid.New() provisionerKeyID := uuid.New() @@ -186,8 +186,8 @@ func TestCoderProvisionerReconciler_BasicCreate(t *testing.T) { }) namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} - reconcileProvisioner(t, ctx, reconciler, namespacedName) - reconcileProvisioner(t, ctx, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) reconciledProvisioner := &coderv1alpha1.CoderProvisioner{} require.NoError(t, k8sClient.Get(ctx, namespacedName, reconciledProvisioner)) @@ -275,9 +275,9 @@ func TestCoderProvisionerReconciler_ExistingSecret(t *testing.T) { t.Parallel() ctx := context.Background() - namespace := createTestNamespace(t, ctx, "coderprov-existing") - controlPlane := createTestControlPlane(t, ctx, namespace, "controlplane-existing", "https://coder.example.com") - bootstrapSecret := createBootstrapSecret(t, ctx, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + namespace := createTestNamespace(ctx, t, "coderprov-existing") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-existing", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") provisionerName := "provisioner-existing" secretName := fmt.Sprintf("%s-provisioner-key", provisionerName) @@ -319,8 +319,8 @@ func TestCoderProvisionerReconciler_ExistingSecret(t *testing.T) { reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} - reconcileProvisioner(t, ctx, reconciler, namespacedName) - reconcileProvisioner(t, ctx, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) require.Equal(t, 0, bootstrapClient.deleteKeyCalls) @@ -357,9 +357,9 @@ func TestCoderProvisionerReconciler_Deletion(t *testing.T) { t.Parallel() ctx := context.Background() - namespace := createTestNamespace(t, ctx, "coderprov-delete") - controlPlane := createTestControlPlane(t, ctx, namespace, "controlplane-delete", "https://coder.example.com") - bootstrapSecret := createBootstrapSecret(t, ctx, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + namespace := createTestNamespace(ctx, t, "coderprov-delete") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-delete", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") provisioner := &coderv1alpha1.CoderProvisioner{ ObjectMeta: metav1.ObjectMeta{Name: "provisioner-delete", Namespace: namespace}, @@ -382,8 +382,8 @@ func TestCoderProvisionerReconciler_Deletion(t *testing.T) { reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} - reconcileProvisioner(t, ctx, reconciler, namespacedName) - reconcileProvisioner(t, ctx, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) require.Equal(t, 1, bootstrapClient.provisionerKeyCalls) latest := &coderv1alpha1.CoderProvisioner{} @@ -395,7 +395,7 @@ func TestCoderProvisionerReconciler_Deletion(t *testing.T) { require.NoError(t, k8sClient.Get(ctx, namespacedName, markedForDeletion)) require.False(t, markedForDeletion.DeletionTimestamp.IsZero()) - reconcileProvisioner(t, ctx, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) require.Equal(t, 1, bootstrapClient.deleteKeyCalls) require.Eventually(t, func() bool { @@ -478,9 +478,9 @@ func TestCoderProvisionerReconciler_ControlPlaneNotReady(t *testing.T) { t.Parallel() ctx := context.Background() - namespace := createTestNamespace(t, ctx, "coderprov-cpnotready") - controlPlane := createTestControlPlane(t, ctx, namespace, "controlplane-notready", "") - bootstrapSecret := createBootstrapSecret(t, ctx, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + namespace := createTestNamespace(ctx, t, "coderprov-cpnotready") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-notready", "") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") provisioner := &coderv1alpha1.CoderProvisioner{ ObjectMeta: metav1.ObjectMeta{Name: "provisioner-notready", Namespace: namespace}, @@ -500,7 +500,7 @@ func TestCoderProvisionerReconciler_ControlPlaneNotReady(t *testing.T) { reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} - reconcileProvisioner(t, ctx, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) require.ErrorContains(t, err, fmt.Sprintf("codercontrolplane %s/%s status.url is empty", controlPlane.Namespace, controlPlane.Name)) From 59ba90c759e25f417b86637aaf937a32bef27516 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 09:08:20 +0000 Subject: [PATCH 08/38] chore: regenerate RBAC manifest for provisioner controller --- config/rbac/role.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6a688a75..58168493 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -15,6 +15,7 @@ rules: - "" resources: - secrets + - serviceaccounts - services verbs: - create @@ -40,6 +41,7 @@ rules: - coder.com resources: - codercontrolplanes + - coderprovisioners - workspaceproxies verbs: - create @@ -53,6 +55,7 @@ rules: - coder.com resources: - codercontrolplanes/finalizers + - coderprovisioners/finalizers - workspaceproxies/finalizers verbs: - update @@ -60,6 +63,7 @@ rules: - coder.com resources: - codercontrolplanes/status + - coderprovisioners/status - workspaceproxies/status verbs: - get @@ -77,3 +81,16 @@ rules: - patch - update - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + - roles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch From d332da7deb869b2d330071ff485bd63613405cac Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 09:10:30 +0000 Subject: [PATCH 09/38] chore: add provisioner terms to cspell dictionary --- .cspell.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.cspell.json b/.cspell.json index 3075f298..76865b83 100644 --- a/.cspell.json +++ b/.cspell.json @@ -18,7 +18,10 @@ "codertemplates", "coderworkspace", "coderworkspaces", + "coderprovisioner", + "coderprovisioners", "controllerapp", + "provisionerd", "workspaceproxy", "workspaceproxies", "derp", From f1eb5bd8cbc0cd023b334e74989addc4f1d3b836 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 09:18:28 +0000 Subject: [PATCH 10/38] fix: make deletion finalizer best-effort and handle key rotation on secret loss Addresses Codex review feedback: - Deletion finalizer now gracefully handles missing CoderControlPlane or bootstrap credentials (common during namespace teardown). Logs a warning and proceeds to finalizer removal instead of blocking. - When the provisioner key secret is deleted but the key already exists in coderd, the controller now rotates the key (delete + recreate) to obtain fresh plaintext material for secret recovery. - Added test for best-effort deletion when control plane is deleted first. --- .../controller/coderprovisioner_controller.go | 89 +++++++++++++++---- .../coderprovisioner_controller_test.go | 60 +++++++++++++ 2 files changed, 130 insertions(+), 19 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index a01e52cd..f79ee70d 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -137,8 +137,43 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req provisionerKeyName = response.KeyName } keyMaterial = response.Key + + // If the key already exists in coderd (e.g. the K8s secret was + // deleted), coderd won't return plaintext again. Rotate the key + // by deleting and recreating it to obtain fresh material. if keyMaterial == "" { - return ctrl.Result{}, fmt.Errorf("provisioner key %q exists in coderd but no key material is available to create secret %q", keyName, keySecretName) + log := ctrl.LoggerFrom(ctx) + log.Info("provisioner key exists in coderd but secret is missing, rotating key to recover", + "keyName", keyName, "secretName", keySecretName) + + if deleteErr := r.BootstrapClient.DeleteProvisionerKey( + ctx, controlPlane.Status.URL, sessionToken, organizationName, keyName, + ); deleteErr != nil { + return ctrl.Result{}, fmt.Errorf("delete stale provisioner key %q for rotation: %w", keyName, deleteErr) + } + rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if rotateErr != nil { + return ctrl.Result{}, fmt.Errorf("recreate provisioner key %q after rotation: %w", keyName, rotateErr) + } + if rotated.OrganizationID != uuid.Nil { + organizationID = rotated.OrganizationID.String() + } + if rotated.KeyID != uuid.Nil { + provisionerKeyID = rotated.KeyID.String() + } + if rotated.KeyName != "" { + provisionerKeyName = rotated.KeyName + } + keyMaterial = rotated.Key + if keyMaterial == "" { + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key %q returned empty key material after rotation", keyName) + } } } @@ -196,26 +231,42 @@ func (r *CoderProvisionerReconciler) reconcileDeletion(ctx context.Context, prov return ctrl.Result{}, nil } - controlPlane, err := r.fetchControlPlane(ctx, provisioner) - if err != nil { - return ctrl.Result{}, err - } - - sessionToken, err := r.readBootstrapSessionToken(ctx, provisioner) - if err != nil { - return ctrl.Result{}, err - } - + log := ctrl.LoggerFrom(ctx) organizationName := provisionerOrganizationName(provisioner.Spec.OrganizationName) keyName, _, _ := provisionerKeyConfig(provisioner) - if err := r.BootstrapClient.DeleteProvisionerKey( - ctx, - controlPlane.Status.URL, - sessionToken, - organizationName, - keyName, - ); err != nil { - return ctrl.Result{}, fmt.Errorf("delete provisioner key %q: %w", keyName, err) + + // Best-effort remote key cleanup: if the referenced control plane or + // bootstrap credentials are already gone (common during namespace + // teardown), log a warning and proceed to finalizer removal so the CR + // does not get stuck in Terminating. + controlPlane, err := r.fetchControlPlane(ctx, provisioner) + if err != nil { + if apierrors.IsNotFound(err) { + log.Info("referenced CoderControlPlane not found during deletion, skipping remote key cleanup", + "controlPlaneRef", provisioner.Spec.ControlPlaneRef.Name) + } else { + return ctrl.Result{}, err + } + } else { + sessionToken, tokenErr := r.readBootstrapSessionToken(ctx, provisioner) + if tokenErr != nil { + if apierrors.IsNotFound(tokenErr) { + log.Info("bootstrap credentials secret not found during deletion, skipping remote key cleanup", + "credentialsSecretRef", provisioner.Spec.Bootstrap.CredentialsSecretRef.Name) + } else { + return ctrl.Result{}, tokenErr + } + } else { + if deleteErr := r.BootstrapClient.DeleteProvisionerKey( + ctx, + controlPlane.Status.URL, + sessionToken, + organizationName, + keyName, + ); deleteErr != nil { + return ctrl.Result{}, fmt.Errorf("delete provisioner key %q: %w", keyName, deleteErr) + } + } } controllerutil.RemoveFinalizer(provisioner, coderv1alpha1.ProvisionerKeyCleanupFinalizer) diff --git a/internal/controller/coderprovisioner_controller_test.go b/internal/controller/coderprovisioner_controller_test.go index b1eebe94..4d53f76e 100644 --- a/internal/controller/coderprovisioner_controller_test.go +++ b/internal/controller/coderprovisioner_controller_test.go @@ -413,6 +413,66 @@ func TestCoderProvisionerReconciler_Deletion(t *testing.T) { }, 5*time.Second, 100*time.Millisecond) } +func TestCoderProvisionerReconciler_DeletionControlPlaneGone(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-delete-cpgone") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-cpgone", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-cpgone", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Image: "provisioner-image:test", + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponse: coderbootstrap.EnsureProvisionerKeyResponse{Key: "provisioner-key-material"}, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + reconcileProvisioner(ctx, t, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) + + // Delete the control plane first (common in namespace teardown). + require.NoError(t, k8sClient.Delete(ctx, controlPlane)) + + // Now delete the provisioner — the finalizer should still be removed + // even though the control plane is gone (best-effort cleanup). + latest := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, latest)) + require.NoError(t, k8sClient.Delete(ctx, latest)) + + reconcileProvisioner(ctx, t, reconciler, namespacedName) + + // DeleteProvisionerKey should NOT have been called since the control + // plane was already gone. + require.Equal(t, 0, bootstrapClient.deleteKeyCalls) + + // The finalizer should still be removed. + require.Eventually(t, func() bool { + reconciled := &coderv1alpha1.CoderProvisioner{} + err := k8sClient.Get(ctx, namespacedName, reconciled) + if apierrors.IsNotFound(err) { + return true + } + if err != nil { + t.Logf("get reconciled provisioner: %v", err) + return false + } + + return !controllerutil.ContainsFinalizer(reconciled, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + }, 5*time.Second, 100*time.Millisecond) +} + func TestCoderProvisionerReconciler_NotFound(t *testing.T) { t.Parallel() From 64281fc52c92fe5b4174d697867b1f82216287ea Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 09:23:30 +0000 Subject: [PATCH 11/38] fix: use persisted key name for cleanup and skip deletion on unreachable control plane Addresses additional Codex review feedback: - Finalizer cleanup now uses the key name from status (reflecting what was actually created in coderd) rather than the current spec value, preventing orphaned keys when spec.key.name is edited after creation. - All fetchControlPlane errors during deletion are treated as non-blocking (not just NotFound), handling the case where the control plane exists but has an empty status.url. --- .../controller/coderprovisioner_controller.go | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index f79ee70d..f3a7e555 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -233,19 +233,31 @@ func (r *CoderProvisionerReconciler) reconcileDeletion(ctx context.Context, prov log := ctrl.LoggerFrom(ctx) organizationName := provisionerOrganizationName(provisioner.Spec.OrganizationName) - keyName, _, _ := provisionerKeyConfig(provisioner) - // Best-effort remote key cleanup: if the referenced control plane or - // bootstrap credentials are already gone (common during namespace - // teardown), log a warning and proceed to finalizer removal so the CR - // does not get stuck in Terminating. + // Prefer the key name persisted in status (reflects what was actually + // created in coderd) over the current spec value, which may have been + // edited after initial provisioning. + keyName := provisioner.Status.ProvisionerKeyName + if keyName == "" { + keyName, _, _ = provisionerKeyConfig(provisioner) + } + + // Best-effort remote key cleanup: if the referenced control plane, + // its URL, or the bootstrap credentials are unavailable (common during + // namespace teardown or when the control plane was never ready), log a + // warning and proceed to finalizer removal so the CR does not get + // stuck in Terminating. controlPlane, err := r.fetchControlPlane(ctx, provisioner) if err != nil { + // fetchControlPlane returns a plain error (not apierrors) when + // status.url is empty, and wraps the k8s API error with %w when + // the object is missing. Treat both cases as non-blocking. if apierrors.IsNotFound(err) { log.Info("referenced CoderControlPlane not found during deletion, skipping remote key cleanup", "controlPlaneRef", provisioner.Spec.ControlPlaneRef.Name) } else { - return ctrl.Result{}, err + log.Info("unable to reach referenced CoderControlPlane during deletion, skipping remote key cleanup", + "controlPlaneRef", provisioner.Spec.ControlPlaneRef.Name, "error", err) } } else { sessionToken, tokenErr := r.readBootstrapSessionToken(ctx, provisioner) From 266d39ec004524e390fcf8ba166a80649b537b08 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 09:29:40 +0000 Subject: [PATCH 12/38] fix: fully best-effort deletion cleanup and use persisted org identity Addresses Codex round 3 feedback: - All token-read errors during deletion are now best-effort (not just NotFound). Handles wrong key names, empty values, etc. - Organization identity for finalizer cleanup uses status.OrganizationID (which may contain the resolved UUID) before falling back to spec. --- .../controller/coderprovisioner_controller.go | 38 ++++++++----------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index f3a7e555..405b9f9a 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -232,42 +232,34 @@ func (r *CoderProvisionerReconciler) reconcileDeletion(ctx context.Context, prov } log := ctrl.LoggerFrom(ctx) - organizationName := provisionerOrganizationName(provisioner.Spec.OrganizationName) - // Prefer the key name persisted in status (reflects what was actually - // created in coderd) over the current spec value, which may have been + // Prefer persisted identity from status (reflects what was actually + // created in coderd) over the current spec values, which may have been // edited after initial provisioning. + organizationName := provisioner.Status.OrganizationID + if organizationName == "" { + organizationName = provisionerOrganizationName(provisioner.Spec.OrganizationName) + } keyName := provisioner.Status.ProvisionerKeyName if keyName == "" { keyName, _, _ = provisionerKeyConfig(provisioner) } // Best-effort remote key cleanup: if the referenced control plane, - // its URL, or the bootstrap credentials are unavailable (common during - // namespace teardown or when the control plane was never ready), log a - // warning and proceed to finalizer removal so the CR does not get - // stuck in Terminating. + // its URL, bootstrap credentials, or any other prerequisite is + // unavailable, log a warning and proceed to finalizer removal so the + // CR does not get stuck in Terminating. This is common during + // namespace teardown, when the control plane was never ready, or + // when credentials were misconfigured. controlPlane, err := r.fetchControlPlane(ctx, provisioner) if err != nil { - // fetchControlPlane returns a plain error (not apierrors) when - // status.url is empty, and wraps the k8s API error with %w when - // the object is missing. Treat both cases as non-blocking. - if apierrors.IsNotFound(err) { - log.Info("referenced CoderControlPlane not found during deletion, skipping remote key cleanup", - "controlPlaneRef", provisioner.Spec.ControlPlaneRef.Name) - } else { - log.Info("unable to reach referenced CoderControlPlane during deletion, skipping remote key cleanup", - "controlPlaneRef", provisioner.Spec.ControlPlaneRef.Name, "error", err) - } + log.Info("unable to reach referenced CoderControlPlane during deletion, skipping remote key cleanup", + "controlPlaneRef", provisioner.Spec.ControlPlaneRef.Name, "error", err) } else { sessionToken, tokenErr := r.readBootstrapSessionToken(ctx, provisioner) if tokenErr != nil { - if apierrors.IsNotFound(tokenErr) { - log.Info("bootstrap credentials secret not found during deletion, skipping remote key cleanup", - "credentialsSecretRef", provisioner.Spec.Bootstrap.CredentialsSecretRef.Name) - } else { - return ctrl.Result{}, tokenErr - } + log.Info("unable to read bootstrap credentials during deletion, skipping remote key cleanup", + "credentialsSecretRef", provisioner.Spec.Bootstrap.CredentialsSecretRef.Name, "error", tokenErr) } else { if deleteErr := r.BootstrapClient.DeleteProvisionerKey( ctx, From 009884be9027159022ef5ca2b2d1a3716d2ba17e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 09:37:48 +0000 Subject: [PATCH 13/38] fix: fully best-effort delete call and validate secret data key Codex round 4: - DeleteProvisionerKey errors during deletion are now logged and skipped (best-effort), preventing transient API failures from blocking finalizer removal. - Secret existence check now also validates the configured data key is present and non-empty. A secret with a missing or empty key triggers key rotation/recovery instead of silently proceeding. --- .../controller/coderprovisioner_controller.go | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 405b9f9a..0de4164b 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -97,15 +97,18 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, err } + // Check whether a usable provisioner key secret already exists. + // The secret is considered "usable" only if the Secret object exists + // AND it contains a non-empty value at the configured data key. secretNamespacedName := types.NamespacedName{Name: keySecretName, Namespace: provisioner.Namespace} existingSecret := &corev1.Secret{} - secretExists := true + secretUsable := false if err := r.Get(ctx, secretNamespacedName, existingSecret); err != nil { - if apierrors.IsNotFound(err) { - secretExists = false - } else { + if !apierrors.IsNotFound(err) { return ctrl.Result{}, fmt.Errorf("get provisioner key secret %s: %w", secretNamespacedName, err) } + } else { + secretUsable = len(existingSecret.Data[keySecretKey]) > 0 } organizationID := provisioner.Status.OrganizationID @@ -116,7 +119,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req } keyMaterial := "" - if !secretExists { + if !secretUsable { response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ CoderURL: controlPlane.Status.URL, SessionToken: sessionToken, @@ -268,7 +271,12 @@ func (r *CoderProvisionerReconciler) reconcileDeletion(ctx context.Context, prov organizationName, keyName, ); deleteErr != nil { - return ctrl.Result{}, fmt.Errorf("delete provisioner key %q: %w", keyName, deleteErr) + // Treat key deletion failures as best-effort so the + // finalizer is still removed. Transient errors, auth + // issues, or org-lookup failures should not block CR + // cleanup. + log.Info("failed to delete remote provisioner key during deletion, proceeding with finalizer removal", + "keyName", keyName, "error", deleteErr) } } } From aa09b30edbffb500a4c816a604bcee8e91330da9 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 09:45:28 +0000 Subject: [PATCH 14/38] controller: watch owned Role and RoleBinding --- internal/controller/coderprovisioner_controller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 0de4164b..0ff833f1 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -640,6 +640,8 @@ func (r *CoderProvisionerReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&appsv1.Deployment{}). Owns(&corev1.Secret{}). Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). Named("coderprovisioner"). Complete(r) } From f2118a3c4743d1a62ed49792ec741e70b1d44956 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 09:52:04 +0000 Subject: [PATCH 15/38] fix: use org name for key deletion and truncate SA names --- .../controller/coderprovisioner_controller.go | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 0ff833f1..39903ab7 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -236,13 +236,9 @@ func (r *CoderProvisionerReconciler) reconcileDeletion(ctx context.Context, prov log := ctrl.LoggerFrom(ctx) - // Prefer persisted identity from status (reflects what was actually - // created in coderd) over the current spec values, which may have been - // edited after initial provisioning. - organizationName := provisioner.Status.OrganizationID - if organizationName == "" { - organizationName = provisionerOrganizationName(provisioner.Spec.OrganizationName) - } + // DeleteProvisionerKey resolves organizations by name, so always pass the + // spec-derived organization name here instead of the persisted organization ID. + organizationName := provisionerOrganizationName(provisioner.Spec.OrganizationName) keyName := provisioner.Status.ProvisionerKeyName if keyName == "" { keyName, _, _ = provisionerKeyConfig(provisioner) @@ -688,7 +684,20 @@ func provisionerInstanceLabelValue(name string) string { } func provisionerServiceAccountName(name string) string { - return fmt.Sprintf("%s%s", name, provisionerServiceAccountSuffix) + candidate := fmt.Sprintf("%s%s", name, provisionerServiceAccountSuffix) + if len(candidate) <= 63 { + return candidate + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 63 - len(provisionerServiceAccountSuffix) - len(suffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s-%s%s", name[:available], suffix, provisionerServiceAccountSuffix) } func provisionerOrganizationName(name string) string { From 90a05eb335b0b81340cb18f0ec54731284e88a46 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 11:09:00 +0000 Subject: [PATCH 16/38] api: enrich CoderProvisioner status metadata --- api/v1alpha1/coderprovisioner_types.go | 42 +++++++++++++++---- .../bases/coder.com_coderprovisioners.yaml | 40 +++++++++++++++++- docs/reference/api/coderprovisioner.md | 18 ++++---- 3 files changed, 82 insertions(+), 18 deletions(-) diff --git a/api/v1alpha1/coderprovisioner_types.go b/api/v1alpha1/coderprovisioner_types.go index da79b098..7defd30c 100644 --- a/api/v1alpha1/coderprovisioner_types.go +++ b/api/v1alpha1/coderprovisioner_types.go @@ -11,6 +11,13 @@ const ( // CoderProvisionerPhaseReady indicates at least one provisioner pod is ready. CoderProvisionerPhaseReady = "Ready" + // Condition types for CoderProvisioner status. + CoderProvisionerConditionControlPlaneReady = "ControlPlaneReady" + CoderProvisionerConditionBootstrapSecretReady = "BootstrapSecretReady" + CoderProvisionerConditionProvisionerKeyReady = "ProvisionerKeyReady" + CoderProvisionerConditionProvisionerKeySecretReady = "ProvisionerKeySecretReady" + CoderProvisionerConditionDeploymentReady = "DeploymentReady" + // DefaultProvisionerKeySecretKey is the default data key for provisioner key secrets. DefaultProvisionerKeySecretKey = "key" @@ -28,10 +35,13 @@ type CoderProvisionerBootstrapSpec struct { // CoderProvisionerKeySpec configures provisioner key naming and storage. type CoderProvisionerKeySpec struct { // Name is the provisioner key name in coderd. Defaults to the CR name. + // +kubebuilder:validation:MaxLength=128 Name string `json:"name,omitempty"` // SecretName is the Kubernetes Secret to store the key. Defaults to "{crName}-provisioner-key". + // +kubebuilder:validation:MaxLength=253 SecretName string `json:"secretName,omitempty"` // SecretKey is the data key in the Secret. Defaults to "key". + // +kubebuilder:validation:MaxLength=253 SecretKey string `json:"secretKey,omitempty"` } @@ -40,6 +50,7 @@ type CoderProvisionerSpec struct { // ControlPlaneRef identifies which CoderControlPlane instance to join. ControlPlaneRef corev1.LocalObjectReference `json:"controlPlaneRef"` // OrganizationName is the Coder organization. Defaults to "default". + // +kubebuilder:validation:MaxLength=128 OrganizationName string `json:"organizationName,omitempty"` // Bootstrap configures credentials for provisioner key management. Bootstrap CoderProvisionerBootstrapSpec `json:"bootstrap"` @@ -65,20 +76,35 @@ type CoderProvisionerSpec struct { // CoderProvisionerStatus defines the observed state of a CoderProvisioner. type CoderProvisionerStatus struct { - ObservedGeneration int64 `json:"observedGeneration,omitempty"` - ReadyReplicas int32 `json:"readyReplicas,omitempty"` - Phase string `json:"phase,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` - OrganizationID string `json:"organizationID,omitempty"` - ProvisionerKeyID string `json:"provisionerKeyID,omitempty"` - ProvisionerKeyName string `json:"provisionerKeyName,omitempty"` - SecretRef *SecretKeySelector `json:"secretRef,omitempty"` + // ObservedGeneration tracks the spec generation this status reflects. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + // ReadyReplicas is the number of ready pods observed in the deployment. + ReadyReplicas int32 `json:"readyReplicas,omitempty"` + // Phase is a high-level readiness indicator. + Phase string `json:"phase,omitempty"` + // Conditions are Kubernetes-standard conditions for this resource. + Conditions []metav1.Condition `json:"conditions,omitempty"` + // OrganizationID is the organization ID last applied to the provisioner key. + OrganizationID string `json:"organizationID,omitempty"` + // OrganizationName is the organization name last applied to the provisioner key. + OrganizationName string `json:"organizationName,omitempty"` + // ProvisionerKeyID is the provisioner key ID last applied in coderd. + ProvisionerKeyID string `json:"provisionerKeyID,omitempty"` + // ProvisionerKeyName is the provisioner key name last applied in coderd. + ProvisionerKeyName string `json:"provisionerKeyName,omitempty"` + // TagsHash is a deterministic hash of spec.tags last applied to the provisioner key. + TagsHash string `json:"tagsHash,omitempty"` + // SecretRef references the provisioner key secret data currently in use. + SecretRef *SecretKeySelector `json:"secretRef,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Namespaced // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Replicas",type=integer,JSONPath=`.status.readyReplicas` +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // CoderProvisioner is the schema for Coder external provisioner daemon resources. type CoderProvisioner struct { diff --git a/config/crd/bases/coder.com_coderprovisioners.yaml b/config/crd/bases/coder.com_coderprovisioners.yaml index 37144a19..84113a7f 100644 --- a/config/crd/bases/coder.com_coderprovisioners.yaml +++ b/config/crd/bases/coder.com_coderprovisioners.yaml @@ -14,7 +14,17 @@ spec: singular: coderprovisioner scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.readyReplicas + name: Replicas + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: description: CoderProvisioner is the schema for Coder external provisioner @@ -268,19 +278,23 @@ spec: name: description: Name is the provisioner key name in coderd. Defaults to the CR name. + maxLength: 128 type: string secretKey: description: SecretKey is the data key in the Secret. Defaults to "key". + maxLength: 253 type: string secretName: description: SecretName is the Kubernetes Secret to store the key. Defaults to "{crName}-provisioner-key". + maxLength: 253 type: string type: object organizationName: description: OrganizationName is the Coder organization. Defaults to "default". + maxLength: 128 type: string replicas: description: Replicas is the desired number of provisioner pods. @@ -362,6 +376,8 @@ spec: description: CoderProvisionerStatus defines the observed state of a CoderProvisioner. properties: conditions: + description: Conditions are Kubernetes-standard conditions for this + resource. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -418,21 +434,37 @@ spec: type: object type: array observedGeneration: + description: ObservedGeneration tracks the spec generation this status + reflects. format: int64 type: integer organizationID: + description: OrganizationID is the organization ID last applied to + the provisioner key. + type: string + organizationName: + description: OrganizationName is the organization name last applied + to the provisioner key. type: string phase: + description: Phase is a high-level readiness indicator. type: string provisionerKeyID: + description: ProvisionerKeyID is the provisioner key ID last applied + in coderd. type: string provisionerKeyName: + description: ProvisionerKeyName is the provisioner key name last applied + in coderd. type: string readyReplicas: + description: ReadyReplicas is the number of ready pods observed in + the deployment. format: int32 type: integer secretRef: - description: SecretKeySelector identifies a key in a Secret. + description: SecretRef references the provisioner key secret data + currently in use. properties: key: description: Key is the key inside the Secret data map. @@ -443,6 +475,10 @@ spec: required: - name type: object + tagsHash: + description: TagsHash is a deterministic hash of spec.tags last applied + to the provisioner key. + type: string type: object type: object served: true diff --git a/docs/reference/api/coderprovisioner.md b/docs/reference/api/coderprovisioner.md index 9e2be183..b5754831 100644 --- a/docs/reference/api/coderprovisioner.md +++ b/docs/reference/api/coderprovisioner.md @@ -30,14 +30,16 @@ | Field | Type | Description | | --- | --- | --- | -| `status.observedGeneration` | `int64` | | -| `status.readyReplicas` | `int32` | | -| `status.phase` | `string` | | -| `status.conditions` | `[]metav1.Condition` | | -| `status.organizationID` | `string` | | -| `status.provisionerKeyID` | `string` | | -| `status.provisionerKeyName` | `string` | | -| `status.secretRef` | `github.com/coder/coder-k8s/api/v1alpha1.SecretKeySelector` | | +| `status.observedGeneration` | `int64` | ObservedGeneration tracks the spec generation this status reflects. | +| `status.readyReplicas` | `int32` | ReadyReplicas is the number of ready pods observed in the deployment. | +| `status.phase` | `string` | Phase is a high-level readiness indicator. | +| `status.conditions` | `[]metav1.Condition` | Conditions are Kubernetes-standard conditions for this resource. | +| `status.organizationID` | `string` | OrganizationID is the organization ID last applied to the provisioner key. | +| `status.organizationName` | `string` | OrganizationName is the organization name last applied to the provisioner key. | +| `status.provisionerKeyID` | `string` | ProvisionerKeyID is the provisioner key ID last applied in coderd. | +| `status.provisionerKeyName` | `string` | ProvisionerKeyName is the provisioner key name last applied in coderd. | +| `status.tagsHash` | `string` | TagsHash is a deterministic hash of spec.tags last applied to the provisioner key. | +| `status.secretRef` | `github.com/coder/coder-k8s/api/v1alpha1.SecretKeySelector` | SecretRef references the provisioner key secret data currently in use. | ## Source From 606345e31941a437091829eba8e405a8f6766de3 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 11:16:05 +0000 Subject: [PATCH 17/38] controller: add provisioner drift rotation and conditions --- .../controller/coderprovisioner_controller.go | 194 ++++++++++++++++-- 1 file changed, 175 insertions(+), 19 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 39903ab7..f52102a5 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -6,13 +6,14 @@ import ( "fmt" "hash/fnv" "maps" + "slices" "github.com/google/uuid" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -86,16 +87,53 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req controlPlane, err := r.fetchControlPlane(ctx, provisioner) if err != nil { + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionControlPlaneReady, + metav1.ConditionFalse, + "ControlPlaneUnavailable", + fmt.Sprintf("Failed to fetch control plane: %v", err), + ) + _ = r.Status().Update(ctx, provisioner) return ctrl.Result{}, err } + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionControlPlaneReady, + metav1.ConditionTrue, + "ControlPlaneAvailable", + "Referenced control plane is available and has a URL", + ) organizationName := provisionerOrganizationName(provisioner.Spec.OrganizationName) keyName, keySecretName, keySecretKey := provisionerKeyConfig(provisioner) sessionToken, err := r.readBootstrapSessionToken(ctx, provisioner) if err != nil { + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady, + metav1.ConditionFalse, + "BootstrapSecretUnavailable", + fmt.Sprintf("Failed to read bootstrap credentials: %v", err), + ) + _ = r.Status().Update(ctx, provisioner) return ctrl.Result{}, err } + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady, + metav1.ConditionTrue, + "BootstrapSecretAvailable", + "Bootstrap credentials secret is available", + ) + + desiredTagsHash := hashProvisionerTags(provisioner.Spec.Tags) + status := provisioner.Status + orgDrift := status.OrganizationName != "" && status.OrganizationName != organizationName + keyNameDrift := status.ProvisionerKeyName != "" && status.ProvisionerKeyName != keyName + tagsDrift := status.TagsHash != "" && status.TagsHash != desiredTagsHash + driftDetected := orgDrift || keyNameDrift || tagsDrift // Check whether a usable provisioner key secret already exists. // The secret is considered "usable" only if the Secret object exists @@ -118,8 +156,56 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req provisionerKeyName = keyName } + log := ctrl.LoggerFrom(ctx) keyMaterial := "" - if !secretUsable { + if driftDetected { + log.Info("spec drift detected, rotating provisioner key", + "orgDrift", orgDrift, "keyNameDrift", keyNameDrift, "tagsDrift", tagsDrift) + + oldOrg := provisioner.Status.OrganizationName + if oldOrg == "" { + oldOrg = organizationName + } + oldKeyName := provisioner.Status.ProvisionerKeyName + if oldKeyName == "" { + oldKeyName = keyName + } + + if deleteErr := r.BootstrapClient.DeleteProvisionerKey( + ctx, + controlPlane.Status.URL, + sessionToken, + oldOrg, + oldKeyName, + ); deleteErr != nil { + log.Info("failed to delete old provisioner key during drift rotation, creating new key anyway", + "oldKeyName", oldKeyName, "error", deleteErr) + } + + response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if ensureErr != nil { + return ctrl.Result{}, fmt.Errorf("ensure provisioner key %q: %w", keyName, ensureErr) + } + if response.OrganizationID != uuid.Nil { + organizationID = response.OrganizationID.String() + } + if response.KeyID != uuid.Nil { + provisionerKeyID = response.KeyID.String() + } + if response.KeyName != "" { + provisionerKeyName = response.KeyName + } + keyMaterial = response.Key + if keyMaterial == "" { + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key returned empty material after drift rotation") + } + } else if !secretUsable { response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ CoderURL: controlPlane.Status.URL, SessionToken: sessionToken, @@ -145,7 +231,6 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req // deleted), coderd won't return plaintext again. Rotate the key // by deleting and recreating it to obtain fresh material. if keyMaterial == "" { - log := ctrl.LoggerFrom(ctx) log.Info("provisioner key exists in coderd but secret is missing, rotating key to recover", "keyName", keyName, "secretName", keySecretName) @@ -179,6 +264,13 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req } } } + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionTrue, + "ProvisionerKeyReady", + "Provisioner key is available in coderd", + ) provisionerKeySecret, err := r.ensureProvisionerKeySecret(ctx, provisioner, keySecretName, keySecretKey, keyMaterial) if err != nil { @@ -194,6 +286,14 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req } secretChecksum := hashProvisionerSecret(secretValue) + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionProvisionerKeySecretReady, + metav1.ConditionTrue, + "SecretReady", + "Provisioner key secret is available", + ) + serviceAccountName := provisionerServiceAccountName(provisioner.Name) if _, err := r.reconcileServiceAccount(ctx, provisioner, serviceAccountName); err != nil { return ctrl.Result{}, err @@ -222,7 +322,17 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, err } - if err := r.reconcileStatus(ctx, provisioner, deployment, secretRef, organizationID, provisionerKeyID, provisionerKeyName); err != nil { + if err := r.reconcileStatus( + ctx, + provisioner, + deployment, + secretRef, + organizationID, + organizationName, + provisionerKeyID, + provisionerKeyName, + desiredTagsHash, + ); err != nil { return ctrl.Result{}, err } @@ -567,31 +677,47 @@ func (r *CoderProvisionerReconciler) reconcileStatus( deployment *appsv1.Deployment, secretRef *coderv1alpha1.SecretKeySelector, organizationID string, + organizationName string, provisionerKeyID string, provisionerKeyName string, + tagsHash string, ) error { phase := coderv1alpha1.CoderProvisionerPhasePending if deployment.Status.ReadyReplicas > 0 { phase = coderv1alpha1.CoderProvisionerPhaseReady } - nextStatus := coderv1alpha1.CoderProvisionerStatus{ - ObservedGeneration: provisioner.Generation, - ReadyReplicas: deployment.Status.ReadyReplicas, - Phase: phase, - OrganizationID: organizationID, - ProvisionerKeyID: provisionerKeyID, - ProvisionerKeyName: provisionerKeyName, - SecretRef: &coderv1alpha1.SecretKeySelector{ - Name: secretRef.Name, - Key: secretRef.Key, - }, - } - if equality.Semantic.DeepEqual(provisioner.Status, nextStatus) { - return nil + provisioner.Status.ObservedGeneration = provisioner.Generation + provisioner.Status.ReadyReplicas = deployment.Status.ReadyReplicas + provisioner.Status.Phase = phase + provisioner.Status.OrganizationID = organizationID + provisioner.Status.OrganizationName = organizationName + provisioner.Status.ProvisionerKeyID = provisionerKeyID + provisioner.Status.ProvisionerKeyName = provisionerKeyName + provisioner.Status.TagsHash = tagsHash + provisioner.Status.SecretRef = &coderv1alpha1.SecretKeySelector{ + Name: secretRef.Name, + Key: secretRef.Key, + } + + if deployment.Status.ReadyReplicas > 0 { + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionDeploymentReady, + metav1.ConditionTrue, + "MinimumReplicasReady", + "At least one provisioner pod is ready", + ) + } else { + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionDeploymentReady, + metav1.ConditionFalse, + "NoReplicasReady", + "No provisioner pods are ready yet", + ) } - provisioner.Status = nextStatus if err := r.Status().Update(ctx, provisioner); err != nil { return fmt.Errorf("update coderprovisioner status: %w", err) } @@ -727,8 +853,38 @@ func provisionerKeyConfig(provisioner *coderv1alpha1.CoderProvisioner) (string, return keyName, secretName, secretKey } +func hashProvisionerTags(tags map[string]string) string { + keys := slices.Collect(maps.Keys(tags)) + slices.Sort(keys) + hasher := fnv.New32a() + for _, key := range keys { + _, _ = hasher.Write([]byte(key)) + _, _ = hasher.Write([]byte{0}) + _, _ = hasher.Write([]byte(tags[key])) + _, _ = hasher.Write([]byte{0}) + } + + return fmt.Sprintf("%08x", hasher.Sum32()) +} + func hashProvisionerSecret(secretValue []byte) string { hasher := fnv.New32a() _, _ = hasher.Write(secretValue) return fmt.Sprintf("%08x", hasher.Sum32()) } + +func setCondition( + provisioner *coderv1alpha1.CoderProvisioner, + conditionType string, + status metav1.ConditionStatus, + reason string, + message string, +) { + meta.SetStatusCondition(&provisioner.Status.Conditions, metav1.Condition{ + Type: conditionType, + Status: status, + ObservedGeneration: provisioner.Generation, + Reason: reason, + Message: message, + }) +} From 161515b430a9545a5758b7922800e64109e13df7 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 11:17:31 +0000 Subject: [PATCH 18/38] fix: lint condition constant comments and formatting --- .gitignore | 1 + api/v1alpha1/coderprovisioner_types.go | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index bc8e4313..ad32dc54 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ terraform.rc # Backend config (local-only, not committed) terraform/backend.hcl +/coder-k8s diff --git a/api/v1alpha1/coderprovisioner_types.go b/api/v1alpha1/coderprovisioner_types.go index 7defd30c..1df0513a 100644 --- a/api/v1alpha1/coderprovisioner_types.go +++ b/api/v1alpha1/coderprovisioner_types.go @@ -11,12 +11,16 @@ const ( // CoderProvisionerPhaseReady indicates at least one provisioner pod is ready. CoderProvisionerPhaseReady = "Ready" - // Condition types for CoderProvisioner status. - CoderProvisionerConditionControlPlaneReady = "ControlPlaneReady" - CoderProvisionerConditionBootstrapSecretReady = "BootstrapSecretReady" - CoderProvisionerConditionProvisionerKeyReady = "ProvisionerKeyReady" + // CoderProvisionerConditionControlPlaneReady indicates whether the referenced control plane is reachable. + CoderProvisionerConditionControlPlaneReady = "ControlPlaneReady" + // CoderProvisionerConditionBootstrapSecretReady indicates whether the bootstrap credentials secret is available. + CoderProvisionerConditionBootstrapSecretReady = "BootstrapSecretReady" + // CoderProvisionerConditionProvisionerKeyReady indicates whether the provisioner key exists in coderd. + CoderProvisionerConditionProvisionerKeyReady = "ProvisionerKeyReady" + // CoderProvisionerConditionProvisionerKeySecretReady indicates whether the provisioner key secret is populated. CoderProvisionerConditionProvisionerKeySecretReady = "ProvisionerKeySecretReady" - CoderProvisionerConditionDeploymentReady = "DeploymentReady" + // CoderProvisionerConditionDeploymentReady indicates whether the provisioner deployment has ready replicas. + CoderProvisionerConditionDeploymentReady = "DeploymentReady" // DefaultProvisionerKeySecretKey is the default data key for provisioner key secrets. DefaultProvisionerKeySecretKey = "key" From 7de700e0d9c1d99e0030254e6d95643c521fb470 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 11:20:10 +0000 Subject: [PATCH 19/38] chore: improve coderprovisioner sample secret guidance --- .../coder_v1alpha1_coderprovisioner.yaml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/config/samples/coder_v1alpha1_coderprovisioner.yaml b/config/samples/coder_v1alpha1_coderprovisioner.yaml index 307406c1..fddfbfc3 100644 --- a/config/samples/coder_v1alpha1_coderprovisioner.yaml +++ b/config/samples/coder_v1alpha1_coderprovisioner.yaml @@ -1,3 +1,14 @@ +# CoderProvisioner sample manifest. +# +# Prerequisites: +# 1. A CoderControlPlane resource must exist in the same namespace. +# 2. Create the bootstrap credentials Secret with a valid Coder session token: +# +# kubectl create secret generic coder-bootstrap-token \ +# --namespace=default \ +# --from-literal=token= +# +# IMPORTANT: Never commit real tokens to source control. apiVersion: coder.com/v1alpha1 kind: CoderProvisioner metadata: @@ -10,6 +21,18 @@ spec: credentialsSecretRef: name: coder-bootstrap-token key: token + # key: + # name: my-provisioner-key # Provisioner key name in coderd (defaults to CR name) + # secretName: my-key-secret # K8s Secret to store the key (defaults to "{name}-provisioner-key") + # secretKey: key # Data key in the Secret (defaults to "key") + # organizationName: default # Coder organization (defaults to "default") + # image: ghcr.io/coder/coder:latest # Container image (defaults to control plane image) + # replicas: 1 + # extraArgs: + # - --verbose + # extraEnv: + # - name: CODER_VERBOSE + # value: "true" tags: scope: organization replicas: 1 From 3f39c797f041f00172c3cae728f114166689b577 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 11:26:26 +0000 Subject: [PATCH 20/38] test: expand coder provisioner reconciler coverage --- .../coderprovisioner_controller_test.go | 404 +++++++++++++++++- .../workspaceproxy_controller_test.go | 46 +- 2 files changed, 434 insertions(+), 16 deletions(-) diff --git a/internal/controller/coderprovisioner_controller_test.go b/internal/controller/coderprovisioner_controller_test.go index 4d53f76e..19bdefd4 100644 --- a/internal/controller/coderprovisioner_controller_test.go +++ b/internal/controller/coderprovisioner_controller_test.go @@ -100,7 +100,21 @@ func expectedProvisionerResourceName(name string) string { } func expectedProvisionerServiceAccountName(name string) string { - return fmt.Sprintf("%s-provisioner", name) + const suffix = "-provisioner" + candidate := fmt.Sprintf("%s%s", name, suffix) + if len(candidate) <= 63 { + return candidate + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + hashSuffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 63 - len(suffix) - len(hashSuffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s-%s%s", name[:available], hashSuffix, suffix) } func reconcileProvisioner(ctx context.Context, t *testing.T, reconciler *controller.CoderProvisionerReconciler, namespacedName types.NamespacedName) { @@ -126,6 +140,24 @@ func requireOwnerReference(t *testing.T, owner, child metav1.Object) { require.Failf(t, "missing owner reference", "expected %s/%s to own %s/%s", owner.GetNamespace(), owner.GetName(), child.GetNamespace(), child.GetName()) } +func requireCondition(t *testing.T, conditions []metav1.Condition, condType string, status metav1.ConditionStatus) { + t.Helper() + condition := findCondition(t, conditions, condType) + require.Equal(t, status, condition.Status, "condition %s: expected status %v, got %v", condType, status, condition.Status) +} + +func findCondition(t *testing.T, conditions []metav1.Condition, condType string) metav1.Condition { + t.Helper() + for idx := range conditions { + if conditions[idx].Type == condType { + return conditions[idx] + } + } + + require.Failf(t, "condition not found", "expected condition %s to be present", condType) + return metav1.Condition{} +} + func TestCoderProvisionerReconciler_BasicCreate(t *testing.T) { t.Parallel() @@ -137,12 +169,12 @@ func TestCoderProvisionerReconciler_BasicCreate(t *testing.T) { organizationID := uuid.New() provisionerKeyID := uuid.New() bootstrapClient := &fakeBootstrapClient{ - provisionerKeyResponse: coderbootstrap.EnsureProvisionerKeyResponse{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{ OrganizationID: organizationID, KeyID: provisionerKeyID, KeyName: "provisioner-key-name", Key: "provisioner-key-material", - }, + }}, } reconciler := &controller.CoderProvisionerReconciler{ Client: k8sClient, @@ -309,12 +341,12 @@ func TestCoderProvisionerReconciler_ExistingSecret(t *testing.T) { }) bootstrapClient := &fakeBootstrapClient{ - provisionerKeyResponse: coderbootstrap.EnsureProvisionerKeyResponse{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{ OrganizationID: uuid.New(), KeyID: uuid.New(), KeyName: provisionerName, Key: "new-key-material", - }, + }}, } reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} @@ -377,7 +409,7 @@ func TestCoderProvisionerReconciler_Deletion(t *testing.T) { require.NoError(t, k8sClient.Create(ctx, provisioner)) bootstrapClient := &fakeBootstrapClient{ - provisionerKeyResponse: coderbootstrap.EnsureProvisionerKeyResponse{Key: "provisioner-key-material"}, + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{Key: "provisioner-key-material"}}, } reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} @@ -434,7 +466,7 @@ func TestCoderProvisionerReconciler_DeletionControlPlaneGone(t *testing.T) { require.NoError(t, k8sClient.Create(ctx, provisioner)) bootstrapClient := &fakeBootstrapClient{ - provisionerKeyResponse: coderbootstrap.EnsureProvisionerKeyResponse{Key: "provisioner-key-material"}, + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{Key: "provisioner-key-material"}}, } reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} @@ -567,3 +599,361 @@ func TestCoderProvisionerReconciler_ControlPlaneNotReady(t *testing.T) { require.Equal(t, ctrl.Result{}, result) require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) } + +func TestCoderProvisionerReconciler_RotationOnSecretLoss(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-rotation") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-rotation", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-rotation", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Key: coderv1alpha1.CoderProvisionerKeySpec{ + Name: "rotation-key", + SecretName: "provisioner-rotation-key", + SecretKey: "daemon-key", + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{ + {KeyName: "rotation-key", Key: ""}, + {KeyName: "rotation-key", Key: "rotated-key-material"}, + {KeyName: "rotation-key", Key: "rotated-key-material"}, + }, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + + require.Equal(t, 2, bootstrapClient.provisionerKeyCalls) + require.Equal(t, 1, bootstrapClient.deleteKeyCalls) + require.Len(t, bootstrapClient.deleteKeyRequests, 1) + require.Equal(t, "rotation-key", bootstrapClient.deleteKeyRequests[0].KeyName) + + keySecret := &corev1.Secret{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: provisioner.Spec.Key.SecretName, Namespace: provisioner.Namespace}, keySecret)) + require.Equal(t, "rotated-key-material", string(keySecret.Data[provisioner.Spec.Key.SecretKey])) +} + +func TestCoderProvisionerReconciler_TagsDrift(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-tags-drift") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-tags", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-tags", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Key: coderv1alpha1.CoderProvisionerKeySpec{Name: "tags-drift-key"}, + Tags: map[string]string{"region": "us-east"}, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{ + {KeyName: "tags-drift-key", Key: "initial-key-material"}, + {KeyName: "tags-drift-key", Key: "rotated-key-material"}, + }, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + + before := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, before)) + oldTagsHash := before.Status.TagsHash + require.NotEmpty(t, oldTagsHash) + + before.Spec.Tags = map[string]string{"region": "eu-west"} + require.NoError(t, k8sClient.Update(ctx, before)) + + reconcileProvisioner(ctx, t, reconciler, request) + + require.Equal(t, 2, bootstrapClient.provisionerKeyCalls) + require.GreaterOrEqual(t, bootstrapClient.deleteKeyCalls, 1) + + keySecret := &corev1.Secret{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-provisioner-key", provisioner.Name), Namespace: provisioner.Namespace}, keySecret)) + require.Equal(t, "rotated-key-material", string(keySecret.Data[coderv1alpha1.DefaultProvisionerKeySecretKey])) + + after := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, after)) + require.NotEmpty(t, after.Status.TagsHash) + require.NotEqual(t, oldTagsHash, after.Status.TagsHash) +} + +func TestCoderProvisionerReconciler_KeyNameDrift(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-key-drift") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-key", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-key-drift", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Key: coderv1alpha1.CoderProvisionerKeySpec{ + Name: "key-v1", + SecretName: "provisioner-key-drift-secret", + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{ + {KeyName: "key-v1", Key: "key-v1-material"}, + {KeyName: "key-v2", Key: "key-v2-material"}, + }, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + + updated := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, updated)) + updated.Spec.Key.Name = "key-v2" + require.NoError(t, k8sClient.Update(ctx, updated)) + + reconcileProvisioner(ctx, t, reconciler, request) + + require.GreaterOrEqual(t, bootstrapClient.deleteKeyCalls, 1) + require.NotEmpty(t, bootstrapClient.deleteKeyRequests) + lastDelete := bootstrapClient.deleteKeyRequests[len(bootstrapClient.deleteKeyRequests)-1] + require.Equal(t, "key-v1", lastDelete.KeyName) + + require.GreaterOrEqual(t, len(bootstrapClient.provisionerKeyRequests), 2) + lastEnsure := bootstrapClient.provisionerKeyRequests[len(bootstrapClient.provisionerKeyRequests)-1] + require.Equal(t, "key-v2", lastEnsure.KeyName) + + keySecret := &corev1.Secret{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: provisioner.Spec.Key.SecretName, Namespace: provisioner.Namespace}, keySecret)) + require.Equal(t, "key-v2-material", string(keySecret.Data[coderv1alpha1.DefaultProvisionerKeySecretKey])) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, reconciled)) + require.Equal(t, "key-v2", reconciled.Status.ProvisionerKeyName) +} + +func TestCoderProvisionerReconciler_ReadyPhaseAndConditions(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-ready") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-ready", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-ready", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{ + KeyName: provisioner.Name, + Key: "provisioner-key-material", + }}, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + + deployment := &appsv1.Deployment{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: expectedProvisionerResourceName(provisioner.Name), Namespace: provisioner.Namespace}, deployment)) + deployment.Status.ReadyReplicas = 1 + deployment.Status.Replicas = 1 + require.NoError(t, k8sClient.Status().Update(ctx, deployment)) + + reconcileProvisioner(ctx, t, reconciler, request) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, reconciled)) + require.Equal(t, coderv1alpha1.CoderProvisionerPhaseReady, reconciled.Status.Phase) + + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionDeploymentReady, metav1.ConditionTrue) + deploymentReadyCondition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionDeploymentReady) + require.Equal(t, "MinimumReplicasReady", deploymentReadyCondition.Reason) + + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionControlPlaneReady, metav1.ConditionTrue) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady, metav1.ConditionTrue) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, metav1.ConditionTrue) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionProvisionerKeySecretReady, metav1.ConditionTrue) +} + +func TestCoderProvisionerReconciler_ConditionsOnFailure(t *testing.T) { + t.Parallel() + + t.Run("control plane unavailable", func(t *testing.T) { + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-cond-cp") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-cond-cp", "") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-cond-cp", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{} + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: request}) + require.ErrorContains(t, err, "status.url is empty") + require.Equal(t, ctrl.Result{}, result) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, reconciled)) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionControlPlaneReady, metav1.ConditionFalse) + controlPlaneCondition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionControlPlaneReady) + require.Equal(t, "ControlPlaneUnavailable", controlPlaneCondition.Reason) + }) + + t.Run("bootstrap secret unavailable", func(t *testing.T) { + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-cond-bootstrap") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-cond-bootstrap", "https://coder.example.com") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-cond-bootstrap", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: "missing-bootstrap-secret", Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{} + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: request}) + require.ErrorContains(t, err, "read bootstrap credentials secret") + require.Equal(t, ctrl.Result{}, result) + require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, reconciled)) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionControlPlaneReady, metav1.ConditionTrue) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady, metav1.ConditionFalse) + bootstrapCondition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady) + require.Equal(t, "BootstrapSecretUnavailable", bootstrapCondition.Reason) + }) +} + +func TestCoderProvisionerReconciler_LongNameTruncation(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-longname") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-longname", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisionerName := strings.Repeat("a", 70) + deploymentCandidateName := fmt.Sprintf("provisioner-%s", provisionerName) + serviceAccountCandidateName := fmt.Sprintf("%s-provisioner", provisionerName) + require.Greater(t, len(deploymentCandidateName), 63) + require.Greater(t, len(serviceAccountCandidateName), 63) + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: provisionerName, Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{ + KeyName: provisionerName, + Key: "provisioner-key-material", + }}, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + + deploymentName := expectedProvisionerResourceName(provisionerName) + serviceAccountName := expectedProvisionerServiceAccountName(provisionerName) + require.LessOrEqual(t, len(deploymentName), 63) + require.LessOrEqual(t, len(serviceAccountName), 63) + + deployment := &appsv1.Deployment{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: deploymentName, Namespace: namespace}, deployment)) + + serviceAccount := &corev1.ServiceAccount{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: serviceAccountName, Namespace: namespace}, serviceAccount)) +} diff --git a/internal/controller/workspaceproxy_controller_test.go b/internal/controller/workspaceproxy_controller_test.go index 7a0a7797..1ec0e70d 100644 --- a/internal/controller/workspaceproxy_controller_test.go +++ b/internal/controller/workspaceproxy_controller_test.go @@ -23,12 +23,21 @@ type fakeBootstrapClient struct { err error calls int - // Provisioner key support (for interface compliance). - provisionerKeyResponse coderbootstrap.EnsureProvisionerKeyResponse - provisionerKeyErr error - provisionerKeyCalls int - deleteKeyErr error - deleteKeyCalls int + // Provisioner key support. + provisionerKeyResponses []coderbootstrap.EnsureProvisionerKeyResponse + provisionerKeyErr error + provisionerKeyCalls int + provisionerKeyRequests []coderbootstrap.EnsureProvisionerKeyRequest + deleteKeyErr error + deleteKeyCalls int + deleteKeyRequests []deleteKeyRequest +} + +type deleteKeyRequest struct { + CoderURL string + SessionToken string + OrganizationName string + KeyName string } func (f *fakeBootstrapClient) EnsureWorkspaceProxy(_ context.Context, _ coderbootstrap.RegisterWorkspaceProxyRequest) (coderbootstrap.RegisterWorkspaceProxyResponse, error) { @@ -36,13 +45,32 @@ func (f *fakeBootstrapClient) EnsureWorkspaceProxy(_ context.Context, _ coderboo return f.response, f.err } -func (f *fakeBootstrapClient) EnsureProvisionerKey(_ context.Context, _ coderbootstrap.EnsureProvisionerKeyRequest) (coderbootstrap.EnsureProvisionerKeyResponse, error) { +func (f *fakeBootstrapClient) EnsureProvisionerKey(_ context.Context, req coderbootstrap.EnsureProvisionerKeyRequest) (coderbootstrap.EnsureProvisionerKeyResponse, error) { f.provisionerKeyCalls++ - return f.provisionerKeyResponse, f.provisionerKeyErr + f.provisionerKeyRequests = append(f.provisionerKeyRequests, req) + + if f.provisionerKeyErr != nil { + return coderbootstrap.EnsureProvisionerKeyResponse{}, f.provisionerKeyErr + } + if len(f.provisionerKeyResponses) == 0 { + return coderbootstrap.EnsureProvisionerKeyResponse{}, nil + } + idx := f.provisionerKeyCalls - 1 + if idx >= len(f.provisionerKeyResponses) { + idx = len(f.provisionerKeyResponses) - 1 + } + + return f.provisionerKeyResponses[idx], nil } -func (f *fakeBootstrapClient) DeleteProvisionerKey(_ context.Context, _, _, _, _ string) error { +func (f *fakeBootstrapClient) DeleteProvisionerKey(_ context.Context, coderURL, sessionToken, orgName, keyName string) error { f.deleteKeyCalls++ + f.deleteKeyRequests = append(f.deleteKeyRequests, deleteKeyRequest{ + CoderURL: coderURL, + SessionToken: sessionToken, + OrganizationName: orgName, + KeyName: keyName, + }) return f.deleteKeyErr } From 2af668d18a0fd68237fcc404644b468f7c7f9dfb Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 11:42:55 +0000 Subject: [PATCH 21/38] fix: preserve key status on skip path and bound secret name --- .../controller/coderprovisioner_controller.go | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index f52102a5..0b84fd66 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -134,6 +134,8 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req keyNameDrift := status.ProvisionerKeyName != "" && status.ProvisionerKeyName != keyName tagsDrift := status.TagsHash != "" && status.TagsHash != desiredTagsHash driftDetected := orgDrift || keyNameDrift || tagsDrift + appliedOrgName := provisioner.Status.OrganizationName + appliedTagsHash := provisioner.Status.TagsHash // Check whether a usable provisioner key secret already exists. // The secret is considered "usable" only if the Secret object exists @@ -205,6 +207,8 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req if keyMaterial == "" { return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key returned empty material after drift rotation") } + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash } else if !secretUsable { response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ CoderURL: controlPlane.Status.URL, @@ -263,6 +267,8 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key %q returned empty key material after rotation", keyName) } } + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash } setCondition( provisioner, @@ -328,10 +334,10 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req deployment, secretRef, organizationID, - organizationName, + appliedOrgName, provisionerKeyID, provisionerKeyName, - desiredTagsHash, + appliedTagsHash, ); err != nil { return ctrl.Result{}, err } @@ -842,7 +848,20 @@ func provisionerKeyConfig(provisioner *coderv1alpha1.CoderProvisioner) (string, secretName := provisioner.Spec.Key.SecretName if secretName == "" { - secretName = fmt.Sprintf("%s-provisioner-key", provisioner.Name) + const secretNameSuffix = "-provisioner-key" + candidate := provisioner.Name + secretNameSuffix + if len(candidate) <= 253 { + secretName = candidate + } else { + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(provisioner.Name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 253 - len(secretNameSuffix) - len(suffix) - 1 + if available < 1 { + available = 1 + } + secretName = fmt.Sprintf("%s-%s%s", provisioner.Name[:available], suffix, secretNameSuffix) + } } secretKey := provisioner.Spec.Key.SecretKey From ef7bf3ca7d0c08878abf3b3f8b88f87798e0ff69 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 11:51:09 +0000 Subject: [PATCH 22/38] fix: set ProvisionerKeyReady only after coderd verification --- .../controller/coderprovisioner_controller.go | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 0b84fd66..879017e3 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -209,6 +209,13 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req } appliedOrgName = organizationName appliedTagsHash = desiredTagsHash + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionTrue, + "ProvisionerKeyReady", + "Provisioner key is available in coderd", + ) } else if !secretUsable { response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ CoderURL: controlPlane.Status.URL, @@ -269,14 +276,14 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req } appliedOrgName = organizationName appliedTagsHash = desiredTagsHash + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionTrue, + "ProvisionerKeyReady", + "Provisioner key is available in coderd", + ) } - setCondition( - provisioner, - coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, - metav1.ConditionTrue, - "ProvisionerKeyReady", - "Provisioner key is available in coderd", - ) provisionerKeySecret, err := r.ensureProvisionerKeySecret(ctx, provisioner, keySecretName, keySecretKey, keyMaterial) if err != nil { From 2d9f216fd7bb2d83d98ed89b9cf49fff7e19e834 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 11:58:22 +0000 Subject: [PATCH 23/38] fix: use status org for deletion and mark key condition false on failure --- .../controller/coderprovisioner_controller.go | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 879017e3..f4927130 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -192,6 +192,10 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req Tags: provisioner.Spec.Tags, }) if ensureErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to ensure provisioner key %q after drift rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) return ctrl.Result{}, fmt.Errorf("ensure provisioner key %q: %w", keyName, ensureErr) } if response.OrganizationID != uuid.Nil { @@ -205,6 +209,10 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req } keyMaterial = response.Key if keyMaterial == "" { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Provisioner key %q returned empty material after drift rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key returned empty material after drift rotation") } appliedOrgName = organizationName @@ -225,6 +233,10 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req Tags: provisioner.Spec.Tags, }) if ensureErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to ensure provisioner key %q", keyName)) + _ = r.Status().Update(ctx, provisioner) return ctrl.Result{}, fmt.Errorf("ensure provisioner key %q: %w", keyName, ensureErr) } if response.OrganizationID != uuid.Nil { @@ -248,6 +260,10 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req if deleteErr := r.BootstrapClient.DeleteProvisionerKey( ctx, controlPlane.Status.URL, sessionToken, organizationName, keyName, ); deleteErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to delete stale provisioner key %q for rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) return ctrl.Result{}, fmt.Errorf("delete stale provisioner key %q for rotation: %w", keyName, deleteErr) } rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ @@ -258,6 +274,10 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req Tags: provisioner.Spec.Tags, }) if rotateErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to recreate provisioner key %q after rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) return ctrl.Result{}, fmt.Errorf("recreate provisioner key %q after rotation: %w", keyName, rotateErr) } if rotated.OrganizationID != uuid.Nil { @@ -271,6 +291,10 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req } keyMaterial = rotated.Key if keyMaterial == "" { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Provisioner key %q returned empty key material after rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key %q returned empty key material after rotation", keyName) } } @@ -359,9 +383,13 @@ func (r *CoderProvisionerReconciler) reconcileDeletion(ctx context.Context, prov log := ctrl.LoggerFrom(ctx) - // DeleteProvisionerKey resolves organizations by name, so always pass the - // spec-derived organization name here instead of the persisted organization ID. - organizationName := provisionerOrganizationName(provisioner.Spec.OrganizationName) + // Use the last-applied organization name from status so we target the + // correct org even if spec was changed but never successfully rotated. + // Fall back to the spec-derived name only when status is empty. + organizationName := provisioner.Status.OrganizationName + if organizationName == "" { + organizationName = provisionerOrganizationName(provisioner.Spec.OrganizationName) + } keyName := provisioner.Status.ProvisionerKeyName if keyName == "" { keyName, _, _ = provisionerKeyConfig(provisioner) From b568e535241d6b845fe4bfe6682b6fb1e3fbd105 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 12:05:12 +0000 Subject: [PATCH 24/38] fix: populate status metadata when secret exists but status is empty --- .../controller/coderprovisioner_controller.go | 35 +++++++++++++++++++ .../coderprovisioner_controller_test.go | 5 ++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index f4927130..39aea71c 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -307,6 +307,41 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req "ProvisionerKeyReady", "Provisioner key is available in coderd", ) + } else if status.OrganizationName == "" || status.TagsHash == "" { + // Secret is usable and no drift detected, but status metadata + // is empty (e.g. upgrade from older version). Call EnsureProvisionerKey + // to populate IDs/name and establish the baseline for future drift detection. + // The key material will be empty (already exists), but that's OK. + response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if ensureErr != nil { + log.Info("failed to verify provisioner key metadata, will retry", + "keyName", keyName, "error", ensureErr) + } else { + if response.OrganizationID != uuid.Nil { + organizationID = response.OrganizationID.String() + } + if response.KeyID != uuid.Nil { + provisionerKeyID = response.KeyID.String() + } + if response.KeyName != "" { + provisionerKeyName = response.KeyName + } + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionTrue, + "ProvisionerKeyReady", + "Provisioner key is available in coderd", + ) + } } provisionerKeySecret, err := r.ensureProvisionerKeySecret(ctx, provisioner, keySecretName, keySecretKey, keyMaterial) diff --git a/internal/controller/coderprovisioner_controller_test.go b/internal/controller/coderprovisioner_controller_test.go index 19bdefd4..1c185312 100644 --- a/internal/controller/coderprovisioner_controller_test.go +++ b/internal/controller/coderprovisioner_controller_test.go @@ -354,7 +354,10 @@ func TestCoderProvisionerReconciler_ExistingSecret(t *testing.T) { reconcileProvisioner(ctx, t, reconciler, namespacedName) reconcileProvisioner(ctx, t, reconciler, namespacedName) - require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) + // The first real reconcile triggers a metadata-only EnsureProvisionerKey + // call because status.OrganizationName and status.TagsHash are empty. + // The second reconcile skips since metadata is now populated. + require.Equal(t, 1, bootstrapClient.provisionerKeyCalls) require.Equal(t, 0, bootstrapClient.deleteKeyCalls) reconciledSecret := &corev1.Secret{} From f648b7c2a1f046aed856541c7a39c004ab291a3b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 12:14:30 +0000 Subject: [PATCH 25/38] fix: capture key material during metadata-only ensure --- internal/controller/coderprovisioner_controller.go | 8 +++++++- internal/controller/coderprovisioner_controller_test.go | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 39aea71c..fe8e310a 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -311,7 +311,8 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req // Secret is usable and no drift detected, but status metadata // is empty (e.g. upgrade from older version). Call EnsureProvisionerKey // to populate IDs/name and establish the baseline for future drift detection. - // The key material will be empty (already exists), but that's OK. + // Typically the key already exists and Key will be empty, but if the + // remote key was deleted we capture the new key material too. response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ CoderURL: controlPlane.Status.URL, SessionToken: sessionToken, @@ -332,6 +333,11 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req if response.KeyName != "" { provisionerKeyName = response.KeyName } + // If coderd created a new key (remote was missing), capture the + // material so ensureProvisionerKeySecret writes it to the Secret. + if response.Key != "" { + keyMaterial = response.Key + } appliedOrgName = organizationName appliedTagsHash = desiredTagsHash setCondition( diff --git a/internal/controller/coderprovisioner_controller_test.go b/internal/controller/coderprovisioner_controller_test.go index 1c185312..92c98e2f 100644 --- a/internal/controller/coderprovisioner_controller_test.go +++ b/internal/controller/coderprovisioner_controller_test.go @@ -345,7 +345,7 @@ func TestCoderProvisionerReconciler_ExistingSecret(t *testing.T) { OrganizationID: uuid.New(), KeyID: uuid.New(), KeyName: provisionerName, - Key: "new-key-material", + Key: "", // Empty: coderd returns no plaintext for existing keys. }}, } reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} From a25bf6bcbfc2e76ffacd287fa9277d604a649e66 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 12:27:00 +0000 Subject: [PATCH 26/38] fix: harden provisioner key drift recovery --- .../controller/coderprovisioner_controller.go | 60 +++++++++++++++++-- .../coderprovisioner_controller_test.go | 41 +++++++++++-- 2 files changed, 92 insertions(+), 9 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index fe8e310a..3e17955f 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -209,11 +209,49 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req } keyMaterial = response.Key if keyMaterial == "" { - setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, - metav1.ConditionFalse, "ProvisionerKeyFailed", - fmt.Sprintf("Provisioner key %q returned empty material after drift rotation", keyName)) - _ = r.Status().Update(ctx, provisioner) - return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key returned empty material after drift rotation") + log.Info("drift-rotated key exists in coderd but returned no plaintext, rotating to recover", + "keyName", keyName) + + if deleteErr := r.BootstrapClient.DeleteProvisionerKey( + ctx, controlPlane.Status.URL, sessionToken, organizationName, keyName, + ); deleteErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to delete provisioner key %q for drift recovery", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("delete provisioner key %q for drift recovery: %w", keyName, deleteErr) + } + rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if rotateErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to recreate provisioner key %q after drift recovery", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("recreate provisioner key %q after drift recovery: %w", keyName, rotateErr) + } + if rotated.OrganizationID != uuid.Nil { + organizationID = rotated.OrganizationID.String() + } + if rotated.KeyID != uuid.Nil { + provisionerKeyID = rotated.KeyID.String() + } + if rotated.KeyName != "" { + provisionerKeyName = rotated.KeyName + } + keyMaterial = rotated.Key + if keyMaterial == "" { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Provisioner key %q returned empty material after drift recovery rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key %q returned empty material after drift recovery rotation", keyName) + } } appliedOrgName = organizationName appliedTagsHash = desiredTagsHash @@ -922,6 +960,18 @@ func provisionerKeyConfig(provisioner *coderv1alpha1.CoderProvisioner) (string, keyName = provisioner.Name } + const maxKeyNameLength = 128 + if len(keyName) > maxKeyNameLength { + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(keyName)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := maxKeyNameLength - len(suffix) - 1 + if available < 1 { + available = 1 + } + keyName = fmt.Sprintf("%s-%s", keyName[:available], suffix) + } + secretName := provisioner.Spec.Key.SecretName if secretName == "" { const secretNameSuffix = "-provisioner-key" diff --git a/internal/controller/coderprovisioner_controller_test.go b/internal/controller/coderprovisioner_controller_test.go index 92c98e2f..c03ed8c0 100644 --- a/internal/controller/coderprovisioner_controller_test.go +++ b/internal/controller/coderprovisioner_controller_test.go @@ -117,6 +117,23 @@ func expectedProvisionerServiceAccountName(name string) string { return fmt.Sprintf("%s-%s%s", name[:available], hashSuffix, suffix) } +func expectedProvisionerKeyName(name string) string { + const maxKeyNameLength = 128 + if len(name) <= maxKeyNameLength { + return name + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := maxKeyNameLength - len(suffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s-%s", name[:available], suffix) +} + func reconcileProvisioner(ctx context.Context, t *testing.T, reconciler *controller.CoderProvisionerReconciler, namespacedName types.NamespacedName) { t.Helper() @@ -681,6 +698,7 @@ func TestCoderProvisionerReconciler_TagsDrift(t *testing.T) { bootstrapClient := &fakeBootstrapClient{ provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{ {KeyName: "tags-drift-key", Key: "initial-key-material"}, + {KeyName: "tags-drift-key", Key: ""}, {KeyName: "tags-drift-key", Key: "rotated-key-material"}, }, } @@ -700,8 +718,11 @@ func TestCoderProvisionerReconciler_TagsDrift(t *testing.T) { reconcileProvisioner(ctx, t, reconciler, request) - require.Equal(t, 2, bootstrapClient.provisionerKeyCalls) - require.GreaterOrEqual(t, bootstrapClient.deleteKeyCalls, 1) + require.Equal(t, 3, bootstrapClient.provisionerKeyCalls) + require.Equal(t, 2, bootstrapClient.deleteKeyCalls) + require.Len(t, bootstrapClient.deleteKeyRequests, 2) + require.Equal(t, "tags-drift-key", bootstrapClient.deleteKeyRequests[0].KeyName) + require.Equal(t, "tags-drift-key", bootstrapClient.deleteKeyRequests[1].KeyName) keySecret := &corev1.Secret{} require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-provisioner-key", provisioner.Name), Namespace: provisioner.Namespace}, keySecret)) @@ -917,11 +938,15 @@ func TestCoderProvisionerReconciler_LongNameTruncation(t *testing.T) { controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-longname", "https://coder.example.com") bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") - provisionerName := strings.Repeat("a", 70) + provisionerName := strings.Repeat("a", 180) deploymentCandidateName := fmt.Sprintf("provisioner-%s", provisionerName) serviceAccountCandidateName := fmt.Sprintf("%s-provisioner", provisionerName) + keyNameCandidate := provisionerName + expectedKeyName := expectedProvisionerKeyName(provisionerName) require.Greater(t, len(deploymentCandidateName), 63) require.Greater(t, len(serviceAccountCandidateName), 63) + require.Greater(t, len(keyNameCandidate), 128) + require.Len(t, expectedKeyName, 128) provisioner := &coderv1alpha1.CoderProvisioner{ ObjectMeta: metav1.ObjectMeta{Name: provisionerName, Namespace: namespace}, @@ -939,7 +964,7 @@ func TestCoderProvisionerReconciler_LongNameTruncation(t *testing.T) { bootstrapClient := &fakeBootstrapClient{ provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{ - KeyName: provisionerName, + KeyName: expectedKeyName, Key: "provisioner-key-material", }}, } @@ -954,6 +979,14 @@ func TestCoderProvisionerReconciler_LongNameTruncation(t *testing.T) { require.LessOrEqual(t, len(deploymentName), 63) require.LessOrEqual(t, len(serviceAccountName), 63) + require.Len(t, bootstrapClient.provisionerKeyRequests, 1) + require.Equal(t, expectedKeyName, bootstrapClient.provisionerKeyRequests[0].KeyName) + + reconciledProvisioner := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, reconciledProvisioner)) + require.Equal(t, expectedKeyName, reconciledProvisioner.Status.ProvisionerKeyName) + require.LessOrEqual(t, len(reconciledProvisioner.Status.ProvisionerKeyName), 128) + deployment := &appsv1.Deployment{} require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: deploymentName, Namespace: namespace}, deployment)) From 7022044736c89a688c728ca7a479582ee24e60fc Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 12:47:36 +0000 Subject: [PATCH 27/38] fix: rotate existing key during metadata backfill --- .../controller/coderprovisioner_controller.go | 52 +++++++++++++++---- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 3e17955f..63a9db7b 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -346,11 +346,11 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req "Provisioner key is available in coderd", ) } else if status.OrganizationName == "" || status.TagsHash == "" { - // Secret is usable and no drift detected, but status metadata - // is empty (e.g. upgrade from older version). Call EnsureProvisionerKey - // to populate IDs/name and establish the baseline for future drift detection. - // Typically the key already exists and Key will be empty, but if the - // remote key was deleted we capture the new key material too. + // Secret is usable and no drift detected, but status metadata is empty + // (e.g. upgrade from older version). Call EnsureProvisionerKey to populate + // IDs and key name. If coderd reports an existing key (no plaintext key + // returned), rotate it best-effort so desired tags are applied before + // stamping the metadata baseline. response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ CoderURL: controlPlane.Status.URL, SessionToken: sessionToken, @@ -371,13 +371,47 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req if response.KeyName != "" { provisionerKeyName = response.KeyName } - // If coderd created a new key (remote was missing), capture the - // material so ensureProvisionerKeySecret writes it to the Secret. if response.Key != "" { + // Key was freshly created with desired tags; capture material and stamp baseline. keyMaterial = response.Key + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash + } else { + // Key already exists; tags may be stale. Rotate to ensure desired tags are applied. + log.Info("existing key found during metadata backfill, rotating to ensure desired tags", + "keyName", keyName) + if deleteErr := r.BootstrapClient.DeleteProvisionerKey( + ctx, controlPlane.Status.URL, sessionToken, organizationName, keyName, + ); deleteErr != nil { + log.Info("failed to delete key for metadata backfill rotation, will retry", + "keyName", keyName, "error", deleteErr) + } else { + rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if rotateErr != nil { + log.Info("failed to recreate key for metadata backfill rotation, will retry", + "keyName", keyName, "error", rotateErr) + } else { + if rotated.OrganizationID != uuid.Nil { + organizationID = rotated.OrganizationID.String() + } + if rotated.KeyID != uuid.Nil { + provisionerKeyID = rotated.KeyID.String() + } + if rotated.KeyName != "" { + provisionerKeyName = rotated.KeyName + } + keyMaterial = rotated.Key + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash + } + } } - appliedOrgName = organizationName - appliedTagsHash = desiredTagsHash setCondition( provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, From bff1409afb93e4ccea2d7da2f9b463b7f6aa093b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 12:50:41 +0000 Subject: [PATCH 28/38] test: update ExistingSecret rotation expectations --- .../coderprovisioner_controller_test.go | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/internal/controller/coderprovisioner_controller_test.go b/internal/controller/coderprovisioner_controller_test.go index c03ed8c0..f29f1f5a 100644 --- a/internal/controller/coderprovisioner_controller_test.go +++ b/internal/controller/coderprovisioner_controller_test.go @@ -358,12 +358,20 @@ func TestCoderProvisionerReconciler_ExistingSecret(t *testing.T) { }) bootstrapClient := &fakeBootstrapClient{ - provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{ - OrganizationID: uuid.New(), - KeyID: uuid.New(), - KeyName: provisionerName, - Key: "", // Empty: coderd returns no plaintext for existing keys. - }}, + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{ + { + OrganizationID: uuid.New(), + KeyID: uuid.New(), + KeyName: provisionerName, + Key: "", // Empty: coderd returns no plaintext for existing keys. + }, + { + OrganizationID: uuid.New(), + KeyID: uuid.New(), + KeyName: provisionerName, + Key: "rotated-key-material", + }, + }, } reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} @@ -373,13 +381,14 @@ func TestCoderProvisionerReconciler_ExistingSecret(t *testing.T) { // The first real reconcile triggers a metadata-only EnsureProvisionerKey // call because status.OrganizationName and status.TagsHash are empty. + // The empty key response rotates by deleting and recreating the key. // The second reconcile skips since metadata is now populated. - require.Equal(t, 1, bootstrapClient.provisionerKeyCalls) - require.Equal(t, 0, bootstrapClient.deleteKeyCalls) + require.Equal(t, 2, bootstrapClient.provisionerKeyCalls) + require.Equal(t, 1, bootstrapClient.deleteKeyCalls) reconciledSecret := &corev1.Secret{} require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: secretName, Namespace: namespace}, reconciledSecret)) - require.Equal(t, "existing-key-material", string(reconciledSecret.Data[coderv1alpha1.DefaultProvisionerKeySecretKey])) + require.Equal(t, "rotated-key-material", string(reconciledSecret.Data[coderv1alpha1.DefaultProvisionerKeySecretKey])) deployment := &appsv1.Deployment{} resourceName := expectedProvisionerResourceName(provisioner.Name) From 4101c4aaddd9de8bdcdbac75cf968b0501545a96 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 13:01:16 +0000 Subject: [PATCH 29/38] fix: fail reconcile on metadata backfill recreate error --- .../controller/coderprovisioner_controller.go | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 63a9db7b..0499de4a 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -394,22 +394,24 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req Tags: provisioner.Spec.Tags, }) if rotateErr != nil { - log.Info("failed to recreate key for metadata backfill rotation, will retry", - "keyName", keyName, "error", rotateErr) - } else { - if rotated.OrganizationID != uuid.Nil { - organizationID = rotated.OrganizationID.String() - } - if rotated.KeyID != uuid.Nil { - provisionerKeyID = rotated.KeyID.String() - } - if rotated.KeyName != "" { - provisionerKeyName = rotated.KeyName - } - keyMaterial = rotated.Key - appliedOrgName = organizationName - appliedTagsHash = desiredTagsHash + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to recreate provisioner key %q after metadata backfill rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("recreate provisioner key %q after metadata backfill rotation: %w", keyName, rotateErr) } + if rotated.OrganizationID != uuid.Nil { + organizationID = rotated.OrganizationID.String() + } + if rotated.KeyID != uuid.Nil { + provisionerKeyID = rotated.KeyID.String() + } + if rotated.KeyName != "" { + provisionerKeyName = rotated.KeyName + } + keyMaterial = rotated.Key + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash } } setCondition( From dd02b9cba66697d671c299733d9041be55a30974 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 13:17:43 +0000 Subject: [PATCH 30/38] fix: guard coderprovisioner status updates --- internal/controller/coderprovisioner_controller.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 0499de4a..d7be1bda 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -12,6 +12,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -838,11 +839,15 @@ func (r *CoderProvisionerReconciler) reconcileStatus( provisionerKeyName string, tagsHash string, ) error { + // Take a snapshot before mutation so status writes are skipped when no fields changed. + previousStatus := provisioner.Status.DeepCopy() + phase := coderv1alpha1.CoderProvisionerPhasePending if deployment.Status.ReadyReplicas > 0 { phase = coderv1alpha1.CoderProvisionerPhaseReady } + // Update fields individually so conditions set earlier in reconciliation are preserved. provisioner.Status.ObservedGeneration = provisioner.Generation provisioner.Status.ReadyReplicas = deployment.Status.ReadyReplicas provisioner.Status.Phase = phase @@ -874,6 +879,10 @@ func (r *CoderProvisionerReconciler) reconcileStatus( ) } + if previousStatus != nil && equality.Semantic.DeepEqual(*previousStatus, provisioner.Status) { + return nil + } + if err := r.Status().Update(ctx, provisioner); err != nil { return fmt.Errorf("update coderprovisioner status: %w", err) } From 2bf3452f1e274758412f543ee57dcb107691fbb1 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 13:27:34 +0000 Subject: [PATCH 31/38] fix: harden metadata backfill key reconciliation --- .../controller/coderprovisioner_controller.go | 123 +++++++++--------- 1 file changed, 65 insertions(+), 58 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index d7be1bda..e5e901f7 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -7,6 +7,7 @@ import ( "hash/fnv" "maps" "slices" + "time" "github.com/google/uuid" appsv1 "k8s.io/api/apps/v1" @@ -360,69 +361,75 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req Tags: provisioner.Spec.Tags, }) if ensureErr != nil { - log.Info("failed to verify provisioner key metadata, will retry", - "keyName", keyName, "error", ensureErr) + // Return a requeue so metadata population retries. + return ctrl.Result{RequeueAfter: 10 * time.Second}, fmt.Errorf("verify provisioner key metadata: %w", ensureErr) + } + if response.OrganizationID != uuid.Nil { + organizationID = response.OrganizationID.String() + } + if response.KeyID != uuid.Nil { + provisionerKeyID = response.KeyID.String() + } + if response.KeyName != "" { + provisionerKeyName = response.KeyName + } + if response.Key != "" { + // Key was freshly created with desired tags; capture material and stamp baseline. + keyMaterial = response.Key + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash } else { - if response.OrganizationID != uuid.Nil { - organizationID = response.OrganizationID.String() - } - if response.KeyID != uuid.Nil { - provisionerKeyID = response.KeyID.String() - } - if response.KeyName != "" { - provisionerKeyName = response.KeyName - } - if response.Key != "" { - // Key was freshly created with desired tags; capture material and stamp baseline. - keyMaterial = response.Key - appliedOrgName = organizationName - appliedTagsHash = desiredTagsHash + // Key already exists; tags may be stale. Rotate to ensure desired tags are applied. + log.Info("existing key found during metadata backfill, rotating to ensure desired tags", + "keyName", keyName) + if deleteErr := r.BootstrapClient.DeleteProvisionerKey( + ctx, controlPlane.Status.URL, sessionToken, organizationName, keyName, + ); deleteErr != nil { + log.Info("failed to delete key for metadata backfill rotation, will retry", + "keyName", keyName, "error", deleteErr) } else { - // Key already exists; tags may be stale. Rotate to ensure desired tags are applied. - log.Info("existing key found during metadata backfill, rotating to ensure desired tags", - "keyName", keyName) - if deleteErr := r.BootstrapClient.DeleteProvisionerKey( - ctx, controlPlane.Status.URL, sessionToken, organizationName, keyName, - ); deleteErr != nil { - log.Info("failed to delete key for metadata backfill rotation, will retry", - "keyName", keyName, "error", deleteErr) - } else { - rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ - CoderURL: controlPlane.Status.URL, - SessionToken: sessionToken, - OrganizationName: organizationName, - KeyName: keyName, - Tags: provisioner.Spec.Tags, - }) - if rotateErr != nil { - setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, - metav1.ConditionFalse, "ProvisionerKeyFailed", - fmt.Sprintf("Failed to recreate provisioner key %q after metadata backfill rotation", keyName)) - _ = r.Status().Update(ctx, provisioner) - return ctrl.Result{}, fmt.Errorf("recreate provisioner key %q after metadata backfill rotation: %w", keyName, rotateErr) - } - if rotated.OrganizationID != uuid.Nil { - organizationID = rotated.OrganizationID.String() - } - if rotated.KeyID != uuid.Nil { - provisionerKeyID = rotated.KeyID.String() - } - if rotated.KeyName != "" { - provisionerKeyName = rotated.KeyName - } - keyMaterial = rotated.Key - appliedOrgName = organizationName - appliedTagsHash = desiredTagsHash + rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if rotateErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to recreate provisioner key %q after metadata backfill rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("recreate provisioner key %q after metadata backfill rotation: %w", keyName, rotateErr) + } + if rotated.OrganizationID != uuid.Nil { + organizationID = rotated.OrganizationID.String() } + if rotated.KeyID != uuid.Nil { + provisionerKeyID = rotated.KeyID.String() + } + if rotated.KeyName != "" { + provisionerKeyName = rotated.KeyName + } + keyMaterial = rotated.Key + if keyMaterial == "" { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Provisioner key %q returned empty material after metadata backfill rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key %q returned empty material after metadata backfill rotation", keyName) + } + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash } - setCondition( - provisioner, - coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, - metav1.ConditionTrue, - "ProvisionerKeyReady", - "Provisioner key is available in coderd", - ) } + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionTrue, + "ProvisionerKeyReady", + "Provisioner key is available in coderd", + ) } provisionerKeySecret, err := r.ensureProvisionerKeySecret(ctx, provisioner, keySecretName, keySecretKey, keyMaterial) From 936bf183bcbeb5eb8ea9411d075a7daa534a3adf Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 13:40:31 +0000 Subject: [PATCH 32/38] fix: retry metadata backfill key deletion failures --- .../controller/coderprovisioner_controller.go | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index e5e901f7..5c8ff642 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -385,43 +385,41 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req if deleteErr := r.BootstrapClient.DeleteProvisionerKey( ctx, controlPlane.Status.URL, sessionToken, organizationName, keyName, ); deleteErr != nil { - log.Info("failed to delete key for metadata backfill rotation, will retry", - "keyName", keyName, "error", deleteErr) - } else { - rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ - CoderURL: controlPlane.Status.URL, - SessionToken: sessionToken, - OrganizationName: organizationName, - KeyName: keyName, - Tags: provisioner.Spec.Tags, - }) - if rotateErr != nil { - setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, - metav1.ConditionFalse, "ProvisionerKeyFailed", - fmt.Sprintf("Failed to recreate provisioner key %q after metadata backfill rotation", keyName)) - _ = r.Status().Update(ctx, provisioner) - return ctrl.Result{}, fmt.Errorf("recreate provisioner key %q after metadata backfill rotation: %w", keyName, rotateErr) - } - if rotated.OrganizationID != uuid.Nil { - organizationID = rotated.OrganizationID.String() - } - if rotated.KeyID != uuid.Nil { - provisionerKeyID = rotated.KeyID.String() - } - if rotated.KeyName != "" { - provisionerKeyName = rotated.KeyName - } - keyMaterial = rotated.Key - if keyMaterial == "" { - setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, - metav1.ConditionFalse, "ProvisionerKeyFailed", - fmt.Sprintf("Provisioner key %q returned empty material after metadata backfill rotation", keyName)) - _ = r.Status().Update(ctx, provisioner) - return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key %q returned empty material after metadata backfill rotation", keyName) - } - appliedOrgName = organizationName - appliedTagsHash = desiredTagsHash + return ctrl.Result{RequeueAfter: 10 * time.Second}, fmt.Errorf("delete provisioner key %q for metadata backfill rotation: %w", keyName, deleteErr) + } + rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if rotateErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to recreate provisioner key %q after metadata backfill rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("recreate provisioner key %q after metadata backfill rotation: %w", keyName, rotateErr) + } + if rotated.OrganizationID != uuid.Nil { + organizationID = rotated.OrganizationID.String() + } + if rotated.KeyID != uuid.Nil { + provisionerKeyID = rotated.KeyID.String() } + if rotated.KeyName != "" { + provisionerKeyName = rotated.KeyName + } + keyMaterial = rotated.Key + if keyMaterial == "" { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Provisioner key %q returned empty material after metadata backfill rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key %q returned empty material after metadata backfill rotation", keyName) + } + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash } setCondition( provisioner, From 33dbfe78c0f20f8db74a2f624ee83ee53f7c7f9d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 13:49:09 +0000 Subject: [PATCH 33/38] fix: drop RequeueAfter when returning errors in backfill --- internal/controller/coderprovisioner_controller.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 5c8ff642..9ef04b79 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -7,7 +7,6 @@ import ( "hash/fnv" "maps" "slices" - "time" "github.com/google/uuid" appsv1 "k8s.io/api/apps/v1" @@ -362,7 +361,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req }) if ensureErr != nil { // Return a requeue so metadata population retries. - return ctrl.Result{RequeueAfter: 10 * time.Second}, fmt.Errorf("verify provisioner key metadata: %w", ensureErr) + return ctrl.Result{}, fmt.Errorf("verify provisioner key metadata: %w", ensureErr) } if response.OrganizationID != uuid.Nil { organizationID = response.OrganizationID.String() @@ -385,7 +384,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req if deleteErr := r.BootstrapClient.DeleteProvisionerKey( ctx, controlPlane.Status.URL, sessionToken, organizationName, keyName, ); deleteErr != nil { - return ctrl.Result{RequeueAfter: 10 * time.Second}, fmt.Errorf("delete provisioner key %q for metadata backfill rotation: %w", keyName, deleteErr) + return ctrl.Result{}, fmt.Errorf("delete provisioner key %q for metadata backfill rotation: %w", keyName, deleteErr) } rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ CoderURL: controlPlane.Status.URL, From ba890eed05b649c464eb05f94c2fffea8656bd99 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 14:16:58 +0000 Subject: [PATCH 34/38] fix: rotate keys on control plane ref drift --- api/v1alpha1/coderprovisioner_types.go | 2 + .../bases/coder.com_coderprovisioners.yaml | 4 ++ docs/reference/api/coderprovisioner.md | 1 + .../controller/coderprovisioner_controller.go | 37 +++++++------------ 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/api/v1alpha1/coderprovisioner_types.go b/api/v1alpha1/coderprovisioner_types.go index 1df0513a..a85320d9 100644 --- a/api/v1alpha1/coderprovisioner_types.go +++ b/api/v1alpha1/coderprovisioner_types.go @@ -98,6 +98,8 @@ type CoderProvisionerStatus struct { ProvisionerKeyName string `json:"provisionerKeyName,omitempty"` // TagsHash is a deterministic hash of spec.tags last applied to the provisioner key. TagsHash string `json:"tagsHash,omitempty"` + // ControlPlaneRefName is the control plane ref name last applied to the provisioner key. + ControlPlaneRefName string `json:"controlPlaneRefName,omitempty"` // SecretRef references the provisioner key secret data currently in use. SecretRef *SecretKeySelector `json:"secretRef,omitempty"` } diff --git a/config/crd/bases/coder.com_coderprovisioners.yaml b/config/crd/bases/coder.com_coderprovisioners.yaml index 84113a7f..324ca86a 100644 --- a/config/crd/bases/coder.com_coderprovisioners.yaml +++ b/config/crd/bases/coder.com_coderprovisioners.yaml @@ -433,6 +433,10 @@ spec: - type type: object type: array + controlPlaneRefName: + description: ControlPlaneRefName is the control plane ref name last + applied to the provisioner key. + type: string observedGeneration: description: ObservedGeneration tracks the spec generation this status reflects. diff --git a/docs/reference/api/coderprovisioner.md b/docs/reference/api/coderprovisioner.md index b5754831..2c91dc50 100644 --- a/docs/reference/api/coderprovisioner.md +++ b/docs/reference/api/coderprovisioner.md @@ -39,6 +39,7 @@ | `status.provisionerKeyID` | `string` | ProvisionerKeyID is the provisioner key ID last applied in coderd. | | `status.provisionerKeyName` | `string` | ProvisionerKeyName is the provisioner key name last applied in coderd. | | `status.tagsHash` | `string` | TagsHash is a deterministic hash of spec.tags last applied to the provisioner key. | +| `status.controlPlaneRefName` | `string` | ControlPlaneRefName is the control plane ref name last applied to the provisioner key. | | `status.secretRef` | `github.com/coder/coder-k8s/api/v1alpha1.SecretKeySelector` | SecretRef references the provisioner key secret data currently in use. | ## Source diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 9ef04b79..4b2ecdfe 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -130,13 +130,16 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req ) desiredTagsHash := hashProvisionerTags(provisioner.Spec.Tags) + desiredControlPlaneRefName := provisioner.Spec.ControlPlaneRef.Name status := provisioner.Status orgDrift := status.OrganizationName != "" && status.OrganizationName != organizationName keyNameDrift := status.ProvisionerKeyName != "" && status.ProvisionerKeyName != keyName tagsDrift := status.TagsHash != "" && status.TagsHash != desiredTagsHash - driftDetected := orgDrift || keyNameDrift || tagsDrift + controlPlaneRefDrift := status.ControlPlaneRefName != "" && status.ControlPlaneRefName != desiredControlPlaneRefName + driftDetected := orgDrift || keyNameDrift || tagsDrift || controlPlaneRefDrift appliedOrgName := provisioner.Status.OrganizationName appliedTagsHash := provisioner.Status.TagsHash + appliedControlPlaneRefName := provisioner.Status.ControlPlaneRefName // Check whether a usable provisioner key secret already exists. // The secret is considered "usable" only if the Secret object exists @@ -163,7 +166,8 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req keyMaterial := "" if driftDetected { log.Info("spec drift detected, rotating provisioner key", - "orgDrift", orgDrift, "keyNameDrift", keyNameDrift, "tagsDrift", tagsDrift) + "orgDrift", orgDrift, "keyNameDrift", keyNameDrift, "tagsDrift", tagsDrift, + "controlPlaneRefDrift", controlPlaneRefDrift) oldOrg := provisioner.Status.OrganizationName if oldOrg == "" { @@ -256,6 +260,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req } appliedOrgName = organizationName appliedTagsHash = desiredTagsHash + appliedControlPlaneRefName = desiredControlPlaneRefName setCondition( provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, @@ -339,6 +344,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req } appliedOrgName = organizationName appliedTagsHash = desiredTagsHash + appliedControlPlaneRefName = desiredControlPlaneRefName setCondition( provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, @@ -346,7 +352,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req "ProvisionerKeyReady", "Provisioner key is available in coderd", ) - } else if status.OrganizationName == "" || status.TagsHash == "" { + } else if status.OrganizationName == "" || status.TagsHash == "" || status.ControlPlaneRefName == "" { // Secret is usable and no drift detected, but status metadata is empty // (e.g. upgrade from older version). Call EnsureProvisionerKey to populate // IDs and key name. If coderd reports an existing key (no plaintext key @@ -377,6 +383,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req keyMaterial = response.Key appliedOrgName = organizationName appliedTagsHash = desiredTagsHash + appliedControlPlaneRefName = desiredControlPlaneRefName } else { // Key already exists; tags may be stale. Rotate to ensure desired tags are applied. log.Info("existing key found during metadata backfill, rotating to ensure desired tags", @@ -419,6 +426,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req } appliedOrgName = organizationName appliedTagsHash = desiredTagsHash + appliedControlPlaneRefName = desiredControlPlaneRefName } setCondition( provisioner, @@ -489,6 +497,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req provisionerKeyID, provisionerKeyName, appliedTagsHash, + appliedControlPlaneRefName, ); err != nil { return ctrl.Result{}, err } @@ -643,10 +652,6 @@ func (r *CoderProvisionerReconciler) ensureProvisionerKeySecret( return nil, fmt.Errorf("reconcile provisioner key secret %q: %w", secretName, err) } - if err := r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: provisioner.Namespace}, secret); err != nil { - return nil, fmt.Errorf("get reconciled provisioner key secret %q: %w", secretName, err) - } - return secret, nil } @@ -670,10 +675,6 @@ func (r *CoderProvisionerReconciler) reconcileServiceAccount( return nil, fmt.Errorf("reconcile serviceaccount %q: %w", serviceAccountName, err) } - if err := r.Get(ctx, types.NamespacedName{Name: serviceAccount.Name, Namespace: serviceAccount.Namespace}, serviceAccount); err != nil { - return nil, fmt.Errorf("get reconciled serviceaccount %q: %w", serviceAccountName, err) - } - return serviceAccount, nil } @@ -702,10 +703,6 @@ func (r *CoderProvisionerReconciler) reconcileRole( return nil, fmt.Errorf("reconcile role %q: %w", roleName, err) } - if err := r.Get(ctx, types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, role); err != nil { - return nil, fmt.Errorf("get reconciled role %q: %w", roleName, err) - } - return role, nil } @@ -741,10 +738,6 @@ func (r *CoderProvisionerReconciler) reconcileRoleBinding( return nil, fmt.Errorf("reconcile rolebinding %q: %w", roleBindingName, err) } - if err := r.Get(ctx, types.NamespacedName{Name: roleBinding.Name, Namespace: roleBinding.Namespace}, roleBinding); err != nil { - return nil, fmt.Errorf("get reconciled rolebinding %q: %w", roleBindingName, err) - } - return roleBinding, nil } @@ -825,10 +818,6 @@ func (r *CoderProvisionerReconciler) reconcileDeployment( return nil, fmt.Errorf("reconcile provisioner deployment: %w", err) } - if err := r.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment); err != nil { - return nil, fmt.Errorf("get reconciled deployment %q: %w", deployment.Name, err) - } - return deployment, nil } @@ -842,6 +831,7 @@ func (r *CoderProvisionerReconciler) reconcileStatus( provisionerKeyID string, provisionerKeyName string, tagsHash string, + controlPlaneRefName string, ) error { // Take a snapshot before mutation so status writes are skipped when no fields changed. previousStatus := provisioner.Status.DeepCopy() @@ -860,6 +850,7 @@ func (r *CoderProvisionerReconciler) reconcileStatus( provisioner.Status.ProvisionerKeyID = provisionerKeyID provisioner.Status.ProvisionerKeyName = provisionerKeyName provisioner.Status.TagsHash = tagsHash + provisioner.Status.ControlPlaneRefName = controlPlaneRefName provisioner.Status.SecretRef = &coderv1alpha1.SecretKeySelector{ Name: secretRef.Name, Key: secretRef.Key, From 5dd24ca18ac50fb7ce97dfc647e36ad49904162b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 14:28:17 +0000 Subject: [PATCH 35/38] fix: persist provisioner control plane URL in status --- api/v1alpha1/coderprovisioner_types.go | 2 ++ .../bases/coder.com_coderprovisioners.yaml | 4 ++++ docs/reference/api/coderprovisioner.md | 1 + .../controller/coderprovisioner_controller.go | 20 +++++++++++++++---- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/api/v1alpha1/coderprovisioner_types.go b/api/v1alpha1/coderprovisioner_types.go index a85320d9..339fc62f 100644 --- a/api/v1alpha1/coderprovisioner_types.go +++ b/api/v1alpha1/coderprovisioner_types.go @@ -100,6 +100,8 @@ type CoderProvisionerStatus struct { TagsHash string `json:"tagsHash,omitempty"` // ControlPlaneRefName is the control plane ref name last applied to the provisioner key. ControlPlaneRefName string `json:"controlPlaneRefName,omitempty"` + // ControlPlaneURL is the control plane URL last applied to the provisioner key. + ControlPlaneURL string `json:"controlPlaneURL,omitempty"` // SecretRef references the provisioner key secret data currently in use. SecretRef *SecretKeySelector `json:"secretRef,omitempty"` } diff --git a/config/crd/bases/coder.com_coderprovisioners.yaml b/config/crd/bases/coder.com_coderprovisioners.yaml index 324ca86a..7f857bbe 100644 --- a/config/crd/bases/coder.com_coderprovisioners.yaml +++ b/config/crd/bases/coder.com_coderprovisioners.yaml @@ -437,6 +437,10 @@ spec: description: ControlPlaneRefName is the control plane ref name last applied to the provisioner key. type: string + controlPlaneURL: + description: ControlPlaneURL is the control plane URL last applied + to the provisioner key. + type: string observedGeneration: description: ObservedGeneration tracks the spec generation this status reflects. diff --git a/docs/reference/api/coderprovisioner.md b/docs/reference/api/coderprovisioner.md index 2c91dc50..a84b7ba9 100644 --- a/docs/reference/api/coderprovisioner.md +++ b/docs/reference/api/coderprovisioner.md @@ -40,6 +40,7 @@ | `status.provisionerKeyName` | `string` | ProvisionerKeyName is the provisioner key name last applied in coderd. | | `status.tagsHash` | `string` | TagsHash is a deterministic hash of spec.tags last applied to the provisioner key. | | `status.controlPlaneRefName` | `string` | ControlPlaneRefName is the control plane ref name last applied to the provisioner key. | +| `status.controlPlaneURL` | `string` | ControlPlaneURL is the control plane URL last applied to the provisioner key. | | `status.secretRef` | `github.com/coder/coder-k8s/api/v1alpha1.SecretKeySelector` | SecretRef references the provisioner key secret data currently in use. | ## Source diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 4b2ecdfe..a854889b 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -78,6 +78,8 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req return r.reconcileDeletion(ctx, provisioner) } + statusSnapshot := provisioner.Status.DeepCopy() + finalizerAdded, err := r.ensureCleanupFinalizer(ctx, provisioner) if err != nil { return ctrl.Result{}, err @@ -177,10 +179,14 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req if oldKeyName == "" { oldKeyName = keyName } + oldControlPlaneURL := provisioner.Status.ControlPlaneURL + if oldControlPlaneURL == "" { + oldControlPlaneURL = controlPlane.Status.URL + } if deleteErr := r.BootstrapClient.DeleteProvisionerKey( ctx, - controlPlane.Status.URL, + oldControlPlaneURL, sessionToken, oldOrg, oldKeyName, @@ -498,6 +504,8 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req provisionerKeyName, appliedTagsHash, appliedControlPlaneRefName, + controlPlane.Status.URL, + statusSnapshot, ); err != nil { return ctrl.Result{}, err } @@ -832,9 +840,12 @@ func (r *CoderProvisionerReconciler) reconcileStatus( provisionerKeyName string, tagsHash string, controlPlaneRefName string, + controlPlaneURL string, + statusSnapshot *coderv1alpha1.CoderProvisionerStatus, ) error { - // Take a snapshot before mutation so status writes are skipped when no fields changed. - previousStatus := provisioner.Status.DeepCopy() + if statusSnapshot == nil { + return fmt.Errorf("assertion failed: status snapshot must not be nil") + } phase := coderv1alpha1.CoderProvisionerPhasePending if deployment.Status.ReadyReplicas > 0 { @@ -851,6 +862,7 @@ func (r *CoderProvisionerReconciler) reconcileStatus( provisioner.Status.ProvisionerKeyName = provisionerKeyName provisioner.Status.TagsHash = tagsHash provisioner.Status.ControlPlaneRefName = controlPlaneRefName + provisioner.Status.ControlPlaneURL = controlPlaneURL provisioner.Status.SecretRef = &coderv1alpha1.SecretKeySelector{ Name: secretRef.Name, Key: secretRef.Key, @@ -874,7 +886,7 @@ func (r *CoderProvisionerReconciler) reconcileStatus( ) } - if previousStatus != nil && equality.Semantic.DeepEqual(*previousStatus, provisioner.Status) { + if equality.Semantic.DeepEqual(*statusSnapshot, provisioner.Status) { return nil } From 2241f64aa8c4df3649fbdd04be33490467a4159d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 14:39:39 +0000 Subject: [PATCH 36/38] fix: track control-plane URL drift for provisioner keys --- internal/controller/coderprovisioner_controller.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index a854889b..5ea3ad8c 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -138,10 +138,12 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req keyNameDrift := status.ProvisionerKeyName != "" && status.ProvisionerKeyName != keyName tagsDrift := status.TagsHash != "" && status.TagsHash != desiredTagsHash controlPlaneRefDrift := status.ControlPlaneRefName != "" && status.ControlPlaneRefName != desiredControlPlaneRefName - driftDetected := orgDrift || keyNameDrift || tagsDrift || controlPlaneRefDrift + controlPlaneURLDrift := status.ControlPlaneURL != "" && status.ControlPlaneURL != controlPlane.Status.URL + driftDetected := orgDrift || keyNameDrift || tagsDrift || controlPlaneRefDrift || controlPlaneURLDrift appliedOrgName := provisioner.Status.OrganizationName appliedTagsHash := provisioner.Status.TagsHash appliedControlPlaneRefName := provisioner.Status.ControlPlaneRefName + appliedControlPlaneURL := provisioner.Status.ControlPlaneURL // Check whether a usable provisioner key secret already exists. // The secret is considered "usable" only if the Secret object exists @@ -169,7 +171,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req if driftDetected { log.Info("spec drift detected, rotating provisioner key", "orgDrift", orgDrift, "keyNameDrift", keyNameDrift, "tagsDrift", tagsDrift, - "controlPlaneRefDrift", controlPlaneRefDrift) + "controlPlaneRefDrift", controlPlaneRefDrift, "controlPlaneURLDrift", controlPlaneURLDrift) oldOrg := provisioner.Status.OrganizationName if oldOrg == "" { @@ -267,6 +269,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req appliedOrgName = organizationName appliedTagsHash = desiredTagsHash appliedControlPlaneRefName = desiredControlPlaneRefName + appliedControlPlaneURL = controlPlane.Status.URL setCondition( provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, @@ -351,6 +354,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req appliedOrgName = organizationName appliedTagsHash = desiredTagsHash appliedControlPlaneRefName = desiredControlPlaneRefName + appliedControlPlaneURL = controlPlane.Status.URL setCondition( provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, @@ -390,6 +394,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req appliedOrgName = organizationName appliedTagsHash = desiredTagsHash appliedControlPlaneRefName = desiredControlPlaneRefName + appliedControlPlaneURL = controlPlane.Status.URL } else { // Key already exists; tags may be stale. Rotate to ensure desired tags are applied. log.Info("existing key found during metadata backfill, rotating to ensure desired tags", @@ -433,6 +438,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req appliedOrgName = organizationName appliedTagsHash = desiredTagsHash appliedControlPlaneRefName = desiredControlPlaneRefName + appliedControlPlaneURL = controlPlane.Status.URL } setCondition( provisioner, @@ -504,7 +510,7 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req provisionerKeyName, appliedTagsHash, appliedControlPlaneRefName, - controlPlane.Status.URL, + appliedControlPlaneURL, statusSnapshot, ); err != nil { return ctrl.Result{}, err From 9c5b16629b6882a4f9160f89e9fe385696294a9b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 14:52:03 +0000 Subject: [PATCH 37/38] controller: prefer status URL for deletion key cleanup --- .../controller/coderprovisioner_controller.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 5ea3ad8c..6bdd184c 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -544,11 +544,18 @@ func (r *CoderProvisionerReconciler) reconcileDeletion(ctx context.Context, prov // CR does not get stuck in Terminating. This is common during // namespace teardown, when the control plane was never ready, or // when credentials were misconfigured. - controlPlane, err := r.fetchControlPlane(ctx, provisioner) - if err != nil { - log.Info("unable to reach referenced CoderControlPlane during deletion, skipping remote key cleanup", - "controlPlaneRef", provisioner.Spec.ControlPlaneRef.Name, "error", err) - } else { + controlPlaneURL := provisioner.Status.ControlPlaneURL + if controlPlaneURL == "" { + controlPlane, err := r.fetchControlPlane(ctx, provisioner) + if err != nil { + log.Info("unable to reach referenced CoderControlPlane during deletion, skipping remote key cleanup", + "controlPlaneRef", provisioner.Spec.ControlPlaneRef.Name, "error", err) + } else { + controlPlaneURL = controlPlane.Status.URL + } + } + + if controlPlaneURL != "" { sessionToken, tokenErr := r.readBootstrapSessionToken(ctx, provisioner) if tokenErr != nil { log.Info("unable to read bootstrap credentials during deletion, skipping remote key cleanup", @@ -556,7 +563,7 @@ func (r *CoderProvisionerReconciler) reconcileDeletion(ctx context.Context, prov } else { if deleteErr := r.BootstrapClient.DeleteProvisionerKey( ctx, - controlPlane.Status.URL, + controlPlaneURL, sessionToken, organizationName, keyName, From 117a6a719f8d53ab1dcf316d4f6fd0bcc1e20b47 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 11 Feb 2026 14:54:20 +0000 Subject: [PATCH 38/38] test: update deletion cleanup expectation when control plane is gone --- internal/controller/coderprovisioner_controller_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/controller/coderprovisioner_controller_test.go b/internal/controller/coderprovisioner_controller_test.go index f29f1f5a..8ed3f2f2 100644 --- a/internal/controller/coderprovisioner_controller_test.go +++ b/internal/controller/coderprovisioner_controller_test.go @@ -514,9 +514,9 @@ func TestCoderProvisionerReconciler_DeletionControlPlaneGone(t *testing.T) { reconcileProvisioner(ctx, t, reconciler, namespacedName) - // DeleteProvisionerKey should NOT have been called since the control - // plane was already gone. - require.Equal(t, 0, bootstrapClient.deleteKeyCalls) + // DeleteProvisionerKey should still be called once using the persisted + // status.ControlPlaneURL even when the control plane object is already gone. + require.Equal(t, 1, bootstrapClient.deleteKeyCalls) // The finalizer should still be removed. require.Eventually(t, func() bool {