diff --git a/.cspell.json b/.cspell.json index 3075f298..76865b83 100644 --- a/.cspell.json +++ b/.cspell.json @@ -18,7 +18,10 @@ "codertemplates", "coderworkspace", "coderworkspaces", + "coderprovisioner", + "coderprovisioners", "controllerapp", + "provisionerd", "workspaceproxy", "workspaceproxies", "derp", diff --git a/.gitignore b/.gitignore index bc8e4313..ad32dc54 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ terraform.rc # Backend config (local-only, not committed) terraform/backend.hcl +/coder-k8s diff --git a/api/v1alpha1/coderprovisioner_types.go b/api/v1alpha1/coderprovisioner_types.go new file mode 100644 index 00000000..339fc62f --- /dev/null +++ b/api/v1alpha1/coderprovisioner_types.go @@ -0,0 +1,138 @@ +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // CoderProvisionerPhasePending indicates the provisioner deployment is not ready. + CoderProvisionerPhasePending = "Pending" + // CoderProvisionerPhaseReady indicates at least one provisioner pod is ready. + CoderProvisionerPhaseReady = "Ready" + + // CoderProvisionerConditionControlPlaneReady indicates whether the referenced control plane is reachable. + CoderProvisionerConditionControlPlaneReady = "ControlPlaneReady" + // CoderProvisionerConditionBootstrapSecretReady indicates whether the bootstrap credentials secret is available. + CoderProvisionerConditionBootstrapSecretReady = "BootstrapSecretReady" + // CoderProvisionerConditionProvisionerKeyReady indicates whether the provisioner key exists in coderd. + CoderProvisionerConditionProvisionerKeyReady = "ProvisionerKeyReady" + // CoderProvisionerConditionProvisionerKeySecretReady indicates whether the provisioner key secret is populated. + CoderProvisionerConditionProvisionerKeySecretReady = "ProvisionerKeySecretReady" + // CoderProvisionerConditionDeploymentReady indicates whether the provisioner deployment has ready replicas. + CoderProvisionerConditionDeploymentReady = "DeploymentReady" + + // DefaultProvisionerKeySecretKey is the default data key for provisioner key secrets. + DefaultProvisionerKeySecretKey = "key" + + // ProvisionerKeyCleanupFinalizer is applied to ensure coderd key cleanup on deletion. + ProvisionerKeyCleanupFinalizer = "coder.com/provisioner-key-cleanup" +) + +// CoderProvisionerBootstrapSpec configures credentials for provisioner key management. +type CoderProvisionerBootstrapSpec struct { + // CredentialsSecretRef points to a Secret containing a Coder session token + // with permission to manage provisioner keys. + CredentialsSecretRef SecretKeySelector `json:"credentialsSecretRef"` +} + +// CoderProvisionerKeySpec configures provisioner key naming and storage. +type CoderProvisionerKeySpec struct { + // Name is the provisioner key name in coderd. Defaults to the CR name. + // +kubebuilder:validation:MaxLength=128 + Name string `json:"name,omitempty"` + // SecretName is the Kubernetes Secret to store the key. Defaults to "{crName}-provisioner-key". + // +kubebuilder:validation:MaxLength=253 + SecretName string `json:"secretName,omitempty"` + // SecretKey is the data key in the Secret. Defaults to "key". + // +kubebuilder:validation:MaxLength=253 + SecretKey string `json:"secretKey,omitempty"` +} + +// CoderProvisionerSpec defines the desired state of a CoderProvisioner. +type CoderProvisionerSpec struct { + // ControlPlaneRef identifies which CoderControlPlane instance to join. + ControlPlaneRef corev1.LocalObjectReference `json:"controlPlaneRef"` + // OrganizationName is the Coder organization. Defaults to "default". + // +kubebuilder:validation:MaxLength=128 + OrganizationName string `json:"organizationName,omitempty"` + // Bootstrap configures credentials for provisioner key management. + Bootstrap CoderProvisionerBootstrapSpec `json:"bootstrap"` + // Key configures provisioner key naming and secret storage. + Key CoderProvisionerKeySpec `json:"key,omitempty"` + // Replicas is the desired number of provisioner pods. + Replicas *int32 `json:"replicas,omitempty"` + // Tags are attached to the provisioner key for job routing. + Tags map[string]string `json:"tags,omitempty"` + // Image is the container image. Defaults to the control plane image. + Image string `json:"image,omitempty"` + // ExtraArgs are appended after "provisionerd start". + ExtraArgs []string `json:"extraArgs,omitempty"` + // ExtraEnv are injected into the provisioner container. + ExtraEnv []corev1.EnvVar `json:"extraEnv,omitempty"` + // Resources for the provisioner container. + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + // ImagePullSecrets are used by the pod to pull private images. + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + // TerminationGracePeriodSeconds for the provisioner pods. + TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"` +} + +// CoderProvisionerStatus defines the observed state of a CoderProvisioner. +type CoderProvisionerStatus struct { + // ObservedGeneration tracks the spec generation this status reflects. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + // ReadyReplicas is the number of ready pods observed in the deployment. + ReadyReplicas int32 `json:"readyReplicas,omitempty"` + // Phase is a high-level readiness indicator. + Phase string `json:"phase,omitempty"` + // Conditions are Kubernetes-standard conditions for this resource. + Conditions []metav1.Condition `json:"conditions,omitempty"` + // OrganizationID is the organization ID last applied to the provisioner key. + OrganizationID string `json:"organizationID,omitempty"` + // OrganizationName is the organization name last applied to the provisioner key. + OrganizationName string `json:"organizationName,omitempty"` + // ProvisionerKeyID is the provisioner key ID last applied in coderd. + ProvisionerKeyID string `json:"provisionerKeyID,omitempty"` + // ProvisionerKeyName is the provisioner key name last applied in coderd. + ProvisionerKeyName string `json:"provisionerKeyName,omitempty"` + // TagsHash is a deterministic hash of spec.tags last applied to the provisioner key. + TagsHash string `json:"tagsHash,omitempty"` + // ControlPlaneRefName is the control plane ref name last applied to the provisioner key. + ControlPlaneRefName string `json:"controlPlaneRefName,omitempty"` + // ControlPlaneURL is the control plane URL last applied to the provisioner key. + ControlPlaneURL string `json:"controlPlaneURL,omitempty"` + // SecretRef references the provisioner key secret data currently in use. + SecretRef *SecretKeySelector `json:"secretRef,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Replicas",type=integer,JSONPath=`.status.readyReplicas` +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// CoderProvisioner is the schema for Coder external provisioner daemon resources. +type CoderProvisioner struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CoderProvisionerSpec `json:"spec,omitempty"` + Status CoderProvisionerStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// CoderProvisionerList contains a list of CoderProvisioner objects. +type CoderProvisionerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CoderProvisioner `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CoderProvisioner{}, &CoderProvisionerList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9c6c1e9f..f231cf81 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -140,6 +140,182 @@ func (in *CoderControlPlaneStatus) DeepCopy() *CoderControlPlaneStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisioner) DeepCopyInto(out *CoderProvisioner) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisioner. +func (in *CoderProvisioner) DeepCopy() *CoderProvisioner { + if in == nil { + return nil + } + out := new(CoderProvisioner) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CoderProvisioner) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisionerBootstrapSpec) DeepCopyInto(out *CoderProvisionerBootstrapSpec) { + *out = *in + out.CredentialsSecretRef = in.CredentialsSecretRef + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisionerBootstrapSpec. +func (in *CoderProvisionerBootstrapSpec) DeepCopy() *CoderProvisionerBootstrapSpec { + if in == nil { + return nil + } + out := new(CoderProvisionerBootstrapSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisionerKeySpec) DeepCopyInto(out *CoderProvisionerKeySpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisionerKeySpec. +func (in *CoderProvisionerKeySpec) DeepCopy() *CoderProvisionerKeySpec { + if in == nil { + return nil + } + out := new(CoderProvisionerKeySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisionerList) DeepCopyInto(out *CoderProvisionerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CoderProvisioner, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisionerList. +func (in *CoderProvisionerList) DeepCopy() *CoderProvisionerList { + if in == nil { + return nil + } + out := new(CoderProvisionerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CoderProvisionerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisionerSpec) DeepCopyInto(out *CoderProvisionerSpec) { + *out = *in + out.ControlPlaneRef = in.ControlPlaneRef + out.Bootstrap = in.Bootstrap + out.Key = in.Key + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ExtraArgs != nil { + in, out := &in.ExtraArgs, &out.ExtraArgs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExtraEnv != nil { + in, out := &in.ExtraEnv, &out.ExtraEnv + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Resources.DeepCopyInto(&out.Resources) + if in.ImagePullSecrets != nil { + in, out := &in.ImagePullSecrets, &out.ImagePullSecrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } + if in.TerminationGracePeriodSeconds != nil { + in, out := &in.TerminationGracePeriodSeconds, &out.TerminationGracePeriodSeconds + *out = new(int64) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisionerSpec. +func (in *CoderProvisionerSpec) DeepCopy() *CoderProvisionerSpec { + if in == nil { + return nil + } + out := new(CoderProvisionerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoderProvisionerStatus) DeepCopyInto(out *CoderProvisionerStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(SecretKeySelector) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoderProvisionerStatus. +func (in *CoderProvisionerStatus) DeepCopy() *CoderProvisionerStatus { + if in == nil { + return nil + } + out := new(CoderProvisionerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorAccessSpec) DeepCopyInto(out *OperatorAccessSpec) { *out = *in diff --git a/config/crd/bases/coder.com_coderprovisioners.yaml b/config/crd/bases/coder.com_coderprovisioners.yaml new file mode 100644 index 00000000..7f857bbe --- /dev/null +++ b/config/crd/bases/coder.com_coderprovisioners.yaml @@ -0,0 +1,495 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: coderprovisioners.coder.com +spec: + group: coder.com + names: + kind: CoderProvisioner + listKind: CoderProvisionerList + plural: coderprovisioners + singular: coderprovisioner + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.readyReplicas + name: Replicas + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CoderProvisioner is the schema for Coder external provisioner + daemon resources. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CoderProvisionerSpec defines the desired state of a CoderProvisioner. + properties: + bootstrap: + description: Bootstrap configures credentials for provisioner key + management. + properties: + credentialsSecretRef: + description: |- + CredentialsSecretRef points to a Secret containing a Coder session token + with permission to manage provisioner keys. + properties: + key: + description: Key is the key inside the Secret data map. + type: string + name: + description: Name is the Kubernetes Secret name. + type: string + required: + - name + type: object + required: + - credentialsSecretRef + type: object + controlPlaneRef: + description: ControlPlaneRef identifies which CoderControlPlane instance + to join. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + extraArgs: + description: ExtraArgs are appended after "provisionerd start". + items: + type: string + type: array + extraEnv: + description: ExtraEnv are injected into the provisioner container. + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + description: Image is the container image. Defaults to the control + plane image. + type: string + imagePullSecrets: + description: ImagePullSecrets are used by the pod to pull private + images. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + key: + description: Key configures provisioner key naming and secret storage. + properties: + name: + description: Name is the provisioner key name in coderd. Defaults + to the CR name. + maxLength: 128 + type: string + secretKey: + description: SecretKey is the data key in the Secret. Defaults + to "key". + maxLength: 253 + type: string + secretName: + description: SecretName is the Kubernetes Secret to store the + key. Defaults to "{crName}-provisioner-key". + maxLength: 253 + type: string + type: object + organizationName: + description: OrganizationName is the Coder organization. Defaults + to "default". + maxLength: 128 + type: string + replicas: + description: Replicas is the desired number of provisioner pods. + format: int32 + type: integer + resources: + description: Resources for the provisioner container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tags: + additionalProperties: + type: string + description: Tags are attached to the provisioner key for job routing. + type: object + terminationGracePeriodSeconds: + description: TerminationGracePeriodSeconds for the provisioner pods. + format: int64 + type: integer + required: + - bootstrap + - controlPlaneRef + type: object + status: + description: CoderProvisionerStatus defines the observed state of a CoderProvisioner. + properties: + conditions: + description: Conditions are Kubernetes-standard conditions for this + resource. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + controlPlaneRefName: + description: ControlPlaneRefName is the control plane ref name last + applied to the provisioner key. + type: string + controlPlaneURL: + description: ControlPlaneURL is the control plane URL last applied + to the provisioner key. + type: string + observedGeneration: + description: ObservedGeneration tracks the spec generation this status + reflects. + format: int64 + type: integer + organizationID: + description: OrganizationID is the organization ID last applied to + the provisioner key. + type: string + organizationName: + description: OrganizationName is the organization name last applied + to the provisioner key. + type: string + phase: + description: Phase is a high-level readiness indicator. + type: string + provisionerKeyID: + description: ProvisionerKeyID is the provisioner key ID last applied + in coderd. + type: string + provisionerKeyName: + description: ProvisionerKeyName is the provisioner key name last applied + in coderd. + type: string + readyReplicas: + description: ReadyReplicas is the number of ready pods observed in + the deployment. + format: int32 + type: integer + secretRef: + description: SecretRef references the provisioner key secret data + currently in use. + properties: + key: + description: Key is the key inside the Secret data map. + type: string + name: + description: Name is the Kubernetes Secret name. + type: string + required: + - name + type: object + tagsHash: + description: TagsHash is a deterministic hash of spec.tags last applied + to the provisioner key. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6a688a75..58168493 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -15,6 +15,7 @@ rules: - "" resources: - secrets + - serviceaccounts - services verbs: - create @@ -40,6 +41,7 @@ rules: - coder.com resources: - codercontrolplanes + - coderprovisioners - workspaceproxies verbs: - create @@ -53,6 +55,7 @@ rules: - coder.com resources: - codercontrolplanes/finalizers + - coderprovisioners/finalizers - workspaceproxies/finalizers verbs: - update @@ -60,6 +63,7 @@ rules: - coder.com resources: - codercontrolplanes/status + - coderprovisioners/status - workspaceproxies/status verbs: - get @@ -77,3 +81,16 @@ rules: - patch - update - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + - roles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/config/samples/coder_v1alpha1_coderprovisioner.yaml b/config/samples/coder_v1alpha1_coderprovisioner.yaml new file mode 100644 index 00000000..fddfbfc3 --- /dev/null +++ b/config/samples/coder_v1alpha1_coderprovisioner.yaml @@ -0,0 +1,38 @@ +# CoderProvisioner sample manifest. +# +# Prerequisites: +# 1. A CoderControlPlane resource must exist in the same namespace. +# 2. Create the bootstrap credentials Secret with a valid Coder session token: +# +# kubectl create secret generic coder-bootstrap-token \ +# --namespace=default \ +# --from-literal=token= +# +# IMPORTANT: Never commit real tokens to source control. +apiVersion: coder.com/v1alpha1 +kind: CoderProvisioner +metadata: + name: coderprovisioner-sample + namespace: default +spec: + controlPlaneRef: + name: codercontrolplane-sample + bootstrap: + credentialsSecretRef: + name: coder-bootstrap-token + key: token + # key: + # name: my-provisioner-key # Provisioner key name in coderd (defaults to CR name) + # secretName: my-key-secret # K8s Secret to store the key (defaults to "{name}-provisioner-key") + # secretKey: key # Data key in the Secret (defaults to "key") + # organizationName: default # Coder organization (defaults to "default") + # image: ghcr.io/coder/coder:latest # Container image (defaults to control plane image) + # replicas: 1 + # extraArgs: + # - --verbose + # extraEnv: + # - name: CODER_VERBOSE + # value: "true" + tags: + scope: organization + replicas: 1 diff --git a/docs/reference/api/coderprovisioner.md b/docs/reference/api/coderprovisioner.md new file mode 100644 index 00000000..a84b7ba9 --- /dev/null +++ b/docs/reference/api/coderprovisioner.md @@ -0,0 +1,49 @@ + + +# `CoderProvisioner` + +## API identity + +- Group/version: `coder.com/v1alpha1` +- Kind: `CoderProvisioner` +- Resource: `coderprovisioners` +- Scope: namespaced + +## Spec + +| Field | Type | Description | +| --- | --- | --- | +| `spec.controlPlaneRef` | `k8s.io/api/core/v1.LocalObjectReference` | ControlPlaneRef identifies which CoderControlPlane instance to join. | +| `spec.organizationName` | `string` | OrganizationName is the Coder organization. Defaults to "default". | +| `spec.bootstrap` | `github.com/coder/coder-k8s/api/v1alpha1.CoderProvisionerBootstrapSpec` | Bootstrap configures credentials for provisioner key management. | +| `spec.key` | `github.com/coder/coder-k8s/api/v1alpha1.CoderProvisionerKeySpec` | Key configures provisioner key naming and secret storage. | +| `spec.replicas` | `int32` | Replicas is the desired number of provisioner pods. | +| `spec.tags` | `map[string]string` | Tags are attached to the provisioner key for job routing. | +| `spec.image` | `string` | Image is the container image. Defaults to the control plane image. | +| `spec.extraArgs` | `[]string` | ExtraArgs are appended after "provisionerd start". | +| `spec.extraEnv` | `[]k8s.io/api/core/v1.EnvVar` | ExtraEnv are injected into the provisioner container. | +| `spec.resources` | `k8s.io/api/core/v1.ResourceRequirements` | Resources for the provisioner container. | +| `spec.imagePullSecrets` | `[]k8s.io/api/core/v1.LocalObjectReference` | ImagePullSecrets are used by the pod to pull private images. | +| `spec.terminationGracePeriodSeconds` | `int64` | TerminationGracePeriodSeconds for the provisioner pods. | + +## Status + +| Field | Type | Description | +| --- | --- | --- | +| `status.observedGeneration` | `int64` | ObservedGeneration tracks the spec generation this status reflects. | +| `status.readyReplicas` | `int32` | ReadyReplicas is the number of ready pods observed in the deployment. | +| `status.phase` | `string` | Phase is a high-level readiness indicator. | +| `status.conditions` | `[]metav1.Condition` | Conditions are Kubernetes-standard conditions for this resource. | +| `status.organizationID` | `string` | OrganizationID is the organization ID last applied to the provisioner key. | +| `status.organizationName` | `string` | OrganizationName is the organization name last applied to the provisioner key. | +| `status.provisionerKeyID` | `string` | ProvisionerKeyID is the provisioner key ID last applied in coderd. | +| `status.provisionerKeyName` | `string` | ProvisionerKeyName is the provisioner key name last applied in coderd. | +| `status.tagsHash` | `string` | TagsHash is a deterministic hash of spec.tags last applied to the provisioner key. | +| `status.controlPlaneRefName` | `string` | ControlPlaneRefName is the control plane ref name last applied to the provisioner key. | +| `status.controlPlaneURL` | `string` | ControlPlaneURL is the control plane URL last applied to the provisioner key. | +| `status.secretRef` | `github.com/coder/coder-k8s/api/v1alpha1.SecretKeySelector` | SecretRef references the provisioner key secret data currently in use. | + +## Source + +- Go type: `api/v1alpha1/coderprovisioner_types.go` +- Generated CRD: `config/crd/bases/coder.com_coderprovisioners.yaml` diff --git a/internal/app/controllerapp/controllerapp.go b/internal/app/controllerapp/controllerapp.go index c99d0dcf..65388326 100644 --- a/internal/app/controllerapp/controllerapp.go +++ b/internal/app/controllerapp/controllerapp.go @@ -103,6 +103,15 @@ func Run(ctx context.Context) error { return fmt.Errorf("unable to create workspace proxy controller: %w", err) } + provisionerReconciler := &controller.CoderProvisionerReconciler{ + Client: client, + Scheme: managerScheme, + BootstrapClient: coderbootstrap.NewSDKClient(), + } + if err := provisionerReconciler.SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create provisioner controller: %w", err) + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { return fmt.Errorf("unable to set up health check: %w", err) } diff --git a/internal/coderbootstrap/client.go b/internal/coderbootstrap/client.go index 536f0d75..6e0b2a0c 100644 --- a/internal/coderbootstrap/client.go +++ b/internal/coderbootstrap/client.go @@ -35,9 +35,11 @@ type RegisterWorkspaceProxyResponse struct { // Client provides optional bootstrap operations against the Coder API. type Client interface { EnsureWorkspaceProxy(context.Context, RegisterWorkspaceProxyRequest) (RegisterWorkspaceProxyResponse, error) + EnsureProvisionerKey(context.Context, EnsureProvisionerKeyRequest) (EnsureProvisionerKeyResponse, error) + DeleteProvisionerKey(ctx context.Context, coderURL, sessionToken, orgName, keyName string) error } -// SDKClient uses codersdk to register workspace proxies. +// SDKClient uses codersdk to perform bootstrap operations. type SDKClient struct{} // NewSDKClient returns a bootstrap client backed by codersdk. diff --git a/internal/coderbootstrap/provisionerkeys.go b/internal/coderbootstrap/provisionerkeys.go new file mode 100644 index 00000000..cb3eead8 --- /dev/null +++ b/internal/coderbootstrap/provisionerkeys.go @@ -0,0 +1,198 @@ +package coderbootstrap + +import ( + "context" + "errors" + "net/http" + "net/url" + + "github.com/google/uuid" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/codersdk" +) + +// EnsureProvisionerKeyRequest describes how to create or look up a provisioner key in Coder. +type EnsureProvisionerKeyRequest struct { + CoderURL string + SessionToken string + OrganizationName string + KeyName string + Tags map[string]string +} + +// EnsureProvisionerKeyResponse contains provisioner key metadata. +type EnsureProvisionerKeyResponse struct { + OrganizationID uuid.UUID + KeyID uuid.UUID + KeyName string + // Key is the plaintext provisioner key. It is only non-empty when a key is created. + Key string +} + +// EnsureProvisionerKey creates a provisioner key if it does not already exist, +// otherwise it returns the existing key metadata. +func (c *SDKClient) EnsureProvisionerKey(ctx context.Context, req EnsureProvisionerKeyRequest) (EnsureProvisionerKeyResponse, error) { + if err := validateProvisionerKeyInputs(req.CoderURL, req.SessionToken, req.KeyName); err != nil { + return EnsureProvisionerKeyResponse{}, err + } + + client, err := newAuthenticatedClient(req.CoderURL, req.SessionToken) + if err != nil { + return EnsureProvisionerKeyResponse{}, err + } + + organizationName := req.OrganizationName + if organizationName == "" { + organizationName = codersdk.DefaultOrganization + } + + organization, err := resolveOrganizationByName(ctx, client, organizationName) + if err != nil { + return EnsureProvisionerKeyResponse{}, err + } + + existing, err := findOrganizationProvisionerKey(ctx, client, organization.ID, req.KeyName) + if err != nil { + return EnsureProvisionerKeyResponse{}, err + } + if existing != nil { + if existing.ID == uuid.Nil { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("assertion failed: provisioner key %q returned an empty ID", req.KeyName) + } + return EnsureProvisionerKeyResponse{ + OrganizationID: organization.ID, + KeyID: existing.ID, + KeyName: existing.Name, + }, nil + } + + created, err := client.CreateProvisionerKey(ctx, organization.ID, codersdk.CreateProvisionerKeyRequest{ + Name: req.KeyName, + Tags: req.Tags, + }) + if err != nil { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("create provisioner key %q: %w", req.KeyName, err) + } + if created.Key == "" { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("assertion failed: created provisioner key %q returned an empty key", req.KeyName) + } + + createdMetadata, err := findOrganizationProvisionerKey(ctx, client, organization.ID, req.KeyName) + if err != nil { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("query created provisioner key %q: %w", req.KeyName, err) + } + if createdMetadata == nil { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("assertion failed: created provisioner key %q was not returned by list", req.KeyName) + } + if createdMetadata.ID == uuid.Nil { + return EnsureProvisionerKeyResponse{}, xerrors.Errorf("assertion failed: created provisioner key %q returned an empty ID", req.KeyName) + } + + return EnsureProvisionerKeyResponse{ + OrganizationID: organization.ID, + KeyID: createdMetadata.ID, + KeyName: createdMetadata.Name, + Key: created.Key, + }, nil +} + +// DeleteProvisionerKey deletes a provisioner key by name. +// A missing key is treated as success for idempotency. +func (c *SDKClient) DeleteProvisionerKey(ctx context.Context, coderURL, sessionToken, orgName, keyName string) error { + if err := validateProvisionerKeyInputs(coderURL, sessionToken, keyName); err != nil { + return err + } + + client, err := newAuthenticatedClient(coderURL, sessionToken) + if err != nil { + return err + } + + organizationName := orgName + if organizationName == "" { + organizationName = codersdk.DefaultOrganization + } + + organization, err := resolveOrganizationByName(ctx, client, organizationName) + if err != nil { + return err + } + + err = client.DeleteProvisionerKey(ctx, organization.ID, keyName) + if err == nil { + return nil + } + + var apiErr *codersdk.Error + if errors.As(err, &apiErr) && apiErr.StatusCode() == http.StatusNotFound { + return nil + } + + return xerrors.Errorf("delete provisioner key %q: %w", keyName, err) +} + +func validateProvisionerKeyInputs(coderURL, sessionToken, keyName string) error { + if coderURL == "" { + return xerrors.New("coder URL is required") + } + if sessionToken == "" { + return xerrors.New("session token is required") + } + if keyName == "" { + return xerrors.New("provisioner key name is required") + } + + return nil +} + +func newAuthenticatedClient(coderURL, sessionToken string) (*codersdk.Client, error) { + coderAPIURL, err := url.Parse(coderURL) + if err != nil { + return nil, xerrors.Errorf("parse coder URL: %w", err) + } + + client := codersdk.New(coderAPIURL) + if client == nil { + return nil, xerrors.New("assertion failed: codersdk client is nil after successful construction") + } + client.SetSessionToken(sessionToken) + if client.HTTPClient == nil { + client.HTTPClient = &http.Client{} + } + client.HTTPClient.Timeout = coderSDKRequestTimeout + + return client, nil +} + +func resolveOrganizationByName(ctx context.Context, client *codersdk.Client, organizationName string) (codersdk.Organization, error) { + organization, err := client.OrganizationByName(ctx, organizationName) + if err != nil { + return codersdk.Organization{}, xerrors.Errorf("query organization %q: %w", organizationName, err) + } + if organization.ID == uuid.Nil { + return codersdk.Organization{}, xerrors.Errorf("assertion failed: organization %q returned an empty ID", organizationName) + } + + return organization, nil +} + +func findOrganizationProvisionerKey(ctx context.Context, client *codersdk.Client, organizationID uuid.UUID, keyName string) (*codersdk.ProvisionerKey, error) { + keys, err := client.ListProvisionerKeys(ctx, organizationID) + if err != nil { + return nil, xerrors.Errorf("list provisioner keys for organization %q: %w", organizationID, err) + } + + var match *codersdk.ProvisionerKey + for i := range keys { + if keys[i].Name != keyName { + continue + } + if match != nil { + return nil, xerrors.Errorf("assertion failed: found multiple provisioner keys named %q in organization %q", keyName, organizationID) + } + match = &keys[i] + } + + return match, nil +} diff --git a/internal/coderbootstrap/provisionerkeys_test.go b/internal/coderbootstrap/provisionerkeys_test.go new file mode 100644 index 00000000..a1032aaf --- /dev/null +++ b/internal/coderbootstrap/provisionerkeys_test.go @@ -0,0 +1,279 @@ +package coderbootstrap_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder-k8s/internal/coderbootstrap" +) + +func TestEnsureProvisionerKey_Create(t *testing.T) { + t.Parallel() + + const keyName = "provisioner-key" + orgID := uuid.New() + keyID := uuid.New() + now := time.Now().UTC().Format(time.RFC3339) + + createCalls := 0 + listCalls := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/default": + writeJSONResponse(t, w, http.StatusOK, map[string]any{ + "id": orgID.String(), + "name": "default", + "created_at": now, + "updated_at": now, + }) + return + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys": + listCalls++ + if createCalls == 0 { + writeJSONResponse(t, w, http.StatusOK, []any{}) + return + } + writeJSONResponse(t, w, http.StatusOK, []map[string]any{{ + "id": keyID.String(), + "name": keyName, + "organization": orgID.String(), + "created_at": now, + "tags": map[string]string{ + "cluster": "dev", + }, + }}) + return + case r.Method == http.MethodPost && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys": + createCalls++ + var payload struct { + Name string `json:"name"` + Tags map[string]string `json:"tags"` + } + err := json.NewDecoder(r.Body).Decode(&payload) + require.NoError(t, err) + require.Equal(t, keyName, payload.Name) + require.Equal(t, map[string]string{"cluster": "dev"}, payload.Tags) + + writeJSONResponse(t, w, http.StatusCreated, map[string]any{ + "key": "plaintext-provisioner-key", + }) + return + default: + writeJSONResponse(t, w, http.StatusNotFound, map[string]any{ + "message": "unexpected route", + }) + return + } + })) + defer server.Close() + + client := coderbootstrap.NewSDKClient() + resp, err := client.EnsureProvisionerKey(context.Background(), coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: server.URL, + SessionToken: "session-token", + KeyName: keyName, + Tags: map[string]string{ + "cluster": "dev", + }, + }) + require.NoError(t, err) + require.Equal(t, 1, createCalls) + require.Equal(t, 2, listCalls) + require.Equal(t, orgID, resp.OrganizationID) + require.Equal(t, keyID, resp.KeyID) + require.Equal(t, keyName, resp.KeyName) + require.Equal(t, "plaintext-provisioner-key", resp.Key) +} + +func TestEnsureProvisionerKey_Exists(t *testing.T) { + t.Parallel() + + const ( + orgName = "engineering" + keyName = "existing-key" + ) + orgID := uuid.New() + keyID := uuid.New() + now := time.Now().UTC().Format(time.RFC3339) + createCalled := false + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/"+orgName: + writeJSONResponse(t, w, http.StatusOK, map[string]any{ + "id": orgID.String(), + "name": orgName, + "created_at": now, + "updated_at": now, + }) + return + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys": + writeJSONResponse(t, w, http.StatusOK, []map[string]any{{ + "id": keyID.String(), + "name": keyName, + "organization": orgID.String(), + "created_at": now, + "tags": map[string]string{"cluster": "prod"}, + }}) + return + case r.Method == http.MethodPost && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys": + createCalled = true + writeJSONResponse(t, w, http.StatusCreated, map[string]any{"key": "should-not-be-created"}) + return + default: + writeJSONResponse(t, w, http.StatusNotFound, map[string]any{"message": "unexpected route"}) + return + } + })) + defer server.Close() + + client := coderbootstrap.NewSDKClient() + resp, err := client.EnsureProvisionerKey(context.Background(), coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: server.URL, + SessionToken: "session-token", + OrganizationName: orgName, + KeyName: keyName, + }) + require.NoError(t, err) + require.False(t, createCalled) + require.Equal(t, orgID, resp.OrganizationID) + require.Equal(t, keyID, resp.KeyID) + require.Equal(t, keyName, resp.KeyName) + require.Empty(t, resp.Key) +} + +func TestEnsureProvisionerKey_ValidationErrors(t *testing.T) { + t.Parallel() + + client := coderbootstrap.NewSDKClient() + tests := []struct { + name string + request coderbootstrap.EnsureProvisionerKeyRequest + errSubstr string + }{ + { + name: "missing coder URL", + request: coderbootstrap.EnsureProvisionerKeyRequest{ + SessionToken: "session-token", + KeyName: "key", + }, + errSubstr: "coder URL is required", + }, + { + name: "missing session token", + request: coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: "https://coder.example.com", + KeyName: "key", + }, + errSubstr: "session token is required", + }, + { + name: "missing key name", + request: coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: "https://coder.example.com", + SessionToken: "session-token", + }, + errSubstr: "provisioner key name is required", + }, + } + + for _, testCase := range tests { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + _, err := client.EnsureProvisionerKey(context.Background(), testCase.request) + require.Error(t, err) + require.Contains(t, err.Error(), testCase.errSubstr) + }) + } +} + +func TestDeleteProvisionerKey_Success(t *testing.T) { + t.Parallel() + + const keyName = "delete-me" + orgID := uuid.New() + now := time.Now().UTC().Format(time.RFC3339) + deleteCalls := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/default": + writeJSONResponse(t, w, http.StatusOK, map[string]any{ + "id": orgID.String(), + "name": "default", + "created_at": now, + "updated_at": now, + }) + return + case r.Method == http.MethodDelete && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys/"+keyName: + deleteCalls++ + w.WriteHeader(http.StatusNoContent) + return + default: + writeJSONResponse(t, w, http.StatusNotFound, map[string]any{"message": "unexpected route"}) + return + } + })) + defer server.Close() + + client := coderbootstrap.NewSDKClient() + err := client.DeleteProvisionerKey(context.Background(), server.URL, "session-token", "", keyName) + require.NoError(t, err) + require.Equal(t, 1, deleteCalls) +} + +func TestDeleteProvisionerKey_NotFound(t *testing.T) { + t.Parallel() + + const keyName = "already-deleted" + orgID := uuid.New() + now := time.Now().UTC().Format(time.RFC3339) + deleteCalls := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/organizations/default": + writeJSONResponse(t, w, http.StatusOK, map[string]any{ + "id": orgID.String(), + "name": "default", + "created_at": now, + "updated_at": now, + }) + return + case r.Method == http.MethodDelete && r.URL.Path == "/api/v2/organizations/"+orgID.String()+"/provisionerkeys/"+keyName: + deleteCalls++ + writeJSONResponse(t, w, http.StatusNotFound, map[string]any{ + "message": "provisioner key not found", + }) + return + default: + writeJSONResponse(t, w, http.StatusNotFound, map[string]any{"message": "unexpected route"}) + return + } + })) + defer server.Close() + + client := coderbootstrap.NewSDKClient() + err := client.DeleteProvisionerKey(context.Background(), server.URL, "session-token", "", keyName) + require.NoError(t, err) + require.Equal(t, 1, deleteCalls) +} + +func writeJSONResponse(t *testing.T, w http.ResponseWriter, statusCode int, payload any) { + t.Helper() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + err := json.NewEncoder(w).Encode(payload) + require.NoError(t, err) +} diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go new file mode 100644 index 00000000..6bdd184c --- /dev/null +++ b/internal/controller/coderprovisioner_controller.go @@ -0,0 +1,1100 @@ +// Package controller contains Kubernetes controllers for coder-k8s resources. +package controller + +import ( + "context" + "fmt" + "hash/fnv" + "maps" + "slices" + + "github.com/google/uuid" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" + "github.com/coder/coder-k8s/internal/coderbootstrap" +) + +const ( + defaultProvisionerReplicas = int32(1) + defaultProvisionerTerminationGracePeriodSeconds = int64(600) + defaultProvisionerOrganizationName = "default" + provisionerNamePrefix = "provisioner-" + provisionerServiceAccountSuffix = "-provisioner" + provisionerKeyChecksumAnnotation = "checksum/provisioner-key" +) + +// CoderProvisionerReconciler reconciles a CoderProvisioner object. +type CoderProvisionerReconciler struct { + client.Client + Scheme *runtime.Scheme + BootstrapClient coderbootstrap.Client +} + +// +kubebuilder:rbac:groups=coder.com,resources=coderprovisioners,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=coder.com,resources=coderprovisioners/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=coder.com,resources=coderprovisioners/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=secrets;serviceaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete + +// Reconcile converges the desired CoderProvisioner spec into Deployment, RBAC, and Secret resources. +func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + if r.Client == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: reconciler client must not be nil") + } + if r.Scheme == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: reconciler scheme must not be nil") + } + if r.BootstrapClient == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: reconciler bootstrap client must not be nil") + } + + provisioner := &coderv1alpha1.CoderProvisioner{} + if err := r.Get(ctx, req.NamespacedName, provisioner); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, fmt.Errorf("get coderprovisioner %s: %w", req.NamespacedName, err) + } + + if provisioner.Name != req.Name || provisioner.Namespace != req.Namespace { + return ctrl.Result{}, fmt.Errorf("assertion failed: fetched object %s/%s does not match request %s/%s", + provisioner.Namespace, provisioner.Name, req.Namespace, req.Name) + } + + if !provisioner.DeletionTimestamp.IsZero() { + return r.reconcileDeletion(ctx, provisioner) + } + + statusSnapshot := provisioner.Status.DeepCopy() + + finalizerAdded, err := r.ensureCleanupFinalizer(ctx, provisioner) + if err != nil { + return ctrl.Result{}, err + } + if finalizerAdded { + return ctrl.Result{}, nil + } + + controlPlane, err := r.fetchControlPlane(ctx, provisioner) + if err != nil { + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionControlPlaneReady, + metav1.ConditionFalse, + "ControlPlaneUnavailable", + fmt.Sprintf("Failed to fetch control plane: %v", err), + ) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, err + } + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionControlPlaneReady, + metav1.ConditionTrue, + "ControlPlaneAvailable", + "Referenced control plane is available and has a URL", + ) + + organizationName := provisionerOrganizationName(provisioner.Spec.OrganizationName) + keyName, keySecretName, keySecretKey := provisionerKeyConfig(provisioner) + + sessionToken, err := r.readBootstrapSessionToken(ctx, provisioner) + if err != nil { + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady, + metav1.ConditionFalse, + "BootstrapSecretUnavailable", + fmt.Sprintf("Failed to read bootstrap credentials: %v", err), + ) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, err + } + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady, + metav1.ConditionTrue, + "BootstrapSecretAvailable", + "Bootstrap credentials secret is available", + ) + + desiredTagsHash := hashProvisionerTags(provisioner.Spec.Tags) + desiredControlPlaneRefName := provisioner.Spec.ControlPlaneRef.Name + status := provisioner.Status + orgDrift := status.OrganizationName != "" && status.OrganizationName != organizationName + keyNameDrift := status.ProvisionerKeyName != "" && status.ProvisionerKeyName != keyName + tagsDrift := status.TagsHash != "" && status.TagsHash != desiredTagsHash + controlPlaneRefDrift := status.ControlPlaneRefName != "" && status.ControlPlaneRefName != desiredControlPlaneRefName + controlPlaneURLDrift := status.ControlPlaneURL != "" && status.ControlPlaneURL != controlPlane.Status.URL + driftDetected := orgDrift || keyNameDrift || tagsDrift || controlPlaneRefDrift || controlPlaneURLDrift + appliedOrgName := provisioner.Status.OrganizationName + appliedTagsHash := provisioner.Status.TagsHash + appliedControlPlaneRefName := provisioner.Status.ControlPlaneRefName + appliedControlPlaneURL := provisioner.Status.ControlPlaneURL + + // Check whether a usable provisioner key secret already exists. + // The secret is considered "usable" only if the Secret object exists + // AND it contains a non-empty value at the configured data key. + secretNamespacedName := types.NamespacedName{Name: keySecretName, Namespace: provisioner.Namespace} + existingSecret := &corev1.Secret{} + secretUsable := false + if err := r.Get(ctx, secretNamespacedName, existingSecret); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("get provisioner key secret %s: %w", secretNamespacedName, err) + } + } else { + secretUsable = len(existingSecret.Data[keySecretKey]) > 0 + } + + organizationID := provisioner.Status.OrganizationID + provisionerKeyID := provisioner.Status.ProvisionerKeyID + provisionerKeyName := provisioner.Status.ProvisionerKeyName + if provisionerKeyName == "" { + provisionerKeyName = keyName + } + + log := ctrl.LoggerFrom(ctx) + keyMaterial := "" + if driftDetected { + log.Info("spec drift detected, rotating provisioner key", + "orgDrift", orgDrift, "keyNameDrift", keyNameDrift, "tagsDrift", tagsDrift, + "controlPlaneRefDrift", controlPlaneRefDrift, "controlPlaneURLDrift", controlPlaneURLDrift) + + oldOrg := provisioner.Status.OrganizationName + if oldOrg == "" { + oldOrg = organizationName + } + oldKeyName := provisioner.Status.ProvisionerKeyName + if oldKeyName == "" { + oldKeyName = keyName + } + oldControlPlaneURL := provisioner.Status.ControlPlaneURL + if oldControlPlaneURL == "" { + oldControlPlaneURL = controlPlane.Status.URL + } + + if deleteErr := r.BootstrapClient.DeleteProvisionerKey( + ctx, + oldControlPlaneURL, + sessionToken, + oldOrg, + oldKeyName, + ); deleteErr != nil { + log.Info("failed to delete old provisioner key during drift rotation, creating new key anyway", + "oldKeyName", oldKeyName, "error", deleteErr) + } + + response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if ensureErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to ensure provisioner key %q after drift rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("ensure provisioner key %q: %w", keyName, ensureErr) + } + if response.OrganizationID != uuid.Nil { + organizationID = response.OrganizationID.String() + } + if response.KeyID != uuid.Nil { + provisionerKeyID = response.KeyID.String() + } + if response.KeyName != "" { + provisionerKeyName = response.KeyName + } + keyMaterial = response.Key + if keyMaterial == "" { + log.Info("drift-rotated key exists in coderd but returned no plaintext, rotating to recover", + "keyName", keyName) + + if deleteErr := r.BootstrapClient.DeleteProvisionerKey( + ctx, controlPlane.Status.URL, sessionToken, organizationName, keyName, + ); deleteErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to delete provisioner key %q for drift recovery", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("delete provisioner key %q for drift recovery: %w", keyName, deleteErr) + } + rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if rotateErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to recreate provisioner key %q after drift recovery", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("recreate provisioner key %q after drift recovery: %w", keyName, rotateErr) + } + if rotated.OrganizationID != uuid.Nil { + organizationID = rotated.OrganizationID.String() + } + if rotated.KeyID != uuid.Nil { + provisionerKeyID = rotated.KeyID.String() + } + if rotated.KeyName != "" { + provisionerKeyName = rotated.KeyName + } + keyMaterial = rotated.Key + if keyMaterial == "" { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Provisioner key %q returned empty material after drift recovery rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key %q returned empty material after drift recovery rotation", keyName) + } + } + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash + appliedControlPlaneRefName = desiredControlPlaneRefName + appliedControlPlaneURL = controlPlane.Status.URL + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionTrue, + "ProvisionerKeyReady", + "Provisioner key is available in coderd", + ) + } else if !secretUsable { + response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if ensureErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to ensure provisioner key %q", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("ensure provisioner key %q: %w", keyName, ensureErr) + } + if response.OrganizationID != uuid.Nil { + organizationID = response.OrganizationID.String() + } + if response.KeyID != uuid.Nil { + provisionerKeyID = response.KeyID.String() + } + if response.KeyName != "" { + provisionerKeyName = response.KeyName + } + keyMaterial = response.Key + + // If the key already exists in coderd (e.g. the K8s secret was + // deleted), coderd won't return plaintext again. Rotate the key + // by deleting and recreating it to obtain fresh material. + if keyMaterial == "" { + log.Info("provisioner key exists in coderd but secret is missing, rotating key to recover", + "keyName", keyName, "secretName", keySecretName) + + if deleteErr := r.BootstrapClient.DeleteProvisionerKey( + ctx, controlPlane.Status.URL, sessionToken, organizationName, keyName, + ); deleteErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to delete stale provisioner key %q for rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("delete stale provisioner key %q for rotation: %w", keyName, deleteErr) + } + rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if rotateErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to recreate provisioner key %q after rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("recreate provisioner key %q after rotation: %w", keyName, rotateErr) + } + if rotated.OrganizationID != uuid.Nil { + organizationID = rotated.OrganizationID.String() + } + if rotated.KeyID != uuid.Nil { + provisionerKeyID = rotated.KeyID.String() + } + if rotated.KeyName != "" { + provisionerKeyName = rotated.KeyName + } + keyMaterial = rotated.Key + if keyMaterial == "" { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Provisioner key %q returned empty key material after rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key %q returned empty key material after rotation", keyName) + } + } + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash + appliedControlPlaneRefName = desiredControlPlaneRefName + appliedControlPlaneURL = controlPlane.Status.URL + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionTrue, + "ProvisionerKeyReady", + "Provisioner key is available in coderd", + ) + } else if status.OrganizationName == "" || status.TagsHash == "" || status.ControlPlaneRefName == "" { + // Secret is usable and no drift detected, but status metadata is empty + // (e.g. upgrade from older version). Call EnsureProvisionerKey to populate + // IDs and key name. If coderd reports an existing key (no plaintext key + // returned), rotate it best-effort so desired tags are applied before + // stamping the metadata baseline. + response, ensureErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if ensureErr != nil { + // Return a requeue so metadata population retries. + return ctrl.Result{}, fmt.Errorf("verify provisioner key metadata: %w", ensureErr) + } + if response.OrganizationID != uuid.Nil { + organizationID = response.OrganizationID.String() + } + if response.KeyID != uuid.Nil { + provisionerKeyID = response.KeyID.String() + } + if response.KeyName != "" { + provisionerKeyName = response.KeyName + } + if response.Key != "" { + // Key was freshly created with desired tags; capture material and stamp baseline. + keyMaterial = response.Key + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash + appliedControlPlaneRefName = desiredControlPlaneRefName + appliedControlPlaneURL = controlPlane.Status.URL + } else { + // Key already exists; tags may be stale. Rotate to ensure desired tags are applied. + log.Info("existing key found during metadata backfill, rotating to ensure desired tags", + "keyName", keyName) + if deleteErr := r.BootstrapClient.DeleteProvisionerKey( + ctx, controlPlane.Status.URL, sessionToken, organizationName, keyName, + ); deleteErr != nil { + return ctrl.Result{}, fmt.Errorf("delete provisioner key %q for metadata backfill rotation: %w", keyName, deleteErr) + } + rotated, rotateErr := r.BootstrapClient.EnsureProvisionerKey(ctx, coderbootstrap.EnsureProvisionerKeyRequest{ + CoderURL: controlPlane.Status.URL, + SessionToken: sessionToken, + OrganizationName: organizationName, + KeyName: keyName, + Tags: provisioner.Spec.Tags, + }) + if rotateErr != nil { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Failed to recreate provisioner key %q after metadata backfill rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("recreate provisioner key %q after metadata backfill rotation: %w", keyName, rotateErr) + } + if rotated.OrganizationID != uuid.Nil { + organizationID = rotated.OrganizationID.String() + } + if rotated.KeyID != uuid.Nil { + provisionerKeyID = rotated.KeyID.String() + } + if rotated.KeyName != "" { + provisionerKeyName = rotated.KeyName + } + keyMaterial = rotated.Key + if keyMaterial == "" { + setCondition(provisioner, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionFalse, "ProvisionerKeyFailed", + fmt.Sprintf("Provisioner key %q returned empty material after metadata backfill rotation", keyName)) + _ = r.Status().Update(ctx, provisioner) + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key %q returned empty material after metadata backfill rotation", keyName) + } + appliedOrgName = organizationName + appliedTagsHash = desiredTagsHash + appliedControlPlaneRefName = desiredControlPlaneRefName + appliedControlPlaneURL = controlPlane.Status.URL + } + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, + metav1.ConditionTrue, + "ProvisionerKeyReady", + "Provisioner key is available in coderd", + ) + } + + provisionerKeySecret, err := r.ensureProvisionerKeySecret(ctx, provisioner, keySecretName, keySecretKey, keyMaterial) + if err != nil { + return ctrl.Result{}, err + } + + secretValue, ok := provisionerKeySecret.Data[keySecretKey] + if !ok { + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key secret %q is missing key %q after reconciliation", keySecretName, keySecretKey) + } + if len(secretValue) == 0 { + return ctrl.Result{}, fmt.Errorf("assertion failed: provisioner key secret %q key %q is empty after reconciliation", keySecretName, keySecretKey) + } + secretChecksum := hashProvisionerSecret(secretValue) + + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionProvisionerKeySecretReady, + metav1.ConditionTrue, + "SecretReady", + "Provisioner key secret is available", + ) + + serviceAccountName := provisionerServiceAccountName(provisioner.Name) + if _, err := r.reconcileServiceAccount(ctx, provisioner, serviceAccountName); err != nil { + return ctrl.Result{}, err + } + + roleName := provisionerResourceName(provisioner.Name) + role, err := r.reconcileRole(ctx, provisioner, roleName) + if err != nil { + return ctrl.Result{}, err + } + if _, err := r.reconcileRoleBinding(ctx, provisioner, roleName, role.Name, serviceAccountName); err != nil { + return ctrl.Result{}, err + } + + image := provisioner.Spec.Image + if image == "" { + image = controlPlane.Spec.Image + } + if image == "" { + image = defaultCoderImage + } + + secretRef := &coderv1alpha1.SecretKeySelector{Name: keySecretName, Key: keySecretKey} + deployment, err := r.reconcileDeployment(ctx, provisioner, image, controlPlane.Status.URL, organizationName, secretRef, serviceAccountName, secretChecksum) + if err != nil { + return ctrl.Result{}, err + } + + if err := r.reconcileStatus( + ctx, + provisioner, + deployment, + secretRef, + organizationID, + appliedOrgName, + provisionerKeyID, + provisionerKeyName, + appliedTagsHash, + appliedControlPlaneRefName, + appliedControlPlaneURL, + statusSnapshot, + ); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func (r *CoderProvisionerReconciler) reconcileDeletion(ctx context.Context, provisioner *coderv1alpha1.CoderProvisioner) (ctrl.Result, error) { + if !controllerutil.ContainsFinalizer(provisioner, coderv1alpha1.ProvisionerKeyCleanupFinalizer) { + return ctrl.Result{}, nil + } + + log := ctrl.LoggerFrom(ctx) + + // Use the last-applied organization name from status so we target the + // correct org even if spec was changed but never successfully rotated. + // Fall back to the spec-derived name only when status is empty. + organizationName := provisioner.Status.OrganizationName + if organizationName == "" { + organizationName = provisionerOrganizationName(provisioner.Spec.OrganizationName) + } + keyName := provisioner.Status.ProvisionerKeyName + if keyName == "" { + keyName, _, _ = provisionerKeyConfig(provisioner) + } + + // Best-effort remote key cleanup: if the referenced control plane, + // its URL, bootstrap credentials, or any other prerequisite is + // unavailable, log a warning and proceed to finalizer removal so the + // CR does not get stuck in Terminating. This is common during + // namespace teardown, when the control plane was never ready, or + // when credentials were misconfigured. + controlPlaneURL := provisioner.Status.ControlPlaneURL + if controlPlaneURL == "" { + controlPlane, err := r.fetchControlPlane(ctx, provisioner) + if err != nil { + log.Info("unable to reach referenced CoderControlPlane during deletion, skipping remote key cleanup", + "controlPlaneRef", provisioner.Spec.ControlPlaneRef.Name, "error", err) + } else { + controlPlaneURL = controlPlane.Status.URL + } + } + + if controlPlaneURL != "" { + sessionToken, tokenErr := r.readBootstrapSessionToken(ctx, provisioner) + if tokenErr != nil { + log.Info("unable to read bootstrap credentials during deletion, skipping remote key cleanup", + "credentialsSecretRef", provisioner.Spec.Bootstrap.CredentialsSecretRef.Name, "error", tokenErr) + } else { + if deleteErr := r.BootstrapClient.DeleteProvisionerKey( + ctx, + controlPlaneURL, + sessionToken, + organizationName, + keyName, + ); deleteErr != nil { + // Treat key deletion failures as best-effort so the + // finalizer is still removed. Transient errors, auth + // issues, or org-lookup failures should not block CR + // cleanup. + log.Info("failed to delete remote provisioner key during deletion, proceeding with finalizer removal", + "keyName", keyName, "error", deleteErr) + } + } + } + + controllerutil.RemoveFinalizer(provisioner, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + if err := r.Update(ctx, provisioner); err != nil { + return ctrl.Result{}, fmt.Errorf("remove finalizer from coderprovisioner %s/%s: %w", provisioner.Namespace, provisioner.Name, err) + } + + return ctrl.Result{}, nil +} + +func (r *CoderProvisionerReconciler) ensureCleanupFinalizer(ctx context.Context, provisioner *coderv1alpha1.CoderProvisioner) (bool, error) { + if controllerutil.ContainsFinalizer(provisioner, coderv1alpha1.ProvisionerKeyCleanupFinalizer) { + return false, nil + } + + controllerutil.AddFinalizer(provisioner, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + if err := r.Update(ctx, provisioner); err != nil { + return false, fmt.Errorf("add finalizer to coderprovisioner %s/%s: %w", provisioner.Namespace, provisioner.Name, err) + } + + return true, nil +} + +func (r *CoderProvisionerReconciler) fetchControlPlane(ctx context.Context, provisioner *coderv1alpha1.CoderProvisioner) (*coderv1alpha1.CoderControlPlane, error) { + controlPlaneName := provisioner.Spec.ControlPlaneRef.Name + if controlPlaneName == "" { + return nil, fmt.Errorf("coderprovisioner %s/%s spec.controlPlaneRef.name is required", provisioner.Namespace, provisioner.Name) + } + + controlPlane := &coderv1alpha1.CoderControlPlane{} + namespacedName := types.NamespacedName{Name: controlPlaneName, Namespace: provisioner.Namespace} + if err := r.Get(ctx, namespacedName, controlPlane); err != nil { + return nil, fmt.Errorf("get referenced codercontrolplane %s for coderprovisioner %s/%s: %w", namespacedName, provisioner.Namespace, provisioner.Name, err) + } + + if controlPlane.Name != controlPlaneName || controlPlane.Namespace != provisioner.Namespace { + return nil, fmt.Errorf("assertion failed: fetched control plane %s/%s does not match expected %s/%s", + controlPlane.Namespace, controlPlane.Name, provisioner.Namespace, controlPlaneName) + } + if controlPlane.Status.URL == "" { + return nil, fmt.Errorf("codercontrolplane %s/%s status.url is empty", controlPlane.Namespace, controlPlane.Name) + } + + return controlPlane, nil +} + +func (r *CoderProvisionerReconciler) readBootstrapSessionToken(ctx context.Context, provisioner *coderv1alpha1.CoderProvisioner) (string, error) { + credentialsRef := provisioner.Spec.Bootstrap.CredentialsSecretRef + if credentialsRef.Name == "" { + return "", fmt.Errorf("coderprovisioner %s/%s spec.bootstrap.credentialsSecretRef.name is required", provisioner.Namespace, provisioner.Name) + } + + credentialsKey := credentialsRef.Key + if credentialsKey == "" { + credentialsKey = coderv1alpha1.DefaultTokenSecretKey + } + + token, err := r.readSecretValue(ctx, provisioner.Namespace, credentialsRef.Name, credentialsKey) + if err != nil { + return "", fmt.Errorf("read bootstrap credentials secret %q/%q key %q: %w", provisioner.Namespace, credentialsRef.Name, credentialsKey, err) + } + + return token, nil +} + +func (r *CoderProvisionerReconciler) ensureProvisionerKeySecret( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + secretName string, + secretKey string, + keyMaterial string, +) (*corev1.Secret, error) { + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: provisioner.Namespace}} + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, secret, func() error { + labels := provisionerLabels(provisioner.Name) + secret.Labels = maps.Clone(labels) + secret.Type = corev1.SecretTypeOpaque + if secret.Data == nil { + secret.Data = make(map[string][]byte) + } + if keyMaterial != "" { + secret.Data[secretKey] = []byte(keyMaterial) + } + if len(secret.Data[secretKey]) == 0 { + return fmt.Errorf("provisioner key secret %q key %q is empty", secretName, secretKey) + } + if err := controllerutil.SetControllerReference(provisioner, secret, r.Scheme); err != nil { + return fmt.Errorf("set controller reference: %w", err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("reconcile provisioner key secret %q: %w", secretName, err) + } + + return secret, nil +} + +func (r *CoderProvisionerReconciler) reconcileServiceAccount( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + serviceAccountName string, +) (*corev1.ServiceAccount, error) { + serviceAccount := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: serviceAccountName, Namespace: provisioner.Namespace}} + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, serviceAccount, func() error { + labels := provisionerLabels(provisioner.Name) + serviceAccount.Labels = maps.Clone(labels) + if err := controllerutil.SetControllerReference(provisioner, serviceAccount, r.Scheme); err != nil { + return fmt.Errorf("set controller reference: %w", err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("reconcile serviceaccount %q: %w", serviceAccountName, err) + } + + return serviceAccount, nil +} + +func (r *CoderProvisionerReconciler) reconcileRole( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + roleName string, +) (*rbacv1.Role, error) { + role := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: roleName, Namespace: provisioner.Namespace}} + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, role, func() error { + labels := provisionerLabels(provisioner.Name) + role.Labels = maps.Clone(labels) + role.Rules = []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"pods", "persistentvolumeclaims"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }} + if err := controllerutil.SetControllerReference(provisioner, role, r.Scheme); err != nil { + return fmt.Errorf("set controller reference: %w", err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("reconcile role %q: %w", roleName, err) + } + + return role, nil +} + +func (r *CoderProvisionerReconciler) reconcileRoleBinding( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + roleBindingName string, + roleName string, + serviceAccountName string, +) (*rbacv1.RoleBinding, error) { + roleBinding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: roleBindingName, Namespace: provisioner.Namespace}} + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, roleBinding, func() error { + labels := provisionerLabels(provisioner.Name) + roleBinding.Labels = maps.Clone(labels) + roleBinding.RoleRef = rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: roleName, + } + roleBinding.Subjects = []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: serviceAccountName, + Namespace: provisioner.Namespace, + }} + if err := controllerutil.SetControllerReference(provisioner, roleBinding, r.Scheme); err != nil { + return fmt.Errorf("set controller reference: %w", err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("reconcile rolebinding %q: %w", roleBindingName, err) + } + + return roleBinding, nil +} + +func (r *CoderProvisionerReconciler) reconcileDeployment( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + image string, + coderURL string, + organizationName string, + secretRef *coderv1alpha1.SecretKeySelector, + serviceAccountName string, + secretChecksum string, +) (*appsv1.Deployment, error) { + deploymentName := provisionerResourceName(provisioner.Name) + deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: provisioner.Namespace}} + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + labels := provisionerLabels(provisioner.Name) + deployment.Labels = maps.Clone(labels) + + if err := controllerutil.SetControllerReference(provisioner, deployment, r.Scheme); err != nil { + return fmt.Errorf("set controller reference: %w", err) + } + + replicas := defaultProvisionerReplicas + if provisioner.Spec.Replicas != nil { + replicas = *provisioner.Spec.Replicas + } + terminationGracePeriodSeconds := defaultProvisionerTerminationGracePeriodSeconds + if provisioner.Spec.TerminationGracePeriodSeconds != nil { + terminationGracePeriodSeconds = *provisioner.Spec.TerminationGracePeriodSeconds + } + + args := []string{"provisionerd", "start"} + args = append(args, provisioner.Spec.ExtraArgs...) + + env := []corev1.EnvVar{ + {Name: "CODER_URL", Value: coderURL}, + { + Name: "CODER_PROVISIONER_DAEMON_KEY", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: secretRef.Name}, + Key: secretRef.Key, + }}, + }, + } + if organizationName != "" && organizationName != defaultProvisionerOrganizationName { + env = append(env, corev1.EnvVar{Name: "CODER_ORGANIZATION", Value: organizationName}) + } + env = append(env, provisioner.Spec.ExtraEnv...) + + deployment.Spec.Replicas = &replicas + deployment.Spec.Selector = &metav1.LabelSelector{MatchLabels: maps.Clone(labels)} + deployment.Spec.Template = corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: maps.Clone(labels), + Annotations: map[string]string{ + provisionerKeyChecksumAnnotation: secretChecksum, + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccountName, + ImagePullSecrets: provisioner.Spec.ImagePullSecrets, + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + Containers: []corev1.Container{{ + Name: "provisioner", + Image: image, + Args: args, + Env: env, + Resources: provisioner.Spec.Resources, + }}, + }, + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("reconcile provisioner deployment: %w", err) + } + + return deployment, nil +} + +func (r *CoderProvisionerReconciler) reconcileStatus( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + deployment *appsv1.Deployment, + secretRef *coderv1alpha1.SecretKeySelector, + organizationID string, + organizationName string, + provisionerKeyID string, + provisionerKeyName string, + tagsHash string, + controlPlaneRefName string, + controlPlaneURL string, + statusSnapshot *coderv1alpha1.CoderProvisionerStatus, +) error { + if statusSnapshot == nil { + return fmt.Errorf("assertion failed: status snapshot must not be nil") + } + + phase := coderv1alpha1.CoderProvisionerPhasePending + if deployment.Status.ReadyReplicas > 0 { + phase = coderv1alpha1.CoderProvisionerPhaseReady + } + + // Update fields individually so conditions set earlier in reconciliation are preserved. + provisioner.Status.ObservedGeneration = provisioner.Generation + provisioner.Status.ReadyReplicas = deployment.Status.ReadyReplicas + provisioner.Status.Phase = phase + provisioner.Status.OrganizationID = organizationID + provisioner.Status.OrganizationName = organizationName + provisioner.Status.ProvisionerKeyID = provisionerKeyID + provisioner.Status.ProvisionerKeyName = provisionerKeyName + provisioner.Status.TagsHash = tagsHash + provisioner.Status.ControlPlaneRefName = controlPlaneRefName + provisioner.Status.ControlPlaneURL = controlPlaneURL + provisioner.Status.SecretRef = &coderv1alpha1.SecretKeySelector{ + Name: secretRef.Name, + Key: secretRef.Key, + } + + if deployment.Status.ReadyReplicas > 0 { + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionDeploymentReady, + metav1.ConditionTrue, + "MinimumReplicasReady", + "At least one provisioner pod is ready", + ) + } else { + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionDeploymentReady, + metav1.ConditionFalse, + "NoReplicasReady", + "No provisioner pods are ready yet", + ) + } + + if equality.Semantic.DeepEqual(*statusSnapshot, provisioner.Status) { + return nil + } + + if err := r.Status().Update(ctx, provisioner); err != nil { + return fmt.Errorf("update coderprovisioner status: %w", err) + } + + return nil +} + +func (r *CoderProvisionerReconciler) readSecretValue(ctx context.Context, namespace, name, key string) (string, error) { + secret := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, secret); err != nil { + return "", err + } + + value, ok := secret.Data[key] + if !ok { + return "", fmt.Errorf("secret %q does not contain key %q", name, key) + } + if len(value) == 0 { + return "", fmt.Errorf("secret %q key %q is empty", name, key) + } + + return string(value), nil +} + +// SetupWithManager wires the reconciler into controller-runtime. +func (r *CoderProvisionerReconciler) SetupWithManager(mgr ctrl.Manager) error { + if mgr == nil { + return fmt.Errorf("assertion failed: manager must not be nil") + } + if r.Client == nil { + return fmt.Errorf("assertion failed: reconciler client must not be nil") + } + if r.Scheme == nil { + return fmt.Errorf("assertion failed: reconciler scheme must not be nil") + } + if r.BootstrapClient == nil { + return fmt.Errorf("assertion failed: reconciler bootstrap client must not be nil") + } + + return ctrl.NewControllerManagedBy(mgr). + For(&coderv1alpha1.CoderProvisioner{}). + Owns(&appsv1.Deployment{}). + Owns(&corev1.Secret{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + Named("coderprovisioner"). + Complete(r) +} + +func provisionerResourceName(name string) string { + candidate := provisionerNamePrefix + name + if len(candidate) <= 63 { + return candidate + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 63 - len(provisionerNamePrefix) - len(suffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s%s-%s", provisionerNamePrefix, name[:available], suffix) +} + +func provisionerLabels(name string) map[string]string { + return map[string]string{ + "app.kubernetes.io/name": "coder-provisioner", + "app.kubernetes.io/instance": provisionerInstanceLabelValue(name), + "app.kubernetes.io/managed-by": "coder-k8s", + } +} + +func provisionerInstanceLabelValue(name string) string { + if len(name) <= 63 { + return name + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 63 - len(suffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s-%s", name[:available], suffix) +} + +func provisionerServiceAccountName(name string) string { + candidate := fmt.Sprintf("%s%s", name, provisionerServiceAccountSuffix) + if len(candidate) <= 63 { + return candidate + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 63 - len(provisionerServiceAccountSuffix) - len(suffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s-%s%s", name[:available], suffix, provisionerServiceAccountSuffix) +} + +func provisionerOrganizationName(name string) string { + if name == "" { + return defaultProvisionerOrganizationName + } + + return name +} + +func provisionerKeyConfig(provisioner *coderv1alpha1.CoderProvisioner) (string, string, string) { + keyName := provisioner.Spec.Key.Name + if keyName == "" { + keyName = provisioner.Name + } + + const maxKeyNameLength = 128 + if len(keyName) > maxKeyNameLength { + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(keyName)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := maxKeyNameLength - len(suffix) - 1 + if available < 1 { + available = 1 + } + keyName = fmt.Sprintf("%s-%s", keyName[:available], suffix) + } + + secretName := provisioner.Spec.Key.SecretName + if secretName == "" { + const secretNameSuffix = "-provisioner-key" + candidate := provisioner.Name + secretNameSuffix + if len(candidate) <= 253 { + secretName = candidate + } else { + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(provisioner.Name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 253 - len(secretNameSuffix) - len(suffix) - 1 + if available < 1 { + available = 1 + } + secretName = fmt.Sprintf("%s-%s%s", provisioner.Name[:available], suffix, secretNameSuffix) + } + } + + secretKey := provisioner.Spec.Key.SecretKey + if secretKey == "" { + secretKey = coderv1alpha1.DefaultProvisionerKeySecretKey + } + + return keyName, secretName, secretKey +} + +func hashProvisionerTags(tags map[string]string) string { + keys := slices.Collect(maps.Keys(tags)) + slices.Sort(keys) + hasher := fnv.New32a() + for _, key := range keys { + _, _ = hasher.Write([]byte(key)) + _, _ = hasher.Write([]byte{0}) + _, _ = hasher.Write([]byte(tags[key])) + _, _ = hasher.Write([]byte{0}) + } + + return fmt.Sprintf("%08x", hasher.Sum32()) +} + +func hashProvisionerSecret(secretValue []byte) string { + hasher := fnv.New32a() + _, _ = hasher.Write(secretValue) + return fmt.Sprintf("%08x", hasher.Sum32()) +} + +func setCondition( + provisioner *coderv1alpha1.CoderProvisioner, + conditionType string, + status metav1.ConditionStatus, + reason string, + message string, +) { + meta.SetStatusCondition(&provisioner.Status.Conditions, metav1.Condition{ + Type: conditionType, + Status: status, + ObservedGeneration: provisioner.Generation, + Reason: reason, + Message: message, + }) +} diff --git a/internal/controller/coderprovisioner_controller_test.go b/internal/controller/coderprovisioner_controller_test.go new file mode 100644 index 00000000..8ed3f2f2 --- /dev/null +++ b/internal/controller/coderprovisioner_controller_test.go @@ -0,0 +1,1004 @@ +package controller_test + +import ( + "context" + "fmt" + "hash/fnv" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" + "github.com/coder/coder-k8s/internal/coderbootstrap" + "github.com/coder/coder-k8s/internal/controller" +) + +func createTestNamespace(ctx context.Context, t *testing.T, prefix string) string { + t.Helper() + + namespaceName := fmt.Sprintf("%s-%s", prefix, strings.ToLower(uuid.NewString()[:8])) + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}} + require.NoError(t, k8sClient.Create(ctx, ns)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), ns) + }) + + return namespaceName +} + +// createTestControlPlane creates a test CoderControlPlane and optionally sets status.url. +func createTestControlPlane(ctx context.Context, t *testing.T, namespace, name, url string) *coderv1alpha1.CoderControlPlane { + t.Helper() + + controlPlane := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + Image: "coder-control-plane:test", + }, + } + require.NoError(t, k8sClient.Create(ctx, controlPlane)) + if url != "" { + controlPlane.Status.URL = url + require.NoError(t, k8sClient.Status().Update(ctx, controlPlane)) + } + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), controlPlane) + }) + + return controlPlane +} + +// createBootstrapSecret creates the bootstrap credentials secret used by provisioner reconciliation. +func createBootstrapSecret(ctx context.Context, t *testing.T, namespace, name, key, value string) *corev1.Secret { + t.Helper() + + if key == "" { + key = coderv1alpha1.DefaultTokenSecretKey + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + key: []byte(value), + }, + } + require.NoError(t, k8sClient.Create(ctx, secret)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), secret) + }) + + return secret +} + +func expectedProvisionerResourceName(name string) string { + const prefix = "provisioner-" + candidate := prefix + name + if len(candidate) <= 63 { + return candidate + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 63 - len(prefix) - len(suffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s%s-%s", prefix, name[:available], suffix) +} + +func expectedProvisionerServiceAccountName(name string) string { + const suffix = "-provisioner" + candidate := fmt.Sprintf("%s%s", name, suffix) + if len(candidate) <= 63 { + return candidate + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + hashSuffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := 63 - len(suffix) - len(hashSuffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s-%s%s", name[:available], hashSuffix, suffix) +} + +func expectedProvisionerKeyName(name string) string { + const maxKeyNameLength = 128 + if len(name) <= maxKeyNameLength { + return name + } + + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(name)) + suffix := fmt.Sprintf("%08x", hasher.Sum32()) + available := maxKeyNameLength - len(suffix) - 1 + if available < 1 { + available = 1 + } + + return fmt.Sprintf("%s-%s", name[:available], suffix) +} + +func reconcileProvisioner(ctx context.Context, t *testing.T, reconciler *controller.CoderProvisionerReconciler, namespacedName types.NamespacedName) { + t.Helper() + + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) +} + +func requireOwnerReference(t *testing.T, owner, child metav1.Object) { + t.Helper() + + ownerReferences := child.GetOwnerReferences() + require.NotEmpty(t, ownerReferences) + + for _, ownerReference := range ownerReferences { + if ownerReference.Name == owner.GetName() && ownerReference.UID == owner.GetUID() { + return + } + } + + require.Failf(t, "missing owner reference", "expected %s/%s to own %s/%s", owner.GetNamespace(), owner.GetName(), child.GetNamespace(), child.GetName()) +} + +func requireCondition(t *testing.T, conditions []metav1.Condition, condType string, status metav1.ConditionStatus) { + t.Helper() + condition := findCondition(t, conditions, condType) + require.Equal(t, status, condition.Status, "condition %s: expected status %v, got %v", condType, status, condition.Status) +} + +func findCondition(t *testing.T, conditions []metav1.Condition, condType string) metav1.Condition { + t.Helper() + for idx := range conditions { + if conditions[idx].Type == condType { + return conditions[idx] + } + } + + require.Failf(t, "condition not found", "expected condition %s to be present", condType) + return metav1.Condition{} +} + +func TestCoderProvisionerReconciler_BasicCreate(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-basic") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-basic", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + organizationID := uuid.New() + provisionerKeyID := uuid.New() + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{ + OrganizationID: organizationID, + KeyID: provisionerKeyID, + KeyName: "provisioner-key-name", + Key: "provisioner-key-material", + }}, + } + reconciler := &controller.CoderProvisionerReconciler{ + Client: k8sClient, + Scheme: scheme, + BootstrapClient: bootstrapClient, + } + + replicas := int32(2) + terminationGracePeriodSeconds := int64(120) + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provisioner-basic", + Namespace: namespace, + }, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + OrganizationName: "acme", + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{ + Name: bootstrapSecret.Name, + Key: coderv1alpha1.DefaultTokenSecretKey, + }, + }, + Key: coderv1alpha1.CoderProvisionerKeySpec{ + Name: "provisioner-key-name", + SecretName: "provisioner-basic-key", + SecretKey: "daemon-key", + }, + Replicas: &replicas, + Tags: map[string]string{"region": "test"}, + Image: "provisioner-image:test", + ExtraArgs: []string{"--test-mode=true"}, + ExtraEnv: []corev1.EnvVar{{Name: "EXTRA_ENV", Value: "extra-value"}}, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "regcred"}}, + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + reconcileProvisioner(ctx, t, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) + + reconciledProvisioner := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, reconciledProvisioner)) + require.Contains(t, reconciledProvisioner.Finalizers, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + + require.Equal(t, 1, bootstrapClient.provisionerKeyCalls) + require.Equal(t, 0, bootstrapClient.deleteKeyCalls) + + keySecret := &corev1.Secret{} + keySecretName := types.NamespacedName{Name: provisioner.Spec.Key.SecretName, Namespace: provisioner.Namespace} + require.NoError(t, k8sClient.Get(ctx, keySecretName, keySecret)) + require.Equal(t, "provisioner-key-material", string(keySecret.Data[provisioner.Spec.Key.SecretKey])) + requireOwnerReference(t, reconciledProvisioner, keySecret) + + serviceAccount := &corev1.ServiceAccount{} + saNamespacedName := types.NamespacedName{Name: expectedProvisionerServiceAccountName(provisioner.Name), Namespace: provisioner.Namespace} + require.NoError(t, k8sClient.Get(ctx, saNamespacedName, serviceAccount)) + requireOwnerReference(t, reconciledProvisioner, serviceAccount) + + roleName := expectedProvisionerResourceName(provisioner.Name) + role := &rbacv1.Role{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: roleName, Namespace: provisioner.Namespace}, role)) + requireOwnerReference(t, reconciledProvisioner, role) + require.Len(t, role.Rules, 1) + require.ElementsMatch(t, []string{""}, role.Rules[0].APIGroups) + require.ElementsMatch(t, []string{"pods", "persistentvolumeclaims"}, role.Rules[0].Resources) + require.ElementsMatch(t, []string{"get", "list", "watch", "create", "update", "patch", "delete"}, role.Rules[0].Verbs) + + roleBinding := &rbacv1.RoleBinding{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: roleName, Namespace: provisioner.Namespace}, roleBinding)) + requireOwnerReference(t, reconciledProvisioner, roleBinding) + require.Equal(t, rbacv1.GroupName, roleBinding.RoleRef.APIGroup) + require.Equal(t, "Role", roleBinding.RoleRef.Kind) + require.Equal(t, role.Name, roleBinding.RoleRef.Name) + require.Len(t, roleBinding.Subjects, 1) + require.Equal(t, rbacv1.ServiceAccountKind, roleBinding.Subjects[0].Kind) + require.Equal(t, serviceAccount.Name, roleBinding.Subjects[0].Name) + require.Equal(t, provisioner.Namespace, roleBinding.Subjects[0].Namespace) + + deployment := &appsv1.Deployment{} + deploymentName := types.NamespacedName{Name: roleName, Namespace: provisioner.Namespace} + require.NoError(t, k8sClient.Get(ctx, deploymentName, deployment)) + requireOwnerReference(t, reconciledProvisioner, deployment) + + require.NotNil(t, deployment.Spec.Replicas) + require.Equal(t, replicas, *deployment.Spec.Replicas) + require.Equal(t, expectedProvisionerServiceAccountName(provisioner.Name), deployment.Spec.Template.Spec.ServiceAccountName) + require.Equal(t, []corev1.LocalObjectReference{{Name: "regcred"}}, deployment.Spec.Template.Spec.ImagePullSecrets) + require.NotNil(t, deployment.Spec.Template.Spec.TerminationGracePeriodSeconds) + require.Equal(t, terminationGracePeriodSeconds, *deployment.Spec.Template.Spec.TerminationGracePeriodSeconds) + require.NotEmpty(t, deployment.Spec.Template.Annotations["checksum/provisioner-key"]) + + require.Len(t, deployment.Spec.Template.Spec.Containers, 1) + container := deployment.Spec.Template.Spec.Containers[0] + require.Equal(t, "provisioner", container.Name) + require.Equal(t, "provisioner-image:test", container.Image) + require.Equal(t, []string{"provisionerd", "start", "--test-mode=true"}, container.Args) + + envByName := make(map[string]corev1.EnvVar, len(container.Env)) + for _, envVar := range container.Env { + envByName[envVar.Name] = envVar + } + require.Equal(t, "https://coder.example.com", envByName["CODER_URL"].Value) + require.Equal(t, "acme", envByName["CODER_ORGANIZATION"].Value) + require.Equal(t, "extra-value", envByName["EXTRA_ENV"].Value) + keyEnv, ok := envByName["CODER_PROVISIONER_DAEMON_KEY"] + require.True(t, ok) + require.NotNil(t, keyEnv.ValueFrom) + require.NotNil(t, keyEnv.ValueFrom.SecretKeyRef) + require.Equal(t, provisioner.Spec.Key.SecretName, keyEnv.ValueFrom.SecretKeyRef.Name) + require.Equal(t, provisioner.Spec.Key.SecretKey, keyEnv.ValueFrom.SecretKeyRef.Key) + + require.Equal(t, reconciledProvisioner.Generation, reconciledProvisioner.Status.ObservedGeneration) + require.Equal(t, int32(0), reconciledProvisioner.Status.ReadyReplicas) + require.Equal(t, coderv1alpha1.CoderProvisionerPhasePending, reconciledProvisioner.Status.Phase) + require.Equal(t, organizationID.String(), reconciledProvisioner.Status.OrganizationID) + require.Equal(t, provisionerKeyID.String(), reconciledProvisioner.Status.ProvisionerKeyID) + require.Equal(t, "provisioner-key-name", reconciledProvisioner.Status.ProvisionerKeyName) + require.NotNil(t, reconciledProvisioner.Status.SecretRef) + require.Equal(t, provisioner.Spec.Key.SecretName, reconciledProvisioner.Status.SecretRef.Name) + require.Equal(t, provisioner.Spec.Key.SecretKey, reconciledProvisioner.Status.SecretRef.Key) +} + +func TestCoderProvisionerReconciler_ExistingSecret(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-existing") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-existing", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisionerName := "provisioner-existing" + secretName := fmt.Sprintf("%s-provisioner-key", provisionerName) + existingSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: namespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + coderv1alpha1.DefaultProvisionerKeySecretKey: []byte("existing-key-material"), + }, + } + require.NoError(t, k8sClient.Create(ctx, existingSecret)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), existingSecret) + }) + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: provisionerName, Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Image: "provisioner-image:test", + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{ + { + OrganizationID: uuid.New(), + KeyID: uuid.New(), + KeyName: provisionerName, + Key: "", // Empty: coderd returns no plaintext for existing keys. + }, + { + OrganizationID: uuid.New(), + KeyID: uuid.New(), + KeyName: provisionerName, + Key: "rotated-key-material", + }, + }, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + reconcileProvisioner(ctx, t, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) + + // The first real reconcile triggers a metadata-only EnsureProvisionerKey + // call because status.OrganizationName and status.TagsHash are empty. + // The empty key response rotates by deleting and recreating the key. + // The second reconcile skips since metadata is now populated. + require.Equal(t, 2, bootstrapClient.provisionerKeyCalls) + require.Equal(t, 1, bootstrapClient.deleteKeyCalls) + + reconciledSecret := &corev1.Secret{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: secretName, Namespace: namespace}, reconciledSecret)) + require.Equal(t, "rotated-key-material", string(reconciledSecret.Data[coderv1alpha1.DefaultProvisionerKeySecretKey])) + + deployment := &appsv1.Deployment{} + resourceName := expectedProvisionerResourceName(provisioner.Name) + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: resourceName, Namespace: namespace}, deployment)) + require.Len(t, deployment.Spec.Template.Spec.Containers, 1) + + envByName := make(map[string]corev1.EnvVar, len(deployment.Spec.Template.Spec.Containers[0].Env)) + for _, envVar := range deployment.Spec.Template.Spec.Containers[0].Env { + envByName[envVar.Name] = envVar + } + keyEnv, ok := envByName["CODER_PROVISIONER_DAEMON_KEY"] + require.True(t, ok) + require.NotNil(t, keyEnv.ValueFrom) + require.NotNil(t, keyEnv.ValueFrom.SecretKeyRef) + require.Equal(t, secretName, keyEnv.ValueFrom.SecretKeyRef.Name) + require.Equal(t, coderv1alpha1.DefaultProvisionerKeySecretKey, keyEnv.ValueFrom.SecretKeyRef.Key) + + reconciledProvisioner := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, reconciledProvisioner)) + require.Equal(t, provisioner.Name, reconciledProvisioner.Status.ProvisionerKeyName) + require.NotNil(t, reconciledProvisioner.Status.SecretRef) + require.Equal(t, secretName, reconciledProvisioner.Status.SecretRef.Name) + require.Equal(t, coderv1alpha1.DefaultProvisionerKeySecretKey, reconciledProvisioner.Status.SecretRef.Key) +} + +func TestCoderProvisionerReconciler_Deletion(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-delete") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-delete", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-delete", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Image: "provisioner-image:test", + Key: coderv1alpha1.CoderProvisionerKeySpec{ + Name: "cleanup-key", + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{Key: "provisioner-key-material"}}, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + reconcileProvisioner(ctx, t, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) + require.Equal(t, 1, bootstrapClient.provisionerKeyCalls) + + latest := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, latest)) + require.Contains(t, latest.Finalizers, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + + require.NoError(t, k8sClient.Delete(ctx, latest)) + markedForDeletion := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, markedForDeletion)) + require.False(t, markedForDeletion.DeletionTimestamp.IsZero()) + + reconcileProvisioner(ctx, t, reconciler, namespacedName) + require.Equal(t, 1, bootstrapClient.deleteKeyCalls) + + require.Eventually(t, func() bool { + reconciled := &coderv1alpha1.CoderProvisioner{} + err := k8sClient.Get(ctx, namespacedName, reconciled) + if apierrors.IsNotFound(err) { + return true + } + if err != nil { + t.Logf("get reconciled provisioner: %v", err) + return false + } + + return !controllerutil.ContainsFinalizer(reconciled, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + }, 5*time.Second, 100*time.Millisecond) +} + +func TestCoderProvisionerReconciler_DeletionControlPlaneGone(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-delete-cpgone") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-cpgone", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-cpgone", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Image: "provisioner-image:test", + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{Key: "provisioner-key-material"}}, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + reconcileProvisioner(ctx, t, reconciler, namespacedName) + reconcileProvisioner(ctx, t, reconciler, namespacedName) + + // Delete the control plane first (common in namespace teardown). + require.NoError(t, k8sClient.Delete(ctx, controlPlane)) + + // Now delete the provisioner — the finalizer should still be removed + // even though the control plane is gone (best-effort cleanup). + latest := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, latest)) + require.NoError(t, k8sClient.Delete(ctx, latest)) + + reconcileProvisioner(ctx, t, reconciler, namespacedName) + + // DeleteProvisionerKey should still be called once using the persisted + // status.ControlPlaneURL even when the control plane object is already gone. + require.Equal(t, 1, bootstrapClient.deleteKeyCalls) + + // The finalizer should still be removed. + require.Eventually(t, func() bool { + reconciled := &coderv1alpha1.CoderProvisioner{} + err := k8sClient.Get(ctx, namespacedName, reconciled) + if apierrors.IsNotFound(err) { + return true + } + if err != nil { + t.Logf("get reconciled provisioner: %v", err) + return false + } + + return !controllerutil.ContainsFinalizer(reconciled, coderv1alpha1.ProvisionerKeyCleanupFinalizer) + }, 5*time.Second, 100*time.Millisecond) +} + +func TestCoderProvisionerReconciler_NotFound(t *testing.T) { + t.Parallel() + + reconciler := &controller.CoderProvisionerReconciler{ + Client: k8sClient, + Scheme: scheme, + BootstrapClient: &fakeBootstrapClient{}, + } + + result, err := reconciler.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "does-not-exist", Namespace: "default"}, + }) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) +} + +func TestCoderProvisionerReconciler_NilChecks(t *testing.T) { + t.Parallel() + + request := ctrl.Request{NamespacedName: types.NamespacedName{Name: "test", Namespace: "default"}} + + t.Run("nil client", func(t *testing.T) { + t.Parallel() + + reconciler := &controller.CoderProvisionerReconciler{ + Client: nil, + Scheme: scheme, + BootstrapClient: &fakeBootstrapClient{}, + } + + _, err := reconciler.Reconcile(context.Background(), request) + require.ErrorContains(t, err, "assertion failed: reconciler client must not be nil") + }) + + t.Run("nil scheme", func(t *testing.T) { + t.Parallel() + + reconciler := &controller.CoderProvisionerReconciler{ + Client: k8sClient, + Scheme: nil, + BootstrapClient: &fakeBootstrapClient{}, + } + + _, err := reconciler.Reconcile(context.Background(), request) + require.ErrorContains(t, err, "assertion failed: reconciler scheme must not be nil") + }) + + t.Run("nil bootstrap client", func(t *testing.T) { + t.Parallel() + + reconciler := &controller.CoderProvisionerReconciler{ + Client: k8sClient, + Scheme: scheme, + BootstrapClient: nil, + } + + _, err := reconciler.Reconcile(context.Background(), request) + require.ErrorContains(t, err, "assertion failed: reconciler bootstrap client must not be nil") + }) +} + +func TestCoderProvisionerReconciler_ControlPlaneNotReady(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-cpnotready") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-notready", "") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-notready", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{} + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, namespacedName) + + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.ErrorContains(t, err, fmt.Sprintf("codercontrolplane %s/%s status.url is empty", controlPlane.Namespace, controlPlane.Name)) + require.Equal(t, ctrl.Result{}, result) + require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) +} + +func TestCoderProvisionerReconciler_RotationOnSecretLoss(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-rotation") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-rotation", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-rotation", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Key: coderv1alpha1.CoderProvisionerKeySpec{ + Name: "rotation-key", + SecretName: "provisioner-rotation-key", + SecretKey: "daemon-key", + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{ + {KeyName: "rotation-key", Key: ""}, + {KeyName: "rotation-key", Key: "rotated-key-material"}, + {KeyName: "rotation-key", Key: "rotated-key-material"}, + }, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + + require.Equal(t, 2, bootstrapClient.provisionerKeyCalls) + require.Equal(t, 1, bootstrapClient.deleteKeyCalls) + require.Len(t, bootstrapClient.deleteKeyRequests, 1) + require.Equal(t, "rotation-key", bootstrapClient.deleteKeyRequests[0].KeyName) + + keySecret := &corev1.Secret{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: provisioner.Spec.Key.SecretName, Namespace: provisioner.Namespace}, keySecret)) + require.Equal(t, "rotated-key-material", string(keySecret.Data[provisioner.Spec.Key.SecretKey])) +} + +func TestCoderProvisionerReconciler_TagsDrift(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-tags-drift") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-tags", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-tags", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Key: coderv1alpha1.CoderProvisionerKeySpec{Name: "tags-drift-key"}, + Tags: map[string]string{"region": "us-east"}, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{ + {KeyName: "tags-drift-key", Key: "initial-key-material"}, + {KeyName: "tags-drift-key", Key: ""}, + {KeyName: "tags-drift-key", Key: "rotated-key-material"}, + }, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + + before := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, before)) + oldTagsHash := before.Status.TagsHash + require.NotEmpty(t, oldTagsHash) + + before.Spec.Tags = map[string]string{"region": "eu-west"} + require.NoError(t, k8sClient.Update(ctx, before)) + + reconcileProvisioner(ctx, t, reconciler, request) + + require.Equal(t, 3, bootstrapClient.provisionerKeyCalls) + require.Equal(t, 2, bootstrapClient.deleteKeyCalls) + require.Len(t, bootstrapClient.deleteKeyRequests, 2) + require.Equal(t, "tags-drift-key", bootstrapClient.deleteKeyRequests[0].KeyName) + require.Equal(t, "tags-drift-key", bootstrapClient.deleteKeyRequests[1].KeyName) + + keySecret := &corev1.Secret{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-provisioner-key", provisioner.Name), Namespace: provisioner.Namespace}, keySecret)) + require.Equal(t, "rotated-key-material", string(keySecret.Data[coderv1alpha1.DefaultProvisionerKeySecretKey])) + + after := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, after)) + require.NotEmpty(t, after.Status.TagsHash) + require.NotEqual(t, oldTagsHash, after.Status.TagsHash) +} + +func TestCoderProvisionerReconciler_KeyNameDrift(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-key-drift") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-key", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-key-drift", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + Key: coderv1alpha1.CoderProvisionerKeySpec{ + Name: "key-v1", + SecretName: "provisioner-key-drift-secret", + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{ + {KeyName: "key-v1", Key: "key-v1-material"}, + {KeyName: "key-v2", Key: "key-v2-material"}, + }, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + + updated := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, updated)) + updated.Spec.Key.Name = "key-v2" + require.NoError(t, k8sClient.Update(ctx, updated)) + + reconcileProvisioner(ctx, t, reconciler, request) + + require.GreaterOrEqual(t, bootstrapClient.deleteKeyCalls, 1) + require.NotEmpty(t, bootstrapClient.deleteKeyRequests) + lastDelete := bootstrapClient.deleteKeyRequests[len(bootstrapClient.deleteKeyRequests)-1] + require.Equal(t, "key-v1", lastDelete.KeyName) + + require.GreaterOrEqual(t, len(bootstrapClient.provisionerKeyRequests), 2) + lastEnsure := bootstrapClient.provisionerKeyRequests[len(bootstrapClient.provisionerKeyRequests)-1] + require.Equal(t, "key-v2", lastEnsure.KeyName) + + keySecret := &corev1.Secret{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: provisioner.Spec.Key.SecretName, Namespace: provisioner.Namespace}, keySecret)) + require.Equal(t, "key-v2-material", string(keySecret.Data[coderv1alpha1.DefaultProvisionerKeySecretKey])) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, reconciled)) + require.Equal(t, "key-v2", reconciled.Status.ProvisionerKeyName) +} + +func TestCoderProvisionerReconciler_ReadyPhaseAndConditions(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-ready") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-ready", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-ready", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{ + KeyName: provisioner.Name, + Key: "provisioner-key-material", + }}, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + + deployment := &appsv1.Deployment{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: expectedProvisionerResourceName(provisioner.Name), Namespace: provisioner.Namespace}, deployment)) + deployment.Status.ReadyReplicas = 1 + deployment.Status.Replicas = 1 + require.NoError(t, k8sClient.Status().Update(ctx, deployment)) + + reconcileProvisioner(ctx, t, reconciler, request) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, reconciled)) + require.Equal(t, coderv1alpha1.CoderProvisionerPhaseReady, reconciled.Status.Phase) + + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionDeploymentReady, metav1.ConditionTrue) + deploymentReadyCondition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionDeploymentReady) + require.Equal(t, "MinimumReplicasReady", deploymentReadyCondition.Reason) + + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionControlPlaneReady, metav1.ConditionTrue) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady, metav1.ConditionTrue) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionProvisionerKeyReady, metav1.ConditionTrue) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionProvisionerKeySecretReady, metav1.ConditionTrue) +} + +func TestCoderProvisionerReconciler_ConditionsOnFailure(t *testing.T) { + t.Parallel() + + t.Run("control plane unavailable", func(t *testing.T) { + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-cond-cp") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-cond-cp", "") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-cond-cp", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{} + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: request}) + require.ErrorContains(t, err, "status.url is empty") + require.Equal(t, ctrl.Result{}, result) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, reconciled)) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionControlPlaneReady, metav1.ConditionFalse) + controlPlaneCondition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionControlPlaneReady) + require.Equal(t, "ControlPlaneUnavailable", controlPlaneCondition.Reason) + }) + + t.Run("bootstrap secret unavailable", func(t *testing.T) { + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-cond-bootstrap") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-cond-bootstrap", "https://coder.example.com") + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: "provisioner-cond-bootstrap", Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: "missing-bootstrap-secret", Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{} + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: request}) + require.ErrorContains(t, err, "read bootstrap credentials secret") + require.Equal(t, ctrl.Result{}, result) + require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, reconciled)) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionControlPlaneReady, metav1.ConditionTrue) + requireCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady, metav1.ConditionFalse) + bootstrapCondition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady) + require.Equal(t, "BootstrapSecretUnavailable", bootstrapCondition.Reason) + }) +} + +func TestCoderProvisionerReconciler_LongNameTruncation(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-longname") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-longname", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-creds", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + provisionerName := strings.Repeat("a", 180) + deploymentCandidateName := fmt.Sprintf("provisioner-%s", provisionerName) + serviceAccountCandidateName := fmt.Sprintf("%s-provisioner", provisionerName) + keyNameCandidate := provisionerName + expectedKeyName := expectedProvisionerKeyName(provisionerName) + require.Greater(t, len(deploymentCandidateName), 63) + require.Greater(t, len(serviceAccountCandidateName), 63) + require.Greater(t, len(keyNameCandidate), 128) + require.Len(t, expectedKeyName, 128) + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: provisionerName, Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlane.Name}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{Name: bootstrapSecret.Name, Key: coderv1alpha1.DefaultTokenSecretKey}, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + bootstrapClient := &fakeBootstrapClient{ + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{ + KeyName: expectedKeyName, + Key: "provisioner-key-material", + }}, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + request := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + reconcileProvisioner(ctx, t, reconciler, request) + reconcileProvisioner(ctx, t, reconciler, request) + + deploymentName := expectedProvisionerResourceName(provisionerName) + serviceAccountName := expectedProvisionerServiceAccountName(provisionerName) + require.LessOrEqual(t, len(deploymentName), 63) + require.LessOrEqual(t, len(serviceAccountName), 63) + + require.Len(t, bootstrapClient.provisionerKeyRequests, 1) + require.Equal(t, expectedKeyName, bootstrapClient.provisionerKeyRequests[0].KeyName) + + reconciledProvisioner := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, request, reconciledProvisioner)) + require.Equal(t, expectedKeyName, reconciledProvisioner.Status.ProvisionerKeyName) + require.LessOrEqual(t, len(reconciledProvisioner.Status.ProvisionerKeyName), 128) + + deployment := &appsv1.Deployment{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: deploymentName, Namespace: namespace}, deployment)) + + serviceAccount := &corev1.ServiceAccount{} + require.NoError(t, k8sClient.Get(ctx, types.NamespacedName{Name: serviceAccountName, Namespace: namespace}, serviceAccount)) +} diff --git a/internal/controller/workspaceproxy_controller_test.go b/internal/controller/workspaceproxy_controller_test.go index fcaf9596..1ec0e70d 100644 --- a/internal/controller/workspaceproxy_controller_test.go +++ b/internal/controller/workspaceproxy_controller_test.go @@ -22,6 +22,22 @@ type fakeBootstrapClient struct { response coderbootstrap.RegisterWorkspaceProxyResponse err error calls int + + // Provisioner key support. + provisionerKeyResponses []coderbootstrap.EnsureProvisionerKeyResponse + provisionerKeyErr error + provisionerKeyCalls int + provisionerKeyRequests []coderbootstrap.EnsureProvisionerKeyRequest + deleteKeyErr error + deleteKeyCalls int + deleteKeyRequests []deleteKeyRequest +} + +type deleteKeyRequest struct { + CoderURL string + SessionToken string + OrganizationName string + KeyName string } func (f *fakeBootstrapClient) EnsureWorkspaceProxy(_ context.Context, _ coderbootstrap.RegisterWorkspaceProxyRequest) (coderbootstrap.RegisterWorkspaceProxyResponse, error) { @@ -29,6 +45,35 @@ func (f *fakeBootstrapClient) EnsureWorkspaceProxy(_ context.Context, _ coderboo return f.response, f.err } +func (f *fakeBootstrapClient) EnsureProvisionerKey(_ context.Context, req coderbootstrap.EnsureProvisionerKeyRequest) (coderbootstrap.EnsureProvisionerKeyResponse, error) { + f.provisionerKeyCalls++ + f.provisionerKeyRequests = append(f.provisionerKeyRequests, req) + + if f.provisionerKeyErr != nil { + return coderbootstrap.EnsureProvisionerKeyResponse{}, f.provisionerKeyErr + } + if len(f.provisionerKeyResponses) == 0 { + return coderbootstrap.EnsureProvisionerKeyResponse{}, nil + } + idx := f.provisionerKeyCalls - 1 + if idx >= len(f.provisionerKeyResponses) { + idx = len(f.provisionerKeyResponses) - 1 + } + + return f.provisionerKeyResponses[idx], nil +} + +func (f *fakeBootstrapClient) DeleteProvisionerKey(_ context.Context, coderURL, sessionToken, orgName, keyName string) error { + f.deleteKeyCalls++ + f.deleteKeyRequests = append(f.deleteKeyRequests, deleteKeyRequest{ + CoderURL: coderURL, + SessionToken: sessionToken, + OrganizationName: orgName, + KeyName: keyName, + }) + return f.deleteKeyErr +} + func workspaceProxyResourceName(name string) string { const prefix = "wsproxy-" candidate := prefix + name diff --git a/mkdocs.yml b/mkdocs.yml index 47d437d5..b457ed5f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,6 +44,7 @@ nav: - API: # BEGIN GENERATED API NAV - CoderControlPlane: reference/api/codercontrolplane.md + - CoderProvisioner: reference/api/coderprovisioner.md - WorkspaceProxy: reference/api/workspaceproxy.md - CoderTemplate: reference/api/codertemplate.md - CoderWorkspace: reference/api/coderworkspace.md