diff --git a/api/v1alpha1/codercontrolplane_types.go b/api/v1alpha1/codercontrolplane_types.go index abab9eec..7f4b1630 100644 --- a/api/v1alpha1/codercontrolplane_types.go +++ b/api/v1alpha1/codercontrolplane_types.go @@ -10,6 +10,8 @@ const ( CoderControlPlanePhasePending = "Pending" // CoderControlPlanePhaseReady indicates at least one control plane pod is ready. CoderControlPlanePhaseReady = "Ready" + // CoderControlPlaneConditionLicenseApplied indicates whether the operator uploaded the configured license. + CoderControlPlaneConditionLicenseApplied = "LicenseApplied" ) // CoderControlPlaneSpec defines the desired state of a CoderControlPlane. @@ -32,6 +34,11 @@ type CoderControlPlaneSpec struct { // OperatorAccess configures bootstrap API access to the coderd instance. // +kubebuilder:default={} OperatorAccess OperatorAccessSpec `json:"operatorAccess,omitempty"` + // LicenseSecretRef references a Secret key containing a Coder Enterprise + // license JWT. When set, the controller uploads the license after the + // control plane is ready and re-uploads when the Secret value changes. + // +optional + LicenseSecretRef *SecretKeySelector `json:"licenseSecretRef,omitempty"` } // OperatorAccessSpec configures the controller-managed coderd operator user. @@ -56,6 +63,14 @@ type CoderControlPlaneStatus struct { OperatorTokenSecretRef *SecretKeySelector `json:"operatorTokenSecretRef,omitempty"` // OperatorAccessReady reports whether operator API access bootstrap succeeded. OperatorAccessReady bool `json:"operatorAccessReady,omitempty"` + // LicenseLastApplied is the timestamp of the most recent successful + // operator-managed license upload. + // +optional + LicenseLastApplied *metav1.Time `json:"licenseLastApplied,omitempty"` + // LicenseLastAppliedHash is the SHA-256 hex hash of the trimmed license JWT + // that LicenseLastApplied refers to. + // +optional + LicenseLastAppliedHash string `json:"licenseLastAppliedHash,omitempty"` // Phase is a high-level readiness indicator. Phase string `json:"phase,omitempty"` // Conditions are Kubernetes-standard conditions for this resource. diff --git a/api/v1alpha1/types_shared.go b/api/v1alpha1/types_shared.go index 5d4295cd..1ef264fd 100644 --- a/api/v1alpha1/types_shared.go +++ b/api/v1alpha1/types_shared.go @@ -5,6 +5,8 @@ import corev1 "k8s.io/api/core/v1" const ( // DefaultTokenSecretKey is the default key used for proxy session tokens. DefaultTokenSecretKey = "token" + // DefaultLicenseSecretKey is the default key used for Coder license JWTs. + DefaultLicenseSecretKey = "license" ) // ServiceSpec defines the Service configuration reconciled by the operator. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 59eab117..544cf382 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -99,6 +99,11 @@ func (in *CoderControlPlaneSpec) DeepCopyInto(out *CoderControlPlaneSpec) { copy(*out, *in) } out.OperatorAccess = in.OperatorAccess + if in.LicenseSecretRef != nil { + in, out := &in.LicenseSecretRef, &out.LicenseSecretRef + *out = new(SecretKeySelector) + **out = **in + } return } @@ -120,6 +125,10 @@ func (in *CoderControlPlaneStatus) DeepCopyInto(out *CoderControlPlaneStatus) { *out = new(SecretKeySelector) **out = **in } + if in.LicenseLastApplied != nil { + in, out := &in.LicenseLastApplied, &out.LicenseLastApplied + *out = (*in).DeepCopy() + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) diff --git a/config/crd/bases/coder.com_codercontrolplanes.yaml b/config/crd/bases/coder.com_codercontrolplanes.yaml index 1e9aa82a..20bfcaca 100644 --- a/config/crd/bases/coder.com_codercontrolplanes.yaml +++ b/config/crd/bases/coder.com_codercontrolplanes.yaml @@ -226,6 +226,21 @@ spec: type: object x-kubernetes-map-type: atomic type: array + licenseSecretRef: + description: |- + LicenseSecretRef references a Secret key containing a Coder Enterprise + license JWT. When set, the controller uploads the license after the + control plane is ready and re-uploads when the Secret value changes. + properties: + key: + description: Key is the key inside the Secret data map. + type: string + name: + description: Name is the Kubernetes Secret name. + type: string + required: + - name + type: object operatorAccess: default: {} description: OperatorAccess configures bootstrap API access to the @@ -330,6 +345,17 @@ spec: - type type: object type: array + licenseLastApplied: + description: |- + LicenseLastApplied is the timestamp of the most recent successful + operator-managed license upload. + format: date-time + type: string + licenseLastAppliedHash: + description: |- + LicenseLastAppliedHash is the SHA-256 hex hash of the trimmed license JWT + that LicenseLastApplied refers to. + type: string observedGeneration: description: ObservedGeneration tracks the spec generation this status reflects. diff --git a/config/samples/coder_v1alpha1_codercontrolplane.yaml b/config/samples/coder_v1alpha1_codercontrolplane.yaml index 479efd46..c6e62f75 100644 --- a/config/samples/coder_v1alpha1_codercontrolplane.yaml +++ b/config/samples/coder_v1alpha1_codercontrolplane.yaml @@ -5,3 +5,6 @@ metadata: namespace: default spec: image: "ghcr.io/coder/coder-k8s:main" + licenseSecretRef: + name: coder-license + key: license diff --git a/docs/reference/api/codercontrolplane.md b/docs/reference/api/codercontrolplane.md index fe27ca36..32591365 100644 --- a/docs/reference/api/codercontrolplane.md +++ b/docs/reference/api/codercontrolplane.md @@ -20,6 +20,7 @@ | `extraEnv` | [EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#envvar-v1-core) array | ExtraEnv are injected into the Coder control plane container. | | `imagePullSecrets` | [LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#localobjectreference-v1-core) array | ImagePullSecrets are used by the pod to pull private images. | | `operatorAccess` | [OperatorAccessSpec](#operatoraccessspec) | OperatorAccess configures bootstrap API access to the coderd instance. | +| `licenseSecretRef` | [SecretKeySelector](#secretkeyselector) | LicenseSecretRef references a Secret key containing a Coder Enterprise license JWT. When set, the controller uploads the license after the control plane is ready and re-uploads when the Secret value changes. | ## Status @@ -30,6 +31,8 @@ | `url` | string | URL is the in-cluster URL for the control plane service. | | `operatorTokenSecretRef` | [SecretKeySelector](#secretkeyselector) | OperatorTokenSecretRef points to the Secret key containing the `coder-k8s-operator` API token. | | `operatorAccessReady` | boolean | OperatorAccessReady reports whether operator API access bootstrap succeeded. | +| `licenseLastApplied` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | LicenseLastApplied is the timestamp of the most recent successful operator-managed license upload. | +| `licenseLastAppliedHash` | string | LicenseLastAppliedHash is the SHA-256 hex hash of the trimmed license JWT that LicenseLastApplied refers to. | | `phase` | string | Phase is a high-level readiness indicator. | | `conditions` | [Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#condition-v1-meta) array | Conditions are Kubernetes-standard conditions for this resource. | diff --git a/internal/app/controllerapp/controllerapp.go b/internal/app/controllerapp/controllerapp.go index e54ccf5d..67fb2f1c 100644 --- a/internal/app/controllerapp/controllerapp.go +++ b/internal/app/controllerapp/controllerapp.go @@ -88,8 +88,10 @@ func SetupControllers(mgr manager.Manager) error { reconciler := &controller.CoderControlPlaneReconciler{ Client: client, + APIReader: mgr.GetAPIReader(), Scheme: managerScheme, OperatorAccessProvisioner: coderbootstrap.NewPostgresOperatorAccessProvisioner(), + LicenseUploader: controller.NewSDKLicenseUploader(), } if err := reconciler.SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to create controller: %w", err) diff --git a/internal/controller/codercontrolplane_controller.go b/internal/controller/codercontrolplane_controller.go index cf39b204..34d8a8d1 100644 --- a/internal/controller/codercontrolplane_controller.go +++ b/internal/controller/codercontrolplane_controller.go @@ -3,24 +3,33 @@ package controller import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "hash/fnv" "maps" + "net/http" + "net/url" "strings" "time" + "github.com/coder/coder/v2/codersdk" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" "github.com/coder/coder-k8s/internal/coderbootstrap" @@ -41,6 +50,18 @@ const ( operatorAccessRetryInterval = 30 * time.Second operatorTokenSecretSuffix = "-operator-token" + + // #nosec G101 -- this is a field index key, not a credential. + licenseSecretNameFieldIndex = ".spec.licenseSecretRef.name" + + licenseConditionReasonApplied = "Applied" + licenseConditionReasonPending = "Pending" + licenseConditionReasonSecretMissing = "SecretMissing" + licenseConditionReasonForbidden = "Forbidden" + licenseConditionReasonNotSupported = "NotSupported" + licenseConditionReasonError = "Error" + + licenseUploadRequestTimeout = 30 * time.Second ) var ( @@ -48,12 +69,88 @@ var ( errSecretValueEmpty = errors.New("secret value empty") ) +// LicenseUploader uploads and inspects Coder Enterprise licenses in a coderd instance. +type LicenseUploader interface { + AddLicense(ctx context.Context, coderURL, sessionToken, licenseJWT string) error + HasAnyLicense(ctx context.Context, coderURL, sessionToken string) (bool, error) +} + +// NewSDKLicenseUploader returns a LicenseUploader backed by codersdk. +func NewSDKLicenseUploader() LicenseUploader { + return &sdkLicenseUploader{} +} + +type sdkLicenseUploader struct{} + +func (u *sdkLicenseUploader) AddLicense(ctx context.Context, coderURL, sessionToken, licenseJWT string) error { + if licenseJWT == "" { + return fmt.Errorf("assertion failed: license JWT must not be empty") + } + + sdkClient, err := newSDKLicenseClient(coderURL, sessionToken) + if err != nil { + return err + } + + if _, err := sdkClient.AddLicense(ctx, codersdk.AddLicenseRequest{License: licenseJWT}); err != nil { + return fmt.Errorf("upload coder license: %w", err) + } + + return nil +} + +func (u *sdkLicenseUploader) HasAnyLicense(ctx context.Context, coderURL, sessionToken string) (bool, error) { + sdkClient, err := newSDKLicenseClient(coderURL, sessionToken) + if err != nil { + return false, err + } + + licenses, err := sdkClient.Licenses(ctx) + if err != nil { + return false, fmt.Errorf("list coder licenses: %w", err) + } + + return len(licenses) > 0, nil +} + +func newSDKLicenseClient(coderURL, sessionToken string) (*codersdk.Client, error) { + if strings.TrimSpace(coderURL) == "" { + return nil, fmt.Errorf("assertion failed: coder URL must not be empty") + } + if sessionToken == "" { + return nil, fmt.Errorf("assertion failed: session token must not be empty") + } + + parsedURL, err := url.Parse(coderURL) + if err != nil { + return nil, fmt.Errorf("parse coder URL: %w", err) + } + + sdkClient := codersdk.New(parsedURL) + sdkClient.SetSessionToken(sessionToken) + if sdkClient.HTTPClient == nil { + sdkClient.HTTPClient = &http.Client{} + } + defaultTransport, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return nil, fmt.Errorf("assertion failed: http.DefaultTransport is not *http.Transport") + } + // Use a dedicated transport to avoid sharing http.DefaultTransport's + // connection pool across parallel test servers. + sdkClient.HTTPClient.Transport = defaultTransport.Clone() + sdkClient.HTTPClient.Timeout = licenseUploadRequestTimeout + + return sdkClient, nil +} + // CoderControlPlaneReconciler reconciles a CoderControlPlane object. type CoderControlPlaneReconciler struct { client.Client - Scheme *runtime.Scheme + APIReader client.Reader + Scheme *runtime.Scheme OperatorAccessProvisioner coderbootstrap.OperatorAccessProvisioner + LicenseUploader LicenseUploader } // +kubebuilder:rbac:groups=coder.com,resources=codercontrolplanes,verbs=get;list;watch;create;update;patch;delete @@ -94,6 +191,7 @@ func (r *CoderControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Re return ctrl.Result{}, err } + originalStatus := *coderControlPlane.Status.DeepCopy() nextStatus := r.desiredStatus(coderControlPlane, deployment, service) operatorResult, err := r.reconcileOperatorAccess(ctx, coderControlPlane, &nextStatus) @@ -101,11 +199,16 @@ func (r *CoderControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Re return ctrl.Result{}, err } - if err := r.reconcileStatus(ctx, coderControlPlane, nextStatus); err != nil { + licenseResult, err := r.reconcileLicense(ctx, coderControlPlane, &nextStatus) + if err != nil { + return ctrl.Result{}, err + } + + if err := r.reconcileStatus(ctx, coderControlPlane, originalStatus, nextStatus); err != nil { return ctrl.Result{}, err } - return operatorResult, nil + return mergeResults(operatorResult, licenseResult), nil } func (r *CoderControlPlaneReconciler) reconcileDeployment(ctx context.Context, coderControlPlane *coderv1alpha1.CoderControlPlane) (*appsv1.Deployment, error) { @@ -330,6 +433,300 @@ func (r *CoderControlPlaneReconciler) reconcileOperatorAccess( return ctrl.Result{}, nil } +func (r *CoderControlPlaneReconciler) reconcileLicense( + ctx context.Context, + coderControlPlane *coderv1alpha1.CoderControlPlane, + nextStatus *coderv1alpha1.CoderControlPlaneStatus, +) (ctrl.Result, error) { + if coderControlPlane == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: coder control plane must not be nil") + } + if nextStatus == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: next status must not be nil") + } + + if coderControlPlane.Spec.LicenseSecretRef == nil { + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionUnknown, + licenseConditionReasonPending, + "License Secret reference is not configured.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + if r.LicenseUploader == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: license uploader must not be nil when licenseSecretRef is configured") + } + + if nextStatus.Phase != coderv1alpha1.CoderControlPlanePhaseReady { + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonPending, + "Waiting for control plane readiness before applying license.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + if !nextStatus.OperatorAccessReady || nextStatus.OperatorTokenSecretRef == nil { + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonPending, + "Waiting for operator access credentials before applying license.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + if strings.TrimSpace(nextStatus.URL) == "" { + return ctrl.Result{}, fmt.Errorf("assertion failed: control plane URL must not be empty when licenseSecretRef is configured") + } + + operatorTokenSecretName := strings.TrimSpace(nextStatus.OperatorTokenSecretRef.Name) + if operatorTokenSecretName == "" { + return ctrl.Result{}, fmt.Errorf("assertion failed: operator token secret name must not be empty when operator access is ready") + } + operatorTokenSecretKey := strings.TrimSpace(nextStatus.OperatorTokenSecretRef.Key) + if operatorTokenSecretKey == "" { + operatorTokenSecretKey = coderv1alpha1.DefaultTokenSecretKey + } + + operatorToken, err := r.readSecretValue(ctx, coderControlPlane.Namespace, operatorTokenSecretName, operatorTokenSecretKey) + switch { + case err == nil: + case apierrors.IsNotFound(err), errors.Is(err, errSecretValueMissing), errors.Is(err, errSecretValueEmpty): + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonSecretMissing, + "Operator token Secret is missing or incomplete; retrying license upload.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + default: + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonError, + "Failed to read operator token Secret; retrying license upload.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + } + + licenseSecretName := strings.TrimSpace(coderControlPlane.Spec.LicenseSecretRef.Name) + if licenseSecretName == "" { + return ctrl.Result{}, fmt.Errorf("assertion failed: license secret name must not be empty when licenseSecretRef is configured") + } + licenseSecretKey := strings.TrimSpace(coderControlPlane.Spec.LicenseSecretRef.Key) + if licenseSecretKey == "" { + licenseSecretKey = coderv1alpha1.DefaultLicenseSecretKey + } + + licenseJWT, err := r.readSecretValue(ctx, coderControlPlane.Namespace, licenseSecretName, licenseSecretKey) + switch { + case err == nil: + case apierrors.IsNotFound(err), errors.Is(err, errSecretValueMissing), errors.Is(err, errSecretValueEmpty): + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonSecretMissing, + "License Secret is missing or incomplete; retrying upload.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + default: + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonError, + "Failed to read license Secret; retrying upload.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + } + + licenseJWT = strings.TrimSpace(licenseJWT) + if licenseJWT == "" { + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonSecretMissing, + "License Secret value is empty after trimming whitespace.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + } + + licenseHash, err := hashLicenseJWT(licenseJWT) + if err != nil { + return ctrl.Result{}, err + } + + if nextStatus.LicenseLastApplied != nil && nextStatus.LicenseLastAppliedHash == licenseHash { + hasAnyLicense, hasLicenseErr := r.LicenseUploader.HasAnyLicense(ctx, nextStatus.URL, operatorToken) + if hasLicenseErr != nil { + var sdkErr *codersdk.Error + if errors.As(hasLicenseErr, &sdkErr) { + switch sdkErr.StatusCode() { + case http.StatusNotFound: + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonNotSupported, + "Control plane does not expose the Enterprise licenses API.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + case http.StatusUnauthorized, http.StatusForbidden: + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonForbidden, + "Operator token is not authorized to query configured licenses.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + } + } + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonError, + "Failed to query existing licenses; retrying.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + } + if hasAnyLicense { + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionTrue, + licenseConditionReasonApplied, + "Configured license is already applied.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + } + + if err := r.LicenseUploader.AddLicense(ctx, nextStatus.URL, operatorToken, licenseJWT); err != nil { + if isDuplicateLicenseUploadError(err) { + now := metav1.Now() + nextStatus.LicenseLastApplied = &now + nextStatus.LicenseLastAppliedHash = licenseHash + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionTrue, + licenseConditionReasonApplied, + "Configured license already exists in coderd.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + var sdkErr *codersdk.Error + if errors.As(err, &sdkErr) { + switch sdkErr.StatusCode() { + case http.StatusNotFound: + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonNotSupported, + "Control plane does not expose the Enterprise licenses API.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + case http.StatusUnauthorized, http.StatusForbidden: + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonForbidden, + "Operator token is not authorized to upload the configured license.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + } + } + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionFalse, + licenseConditionReasonError, + "Failed to upload configured license; retrying.", + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + } + + now := metav1.Now() + nextStatus.LicenseLastApplied = &now + nextStatus.LicenseLastAppliedHash = licenseHash + if err := setControlPlaneCondition( + nextStatus, + coderControlPlane.Generation, + coderv1alpha1.CoderControlPlaneConditionLicenseApplied, + metav1.ConditionTrue, + licenseConditionReasonApplied, + "Configured license uploaded successfully.", + ); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + func (r *CoderControlPlaneReconciler) cleanupDisabledOperatorAccess( ctx context.Context, coderControlPlane *coderv1alpha1.CoderControlPlane, @@ -574,17 +971,241 @@ func (r *CoderControlPlaneReconciler) readSecretValue(ctx context.Context, names return string(value), nil } +func setControlPlaneCondition( + nextStatus *coderv1alpha1.CoderControlPlaneStatus, + generation int64, + conditionType string, + status metav1.ConditionStatus, + reason string, + message string, +) error { + if nextStatus == nil { + return fmt.Errorf("assertion failed: next status must not be nil") + } + if strings.TrimSpace(conditionType) == "" { + return fmt.Errorf("assertion failed: condition type must not be empty") + } + if strings.TrimSpace(reason) == "" { + return fmt.Errorf("assertion failed: condition reason must not be empty") + } + + meta.SetStatusCondition(&nextStatus.Conditions, metav1.Condition{ + Type: conditionType, + Status: status, + ObservedGeneration: generation, + Reason: reason, + Message: message, + }) + + return nil +} + +func hashLicenseJWT(licenseJWT string) (string, error) { + if licenseJWT == "" { + return "", fmt.Errorf("assertion failed: license JWT must not be empty") + } + + sum := sha256.Sum256([]byte(licenseJWT)) + return hex.EncodeToString(sum[:]), nil +} + +func mergeResults(results ...ctrl.Result) ctrl.Result { + merged := ctrl.Result{} + for _, result := range results { + if result.RequeueAfter > 0 && (merged.RequeueAfter == 0 || result.RequeueAfter < merged.RequeueAfter) { + merged.RequeueAfter = result.RequeueAfter + } + } + + return merged +} + +func indexByLicenseSecretName(obj client.Object) []string { + coderControlPlane, ok := obj.(*coderv1alpha1.CoderControlPlane) + if !ok || coderControlPlane.Spec.LicenseSecretRef == nil { + return nil + } + + licenseSecretName := strings.TrimSpace(coderControlPlane.Spec.LicenseSecretRef.Name) + if licenseSecretName == "" { + return nil + } + + return []string{licenseSecretName} +} + +func (r *CoderControlPlaneReconciler) reconcileRequestsForLicenseSecret( + ctx context.Context, + obj client.Object, +) []reconcile.Request { + secret, ok := obj.(*corev1.Secret) + if !ok { + return nil + } + if strings.TrimSpace(secret.Name) == "" || strings.TrimSpace(secret.Namespace) == "" { + return nil + } + + var coderControlPlanes coderv1alpha1.CoderControlPlaneList + if err := r.List( + ctx, + &coderControlPlanes, + client.InNamespace(secret.Namespace), + client.MatchingFields{licenseSecretNameFieldIndex: secret.Name}, + ); err != nil { + return nil + } + + requests := make([]reconcile.Request, 0, len(coderControlPlanes.Items)) + for _, coderControlPlane := range coderControlPlanes.Items { + if strings.TrimSpace(coderControlPlane.Name) == "" || strings.TrimSpace(coderControlPlane.Namespace) == "" { + continue + } + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: coderControlPlane.Name, + Namespace: coderControlPlane.Namespace, + }}) + } + + return requests +} + +func isDuplicateLicenseUploadError(err error) bool { + var sdkErr *codersdk.Error + if !errors.As(err, &sdkErr) { + return false + } + + message := strings.ToLower(strings.TrimSpace(sdkErr.Message + " " + sdkErr.Detail)) + if message == "" { + return false + } + + if strings.Contains(message, "licenses_jwt_key") { + return true + } + if strings.Contains(message, "duplicate key") { + return true + } + if strings.Contains(message, "already exists") { + return true + } + + return false +} + +func mergeControlPlaneStatusDelta( + baseStatus coderv1alpha1.CoderControlPlaneStatus, + nextStatus coderv1alpha1.CoderControlPlaneStatus, + latestStatus coderv1alpha1.CoderControlPlaneStatus, +) coderv1alpha1.CoderControlPlaneStatus { + mergedStatus := latestStatus + + if baseStatus.ObservedGeneration != nextStatus.ObservedGeneration { + mergedStatus.ObservedGeneration = nextStatus.ObservedGeneration + } + if baseStatus.ReadyReplicas != nextStatus.ReadyReplicas { + mergedStatus.ReadyReplicas = nextStatus.ReadyReplicas + } + if baseStatus.URL != nextStatus.URL { + mergedStatus.URL = nextStatus.URL + } + if !equality.Semantic.DeepEqual(baseStatus.OperatorTokenSecretRef, nextStatus.OperatorTokenSecretRef) { + mergedStatus.OperatorTokenSecretRef = cloneSecretKeySelector(nextStatus.OperatorTokenSecretRef) + } + if baseStatus.OperatorAccessReady != nextStatus.OperatorAccessReady { + mergedStatus.OperatorAccessReady = nextStatus.OperatorAccessReady + } + if !equality.Semantic.DeepEqual(baseStatus.LicenseLastApplied, nextStatus.LicenseLastApplied) { + mergedStatus.LicenseLastApplied = cloneMetav1Time(nextStatus.LicenseLastApplied) + } + if baseStatus.LicenseLastAppliedHash != nextStatus.LicenseLastAppliedHash { + mergedStatus.LicenseLastAppliedHash = nextStatus.LicenseLastAppliedHash + } + if baseStatus.Phase != nextStatus.Phase { + mergedStatus.Phase = nextStatus.Phase + } + if !equality.Semantic.DeepEqual(baseStatus.Conditions, nextStatus.Conditions) { + mergedStatus.Conditions = append([]metav1.Condition(nil), nextStatus.Conditions...) + } + + return mergedStatus +} + +func cloneSecretKeySelector(selector *coderv1alpha1.SecretKeySelector) *coderv1alpha1.SecretKeySelector { + if selector == nil { + return nil + } + + copied := *selector + return &copied +} + +func cloneMetav1Time(timestamp *metav1.Time) *metav1.Time { + if timestamp == nil { + return nil + } + + return timestamp.DeepCopy() +} + func (r *CoderControlPlaneReconciler) reconcileStatus( ctx context.Context, coderControlPlane *coderv1alpha1.CoderControlPlane, + baseStatus coderv1alpha1.CoderControlPlaneStatus, nextStatus coderv1alpha1.CoderControlPlaneStatus, ) error { - if equality.Semantic.DeepEqual(coderControlPlane.Status, nextStatus) { + if coderControlPlane == nil { + return fmt.Errorf("assertion failed: coder control plane must not be nil") + } + + if equality.Semantic.DeepEqual(baseStatus, nextStatus) { return nil } - coderControlPlane.Status = nextStatus - if err := r.Status().Update(ctx, coderControlPlane); err != nil { + namespacedName := types.NamespacedName{Name: coderControlPlane.Name, Namespace: coderControlPlane.Namespace} + if strings.TrimSpace(namespacedName.Name) == "" || strings.TrimSpace(namespacedName.Namespace) == "" { + return fmt.Errorf("assertion failed: coder control plane namespaced name must not be empty") + } + + statusReader := r.APIReader + if statusReader == nil { + statusReader = r.Client + } + if statusReader == nil { + return fmt.Errorf("assertion failed: status reader must not be nil") + } + + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &coderv1alpha1.CoderControlPlane{} + if err := statusReader.Get(ctx, namespacedName, latest); err != nil { + return err + } + if latest.Name != namespacedName.Name || latest.Namespace != namespacedName.Namespace { + return fmt.Errorf("assertion failed: fetched object %s/%s does not match expected %s/%s", + latest.Namespace, latest.Name, namespacedName.Namespace, namespacedName.Name) + } + if nextStatus.ObservedGeneration > 0 && latest.Generation != nextStatus.ObservedGeneration { + // A newer reconcile has observed a newer generation. Avoid overwriting + // status with stale data from an older reconcile attempt. + coderControlPlane.Status = latest.Status + return nil + } + + mergedStatus := mergeControlPlaneStatusDelta(baseStatus, nextStatus, latest.Status) + if equality.Semantic.DeepEqual(latest.Status, mergedStatus) { + coderControlPlane.Status = latest.Status + return nil + } + + latest.Status = mergedStatus + if err := r.Status().Update(ctx, latest); err != nil { + return err + } + + coderControlPlane.Status = mergedStatus + return nil + }); err != nil { return fmt.Errorf("update control plane status: %w", err) } @@ -603,11 +1224,24 @@ func (r *CoderControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error { return fmt.Errorf("assertion failed: reconciler scheme must not be nil") } + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &coderv1alpha1.CoderControlPlane{}, + licenseSecretNameFieldIndex, + indexByLicenseSecretName, + ); err != nil { + return fmt.Errorf("index coder control planes by license secret name: %w", err) + } + return ctrl.NewControllerManagedBy(mgr). For(&coderv1alpha1.CoderControlPlane{}). Owns(&appsv1.Deployment{}). Owns(&corev1.Service{}). Owns(&corev1.Secret{}). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.reconcileRequestsForLicenseSecret), + ). Named("codercontrolplane"). Complete(r) } diff --git a/internal/controller/codercontrolplane_controller_test.go b/internal/controller/codercontrolplane_controller_test.go index 2fe41611..f6490b99 100644 --- a/internal/controller/codercontrolplane_controller_test.go +++ b/internal/controller/codercontrolplane_controller_test.go @@ -3,10 +3,12 @@ package controller_test import ( "context" "errors" + "net/http" "reflect" "strings" "testing" + "github.com/coder/coder/v2/codersdk" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -41,6 +43,47 @@ func (f *fakeOperatorAccessProvisioner) RevokeOperatorToken(_ context.Context, r return f.revokeErr } +type licenseUploadCall struct { + coderURL string + sessionToken string + licenseJWT string +} + +type fakeLicenseUploader struct { + err error + addLicenseErrs []error + hasAnyLicenseErr error + hasAnyLicense *bool + hasAnyLicenseCall int + calls []licenseUploadCall +} + +func (f *fakeLicenseUploader) AddLicense(_ context.Context, coderURL, sessionToken, licenseJWT string) error { + f.calls = append(f.calls, licenseUploadCall{ + coderURL: coderURL, + sessionToken: sessionToken, + licenseJWT: licenseJWT, + }) + if len(f.addLicenseErrs) > 0 { + err := f.addLicenseErrs[0] + f.addLicenseErrs = f.addLicenseErrs[1:] + return err + } + return f.err +} + +func (f *fakeLicenseUploader) HasAnyLicense(_ context.Context, _, _ string) (bool, error) { + f.hasAnyLicenseCall++ + if f.hasAnyLicenseErr != nil { + return false, f.hasAnyLicenseErr + } + if f.hasAnyLicense != nil { + return *f.hasAnyLicense, nil + } + + return len(f.calls) > 0, nil +} + func TestReconcile_NotFound(t *testing.T) { r := &controller.CoderControlPlaneReconciler{ Client: k8sClient, @@ -348,6 +391,611 @@ func TestReconcile_PhaseTransitionToReady(t *testing.T) { } } +func TestReconcile_LicenseSecretRefNil_DoesNotUpload(t *testing.T) { + ctx := context.Background() + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-license-no-ref", + Namespace: "default", + }, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + ExtraEnv: []corev1.EnvVar{{ + Name: "CODER_PG_CONNECTION_URL", + Value: "postgres://example/coder", + }}, + }, + } + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("failed to create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + provisioner := &fakeOperatorAccessProvisioner{token: "operator-token-no-license-ref"} + uploader := &fakeLicenseUploader{} + r := &controller.CoderControlPlaneReconciler{ + Client: k8sClient, + Scheme: scheme, + OperatorAccessProvisioner: provisioner, + LicenseUploader: uploader, + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("first reconcile control plane: %v", err) + } + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, deployment); err != nil { + t.Fatalf("get reconciled deployment: %v", err) + } + deployment.Status.ReadyReplicas = 1 + deployment.Status.Replicas = 1 + if err := k8sClient.Status().Update(ctx, deployment); err != nil { + t.Fatalf("update deployment status: %v", err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("second reconcile control plane: %v", err) + } + + if len(uploader.calls) != 0 { + t.Fatalf("expected no license upload calls when licenseSecretRef is not configured, got %d", len(uploader.calls)) + } + + reconciled := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciled); err != nil { + t.Fatalf("get reconciled control plane: %v", err) + } + licenseCondition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderControlPlaneConditionLicenseApplied) + if licenseCondition.Status != metav1.ConditionUnknown { + t.Fatalf("expected license condition status %q, got %q", metav1.ConditionUnknown, licenseCondition.Status) + } + if licenseCondition.Reason != "Pending" { + t.Fatalf("expected license condition reason %q, got %q", "Pending", licenseCondition.Reason) + } +} + +func TestReconcile_LicensePendingUntilControlPlaneReady(t *testing.T) { + ctx := context.Background() + + licenseSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-pending-secret", Namespace: "default"}, + Data: map[string][]byte{ + coderv1alpha1.DefaultLicenseSecretKey: []byte("license-pending"), + }, + } + if err := k8sClient.Create(ctx, licenseSecret); err != nil { + t.Fatalf("create license secret: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, licenseSecret) + }) + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-pending", Namespace: "default"}, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + ExtraEnv: []corev1.EnvVar{{ + Name: "CODER_PG_CONNECTION_URL", + Value: "postgres://example/pending", + }}, + LicenseSecretRef: &coderv1alpha1.SecretKeySelector{Name: licenseSecret.Name}, + }, + } + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + provisioner := &fakeOperatorAccessProvisioner{token: "operator-token-pending"} + uploader := &fakeLicenseUploader{} + r := &controller.CoderControlPlaneReconciler{ + Client: k8sClient, + Scheme: scheme, + OperatorAccessProvisioner: provisioner, + LicenseUploader: uploader, + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("reconcile control plane: %v", err) + } + if len(uploader.calls) != 0 { + t.Fatalf("expected no license upload calls before deployment readiness, got %d", len(uploader.calls)) + } + + reconciled := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciled); err != nil { + t.Fatalf("get reconciled control plane: %v", err) + } + licenseCondition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderControlPlaneConditionLicenseApplied) + if licenseCondition.Status != metav1.ConditionFalse { + t.Fatalf("expected license condition status %q, got %q", metav1.ConditionFalse, licenseCondition.Status) + } + if licenseCondition.Reason != "Pending" { + t.Fatalf("expected license condition reason %q, got %q", "Pending", licenseCondition.Reason) + } +} + +func TestReconcile_LicenseAppliesOnceAndTracksHash(t *testing.T) { + ctx := context.Background() + + licenseSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-apply-secret", Namespace: "default"}, + Data: map[string][]byte{ + coderv1alpha1.DefaultLicenseSecretKey: []byte(" license-jwt-initial \n"), + }, + } + if err := k8sClient.Create(ctx, licenseSecret); err != nil { + t.Fatalf("create license secret: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, licenseSecret) + }) + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-apply", Namespace: "default"}, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + ExtraEnv: []corev1.EnvVar{{ + Name: "CODER_PG_CONNECTION_URL", + Value: "postgres://example/license-apply", + }}, + LicenseSecretRef: &coderv1alpha1.SecretKeySelector{Name: licenseSecret.Name}, + }, + } + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + provisioner := &fakeOperatorAccessProvisioner{token: "operator-token-license-apply"} + uploader := &fakeLicenseUploader{} + r := &controller.CoderControlPlaneReconciler{ + Client: k8sClient, + Scheme: scheme, + OperatorAccessProvisioner: provisioner, + LicenseUploader: uploader, + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("first reconcile control plane: %v", err) + } + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, deployment); err != nil { + t.Fatalf("get reconciled deployment: %v", err) + } + deployment.Status.ReadyReplicas = 1 + deployment.Status.Replicas = 1 + if err := k8sClient.Status().Update(ctx, deployment); err != nil { + t.Fatalf("update deployment status: %v", err) + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("second reconcile control plane: %v", err) + } + if len(uploader.calls) != 1 { + t.Fatalf("expected one license upload call, got %d", len(uploader.calls)) + } + if uploader.calls[0].sessionToken != "operator-token-license-apply" { + t.Fatalf("expected license upload session token %q, got %q", "operator-token-license-apply", uploader.calls[0].sessionToken) + } + if uploader.calls[0].licenseJWT != "license-jwt-initial" { + t.Fatalf("expected trimmed license JWT %q, got %q", "license-jwt-initial", uploader.calls[0].licenseJWT) + } + + reconciled := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciled); err != nil { + t.Fatalf("get reconciled control plane: %v", err) + } + if reconciled.Status.LicenseLastApplied == nil { + t.Fatalf("expected licenseLastApplied to be set after successful upload") + } + if reconciled.Status.LicenseLastAppliedHash == "" { + t.Fatalf("expected licenseLastAppliedHash to be set after successful upload") + } + licenseCondition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderControlPlaneConditionLicenseApplied) + if licenseCondition.Status != metav1.ConditionTrue { + t.Fatalf("expected license condition status %q, got %q", metav1.ConditionTrue, licenseCondition.Status) + } + if licenseCondition.Reason != "Applied" { + t.Fatalf("expected license condition reason %q, got %q", "Applied", licenseCondition.Reason) + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("third reconcile control plane: %v", err) + } + if len(uploader.calls) != 1 { + t.Fatalf("expected license upload call count to remain 1 for idempotent reconcile, got %d", len(uploader.calls)) + } +} + +func TestReconcile_LicenseReuploadsWhenBackendHasNoLicenses(t *testing.T) { + ctx := context.Background() + + licenseSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-backend-reset-secret", Namespace: "default"}, + Data: map[string][]byte{ + coderv1alpha1.DefaultLicenseSecretKey: []byte("license-jwt-backend-reset"), + }, + } + if err := k8sClient.Create(ctx, licenseSecret); err != nil { + t.Fatalf("create license secret: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, licenseSecret) + }) + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-backend-reset", Namespace: "default"}, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + ExtraEnv: []corev1.EnvVar{{ + Name: "CODER_PG_CONNECTION_URL", + Value: "postgres://example/license-backend-reset", + }}, + LicenseSecretRef: &coderv1alpha1.SecretKeySelector{Name: licenseSecret.Name}, + }, + } + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + provisioner := &fakeOperatorAccessProvisioner{token: "operator-token-backend-reset"} + uploader := &fakeLicenseUploader{} + r := &controller.CoderControlPlaneReconciler{ + Client: k8sClient, + Scheme: scheme, + OperatorAccessProvisioner: provisioner, + LicenseUploader: uploader, + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("first reconcile control plane: %v", err) + } + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, deployment); err != nil { + t.Fatalf("get reconciled deployment: %v", err) + } + deployment.Status.ReadyReplicas = 1 + deployment.Status.Replicas = 1 + if err := k8sClient.Status().Update(ctx, deployment); err != nil { + t.Fatalf("update deployment status: %v", err) + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("second reconcile control plane: %v", err) + } + if len(uploader.calls) != 1 { + t.Fatalf("expected initial upload call count 1, got %d", len(uploader.calls)) + } + + backendHasNoLicenses := false + uploader.hasAnyLicense = &backendHasNoLicenses + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("third reconcile control plane: %v", err) + } + if uploader.hasAnyLicenseCall == 0 { + t.Fatalf("expected reconcile to query existing licenses when hash matches") + } + if len(uploader.calls) != 2 { + t.Fatalf("expected license to be re-uploaded when backend has no licenses, got %d upload calls", len(uploader.calls)) + } +} + +func TestReconcile_LicenseRotationUploadsNewSecretValue(t *testing.T) { + ctx := context.Background() + + licenseSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-rotation-secret", Namespace: "default"}, + Data: map[string][]byte{ + coderv1alpha1.DefaultLicenseSecretKey: []byte("license-jwt-v1"), + }, + } + if err := k8sClient.Create(ctx, licenseSecret); err != nil { + t.Fatalf("create license secret: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, licenseSecret) + }) + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-rotation", Namespace: "default"}, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + ExtraEnv: []corev1.EnvVar{{ + Name: "CODER_PG_CONNECTION_URL", + Value: "postgres://example/license-rotation", + }}, + LicenseSecretRef: &coderv1alpha1.SecretKeySelector{Name: licenseSecret.Name}, + }, + } + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + provisioner := &fakeOperatorAccessProvisioner{token: "operator-token-license-rotation"} + uploader := &fakeLicenseUploader{} + r := &controller.CoderControlPlaneReconciler{ + Client: k8sClient, + Scheme: scheme, + OperatorAccessProvisioner: provisioner, + LicenseUploader: uploader, + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("first reconcile control plane: %v", err) + } + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, deployment); err != nil { + t.Fatalf("get reconciled deployment: %v", err) + } + deployment.Status.ReadyReplicas = 1 + deployment.Status.Replicas = 1 + if err := k8sClient.Status().Update(ctx, deployment); err != nil { + t.Fatalf("update deployment status: %v", err) + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("second reconcile control plane: %v", err) + } + if len(uploader.calls) != 1 { + t.Fatalf("expected first license upload call, got %d", len(uploader.calls)) + } + + reconciled := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciled); err != nil { + t.Fatalf("get reconciled control plane: %v", err) + } + initialHash := reconciled.Status.LicenseLastAppliedHash + if initialHash == "" { + t.Fatalf("expected initial license hash to be set") + } + + secretToRotate := &corev1.Secret{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: licenseSecret.Name, Namespace: licenseSecret.Namespace}, secretToRotate); err != nil { + t.Fatalf("get license secret for update: %v", err) + } + secretToRotate.Data[coderv1alpha1.DefaultLicenseSecretKey] = []byte("license-jwt-v2") + if err := k8sClient.Update(ctx, secretToRotate); err != nil { + t.Fatalf("update license secret: %v", err) + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("third reconcile control plane: %v", err) + } + if len(uploader.calls) != 2 { + t.Fatalf("expected rotated license to trigger second upload call, got %d", len(uploader.calls)) + } + if uploader.calls[1].licenseJWT != "license-jwt-v2" { + t.Fatalf("expected rotated license JWT %q, got %q", "license-jwt-v2", uploader.calls[1].licenseJWT) + } + + reconciledAfterRotation := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciledAfterRotation); err != nil { + t.Fatalf("get reconciled control plane after rotation: %v", err) + } + if reconciledAfterRotation.Status.LicenseLastAppliedHash == initialHash { + t.Fatalf("expected license hash to change after rotation") + } +} + +func TestReconcile_LicenseRollbackDuplicateUploadConverges(t *testing.T) { + ctx := context.Background() + + licenseSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-rollback-secret", Namespace: "default"}, + Data: map[string][]byte{ + coderv1alpha1.DefaultLicenseSecretKey: []byte("license-jwt-a"), + }, + } + if err := k8sClient.Create(ctx, licenseSecret); err != nil { + t.Fatalf("create license secret: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, licenseSecret) + }) + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-rollback", Namespace: "default"}, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + ExtraEnv: []corev1.EnvVar{{ + Name: "CODER_PG_CONNECTION_URL", + Value: "postgres://example/license-rollback", + }}, + LicenseSecretRef: &coderv1alpha1.SecretKeySelector{Name: licenseSecret.Name}, + }, + } + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + duplicateErr := codersdk.NewTestError(http.StatusInternalServerError, http.MethodPost, "/api/v2/licenses") + duplicateErr.Message = "duplicate key value violates unique constraint \"licenses_jwt_key\"" + uploader := &fakeLicenseUploader{addLicenseErrs: []error{nil, nil, duplicateErr}} + provisioner := &fakeOperatorAccessProvisioner{token: "operator-token-license-rollback"} + r := &controller.CoderControlPlaneReconciler{ + Client: k8sClient, + Scheme: scheme, + OperatorAccessProvisioner: provisioner, + LicenseUploader: uploader, + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("first reconcile control plane: %v", err) + } + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, deployment); err != nil { + t.Fatalf("get reconciled deployment: %v", err) + } + deployment.Status.ReadyReplicas = 1 + deployment.Status.Replicas = 1 + if err := k8sClient.Status().Update(ctx, deployment); err != nil { + t.Fatalf("update deployment status: %v", err) + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("second reconcile control plane: %v", err) + } + if len(uploader.calls) != 1 { + t.Fatalf("expected initial upload call count 1, got %d", len(uploader.calls)) + } + + reconciledAfterInitial := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciledAfterInitial); err != nil { + t.Fatalf("get reconciled control plane after initial apply: %v", err) + } + hashA := reconciledAfterInitial.Status.LicenseLastAppliedHash + if hashA == "" { + t.Fatalf("expected hash after initial apply") + } + + secretToRotate := &corev1.Secret{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: licenseSecret.Name, Namespace: licenseSecret.Namespace}, secretToRotate); err != nil { + t.Fatalf("get license secret: %v", err) + } + secretToRotate.Data[coderv1alpha1.DefaultLicenseSecretKey] = []byte("license-jwt-b") + if err := k8sClient.Update(ctx, secretToRotate); err != nil { + t.Fatalf("rotate license to B: %v", err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("third reconcile control plane: %v", err) + } + + reconciledAfterB := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciledAfterB); err != nil { + t.Fatalf("get reconciled control plane after B apply: %v", err) + } + if reconciledAfterB.Status.LicenseLastAppliedHash == hashA { + t.Fatalf("expected hash to change after applying B") + } + + secretToRotateBack := &corev1.Secret{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: licenseSecret.Name, Namespace: licenseSecret.Namespace}, secretToRotateBack); err != nil { + t.Fatalf("get license secret for rollback: %v", err) + } + secretToRotateBack.Data[coderv1alpha1.DefaultLicenseSecretKey] = []byte("license-jwt-a") + if err := k8sClient.Update(ctx, secretToRotateBack); err != nil { + t.Fatalf("rollback license to A: %v", err) + } + + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}) + if err != nil { + t.Fatalf("fourth reconcile control plane: %v", err) + } + if result.RequeueAfter > 0 { + t.Fatalf("expected duplicate rollback upload handling to converge without requeue, got %+v", result) + } + if len(uploader.calls) != 3 { + t.Fatalf("expected three upload attempts across A->B->A rollback, got %d", len(uploader.calls)) + } + + reconciledAfterRollback := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciledAfterRollback); err != nil { + t.Fatalf("get reconciled control plane after rollback: %v", err) + } + if reconciledAfterRollback.Status.LicenseLastAppliedHash != hashA { + t.Fatalf("expected rollback to converge to original hash %q, got %q", hashA, reconciledAfterRollback.Status.LicenseLastAppliedHash) + } + licenseCondition := findCondition(t, reconciledAfterRollback.Status.Conditions, coderv1alpha1.CoderControlPlaneConditionLicenseApplied) + if licenseCondition.Status != metav1.ConditionTrue { + t.Fatalf("expected license condition true after duplicate rollback handling, got %q", licenseCondition.Status) + } +} + +func TestReconcile_LicenseNotSupportedSetsConditionWithoutRequeue(t *testing.T) { + ctx := context.Background() + + licenseSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-not-supported-secret", Namespace: "default"}, + Data: map[string][]byte{ + coderv1alpha1.DefaultLicenseSecretKey: []byte("license-oss"), + }, + } + if err := k8sClient.Create(ctx, licenseSecret); err != nil { + t.Fatalf("create license secret: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, licenseSecret) + }) + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{Name: "test-license-not-supported", Namespace: "default"}, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + ExtraEnv: []corev1.EnvVar{{ + Name: "CODER_PG_CONNECTION_URL", + Value: "postgres://example/license-not-supported", + }}, + LicenseSecretRef: &coderv1alpha1.SecretKeySelector{Name: licenseSecret.Name}, + }, + } + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + provisioner := &fakeOperatorAccessProvisioner{token: "operator-token-license-not-supported"} + uploader := &fakeLicenseUploader{err: codersdk.NewTestError(http.StatusNotFound, http.MethodPost, "/api/v2/licenses")} + r := &controller.CoderControlPlaneReconciler{ + Client: k8sClient, + Scheme: scheme, + OperatorAccessProvisioner: provisioner, + LicenseUploader: uploader, + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("first reconcile control plane: %v", err) + } + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, deployment); err != nil { + t.Fatalf("get reconciled deployment: %v", err) + } + deployment.Status.ReadyReplicas = 1 + deployment.Status.Replicas = 1 + if err := k8sClient.Status().Update(ctx, deployment); err != nil { + t.Fatalf("update deployment status: %v", err) + } + + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}) + if err != nil { + t.Fatalf("second reconcile control plane: %v", err) + } + if result.RequeueAfter > 0 { + t.Fatalf("expected no requeue for not-supported license API, got %+v", result) + } + if len(uploader.calls) != 1 { + t.Fatalf("expected one attempted license upload call, got %d", len(uploader.calls)) + } + + reconciled := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciled); err != nil { + t.Fatalf("get reconciled control plane: %v", err) + } + if reconciled.Status.LicenseLastApplied != nil { + t.Fatalf("expected licenseLastApplied to remain nil when API is not supported") + } + if reconciled.Status.LicenseLastAppliedHash != "" { + t.Fatalf("expected licenseLastAppliedHash to remain empty when API is not supported") + } + licenseCondition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderControlPlaneConditionLicenseApplied) + if licenseCondition.Status != metav1.ConditionFalse { + t.Fatalf("expected license condition status %q, got %q", metav1.ConditionFalse, licenseCondition.Status) + } + if licenseCondition.Reason != "NotSupported" { + t.Fatalf("expected license condition reason %q, got %q", "NotSupported", licenseCondition.Reason) + } +} + func TestReconcile_DefaultsApplied(t *testing.T) { ctx := context.Background()