diff --git a/api/v1alpha1/codercontrolplane_types.go b/api/v1alpha1/codercontrolplane_types.go index 7f4b1630..d1558622 100644 --- a/api/v1alpha1/codercontrolplane_types.go +++ b/api/v1alpha1/codercontrolplane_types.go @@ -12,6 +12,20 @@ const ( CoderControlPlanePhaseReady = "Ready" // CoderControlPlaneConditionLicenseApplied indicates whether the operator uploaded the configured license. CoderControlPlaneConditionLicenseApplied = "LicenseApplied" + + // CoderControlPlaneLicenseTierNone indicates no license is currently installed. + CoderControlPlaneLicenseTierNone = "none" + // CoderControlPlaneLicenseTierTrial indicates a trial license is currently installed. + CoderControlPlaneLicenseTierTrial = "trial" + // CoderControlPlaneLicenseTierEnterprise indicates an enterprise license is currently installed. + CoderControlPlaneLicenseTierEnterprise = "enterprise" + // CoderControlPlaneLicenseTierPremium indicates a premium license is currently installed. + CoderControlPlaneLicenseTierPremium = "premium" + // CoderControlPlaneLicenseTierUnknown indicates the controller could not determine the current license tier. + CoderControlPlaneLicenseTierUnknown = "unknown" + + // CoderControlPlaneEntitlementUnknown indicates the controller could not determine a feature entitlement. + CoderControlPlaneEntitlementUnknown = "unknown" ) // CoderControlPlaneSpec defines the desired state of a CoderControlPlane. @@ -71,6 +85,18 @@ type CoderControlPlaneStatus struct { // that LicenseLastApplied refers to. // +optional LicenseLastAppliedHash string `json:"licenseLastAppliedHash,omitempty"` + // LicenseTier is a best-effort classification of the currently applied license. + // Values: none, trial, enterprise, premium, unknown. + // +optional + LicenseTier string `json:"licenseTier,omitempty"` + // EntitlementsLastChecked is when the operator last queried coderd entitlements. + // +optional + EntitlementsLastChecked *metav1.Time `json:"entitlementsLastChecked,omitempty"` + // ExternalProvisionerDaemonsEntitlement is the entitlement value for feature + // "external_provisioner_daemons". + // Values: entitled, grace_period, not_entitled, unknown. + // +optional + ExternalProvisionerDaemonsEntitlement string `json:"externalProvisionerDaemonsEntitlement,omitempty"` // Phase is a high-level readiness indicator. Phase string `json:"phase,omitempty"` // Conditions are Kubernetes-standard conditions for this resource. diff --git a/api/v1alpha1/coderprovisioner_types.go b/api/v1alpha1/coderprovisioner_types.go index 339fc62f..3bb27001 100644 --- a/api/v1alpha1/coderprovisioner_types.go +++ b/api/v1alpha1/coderprovisioner_types.go @@ -19,6 +19,9 @@ const ( CoderProvisionerConditionProvisionerKeyReady = "ProvisionerKeyReady" // CoderProvisionerConditionProvisionerKeySecretReady indicates whether the provisioner key secret is populated. CoderProvisionerConditionProvisionerKeySecretReady = "ProvisionerKeySecretReady" + // CoderProvisionerConditionExternalProvisionersEntitled indicates whether the + // referenced Coder deployment is entitled to run external provisioner daemons. + CoderProvisionerConditionExternalProvisionersEntitled = "ExternalProvisionersEntitled" // CoderProvisionerConditionDeploymentReady indicates whether the provisioner deployment has ready replicas. CoderProvisionerConditionDeploymentReady = "DeploymentReady" diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 544cf382..51d201c3 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -129,6 +129,10 @@ func (in *CoderControlPlaneStatus) DeepCopyInto(out *CoderControlPlaneStatus) { in, out := &in.LicenseLastApplied, &out.LicenseLastApplied *out = (*in).DeepCopy() } + if in.EntitlementsLastChecked != nil { + in, out := &in.EntitlementsLastChecked, &out.EntitlementsLastChecked + *out = (*in).DeepCopy() + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) diff --git a/config/crd/bases/coder.com_codercontrolplanes.yaml b/config/crd/bases/coder.com_codercontrolplanes.yaml index 20bfcaca..10764eed 100644 --- a/config/crd/bases/coder.com_codercontrolplanes.yaml +++ b/config/crd/bases/coder.com_codercontrolplanes.yaml @@ -345,6 +345,17 @@ spec: - type type: object type: array + entitlementsLastChecked: + description: EntitlementsLastChecked is when the operator last queried + coderd entitlements. + format: date-time + type: string + externalProvisionerDaemonsEntitlement: + description: |- + ExternalProvisionerDaemonsEntitlement is the entitlement value for feature + "external_provisioner_daemons". + Values: entitled, grace_period, not_entitled, unknown. + type: string licenseLastApplied: description: |- LicenseLastApplied is the timestamp of the most recent successful @@ -356,6 +367,11 @@ spec: LicenseLastAppliedHash is the SHA-256 hex hash of the trimmed license JWT that LicenseLastApplied refers to. type: string + licenseTier: + description: |- + LicenseTier is a best-effort classification of the currently applied license. + Values: none, trial, enterprise, premium, unknown. + type: string observedGeneration: description: ObservedGeneration tracks the spec generation this status reflects. diff --git a/docs/reference/api/codercontrolplane.md b/docs/reference/api/codercontrolplane.md index 32591365..c373aeb8 100644 --- a/docs/reference/api/codercontrolplane.md +++ b/docs/reference/api/codercontrolplane.md @@ -33,6 +33,9 @@ | `operatorAccessReady` | boolean | OperatorAccessReady reports whether operator API access bootstrap succeeded. | | `licenseLastApplied` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | LicenseLastApplied is the timestamp of the most recent successful operator-managed license upload. | | `licenseLastAppliedHash` | string | LicenseLastAppliedHash is the SHA-256 hex hash of the trimmed license JWT that LicenseLastApplied refers to. | +| `licenseTier` | string | LicenseTier is a best-effort classification of the currently applied license. Values: none, trial, enterprise, premium, unknown. | +| `entitlementsLastChecked` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | EntitlementsLastChecked is when the operator last queried coderd entitlements. | +| `externalProvisionerDaemonsEntitlement` | string | ExternalProvisionerDaemonsEntitlement is the entitlement value for feature "external_provisioner_daemons". Values: entitled, grace_period, not_entitled, unknown. | | `phase` | string | Phase is a high-level readiness indicator. | | `conditions` | [Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#condition-v1-meta) array | Conditions are Kubernetes-standard conditions for this resource. | diff --git a/internal/app/controllerapp/controllerapp.go b/internal/app/controllerapp/controllerapp.go index 67fb2f1c..c7c34b3d 100644 --- a/internal/app/controllerapp/controllerapp.go +++ b/internal/app/controllerapp/controllerapp.go @@ -92,6 +92,7 @@ func SetupControllers(mgr manager.Manager) error { Scheme: managerScheme, OperatorAccessProvisioner: coderbootstrap.NewPostgresOperatorAccessProvisioner(), LicenseUploader: controller.NewSDKLicenseUploader(), + EntitlementsInspector: controller.NewSDKEntitlementsInspector(), } if err := reconciler.SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to create controller: %w", err) diff --git a/internal/coderbootstrap/client.go b/internal/coderbootstrap/client.go index cde090c8..495581c2 100644 --- a/internal/coderbootstrap/client.go +++ b/internal/coderbootstrap/client.go @@ -95,6 +95,7 @@ type Client interface { EnsureWorkspaceProxy(context.Context, RegisterWorkspaceProxyRequest) (RegisterWorkspaceProxyResponse, error) EnsureProvisionerKey(context.Context, EnsureProvisionerKeyRequest) (EnsureProvisionerKeyResponse, error) DeleteProvisionerKey(ctx context.Context, coderURL, sessionToken, orgName, keyName string) error + Entitlements(ctx context.Context, coderURL, sessionToken string) (codersdk.Entitlements, error) } // SDKClient uses codersdk to perform bootstrap operations. diff --git a/internal/coderbootstrap/provisionerkeys.go b/internal/coderbootstrap/provisionerkeys.go index fdc2f387..51f1964f 100644 --- a/internal/coderbootstrap/provisionerkeys.go +++ b/internal/coderbootstrap/provisionerkeys.go @@ -138,6 +138,33 @@ func (c *SDKClient) DeleteProvisionerKey(ctx context.Context, coderURL, sessionT return xerrors.Errorf("delete provisioner key %q: %w", keyName, err) } +// Entitlements returns deployment entitlements for the given coderd instance. +func (c *SDKClient) Entitlements(ctx context.Context, coderURL, sessionToken string) (codersdk.Entitlements, error) { + if coderURL == "" { + return codersdk.Entitlements{}, xerrors.New("coder URL is required") + } + if sessionToken == "" { + return codersdk.Entitlements{}, xerrors.New("session token is required") + } + + client, err := newAuthenticatedClient(coderURL, sessionToken) + if err != nil { + return codersdk.Entitlements{}, err + } + + entitlements, err := withOptionalRateLimitBypass(ctx, func(requestCtx context.Context) (codersdk.Entitlements, error) { + return client.Entitlements(requestCtx) + }) + if err != nil { + return codersdk.Entitlements{}, xerrors.Errorf("get entitlements: %w", err) + } + if entitlements.Features == nil { + return codersdk.Entitlements{}, xerrors.New("assertion failed: entitlements.features is nil") + } + + return entitlements, nil +} + func validateProvisionerKeyInputs(coderURL, sessionToken, keyName string) error { if coderURL == "" { return xerrors.New("coder URL is required") diff --git a/internal/coderbootstrap/provisionerkeys_test.go b/internal/coderbootstrap/provisionerkeys_test.go index 05aa6427..1fdc8985 100644 --- a/internal/coderbootstrap/provisionerkeys_test.go +++ b/internal/coderbootstrap/provisionerkeys_test.go @@ -315,6 +315,55 @@ func TestEnsureProvisionerKey_ValidationErrors(t *testing.T) { } } +func TestEntitlements_Success(t *testing.T) { + t.Parallel() + + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, http.MethodGet, r.Method) + require.Equal(t, "/api/v2/entitlements", r.URL.Path) + requestCount++ + writeJSONResponse(t, w, http.StatusOK, map[string]any{ + "features": map[string]any{ + string(codersdk.FeatureExternalProvisionerDaemons): map[string]any{ + "entitlement": string(codersdk.EntitlementEntitled), + "enabled": true, + }, + }, + "warnings": []string{}, + "errors": []string{}, + "has_license": true, + "trial": false, + "require_telemetry": false, + "refreshed_at": time.Now().UTC().Format(time.RFC3339), + }) + })) + defer server.Close() + + client := coderbootstrap.NewSDKClient() + entitlements, err := client.Entitlements(context.Background(), server.URL, "session-token") + require.NoError(t, err) + require.Equal(t, 1, requestCount) + + feature, ok := entitlements.Features[codersdk.FeatureExternalProvisionerDaemons] + require.True(t, ok) + require.Equal(t, codersdk.EntitlementEntitled, feature.Entitlement) +} + +func TestEntitlements_ValidationErrors(t *testing.T) { + t.Parallel() + + client := coderbootstrap.NewSDKClient() + + _, err := client.Entitlements(context.Background(), "", "session-token") + require.Error(t, err) + require.Contains(t, err.Error(), "coder URL is required") + + _, err = client.Entitlements(context.Background(), "https://coder.example.com", "") + require.Error(t, err) + require.Contains(t, err.Error(), "session token is required") +} + func TestDeleteProvisionerKey_Success(t *testing.T) { t.Parallel() diff --git a/internal/controller/codercontrolplane_controller.go b/internal/controller/codercontrolplane_controller.go index 34d8a8d1..faab84a2 100644 --- a/internal/controller/codercontrolplane_controller.go +++ b/internal/controller/codercontrolplane_controller.go @@ -61,7 +61,8 @@ const ( licenseConditionReasonNotSupported = "NotSupported" licenseConditionReasonError = "Error" - licenseUploadRequestTimeout = 30 * time.Second + licenseUploadRequestTimeout = 30 * time.Second + entitlementsStatusRefreshInterval = 2 * time.Minute ) var ( @@ -75,6 +76,35 @@ type LicenseUploader interface { HasAnyLicense(ctx context.Context, coderURL, sessionToken string) (bool, error) } +// EntitlementsInspector inspects coderd entitlements. +type EntitlementsInspector interface { + Entitlements(ctx context.Context, coderURL, sessionToken string) (codersdk.Entitlements, error) +} + +// NewSDKEntitlementsInspector returns an EntitlementsInspector backed by codersdk. +func NewSDKEntitlementsInspector() EntitlementsInspector { + return &sdkEntitlementsInspector{} +} + +type sdkEntitlementsInspector struct{} + +func (i *sdkEntitlementsInspector) Entitlements(ctx context.Context, coderURL, sessionToken string) (codersdk.Entitlements, error) { + sdkClient, err := newSDKLicenseClient(coderURL, sessionToken) + if err != nil { + return codersdk.Entitlements{}, err + } + + entitlements, err := sdkClient.Entitlements(ctx) + if err != nil { + return codersdk.Entitlements{}, fmt.Errorf("query coder entitlements: %w", err) + } + if entitlements.Features == nil { + return codersdk.Entitlements{}, fmt.Errorf("assertion failed: entitlements features must not be nil") + } + + return entitlements, nil +} + // NewSDKLicenseUploader returns a LicenseUploader backed by codersdk. func NewSDKLicenseUploader() LicenseUploader { return &sdkLicenseUploader{} @@ -151,6 +181,7 @@ type CoderControlPlaneReconciler struct { OperatorAccessProvisioner coderbootstrap.OperatorAccessProvisioner LicenseUploader LicenseUploader + EntitlementsInspector EntitlementsInspector } // +kubebuilder:rbac:groups=coder.com,resources=codercontrolplanes,verbs=get;list;watch;create;update;patch;delete @@ -204,11 +235,16 @@ func (r *CoderControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Re return ctrl.Result{}, err } + entitlementsResult, err := r.reconcileEntitlements(ctx, coderControlPlane, &nextStatus) + if err != nil { + return ctrl.Result{}, err + } + if err := r.reconcileStatus(ctx, coderControlPlane, originalStatus, nextStatus); err != nil { return ctrl.Result{}, err } - return mergeResults(operatorResult, licenseResult), nil + return mergeResults(operatorResult, licenseResult, entitlementsResult), nil } func (r *CoderControlPlaneReconciler) reconcileDeployment(ctx context.Context, coderControlPlane *coderv1alpha1.CoderControlPlane) (*appsv1.Deployment, error) { @@ -727,6 +763,143 @@ func (r *CoderControlPlaneReconciler) reconcileLicense( return ctrl.Result{}, nil } +func (r *CoderControlPlaneReconciler) reconcileEntitlements( + ctx context.Context, + coderControlPlane *coderv1alpha1.CoderControlPlane, + nextStatus *coderv1alpha1.CoderControlPlaneStatus, +) (ctrl.Result, error) { + if coderControlPlane == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: coder control plane must not be nil") + } + if nextStatus == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: next status must not be nil") + } + + if strings.TrimSpace(nextStatus.LicenseTier) == "" { + nextStatus.LicenseTier = coderv1alpha1.CoderControlPlaneLicenseTierUnknown + } + if strings.TrimSpace(nextStatus.ExternalProvisionerDaemonsEntitlement) == "" { + nextStatus.ExternalProvisionerDaemonsEntitlement = coderv1alpha1.CoderControlPlaneEntitlementUnknown + } + + if nextStatus.Phase != coderv1alpha1.CoderControlPlanePhaseReady || + !nextStatus.OperatorAccessReady || + nextStatus.OperatorTokenSecretRef == nil { + return ctrl.Result{}, nil + } + if strings.TrimSpace(nextStatus.URL) == "" { + return ctrl.Result{}, fmt.Errorf("assertion failed: control plane URL must not be empty when querying entitlements") + } + if r.EntitlementsInspector == nil { + return ctrl.Result{}, nil + } + + operatorTokenSecretName := strings.TrimSpace(nextStatus.OperatorTokenSecretRef.Name) + if operatorTokenSecretName == "" { + return ctrl.Result{}, fmt.Errorf("assertion failed: operator token secret name must not be empty when querying entitlements") + } + operatorTokenSecretKey := strings.TrimSpace(nextStatus.OperatorTokenSecretRef.Key) + if operatorTokenSecretKey == "" { + operatorTokenSecretKey = coderv1alpha1.DefaultTokenSecretKey + } + + operatorToken, err := r.readSecretValue(ctx, coderControlPlane.Namespace, operatorTokenSecretName, operatorTokenSecretKey) + switch { + case err == nil: + case apierrors.IsNotFound(err), errors.Is(err, errSecretValueMissing), errors.Is(err, errSecretValueEmpty): + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + default: + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + } + + entitlements, err := r.EntitlementsInspector.Entitlements(ctx, nextStatus.URL, operatorToken) + if err != nil { + var sdkErr *codersdk.Error + if errors.As(err, &sdkErr) { + switch sdkErr.StatusCode() { + case http.StatusNotFound, http.StatusUnauthorized, http.StatusForbidden: + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + } + } + return ctrl.Result{RequeueAfter: operatorAccessRetryInterval}, nil + } + if entitlements.Features == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: entitlements features must not be nil") + } + + previousTier := nextStatus.LicenseTier + previousExternalProvisionerEntitlement := nextStatus.ExternalProvisionerDaemonsEntitlement + + nextStatus.LicenseTier = licenseTierFromEntitlements(entitlements) + nextStatus.ExternalProvisionerDaemonsEntitlement = externalProvisionerDaemonsEntitlement(entitlements) + + shouldRefreshEntitlementsTimestamp := nextStatus.EntitlementsLastChecked == nil + if !shouldRefreshEntitlementsTimestamp { + elapsedSinceLastCheck := time.Since(nextStatus.EntitlementsLastChecked.Time) + shouldRefreshEntitlementsTimestamp = elapsedSinceLastCheck < 0 || elapsedSinceLastCheck >= entitlementsStatusRefreshInterval + } + if previousTier != nextStatus.LicenseTier || + previousExternalProvisionerEntitlement != nextStatus.ExternalProvisionerDaemonsEntitlement { + shouldRefreshEntitlementsTimestamp = true + } + if shouldRefreshEntitlementsTimestamp { + now := metav1.Now() + nextStatus.EntitlementsLastChecked = &now + } + + requeueAfter := entitlementsStatusRefreshInterval + if nextStatus.EntitlementsLastChecked != nil { + elapsedSinceLastCheck := time.Since(nextStatus.EntitlementsLastChecked.Time) + if elapsedSinceLastCheck >= 0 && elapsedSinceLastCheck < entitlementsStatusRefreshInterval { + requeueAfter = entitlementsStatusRefreshInterval - elapsedSinceLastCheck + } + } + + return ctrl.Result{RequeueAfter: requeueAfter}, nil +} + +func externalProvisionerDaemonsEntitlement(entitlements codersdk.Entitlements) string { + feature, ok := entitlements.Features[codersdk.FeatureExternalProvisionerDaemons] + if !ok { + return coderv1alpha1.CoderControlPlaneEntitlementUnknown + } + + return normalizedEntitlementValue(feature.Entitlement) +} + +func normalizedEntitlementValue(entitlement codersdk.Entitlement) string { + switch entitlement { + case codersdk.EntitlementEntitled, codersdk.EntitlementGracePeriod, codersdk.EntitlementNotEntitled: + return string(entitlement) + default: + return coderv1alpha1.CoderControlPlaneEntitlementUnknown + } +} + +func licenseTierFromEntitlements(entitlements codersdk.Entitlements) string { + if !entitlements.HasLicense { + return coderv1alpha1.CoderControlPlaneLicenseTierNone + } + if entitlements.Trial { + return coderv1alpha1.CoderControlPlaneLicenseTierTrial + } + + for _, featureName := range []codersdk.FeatureName{ + codersdk.FeatureCustomRoles, + codersdk.FeatureMultipleOrganizations, + } { + feature, ok := entitlements.Features[featureName] + if !ok { + continue + } + if feature.Entitlement.Entitled() { + return coderv1alpha1.CoderControlPlaneLicenseTierPremium + } + } + + return coderv1alpha1.CoderControlPlaneLicenseTierEnterprise +} + func (r *CoderControlPlaneReconciler) cleanupDisabledOperatorAccess( ctx context.Context, coderControlPlane *coderv1alpha1.CoderControlPlane, @@ -1122,6 +1295,15 @@ func mergeControlPlaneStatusDelta( if baseStatus.LicenseLastAppliedHash != nextStatus.LicenseLastAppliedHash { mergedStatus.LicenseLastAppliedHash = nextStatus.LicenseLastAppliedHash } + if baseStatus.LicenseTier != nextStatus.LicenseTier { + mergedStatus.LicenseTier = nextStatus.LicenseTier + } + if !equality.Semantic.DeepEqual(baseStatus.EntitlementsLastChecked, nextStatus.EntitlementsLastChecked) { + mergedStatus.EntitlementsLastChecked = cloneMetav1Time(nextStatus.EntitlementsLastChecked) + } + if baseStatus.ExternalProvisionerDaemonsEntitlement != nextStatus.ExternalProvisionerDaemonsEntitlement { + mergedStatus.ExternalProvisionerDaemonsEntitlement = nextStatus.ExternalProvisionerDaemonsEntitlement + } if baseStatus.Phase != nextStatus.Phase { mergedStatus.Phase = nextStatus.Phase } diff --git a/internal/controller/codercontrolplane_controller_test.go b/internal/controller/codercontrolplane_controller_test.go index f6490b99..69d67986 100644 --- a/internal/controller/codercontrolplane_controller_test.go +++ b/internal/controller/codercontrolplane_controller_test.go @@ -7,6 +7,7 @@ import ( "reflect" "strings" "testing" + "time" "github.com/coder/coder/v2/codersdk" appsv1 "k8s.io/api/apps/v1" @@ -84,6 +85,30 @@ func (f *fakeLicenseUploader) HasAnyLicense(_ context.Context, _, _ string) (boo return len(f.calls) > 0, nil } +type fakeEntitlementsInspector struct { + response codersdk.Entitlements + err error + calls int + requests []entitlementsInspectCall +} + +type entitlementsInspectCall struct { + coderURL string + sessionToken string +} + +func (f *fakeEntitlementsInspector) Entitlements(_ context.Context, coderURL, sessionToken string) (codersdk.Entitlements, error) { + f.calls++ + f.requests = append(f.requests, entitlementsInspectCall{coderURL: coderURL, sessionToken: sessionToken}) + if f.err != nil { + return codersdk.Entitlements{}, f.err + } + if f.response.Features == nil { + f.response.Features = map[codersdk.FeatureName]codersdk.Feature{} + } + return f.response, nil +} + func TestReconcile_NotFound(t *testing.T) { r := &controller.CoderControlPlaneReconciler{ Client: k8sClient, @@ -1773,6 +1798,154 @@ func TestReconcile_OperatorAccess_ResolvesPostgresURLFromSecretRef(t *testing.T) } } +func TestReconcile_EntitlementsStatusFields(t *testing.T) { + ctx := context.Background() + + testCases := []struct { + name string + entitlements codersdk.Entitlements + expectedTier string + expectedProvisionerFeature string + }{ + { + name: "none", + entitlements: codersdk.Entitlements{ + Features: map[codersdk.FeatureName]codersdk.Feature{ + codersdk.FeatureExternalProvisionerDaemons: {Entitlement: codersdk.EntitlementNotEntitled}, + }, + HasLicense: false, + }, + expectedTier: coderv1alpha1.CoderControlPlaneLicenseTierNone, + expectedProvisionerFeature: string(codersdk.EntitlementNotEntitled), + }, + { + name: "trial", + entitlements: codersdk.Entitlements{ + Features: map[codersdk.FeatureName]codersdk.Feature{ + codersdk.FeatureExternalProvisionerDaemons: {Entitlement: codersdk.EntitlementGracePeriod}, + }, + HasLicense: true, + Trial: true, + }, + expectedTier: coderv1alpha1.CoderControlPlaneLicenseTierTrial, + expectedProvisionerFeature: string(codersdk.EntitlementGracePeriod), + }, + { + name: "premium", + entitlements: codersdk.Entitlements{ + Features: map[codersdk.FeatureName]codersdk.Feature{ + codersdk.FeatureExternalProvisionerDaemons: {Entitlement: codersdk.EntitlementEntitled}, + codersdk.FeatureCustomRoles: {Entitlement: codersdk.EntitlementEntitled}, + }, + HasLicense: true, + }, + expectedTier: coderv1alpha1.CoderControlPlaneLicenseTierPremium, + expectedProvisionerFeature: string(codersdk.EntitlementEntitled), + }, + { + name: "enterprise", + entitlements: codersdk.Entitlements{ + Features: map[codersdk.FeatureName]codersdk.Feature{ + codersdk.FeatureExternalProvisionerDaemons: {Entitlement: codersdk.EntitlementEntitled}, + codersdk.FeatureCustomRoles: {Entitlement: codersdk.EntitlementNotEntitled}, + codersdk.FeatureMultipleOrganizations: {Entitlement: codersdk.EntitlementNotEntitled}, + }, + HasLicense: true, + }, + expectedTier: coderv1alpha1.CoderControlPlaneLicenseTierEnterprise, + expectedProvisionerFeature: string(codersdk.EntitlementEntitled), + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + controlPlaneName := "test-entitlements-" + strings.ToLower(testCase.name) + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: controlPlaneName, + Namespace: "default", + }, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + Image: "test-entitlements:latest", + ExtraEnv: []corev1.EnvVar{{ + Name: "CODER_PG_CONNECTION_URL", + Value: "postgres://example.test/coder", + }}, + }, + } + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("failed to create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + provisioner := &fakeOperatorAccessProvisioner{token: "operator-token-entitlements"} + inspector := &fakeEntitlementsInspector{response: testCase.entitlements} + r := &controller.CoderControlPlaneReconciler{ + Client: k8sClient, + Scheme: scheme, + OperatorAccessProvisioner: provisioner, + EntitlementsInspector: inspector, + } + + namespacedName := types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}); err != nil { + t.Fatalf("reconcile control plane: %v", err) + } + + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, namespacedName, deployment); err != nil { + t.Fatalf("get deployment: %v", err) + } + deployment.Status.Replicas = 1 + deployment.Status.ReadyReplicas = 1 + if err := k8sClient.Status().Update(ctx, deployment); err != nil { + t.Fatalf("update deployment status: %v", err) + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}); err != nil { + t.Fatalf("reconcile control plane after deployment ready: %v", err) + } + + reconciled := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, namespacedName, reconciled); err != nil { + t.Fatalf("get reconciled control plane: %v", err) + } + if reconciled.Status.LicenseTier != testCase.expectedTier { + t.Fatalf("expected license tier %q, got %q", testCase.expectedTier, reconciled.Status.LicenseTier) + } + if reconciled.Status.ExternalProvisionerDaemonsEntitlement != testCase.expectedProvisionerFeature { + t.Fatalf("expected external provisioner entitlement %q, got %q", testCase.expectedProvisionerFeature, reconciled.Status.ExternalProvisionerDaemonsEntitlement) + } + if reconciled.Status.EntitlementsLastChecked == nil { + t.Fatal("expected entitlementsLastChecked to be set") + } + firstCheckedAt := reconciled.Status.EntitlementsLastChecked.DeepCopy() + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}); err != nil { + t.Fatalf("reconcile control plane with unchanged entitlements: %v", err) + } + + reconciledAgain := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, namespacedName, reconciledAgain); err != nil { + t.Fatalf("get reconciled control plane after stable reconcile: %v", err) + } + if reconciledAgain.Status.EntitlementsLastChecked == nil { + t.Fatal("expected entitlementsLastChecked to remain set") + } + if !reconciledAgain.Status.EntitlementsLastChecked.Equal(firstCheckedAt) { + t.Fatalf("expected entitlementsLastChecked to remain %s, got %s", firstCheckedAt.UTC().Format(time.RFC3339Nano), reconciledAgain.Status.EntitlementsLastChecked.UTC().Format(time.RFC3339Nano)) + } + if inspector.calls == 0 { + t.Fatal("expected entitlements inspector to be called") + } + }) + } +} + func ptrTo[T any](value T) *T { return &value } diff --git a/internal/controller/coderprovisioner_controller.go b/internal/controller/coderprovisioner_controller.go index 65bcc8cd..96bbe8e8 100644 --- a/internal/controller/coderprovisioner_controller.go +++ b/internal/controller/coderprovisioner_controller.go @@ -5,13 +5,17 @@ import ( "context" "crypto/rand" "encoding/binary" + "errors" "fmt" "hash/fnv" "maps" + "net/http" "slices" + "strings" "sync" "time" + "github.com/coder/coder/v2/codersdk" "github.com/google/uuid" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -25,6 +29,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" "github.com/coder/coder-k8s/internal/coderbootstrap" @@ -42,6 +48,11 @@ const ( provisionerRateLimitBackoffCap = 2 * time.Minute provisionerRateLimitBackoffFloor = 1 * time.Second provisionerRateLimitJitterRatio = 0.2 + + externalProvisionerEntitlementRetryInterval = 2 * time.Minute + + // #nosec G101 -- this is a field index key, not a credential. + provisionerControlPlaneRefNameFieldIndex = ".spec.controlPlaneRef.name" ) var provisionerRateLimitBackoffAttempts sync.Map // map[string]int @@ -179,6 +190,26 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req "Bootstrap credentials secret is available", ) + entitlementResult, entitlementErr := r.reconcileExternalProvisionerEntitlement(ctx, provisioner, controlPlane, sessionToken) + if entitlementErr != nil { + if statusSnapshot == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: status snapshot must not be nil") + } + if !equality.Semantic.DeepEqual(*statusSnapshot, provisioner.Status) { + _ = r.Status().Update(ctx, provisioner) + } + return ctrl.Result{}, entitlementErr + } + if entitlementResult.RequeueAfter > 0 { + if statusSnapshot == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: status snapshot must not be nil") + } + if !equality.Semantic.DeepEqual(*statusSnapshot, provisioner.Status) { + _ = r.Status().Update(ctx, provisioner) + } + return entitlementResult, nil + } + desiredTagsHash := hashProvisionerTags(provisioner.Spec.Tags) desiredControlPlaneRefName := provisioner.Spec.ControlPlaneRef.Name status := provisioner.Status @@ -689,6 +720,106 @@ func (r *CoderProvisionerReconciler) readBootstrapSessionToken(ctx context.Conte return token, nil } +func (r *CoderProvisionerReconciler) reconcileExternalProvisionerEntitlement( + ctx context.Context, + provisioner *coderv1alpha1.CoderProvisioner, + controlPlane *coderv1alpha1.CoderControlPlane, + sessionToken string, +) (ctrl.Result, error) { + if provisioner == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: coder provisioner must not be nil") + } + if controlPlane == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: coder control plane must not be nil") + } + if strings.TrimSpace(controlPlane.Status.URL) == "" { + return ctrl.Result{}, fmt.Errorf("assertion failed: coder control plane URL must not be empty") + } + if sessionToken == "" { + return ctrl.Result{}, fmt.Errorf("assertion failed: bootstrap session token must not be empty") + } + + if controlPlane.Status.EntitlementsLastChecked != nil { + recheckAfter := durationUntilNextEntitlementsCheck(controlPlane.Status.EntitlementsLastChecked, externalProvisionerEntitlementRetryInterval) + + switch strings.TrimSpace(controlPlane.Status.ExternalProvisionerDaemonsEntitlement) { + case string(codersdk.EntitlementEntitled), string(codersdk.EntitlementGracePeriod): + if recheckAfter > 0 { + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionExternalProvisionersEntitled, + metav1.ConditionTrue, + "Entitled", + "Coder deployment is entitled to external provisioner daemons", + ) + return ctrl.Result{}, nil + } + case string(codersdk.EntitlementNotEntitled): + if recheckAfter > 0 { + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionExternalProvisionersEntitled, + metav1.ConditionFalse, + "NotEntitled", + "Coder deployment is not entitled to external provisioner daemons; install a Premium/Enterprise license to enable external provisioners.", + ) + return ctrl.Result{RequeueAfter: recheckAfter}, nil + } + } + } + + entitlements, err := r.BootstrapClient.Entitlements(ctx, controlPlane.Status.URL, sessionToken) + if err != nil { + reason := "EntitlementsQueryFailed" + message := "Failed to query Coder entitlements; retrying." + + var apiErr *codersdk.Error + if errors.As(err, &apiErr) { + switch apiErr.StatusCode() { + case http.StatusNotFound: + reason = "NotSupported" + message = "Coder deployment does not expose /api/v2/entitlements; cannot verify license." + case http.StatusUnauthorized, http.StatusForbidden: + reason = "Forbidden" + message = "Bootstrap token is not authorized to read entitlements; retrying." + } + } + + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionExternalProvisionersEntitled, + metav1.ConditionFalse, + reason, + message, + ) + return ctrl.Result{RequeueAfter: externalProvisionerEntitlementRetryInterval}, nil + } + if entitlements.Features == nil { + return ctrl.Result{}, fmt.Errorf("assertion failed: entitlements features must not be nil") + } + + feature, ok := entitlements.Features[codersdk.FeatureExternalProvisionerDaemons] + if !ok || !feature.Entitlement.Entitled() { + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionExternalProvisionersEntitled, + metav1.ConditionFalse, + "NotEntitled", + "Coder deployment is not entitled to external provisioner daemons; install a Premium/Enterprise license to enable external provisioners.", + ) + return ctrl.Result{RequeueAfter: externalProvisionerEntitlementRetryInterval}, nil + } + + setCondition( + provisioner, + coderv1alpha1.CoderProvisionerConditionExternalProvisionersEntitled, + metav1.ConditionTrue, + "Entitled", + "Coder deployment is entitled to external provisioner daemons", + ) + return ctrl.Result{}, nil +} + func (r *CoderProvisionerReconciler) ensureProvisionerKeySecret( ctx context.Context, provisioner *coderv1alpha1.CoderProvisioner, @@ -975,6 +1106,56 @@ func (r *CoderProvisionerReconciler) readSecretValue(ctx context.Context, namesp return string(value), nil } +func indexProvisionerByControlPlaneRefName(obj client.Object) []string { + provisioner, ok := obj.(*coderv1alpha1.CoderProvisioner) + if !ok { + return nil + } + + controlPlaneName := strings.TrimSpace(provisioner.Spec.ControlPlaneRef.Name) + if controlPlaneName == "" { + return nil + } + + return []string{controlPlaneName} +} + +func (r *CoderProvisionerReconciler) reconcileRequestsForControlPlane( + ctx context.Context, + obj client.Object, +) []reconcile.Request { + controlPlane, ok := obj.(*coderv1alpha1.CoderControlPlane) + if !ok { + return nil + } + if strings.TrimSpace(controlPlane.Name) == "" || strings.TrimSpace(controlPlane.Namespace) == "" { + return nil + } + + var provisioners coderv1alpha1.CoderProvisionerList + if err := r.List( + ctx, + &provisioners, + client.InNamespace(controlPlane.Namespace), + client.MatchingFields{provisionerControlPlaneRefNameFieldIndex: controlPlane.Name}, + ); err != nil { + return nil + } + + requests := make([]reconcile.Request, 0, len(provisioners.Items)) + for _, provisioner := range provisioners.Items { + if strings.TrimSpace(provisioner.Name) == "" || strings.TrimSpace(provisioner.Namespace) == "" { + continue + } + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: provisioner.Name, + Namespace: provisioner.Namespace, + }}) + } + + return requests +} + // SetupWithManager wires the reconciler into controller-runtime. func (r *CoderProvisionerReconciler) SetupWithManager(mgr ctrl.Manager) error { if mgr == nil { @@ -990,6 +1171,15 @@ func (r *CoderProvisionerReconciler) SetupWithManager(mgr ctrl.Manager) error { return fmt.Errorf("assertion failed: reconciler bootstrap client must not be nil") } + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &coderv1alpha1.CoderProvisioner{}, + provisionerControlPlaneRefNameFieldIndex, + indexProvisionerByControlPlaneRefName, + ); err != nil { + return fmt.Errorf("index coder provisioners by control plane ref name: %w", err) + } + return ctrl.NewControllerManagedBy(mgr). For(&coderv1alpha1.CoderProvisioner{}). Owns(&appsv1.Deployment{}). @@ -997,6 +1187,10 @@ func (r *CoderProvisionerReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Watches( + &coderv1alpha1.CoderControlPlane{}, + handler.EnqueueRequestsFromMapFunc(r.reconcileRequestsForControlPlane), + ). Named("coderprovisioner"). Complete(r) } @@ -1192,6 +1386,25 @@ func provisionerRateLimitJitterMultiplier() float64 { return 1 + jitter } +func durationUntilNextEntitlementsCheck(lastChecked *metav1.Time, interval time.Duration) time.Duration { + if lastChecked == nil { + return 0 + } + if interval <= 0 { + return 0 + } + + elapsed := time.Since(lastChecked.Time) + if elapsed < 0 { + return interval + } + if elapsed >= interval { + return 0 + } + + return interval - elapsed +} + func setCondition( provisioner *coderv1alpha1.CoderProvisioner, conditionType string, diff --git a/internal/controller/coderprovisioner_controller_test.go b/internal/controller/coderprovisioner_controller_test.go index bdbb5782..a271d3ed 100644 --- a/internal/controller/coderprovisioner_controller_test.go +++ b/internal/controller/coderprovisioner_controller_test.go @@ -177,6 +177,229 @@ func findCondition(t *testing.T, conditions []metav1.Condition, condType string) return metav1.Condition{} } +func createTestProvisioner( + ctx context.Context, + t *testing.T, + namespace string, + name string, + controlPlaneName string, + bootstrapSecretName string, +) *coderv1alpha1.CoderProvisioner { + t.Helper() + + provisioner := &coderv1alpha1.CoderProvisioner{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: coderv1alpha1.CoderProvisionerSpec{ + ControlPlaneRef: corev1.LocalObjectReference{Name: controlPlaneName}, + Bootstrap: coderv1alpha1.CoderProvisionerBootstrapSpec{ + CredentialsSecretRef: coderv1alpha1.SecretKeySelector{ + Name: bootstrapSecretName, + Key: coderv1alpha1.DefaultTokenSecretKey, + }, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, provisioner)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), provisioner) + }) + + return provisioner +} + +func TestCoderProvisionerReconciler_EntitlementFastPathNotEntitled(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-ent-fast") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-ent-fast", "https://coder.example.com") + now := metav1.Now() + controlPlane.Status.EntitlementsLastChecked = &now + controlPlane.Status.ExternalProvisionerDaemonsEntitlement = string(codersdk.EntitlementNotEntitled) + require.NoError(t, k8sClient.Status().Update(ctx, controlPlane)) + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-ent-fast", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + bootstrapClient := &fakeBootstrapClient{} + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + provisioner := createTestProvisioner(ctx, t, namespace, "provisioner-ent-fast", controlPlane.Name, bootstrapSecret.Name) + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) + + result, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Greater(t, result.RequeueAfter, time.Duration(0)) + require.Equal(t, 0, bootstrapClient.entitlementsCalls) + require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, reconciled)) + condition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionExternalProvisionersEntitled) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, "NotEntitled", condition.Reason) + + deployment := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: expectedProvisionerResourceName(provisioner.Name), Namespace: provisioner.Namespace}, deployment) + require.Error(t, err) + require.True(t, apierrors.IsNotFound(err)) +} + +func TestCoderProvisionerReconciler_EntitlementFallbackNotEntitled(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-ent-fallback") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-ent-fallback", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-ent-fallback", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + bootstrapClient := &fakeBootstrapClient{ + entitlementsResponse: codersdk.Entitlements{ + Features: map[codersdk.FeatureName]codersdk.Feature{ + codersdk.FeatureExternalProvisionerDaemons: {Entitlement: codersdk.EntitlementNotEntitled}, + }, + }, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + provisioner := createTestProvisioner(ctx, t, namespace, "provisioner-ent-fallback", controlPlane.Name, bootstrapSecret.Name) + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) + + result, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Greater(t, result.RequeueAfter, time.Duration(0)) + require.Equal(t, 1, bootstrapClient.entitlementsCalls) + require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, reconciled)) + condition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionExternalProvisionersEntitled) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, "NotEntitled", condition.Reason) +} + +func TestCoderProvisionerReconciler_EntitlementFastPathNotEntitledStaleRechecks(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-ent-stale") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-ent-stale", "https://coder.example.com") + staleCheckedAt := metav1.NewTime(time.Now().Add(-5 * time.Minute)) + controlPlane.Status.EntitlementsLastChecked = &staleCheckedAt + controlPlane.Status.ExternalProvisionerDaemonsEntitlement = string(codersdk.EntitlementNotEntitled) + require.NoError(t, k8sClient.Status().Update(ctx, controlPlane)) + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-ent-stale", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + bootstrapClient := &fakeBootstrapClient{ + entitlementsResponse: codersdk.Entitlements{ + Features: map[codersdk.FeatureName]codersdk.Feature{ + codersdk.FeatureExternalProvisionerDaemons: {Entitlement: codersdk.EntitlementEntitled}, + }, + }, + provisionerKeyResponses: []coderbootstrap.EnsureProvisionerKeyResponse{{ + OrganizationID: uuid.New(), + KeyID: uuid.New(), + KeyName: "provisioner-ent-stale", + Key: "provisioner-key-material", + }}, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + provisioner := createTestProvisioner(ctx, t, namespace, "provisioner-ent-stale", controlPlane.Name, bootstrapSecret.Name) + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) + + result, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) + require.Equal(t, 1, bootstrapClient.entitlementsCalls) + require.Equal(t, 1, bootstrapClient.provisionerKeyCalls) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, reconciled)) + condition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionExternalProvisionersEntitled) + require.Equal(t, metav1.ConditionTrue, condition.Status) + require.Equal(t, "Entitled", condition.Reason) +} + +func TestCoderProvisionerReconciler_EntitlementFastPathEntitledStaleRechecks(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-entitled-stale") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-entitled-stale", "https://coder.example.com") + staleCheckedAt := metav1.NewTime(time.Now().Add(-5 * time.Minute)) + controlPlane.Status.EntitlementsLastChecked = &staleCheckedAt + controlPlane.Status.ExternalProvisionerDaemonsEntitlement = string(codersdk.EntitlementEntitled) + require.NoError(t, k8sClient.Status().Update(ctx, controlPlane)) + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-entitled-stale", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + bootstrapClient := &fakeBootstrapClient{ + entitlementsResponse: codersdk.Entitlements{ + Features: map[codersdk.FeatureName]codersdk.Feature{ + codersdk.FeatureExternalProvisionerDaemons: {Entitlement: codersdk.EntitlementNotEntitled}, + }, + }, + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + provisioner := createTestProvisioner(ctx, t, namespace, "provisioner-entitled-stale", controlPlane.Name, bootstrapSecret.Name) + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) + + result, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Greater(t, result.RequeueAfter, time.Duration(0)) + require.Equal(t, 1, bootstrapClient.entitlementsCalls) + require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, reconciled)) + condition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionExternalProvisionersEntitled) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, "NotEntitled", condition.Reason) +} + +func TestCoderProvisionerReconciler_EntitlementFallbackForbidden(t *testing.T) { + t.Parallel() + + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "coderprov-ent-forbidden") + controlPlane := createTestControlPlane(ctx, t, namespace, "controlplane-ent-forbidden", "https://coder.example.com") + bootstrapSecret := createBootstrapSecret(ctx, t, namespace, "bootstrap-ent-forbidden", coderv1alpha1.DefaultTokenSecretKey, "session-token") + + bootstrapClient := &fakeBootstrapClient{ + entitlementsErr: codersdk.NewTestError(http.StatusForbidden, http.MethodGet, "/api/v2/entitlements"), + } + reconciler := &controller.CoderProvisionerReconciler{Client: k8sClient, Scheme: scheme, BootstrapClient: bootstrapClient} + provisioner := createTestProvisioner(ctx, t, namespace, "provisioner-ent-forbidden", controlPlane.Name, bootstrapSecret.Name) + namespacedName := types.NamespacedName{Name: provisioner.Name, Namespace: provisioner.Namespace} + + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) + + result, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.Greater(t, result.RequeueAfter, time.Duration(0)) + require.Equal(t, 1, bootstrapClient.entitlementsCalls) + require.Equal(t, 0, bootstrapClient.provisionerKeyCalls) + + reconciled := &coderv1alpha1.CoderProvisioner{} + require.NoError(t, k8sClient.Get(ctx, namespacedName, reconciled)) + condition := findCondition(t, reconciled.Status.Conditions, coderv1alpha1.CoderProvisionerConditionExternalProvisionersEntitled) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, "Forbidden", condition.Reason) +} + func TestCoderProvisionerReconciler_BasicCreate(t *testing.T) { t.Parallel() @@ -320,6 +543,12 @@ func TestCoderProvisionerReconciler_BasicCreate(t *testing.T) { require.NotNil(t, reconciledProvisioner.Status.SecretRef) require.Equal(t, provisioner.Spec.Key.SecretName, reconciledProvisioner.Status.SecretRef.Name) require.Equal(t, provisioner.Spec.Key.SecretKey, reconciledProvisioner.Status.SecretRef.Key) + requireCondition( + t, + reconciledProvisioner.Status.Conditions, + coderv1alpha1.CoderProvisionerConditionExternalProvisionersEntitled, + metav1.ConditionTrue, + ) } func TestCoderProvisionerReconciler_ExistingSecret(t *testing.T) { diff --git a/internal/controller/coderworkspaceproxy_controller_test.go b/internal/controller/coderworkspaceproxy_controller_test.go index 861607b6..7dc3c56f 100644 --- a/internal/controller/coderworkspaceproxy_controller_test.go +++ b/internal/controller/coderworkspaceproxy_controller_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/coder/coder/v2/codersdk" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,6 +32,12 @@ type fakeBootstrapClient struct { deleteKeyErr error deleteKeyCalls int deleteKeyRequests []deleteKeyRequest + + // Entitlements support. + entitlementsResponse codersdk.Entitlements + entitlementsErr error + entitlementsCalls int + entitlementsRequests []entitlementsRequest } type deleteKeyRequest struct { @@ -40,6 +47,11 @@ type deleteKeyRequest struct { KeyName string } +type entitlementsRequest struct { + CoderURL string + SessionToken string +} + func (f *fakeBootstrapClient) EnsureWorkspaceProxy(_ context.Context, _ coderbootstrap.RegisterWorkspaceProxyRequest) (coderbootstrap.RegisterWorkspaceProxyResponse, error) { f.calls++ return f.response, f.err @@ -74,6 +86,23 @@ func (f *fakeBootstrapClient) DeleteProvisionerKey(_ context.Context, coderURL, return f.deleteKeyErr } +func (f *fakeBootstrapClient) Entitlements(_ context.Context, coderURL, sessionToken string) (codersdk.Entitlements, error) { + f.entitlementsCalls++ + f.entitlementsRequests = append(f.entitlementsRequests, entitlementsRequest{ + CoderURL: coderURL, + SessionToken: sessionToken, + }) + if f.entitlementsErr != nil { + return codersdk.Entitlements{}, f.entitlementsErr + } + if f.entitlementsResponse.Features == nil { + f.entitlementsResponse.Features = map[codersdk.FeatureName]codersdk.Feature{ + codersdk.FeatureExternalProvisionerDaemons: {Entitlement: codersdk.EntitlementEntitled}, + } + } + return f.entitlementsResponse, nil +} + func workspaceProxyResourceName(name string) string { const prefix = "wsproxy-" candidate := prefix + name