diff --git a/api/v1alpha1/codercontrolplane_types.go b/api/v1alpha1/codercontrolplane_types.go index cfeacad3..b767792e 100644 --- a/api/v1alpha1/codercontrolplane_types.go +++ b/api/v1alpha1/codercontrolplane_types.go @@ -15,10 +15,13 @@ const ( // CoderControlPlaneSpec defines the desired state of a CoderControlPlane. type CoderControlPlaneSpec struct { // Image is the container image used for the Coder control plane pod. + // +kubebuilder:default="ghcr.io/coder/coder:latest" Image string `json:"image,omitempty"` // Replicas is the desired number of control plane pods. + // +kubebuilder:default=1 Replicas *int32 `json:"replicas,omitempty"` // Service controls the service created in front of the control plane. + // +kubebuilder:default={} Service ServiceSpec `json:"service,omitempty"` // ExtraArgs are appended to the default Coder server arguments. ExtraArgs []string `json:"extraArgs,omitempty"` diff --git a/api/v1alpha1/types_shared.go b/api/v1alpha1/types_shared.go index 81d5d506..5d4295cd 100644 --- a/api/v1alpha1/types_shared.go +++ b/api/v1alpha1/types_shared.go @@ -10,8 +10,10 @@ const ( // ServiceSpec defines the Service configuration reconciled by the operator. type ServiceSpec struct { // Type controls the Kubernetes service type. + // +kubebuilder:default="ClusterIP" Type corev1.ServiceType `json:"type,omitempty"` // Port controls the exposed service port. + // +kubebuilder:default=80 Port int32 `json:"port,omitempty"` // Annotations are applied to the reconciled service object. Annotations map[string]string `json:"annotations,omitempty"` diff --git a/api/v1alpha1/workspaceproxy_types.go b/api/v1alpha1/workspaceproxy_types.go index abb988e3..1f8bc4ed 100644 --- a/api/v1alpha1/workspaceproxy_types.go +++ b/api/v1alpha1/workspaceproxy_types.go @@ -35,6 +35,7 @@ type WorkspaceProxySpec struct { // Replicas is the desired number of proxy pods. Replicas *int32 `json:"replicas,omitempty"` // Service controls the service created in front of the workspace proxy. + // +kubebuilder:default={} Service ServiceSpec `json:"service,omitempty"` // PrimaryAccessURL is the coderd URL the proxy should connect to. PrimaryAccessURL string `json:"primaryAccessURL,omitempty"` diff --git a/config/crd/bases/coder.com_codercontrolplanes.yaml b/config/crd/bases/coder.com_codercontrolplanes.yaml index 59ad8380..310ace90 100644 --- a/config/crd/bases/coder.com_codercontrolplanes.yaml +++ b/config/crd/bases/coder.com_codercontrolplanes.yaml @@ -202,6 +202,7 @@ spec: type: object type: array image: + default: ghcr.io/coder/coder:latest description: Image is the container image used for the Coder control plane pod. type: string @@ -226,10 +227,12 @@ spec: x-kubernetes-map-type: atomic type: array replicas: + default: 1 description: Replicas is the desired number of control plane pods. format: int32 type: integer service: + default: {} description: Service controls the service created in front of the control plane. properties: @@ -240,10 +243,12 @@ spec: object. type: object port: + default: 80 description: Port controls the exposed service port. format: int32 type: integer type: + default: ClusterIP description: Type controls the Kubernetes service type. type: string type: object diff --git a/config/crd/bases/coder.com_workspaceproxies.yaml b/config/crd/bases/coder.com_workspaceproxies.yaml index 1d57b9f9..fe9041d9 100644 --- a/config/crd/bases/coder.com_workspaceproxies.yaml +++ b/config/crd/bases/coder.com_workspaceproxies.yaml @@ -293,6 +293,7 @@ spec: format: int32 type: integer service: + default: {} description: Service controls the service created in front of the workspace proxy. properties: @@ -303,10 +304,12 @@ spec: object. type: object port: + default: 80 description: Port controls the exposed service port. format: int32 type: integer type: + default: ClusterIP description: Type controls the Kubernetes service type. type: string type: object diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 69c7393e..6a688a75 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,13 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - "" resources: @@ -58,3 +65,15 @@ rules: - get - patch - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/internal/app/controllerapp/controllerapp.go b/internal/app/controllerapp/controllerapp.go index dbd8a69a..65bf06ee 100644 --- a/internal/app/controllerapp/controllerapp.go +++ b/internal/app/controllerapp/controllerapp.go @@ -4,6 +4,10 @@ package controllerapp import ( "context" "fmt" + "net/http" + "os" + "strings" + "time" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -19,6 +23,17 @@ import ( const ( // HealthProbeBindAddress exposes /healthz and /readyz checks for kube probes. HealthProbeBindAddress = ":8081" + + // leaderElectionID is the stable identity used for leader-election lease objects. + leaderElectionID = "coder-k8s-controller.coder.com" + + // defaultLeaderElectionNamespace is used when the pod namespace cannot be + // detected (e.g. out-of-cluster development runs). + defaultLeaderElectionNamespace = "kube-system" + + // inClusterNamespacePath is the standard path where Kubernetes injects the + // pod namespace when running inside a cluster. + inClusterNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" ) var setupLog = ctrl.Log.WithName("setup") @@ -31,6 +46,9 @@ func NewScheme() *runtime.Scheme { return scheme } +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch + // Run starts the controller-runtime manager for the controller application mode. func Run(ctx context.Context) error { if ctx == nil { @@ -43,8 +61,12 @@ func Run(ctx context.Context) error { } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - HealthProbeBindAddress: HealthProbeBindAddress, + Scheme: scheme, + HealthProbeBindAddress: HealthProbeBindAddress, + LeaderElection: true, + LeaderElectionID: leaderElectionID, + LeaderElectionNamespace: detectLeaderElectionNamespace(), + LeaderElectionReleaseOnCancel: true, }) if err != nil { return fmt.Errorf("unable to start manager: %w", err) @@ -83,7 +105,14 @@ func Run(ctx context.Context) error { if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { return fmt.Errorf("unable to set up health check: %w", err) } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + if err := mgr.AddReadyzCheck("readyz", func(req *http.Request) error { + ctx, cancel := context.WithTimeout(req.Context(), 2*time.Second) + defer cancel() + if synced := mgr.GetCache().WaitForCacheSync(ctx); !synced { + return fmt.Errorf("informer caches not synced") + } + return nil + }); err != nil { return fmt.Errorf("unable to set up ready check: %w", err) } @@ -93,3 +122,21 @@ func Run(ctx context.Context) error { } return nil } + +// detectLeaderElectionNamespace returns the namespace to use for leader-election +// lease objects. Resolution order: +// 1. POD_NAMESPACE env var (allows explicit override for any environment). +// 2. In-cluster namespace file (standard Kubernetes downward API path). +// 3. defaultLeaderElectionNamespace as a last-resort fallback. +func detectLeaderElectionNamespace() string { + if ns := strings.TrimSpace(os.Getenv("POD_NAMESPACE")); ns != "" { + return ns + } + data, err := os.ReadFile(inClusterNamespacePath) + if err == nil { + if ns := strings.TrimSpace(string(data)); ns != "" { + return ns + } + } + return defaultLeaderElectionNamespace +} diff --git a/internal/controller/codercontrolplane_controller_test.go b/internal/controller/codercontrolplane_controller_test.go index aa5e0885..b2893ec0 100644 --- a/internal/controller/codercontrolplane_controller_test.go +++ b/internal/controller/codercontrolplane_controller_test.go @@ -106,6 +106,290 @@ func TestReconcile_ExistingResource(t *testing.T) { } } +func TestReconcile_StatusPersistence(t *testing.T) { + ctx := context.Background() + replicas := int32(1) + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-status-persistence", + Namespace: "default", + }, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + Image: "test-status-image:latest", + Replicas: &replicas, + Service: coderv1alpha1.ServiceSpec{ + Port: 8080, + }, + }, + } + + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("failed to create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + r := &controller.CoderControlPlaneReconciler{Client: k8sClient, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("reconcile control plane: %v", err) + } + + reconciled := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciled); err != nil { + t.Fatalf("get reconciled control plane: %v", err) + } + + if reconciled.Status.ObservedGeneration != reconciled.Generation { + t.Fatalf("expected observed generation %d, got %d", reconciled.Generation, reconciled.Status.ObservedGeneration) + } + expectedURL := "http://" + cp.Name + "." + cp.Namespace + ".svc.cluster.local:8080" + if reconciled.Status.URL != expectedURL { + t.Fatalf("expected status URL %q, got %q", expectedURL, reconciled.Status.URL) + } + if reconciled.Status.ReadyReplicas != 0 { + t.Fatalf("expected ready replicas 0, got %d", reconciled.Status.ReadyReplicas) + } + if reconciled.Status.Phase != coderv1alpha1.CoderControlPlanePhasePending { + t.Fatalf("expected phase %q, got %q", coderv1alpha1.CoderControlPlanePhasePending, reconciled.Status.Phase) + } +} + +func TestReconcile_OwnerReferences(t *testing.T) { + ctx := context.Background() + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-owner-references", + Namespace: "default", + }, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + Image: "test-owner-image:latest", + }, + } + + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("failed to create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + r := &controller.CoderControlPlaneReconciler{Client: k8sClient, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("reconcile control plane: %v", err) + } + + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, deployment); err != nil { + t.Fatalf("get reconciled deployment: %v", err) + } + assertSingleControllerOwnerReference(t, deployment.OwnerReferences, cp.Name) + + service := &corev1.Service{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, service); err != nil { + t.Fatalf("get reconciled service: %v", err) + } + assertSingleControllerOwnerReference(t, service.OwnerReferences, cp.Name) +} + +func TestReconcile_SpecUpdatePropagates(t *testing.T) { + ctx := context.Background() + initialReplicas := int32(1) + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-spec-update-propagates", + Namespace: "default", + }, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + Image: "img:v1", + Replicas: &initialReplicas, + Service: coderv1alpha1.ServiceSpec{ + Port: 8080, + }, + }, + } + + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("failed to create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + r := &controller.CoderControlPlaneReconciler{Client: k8sClient, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("first reconcile control plane: %v", err) + } + + reconciled := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciled); err != nil { + t.Fatalf("get reconciled control plane: %v", err) + } + + updatedReplicas := int32(3) + reconciled.Spec.Replicas = &updatedReplicas + reconciled.Spec.Image = "img:v2" + reconciled.Spec.Service.Port = 9090 + if err := k8sClient.Update(ctx, reconciled); err != nil { + t.Fatalf("update control plane spec: %v", err) + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("second reconcile control plane: %v", err) + } + + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, deployment); err != nil { + t.Fatalf("get reconciled deployment: %v", err) + } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != updatedReplicas { + t.Fatalf("expected deployment replicas %d, got %#v", updatedReplicas, deployment.Spec.Replicas) + } + if len(deployment.Spec.Template.Spec.Containers) != 1 { + t.Fatalf("expected one container in deployment pod spec, got %d", len(deployment.Spec.Template.Spec.Containers)) + } + if deployment.Spec.Template.Spec.Containers[0].Image != "img:v2" { + t.Fatalf("expected container image %q, got %q", "img:v2", deployment.Spec.Template.Spec.Containers[0].Image) + } + + service := &corev1.Service{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, service); err != nil { + t.Fatalf("get reconciled service: %v", err) + } + if len(service.Spec.Ports) != 1 || service.Spec.Ports[0].Port != 9090 { + t.Fatalf("expected service port 9090, got %+v", service.Spec.Ports) + } +} + +func TestReconcile_PhaseTransitionToReady(t *testing.T) { + ctx := context.Background() + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-phase-transition-ready", + Namespace: "default", + }, + Spec: coderv1alpha1.CoderControlPlaneSpec{ + Image: "test-phase-image:latest", + }, + } + + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("failed to create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + r := &controller.CoderControlPlaneReconciler{Client: k8sClient, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("first reconcile control plane: %v", err) + } + + reconciled := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciled); err != nil { + t.Fatalf("get reconciled control plane: %v", err) + } + if reconciled.Status.Phase != coderv1alpha1.CoderControlPlanePhasePending { + t.Fatalf("expected phase %q before deployment ready, got %q", coderv1alpha1.CoderControlPlanePhasePending, reconciled.Status.Phase) + } + + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, deployment); err != nil { + t.Fatalf("get reconciled deployment: %v", err) + } + deployment.Status.ReadyReplicas = 1 + deployment.Status.Replicas = 1 + if err := k8sClient.Status().Update(ctx, deployment); err != nil { + t.Fatalf("update deployment status: %v", err) + } + + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("second reconcile control plane: %v", err) + } + + reconciledAfterReady := &coderv1alpha1.CoderControlPlane{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, reconciledAfterReady); err != nil { + t.Fatalf("get reconciled control plane after deployment ready: %v", err) + } + if reconciledAfterReady.Status.Phase != coderv1alpha1.CoderControlPlanePhaseReady { + t.Fatalf("expected phase %q after deployment ready, got %q", coderv1alpha1.CoderControlPlanePhaseReady, reconciledAfterReady.Status.Phase) + } + if reconciledAfterReady.Status.ReadyReplicas != 1 { + t.Fatalf("expected ready replicas 1 after deployment ready, got %d", reconciledAfterReady.Status.ReadyReplicas) + } +} + +func TestReconcile_DefaultsApplied(t *testing.T) { + ctx := context.Background() + + cp := &coderv1alpha1.CoderControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-defaults-applied", + Namespace: "default", + }, + Spec: coderv1alpha1.CoderControlPlaneSpec{}, + } + + if err := k8sClient.Create(ctx, cp); err != nil { + t.Fatalf("failed to create test CoderControlPlane: %v", err) + } + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, cp) + }) + + r := &controller.CoderControlPlaneReconciler{Client: k8sClient, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}}); err != nil { + t.Fatalf("reconcile control plane: %v", err) + } + + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, deployment); err != nil { + t.Fatalf("get reconciled deployment: %v", err) + } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 { + t.Fatalf("expected default deployment replicas 1, got %#v", deployment.Spec.Replicas) + } + if len(deployment.Spec.Template.Spec.Containers) != 1 { + t.Fatalf("expected one container in deployment pod spec, got %d", len(deployment.Spec.Template.Spec.Containers)) + } + if deployment.Spec.Template.Spec.Containers[0].Image != "ghcr.io/coder/coder:latest" { + t.Fatalf("expected default image %q, got %q", "ghcr.io/coder/coder:latest", deployment.Spec.Template.Spec.Containers[0].Image) + } + + service := &corev1.Service{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, service); err != nil { + t.Fatalf("get reconciled service: %v", err) + } + if service.Spec.Type != corev1.ServiceTypeClusterIP { + t.Fatalf("expected default service type %q, got %q", corev1.ServiceTypeClusterIP, service.Spec.Type) + } + if len(service.Spec.Ports) != 1 || service.Spec.Ports[0].Port != 80 { + t.Fatalf("expected default service port 80, got %+v", service.Spec.Ports) + } +} + +func assertSingleControllerOwnerReference(t *testing.T, ownerReferences []metav1.OwnerReference, ownerName string) { + t.Helper() + + if len(ownerReferences) != 1 { + t.Fatalf("expected one owner reference, got %d", len(ownerReferences)) + } + ownerReference := ownerReferences[0] + if ownerReference.Name != ownerName { + t.Fatalf("expected owner reference name %q, got %q", ownerName, ownerReference.Name) + } + if ownerReference.Kind != "CoderControlPlane" { + t.Fatalf("expected owner reference kind %q, got %q", "CoderControlPlane", ownerReference.Kind) + } + if ownerReference.Controller == nil || !*ownerReference.Controller { + t.Fatalf("expected owner reference controller=true, got %#v", ownerReference.Controller) + } +} + func TestReconcile_NilClient(t *testing.T) { r := &controller.CoderControlPlaneReconciler{ Client: nil,