From 3bda57390ac551e676fb87fb04f888e7adbd854f Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Tue, 21 Oct 2025 13:58:34 +0000 Subject: [PATCH 01/26] Update golangci lint --- Makefile | 2 +- deploy/operator/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 184e6ae6..c8d98159 100644 --- a/Makefile +++ b/Makefile @@ -185,7 +185,7 @@ GRPCURL = $(LOCALBIN)/grpcurl KUSTOMIZE_VERSION ?= v5.4.1 CONTROLLER_TOOLS_VERSION ?= v0.16.3 ENVTEST_VERSION ?= release-0.18 -GOLANGCI_LINT_VERSION ?= v2.1.2 +GOLANGCI_LINT_VERSION ?= v2.5.0 KIND_VERSION ?= v0.27.0 GRPCURL_VERSION ?= v1.9.2 diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 5c5efa97..45314b84 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -242,7 +242,7 @@ CONTROLLER_TOOLS_VERSION ?= v0.18.0 ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') #ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}') -GOLANGCI_LINT_VERSION ?= v2.1.0 +GOLANGCI_LINT_VERSION ?= v2.5.0 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. From 8c13c623d284ad3b8196e96bb0090b3ddfc6151d Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 13 Oct 2025 14:42:32 +0200 Subject: [PATCH 02/26] operator: add RBAC permissions for the operator --- .../api/v1alpha1/zz_generated.deepcopy.go | 2 +- deploy/operator/config/rbac/role.yaml | 127 ++++++++++++++++++ .../controller/jumpstarter_controller.go | 33 ++++- 3 files changed, 154 insertions(+), 8 deletions(-) diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 1be37fc2..0d6c3739 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ //go:build !ignore_autogenerated /* -Copyright 2024. +Copyright 2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/deploy/operator/config/rbac/role.yaml b/deploy/operator/config/rbac/role.yaml index a3a3c3ea..f531186b 100644 --- a/deploy/operator/config/rbac/role.yaml +++ b/deploy/operator/config/rbac/role.yaml @@ -4,6 +4,100 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + - serviceaccounts + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - services/status + verbs: + - get + - patch + - update +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments/status + verbs: + - get + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - get + - patch + - update - apiGroups: - operator.jumpstarter.dev resources: @@ -30,3 +124,36 @@ rules: - get - patch - update +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + - roles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - route.openshift.io + resources: + - routes/status + verbs: + - get + - patch + - update diff --git a/deploy/operator/internal/controller/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter_controller.go index 95a3d516..c8f62501 100644 --- a/deploy/operator/internal/controller/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter_controller.go @@ -37,13 +37,32 @@ type JumpstarterReconciler struct { // +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters/finalizers,verbs=update -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the Jumpstarter object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// +// Core Kubernetes resources +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=services/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch + +// RBAC resources +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete + +// Leader election +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete + +// Networking resources +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes/status,verbs=get;update;patch + +// Monitoring resources +// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create;update;patch;delete + // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile func (r *JumpstarterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { From 3d616113b512701c1097e15913892bdb4ca5ae64 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 13 Oct 2025 18:24:02 +0200 Subject: [PATCH 03/26] operator: basic controller and endpoint reconciler Creates the basic structure on the controller, and a package for reconciling endpoints which should take care of the service, next commit should handle the ingress or router. --- deploy/operator/cmd/main.go | 10 +- .../jumpstarter/endpoints/endpoints.go | 169 +++++++++ .../jumpstarter/endpoints/endpoints_test.go | 233 ++++++++++++ .../jumpstarter/endpoints/suite_test.go | 116 ++++++ .../jumpstarter/jumpstarter_controller.go | 359 ++++++++++++++++++ .../jumpstarter_controller_test.go | 54 ++- .../{ => jumpstarter}/suite_test.go | 6 +- .../controller/jumpstarter_controller.go | 82 ---- deploy/operator/internal/utils/maps.go | 30 ++ 9 files changed, 964 insertions(+), 95 deletions(-) create mode 100644 deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go create mode 100644 deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go create mode 100644 deploy/operator/internal/controller/jumpstarter/endpoints/suite_test.go create mode 100644 deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go rename deploy/operator/internal/controller/{ => jumpstarter}/jumpstarter_controller_test.go (61%) rename deploy/operator/internal/controller/{ => jumpstarter}/suite_test.go (94%) delete mode 100644 deploy/operator/internal/controller/jumpstarter_controller.go create mode 100644 deploy/operator/internal/utils/maps.go diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index c404f2e2..d6dedef6 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -38,7 +38,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" - "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/jumpstarter" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/jumpstarter/endpoints" // +kubebuilder:scaffold:imports ) @@ -202,9 +203,10 @@ func main() { os.Exit(1) } - if err := (&controller.JumpstarterReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + if err := (&jumpstarter.JumpstarterReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + EndpointReconciler: endpoints.NewReconciler(mgr.GetClient()), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Jumpstarter") os.Exit(1) diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go new file mode 100644 index 00000000..d95e5d56 --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go @@ -0,0 +1,169 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endpoints + +import ( + "context" + "errors" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/utils" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Reconciler provides endpoint reconciliation functionality +type Reconciler struct { + Client client.Client +} + +// serviceDetails contains the configuration details for creating a service +type serviceDetails struct { + ServiceType corev1.ServiceType + Annotations map[string]string + Labels map[string]string + Suffix string +} + +// NewReconciler creates a new endpoint reconciler +func NewReconciler(client client.Client) *Reconciler { + return &Reconciler{ + Client: client, + } +} + +// ReconcileEndpoint creates or updates a service for the given endpoint +func (r *Reconciler) ReconcileEndpoint(ctx context.Context, namespace string, endpoint *operatorv1alpha1.Endpoint, endpointName string, svcPort corev1.ServicePort) error { + details, err := serviceDetailsForEndpoint(*endpoint) + if err != nil { + return fmt.Errorf("reconcileEndpoint: failed calculate service type for endpoint %q: %w", endpointName, err) + } + // add app label to the service + if details.Labels == nil { + details.Labels = make(map[string]string) + } + details.Labels["app"] = endpointName + + // ensure annotations is not nil + if details.Annotations == nil { + details.Annotations = make(map[string]string) + } + + // create the service for the endpoint + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: endpointName, + Namespace: namespace, + Annotations: details.Annotations, + Labels: details.Labels, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": endpointName, + }, + Ports: []corev1.ServicePort{svcPort}, + Type: details.ServiceType, + }, + } + + // Create or update the service using controller-runtime's CreateOrUpdate + _, err = r.createOrUpdateService(ctx, service) + return err +} + +// createOrUpdateService creates or updates a service using controller-runtime pattern +func (r *Reconciler) createOrUpdateService(ctx context.Context, desiredService *corev1.Service) (bool, error) { + existingService := &corev1.Service{} + err := r.Client.Get(ctx, client.ObjectKeyFromObject(desiredService), existingService) + if err != nil { + if client.IgnoreNotFound(err) != nil { + return false, err + } + // Service doesn't exist, create it + if err := r.Client.Create(ctx, desiredService); err != nil { + return false, err + } + return true, nil + } + + // Service exists, check if it needs updating + if r.serviceNeedsUpdate(existingService, desiredService) { + existingService.Spec = desiredService.Spec + existingService.Annotations = desiredService.Annotations + existingService.Labels = desiredService.Labels + + if err := r.Client.Update(ctx, existingService); err != nil { + return false, err + } + return true, nil + } + + return false, nil +} + +// serviceNeedsUpdate checks if the service needs to be updated +func (r *Reconciler) serviceNeedsUpdate(existing, desired *corev1.Service) bool { + // Check if specs are different + if existing.Spec.Type != desired.Spec.Type || + len(existing.Spec.Ports) != len(desired.Spec.Ports) || + !utils.MapsEqual(existing.Spec.Selector, desired.Spec.Selector) { + return true + } + + // Check if annotations or labels are different + if !utils.MapsEqual(existing.Annotations, desired.Annotations) || + !utils.MapsEqual(existing.Labels, desired.Labels) { + return true + } + + return false +} + +// serviceDetailsForEndpoint returns the service configuration details for the endpoint. +// It returns an error if both LoadBalancer and NodePort are enabled for the same endpoint. +func serviceDetailsForEndpoint(endpoint operatorv1alpha1.Endpoint) (*serviceDetails, error) { + if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled && + endpoint.NodePort != nil && endpoint.NodePort.Enabled { + return nil, errors.New("both LoadBalancer and NodePort are enabled for the same endpoint") + } + if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { + return &serviceDetails{ + ServiceType: corev1.ServiceTypeLoadBalancer, + Annotations: endpoint.LoadBalancer.Annotations, + Labels: endpoint.LoadBalancer.Labels, + Suffix: "-lb", + }, nil + } + if endpoint.NodePort != nil && endpoint.NodePort.Enabled { + return &serviceDetails{ + ServiceType: corev1.ServiceTypeNodePort, + Annotations: endpoint.NodePort.Annotations, + Labels: endpoint.NodePort.Labels, + Suffix: "-nodeport", + }, nil + } + + return &serviceDetails{ + ServiceType: corev1.ServiceTypeClusterIP, + Annotations: nil, + Labels: nil, + Suffix: "", + }, nil +} diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go new file mode 100644 index 00000000..3486439c --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go @@ -0,0 +1,233 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endpoints + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" +) + +var _ = Describe("Endpoints Reconciler", func() { + Context("When reconciling an endpoint", func() { + const ( + namespace = "test-namespace" + endpointName = "test-endpoint" + ) + + ctx := context.Background() + var reconciler *Reconciler + + BeforeEach(func() { + reconciler = NewReconciler(k8sClient) + + // Create the test namespace + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + err := k8sClient.Create(ctx, ns) + if err != nil && !errors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + }) + + Context("with ClusterIP service type", func() { + It("should create a ClusterIP service successfully", func() { + endpoint := &operatorv1alpha1.Endpoint{ + Hostname: endpointName, + } + + svcPort := corev1.ServicePort{ + Name: "grpc", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify the service was created + service := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: endpointName, + Namespace: namespace, + }, service) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) + Expect(service.Labels["app"]).To(Equal(endpointName)) + }) + }) + + Context("with LoadBalancer service type", func() { + It("should create a LoadBalancer service successfully", func() { + endpoint := &operatorv1alpha1.Endpoint{ + Hostname: endpointName, + LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ + Enabled: true, + Annotations: map[string]string{"service.beta.kubernetes.io/aws-load-balancer-type": "nlb"}, + Labels: map[string]string{"environment": "production"}, + }, + } + + svcPort := corev1.ServicePort{ + Name: "grpc", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify the service was created + service := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: endpointName, + Namespace: namespace, + }, service) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Spec.Type).To(Equal(corev1.ServiceTypeLoadBalancer)) + Expect(service.Annotations["service.beta.kubernetes.io/aws-load-balancer-type"]).To(Equal("nlb")) + Expect(service.Labels["environment"]).To(Equal("production")) + }) + }) + + Context("with NodePort service type", func() { + It("should create a NodePort service successfully", func() { + endpoint := &operatorv1alpha1.Endpoint{ + Hostname: endpointName, + NodePort: &operatorv1alpha1.NodePortConfig{ + Enabled: true, + Port: 30090, + Annotations: map[string]string{"nodeport.kubernetes.io/port": "30090"}, + Labels: map[string]string{"nodeport": "true"}, + }, + } + + svcPort := corev1.ServicePort{ + Name: "grpc", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify the service was created + service := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: endpointName, + Namespace: namespace, + }, service) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Spec.Type).To(Equal(corev1.ServiceTypeNodePort)) + Expect(service.Annotations["nodeport.kubernetes.io/port"]).To(Equal("30090")) + Expect(service.Labels["nodeport"]).To(Equal("true")) + }) + }) + + Context("with invalid configuration", func() { + It("should return an error when both LoadBalancer and NodePort are enabled", func() { + endpoint := &operatorv1alpha1.Endpoint{ + Hostname: endpointName, + LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ + Enabled: true, + }, + NodePort: &operatorv1alpha1.NodePortConfig{ + Enabled: true, + }, + } + + svcPort := corev1.ServicePort{ + Name: "grpc", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("both LoadBalancer and NodePort are enabled")) + }) + }) + + Context("when updating an existing service", func() { + It("should update the service when configuration changes", func() { + // Create initial service + endpoint := &operatorv1alpha1.Endpoint{ + Hostname: endpointName, + LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ + Enabled: true, + Annotations: map[string]string{"initial": "annotation"}, + Labels: map[string]string{"initial": "label"}, + }, + } + + svcPort := corev1.ServicePort{ + Name: "grpc", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Update the endpoint configuration + endpoint.LoadBalancer.Annotations["updated"] = "annotation" + endpoint.LoadBalancer.Labels["updated"] = "label" + + err = reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify the service was updated + service := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: endpointName, + Namespace: namespace, + }, service) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Annotations["updated"]).To(Equal("annotation")) + Expect(service.Labels["updated"]).To(Equal("label")) + }) + }) + + AfterEach(func() { + // Clean up created services + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: endpointName, + Namespace: namespace, + }, + } + _ = k8sClient.Delete(ctx, service) + }) + }) + +}) diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/suite_test.go b/deploy/operator/internal/controller/jumpstarter/endpoints/suite_test.go new file mode 100644 index 00000000..b36dfed1 --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/suite_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endpoints + +import ( + "context" + "os" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + ctx context.Context + cancel context.CancelFunc + testEnv *envtest.Environment + cfg *rest.Config + k8sClient client.Client +) + +func TestEndpoints(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Endpoints Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + var err error + err = operatorv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + // Retrieve the first found binary directory to allow running tests from IDEs + if getFirstFoundEnvTestBinaryDir() != "" { + testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir() + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. +// ENVTEST-based tests depend on specific binaries, usually located in paths set by +// controller-runtime. When running tests directly (e.g., via an IDE) without using +// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. +// +// This function streamlines the process by finding the required binaries, similar to +// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are +// properly set up, run 'make setup-envtest' beforehand. +func getFirstFoundEnvTestBinaryDir() string { + basePath := filepath.Join("..", "..", "..", "..", "..", "..", "bin", "k8s") + entries, err := os.ReadDir(basePath) + if err != nil { + logf.Log.Error(err, "Failed to read directory", "path", basePath) + return "" + } + for _, entry := range entries { + if entry.IsDir() { + return filepath.Join(basePath, entry.Name()) + } + } + return "" +} diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go new file mode 100644 index 00000000..67249d3a --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -0,0 +1,359 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jumpstarter + +import ( + "context" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/jumpstarter/endpoints" +) + +// JumpstarterReconciler reconciles a Jumpstarter object +type JumpstarterReconciler struct { + client.Client + Scheme *runtime.Scheme + EndpointReconciler *endpoints.Reconciler +} + +// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters/finalizers,verbs=update + +// Core Kubernetes resources +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=services/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch + +// RBAC resources +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete + +// Leader election +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete + +// Networking resources +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes/status,verbs=get;update;patch + +// Monitoring resources +// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create;update;patch;delete + +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile +func (r *JumpstarterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + // Fetch the Jumpstarter instance + var jumpstarter operatorv1alpha1.Jumpstarter + if err := r.Get(ctx, req.NamespacedName, &jumpstarter); err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + log.Info("Jumpstarter resource not found. Ignoring since object must be deleted.") + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + log.Error(err, "Failed to get Jumpstarter") + return ctrl.Result{}, err + } + + // Check if the instance is marked to be deleted + if jumpstarter.GetDeletionTimestamp() != nil { + // Handle finalizer logic here if needed + return ctrl.Result{}, nil + } + + // Reconcile Controller Deployment + if err := r.reconcileControllerDeployment(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile Controller Deployment") + return ctrl.Result{}, err + } + + // Reconcile Router Deployment + if err := r.reconcileRouterDeployment(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile Router Deployment") + return ctrl.Result{}, err + } + + // Reconcile Services + if err := r.reconcileServices(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile Services") + return ctrl.Result{}, err + } + + // Reconcile ConfigMaps + if err := r.reconcileConfigMaps(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile ConfigMaps") + return ctrl.Result{}, err + } + + // Reconcile Secrets + r.reconcileSecrets(ctx, &jumpstarter) + + // Update status + if err := r.updateStatus(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to update status") + return ctrl.Result{}, err + } + + // Requeue after 10 seconds to check for changes + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil +} + +// reconcileControllerDeployment reconciles the controller deployment +func (r *JumpstarterReconciler) reconcileControllerDeployment(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + deployment := r.createControllerDeployment(jumpstarter) + + // Set the owner reference + if err := controllerutil.SetControllerReference(jumpstarter, deployment, r.Scheme); err != nil { + return err + } + + // Create or update the deployment + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + // Update deployment spec if needed + return nil + }) + + return err +} + +// reconcileRouterDeployment reconciles the router deployment +func (r *JumpstarterReconciler) reconcileRouterDeployment(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + deployment := r.createRouterDeployment(jumpstarter) + + // Set the owner reference + if err := controllerutil.SetControllerReference(jumpstarter, deployment, r.Scheme); err != nil { + return err + } + + // Create or update the deployment + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + // Update deployment spec if needed + return nil + }) + + return err +} + +// reconcileServices reconciles all services +func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + // Reconcile controller services + for _, endpoint := range jumpstarter.Spec.Controller.GRPC.Endpoints { + if err := r.reconcileEndpointService(ctx, jumpstarter, &endpoint, "controller-grpc"); err != nil { + return err + } + } + + // Reconcile router services + for _, endpoint := range jumpstarter.Spec.Routers.GRPC.Endpoints { + if err := r.reconcileEndpointService(ctx, jumpstarter, &endpoint, "router-grpc"); err != nil { + return err + } + } + + return nil +} + +// reconcileConfigMaps reconciles all configmaps +func (r *JumpstarterReconciler) reconcileConfigMaps(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + configMap := r.createConfigMap(jumpstarter) + + // Set the owner reference + if err := controllerutil.SetControllerReference(jumpstarter, configMap, r.Scheme); err != nil { + return err + } + + // Create or update the configmap + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, configMap, func() error { + // Update configmap data if needed + return nil + }) + + return err +} + +// reconcileSecrets reconciles all secrets +func (r *JumpstarterReconciler) reconcileSecrets(_ context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) { + // Create TLS secrets for endpoints if cert-manager is not used + // This is a placeholder - actual implementation would generate certificates + _ = jumpstarter.Spec.UseCertManager +} + +// reconcileEndpointService reconciles a single endpoint service +func (r *JumpstarterReconciler) reconcileEndpointService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, endpoint *operatorv1alpha1.Endpoint, endpointName string) error { + // Create service port + svcPort := corev1.ServicePort{ + Name: "grpc", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + + // Use the endpoint reconciler to create/update the service + return r.EndpointReconciler.ReconcileEndpoint(ctx, jumpstarter.Namespace, endpoint, endpointName, svcPort) +} + +// updateStatus updates the status of the Jumpstarter resource +func (r *JumpstarterReconciler) updateStatus(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + // Update status fields based on current state + // This is a placeholder - actual implementation would check deployment status, etc. + // TODO: Add status fields to JumpstarterStatus in the API types + + return nil +} + +// createControllerDeployment creates a deployment for the controller +func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operatorv1alpha1.Jumpstarter) *appsv1.Deployment { + labels := map[string]string{ + "app": "jumpstarter-controller", + "controller": jumpstarter.Name, + } + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-controller", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &jumpstarter.Spec.Controller.Replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "controller", + Image: jumpstarter.Spec.Controller.Image, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 9090, + Name: "grpc", + }, + { + ContainerPort: 8080, + Name: "http", + }, + }, + Resources: jumpstarter.Spec.Controller.Resources, + }, + }, + }, + }, + }, + } +} + +// createRouterDeployment creates a deployment for the router +func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1alpha1.Jumpstarter) *appsv1.Deployment { + labels := map[string]string{ + "app": "jumpstarter-router", + "router": jumpstarter.Name, + } + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-router", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &jumpstarter.Spec.Routers.Replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "router", + Image: jumpstarter.Spec.Routers.Image, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 9090, + Name: "grpc", + }, + }, + Resources: jumpstarter.Spec.Routers.Resources, + }, + }, + TopologySpreadConstraints: jumpstarter.Spec.Routers.TopologySpreadConstraints, + }, + }, + }, + } +} + +// createConfigMap creates a configmap for jumpstarter configuration +func (r *JumpstarterReconciler) createConfigMap(jumpstarter *operatorv1alpha1.Jumpstarter) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-config", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: map[string]string{ + "app": jumpstarter.Name, + }, + }, + Data: map[string]string{ + "baseDomain": jumpstarter.Spec.BaseDomain, + "useCertManager": fmt.Sprintf("%t", jumpstarter.Spec.UseCertManager), + "controllerImage": jumpstarter.Spec.Controller.Image, + "routerImage": jumpstarter.Spec.Routers.Image, + }, + } +} + +// SetupWithManager sets up the controller with the Manager. +func (r *JumpstarterReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&operatorv1alpha1.Jumpstarter{}). + Named("jumpstarter"). + Owns(&appsv1.Deployment{}). + Owns(&corev1.Service{}). + Owns(&corev1.Secret{}). + Owns(&corev1.ConfigMap{}). + Complete(r) +} diff --git a/deploy/operator/internal/controller/jumpstarter_controller_test.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go similarity index 61% rename from deploy/operator/internal/controller/jumpstarter_controller_test.go rename to deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go index a66ca3ed..5e8b4a65 100644 --- a/deploy/operator/internal/controller/jumpstarter_controller_test.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go @@ -14,20 +14,22 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package jumpstarter import ( "context" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/jumpstarter/endpoints" ) var _ = Describe("Jumpstarter Controller", func() { @@ -51,7 +53,46 @@ var _ = Describe("Jumpstarter Controller", func() { Name: resourceName, Namespace: "default", }, - // TODO(user): Specify other spec details if needed. + Spec: operatorv1alpha1.JumpstarterSpec{ + BaseDomain: "example.com", + UseCertManager: true, + Controller: operatorv1alpha1.ControllerConfig{ + Image: "quay.io/jumpstarter/jumpstarter:latest", + ImagePullPolicy: "IfNotPresent", + Replicas: 1, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + GRPC: operatorv1alpha1.GRPCConfig{ + Endpoints: []operatorv1alpha1.Endpoint{ + { + Hostname: "controller", + }, + }, + }, + }, + Routers: operatorv1alpha1.RoutersConfig{ + Image: "quay.io/jumpstarter/jumpstarter:latest", + ImagePullPolicy: "IfNotPresent", + Replicas: 1, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + GRPC: operatorv1alpha1.GRPCConfig{ + Endpoints: []operatorv1alpha1.Endpoint{ + { + Hostname: "router", + }, + }, + }, + }, + }, } Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } @@ -69,8 +110,9 @@ var _ = Describe("Jumpstarter Controller", func() { It("should successfully reconcile the resource", func() { By("Reconciling the created resource") controllerReconciler := &JumpstarterReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), + Client: k8sClient, + Scheme: k8sClient.Scheme(), + EndpointReconciler: endpoints.NewReconciler(k8sClient), } _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ diff --git a/deploy/operator/internal/controller/suite_test.go b/deploy/operator/internal/controller/jumpstarter/suite_test.go similarity index 94% rename from deploy/operator/internal/controller/suite_test.go rename to deploy/operator/internal/controller/jumpstarter/suite_test.go index 746a5b30..aaff5d6d 100644 --- a/deploy/operator/internal/controller/suite_test.go +++ b/deploy/operator/internal/controller/jumpstarter/suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package jumpstarter import ( "context" @@ -66,7 +66,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, } @@ -101,7 +101,7 @@ var _ = AfterSuite(func() { // setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are // properly set up, run 'make setup-envtest' beforehand. func getFirstFoundEnvTestBinaryDir() string { - basePath := filepath.Join("..", "..", "bin", "k8s") + basePath := filepath.Join("..", "..", "..", "..", "..", "bin", "k8s") entries, err := os.ReadDir(basePath) if err != nil { logf.Log.Error(err, "Failed to read directory", "path", basePath) diff --git a/deploy/operator/internal/controller/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter_controller.go deleted file mode 100644 index c8f62501..00000000 --- a/deploy/operator/internal/controller/jumpstarter_controller.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" -) - -// JumpstarterReconciler reconciles a Jumpstarter object -type JumpstarterReconciler struct { - client.Client - Scheme *runtime.Scheme -} - -// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters/finalizers,verbs=update - -// Core Kubernetes resources -// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch -// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=services/status,verbs=get;update;patch -// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch - -// RBAC resources -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete - -// Leader election -// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete - -// Networking resources -// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=route.openshift.io,resources=routes/status,verbs=get;update;patch - -// Monitoring resources -// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create;update;patch;delete - -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile -func (r *JumpstarterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = logf.FromContext(ctx) - - // TODO(user): your logic here - - return ctrl.Result{}, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *JumpstarterReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&operatorv1alpha1.Jumpstarter{}). - Named("jumpstarter"). - Complete(r) -} diff --git a/deploy/operator/internal/utils/maps.go b/deploy/operator/internal/utils/maps.go new file mode 100644 index 00000000..aadc58f9 --- /dev/null +++ b/deploy/operator/internal/utils/maps.go @@ -0,0 +1,30 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +// MapsEqual compares two string maps for equality +func MapsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} From 8c83b11e506a97fcf4635760e1e378f8f75e28ea Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 15 Oct 2025 12:17:07 +0200 Subject: [PATCH 04/26] operator: build and deploy of operator from main Makefile deploy_from_helm.sh is refactored to reuse as much as possible and avoid divergence. a new makefile target is added for deploy-from-operator. --- Makefile | 6 + deploy/operator/Makefile | 4 +- .../config/manager/kustomization.yaml | 6 + deploy/operator/config/manager/manager.yaml | 2 +- hack/deploy_vars | 36 ++++ hack/deploy_with_helm.sh | 98 ++-------- hack/deploy_with_operator.sh | 150 +++++++++++++++ hack/get_ext_ip.sh | 8 - hack/utils | 175 ++++++++++++++++++ 9 files changed, 394 insertions(+), 91 deletions(-) create mode 100755 hack/deploy_vars create mode 100755 hack/deploy_with_operator.sh delete mode 100755 hack/get_ext_ip.sh create mode 100755 hack/utils diff --git a/Makefile b/Makefile index c8d98159..cb734174 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,9 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes $(GOLANGCI_LINT) run --fix ##@ Build +.PHONY: build-operator +build-operator: + make -C deploy/operator build-installer docker-build .PHONY: build build: manifests generate fmt vet ## Build manager binary. @@ -152,6 +155,9 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified deploy: docker-build cluster grpcurl ./hack/deploy_with_helm.sh +deploy-with-operator: build-operator docker-build cluster grpcurl + ./hack/deploy_with_operator.sh + .PHONY: deploy-exporters deploy-exporters: ./hack/demoenv/prepare_exporters.sh diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 45314b84..0c9fb348 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -29,7 +29,7 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) # # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both # jumpstarter.dev/jumpstarter-operator-bundle:$VERSION and jumpstarter.dev/jumpstarter-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= jumpstarter.dev/jumpstarter-operator +IMAGE_TAG_BASE ?= quay.io/jumpstarter-dev/jumpstarter-operator # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) @@ -50,7 +50,7 @@ endif # This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. OPERATOR_SDK_VERSION ?= v1.41.1 # Image URL to use all building/pushing image targets -IMG ?= controller:latest +IMG ?= quay.io/jumpstarter-dev/jumpstarter-operator:latest # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) diff --git a/deploy/operator/config/manager/kustomization.yaml b/deploy/operator/config/manager/kustomization.yaml index 5c5f0b84..9c94df03 100644 --- a/deploy/operator/config/manager/kustomization.yaml +++ b/deploy/operator/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/jumpstarter-dev/jumpstarter-operator + newTag: latest diff --git a/deploy/operator/config/manager/manager.yaml b/deploy/operator/config/manager/manager.yaml index 9c429c08..677b37d4 100644 --- a/deploy/operator/config/manager/manager.yaml +++ b/deploy/operator/config/manager/manager.yaml @@ -63,7 +63,7 @@ spec: args: - --leader-elect - --health-probe-bind-address=:8081 - image: controller:latest + image: quay.io/jumpstarter-dev/jumpstarter-operator:latest name: manager ports: [] securityContext: diff --git a/hack/deploy_vars b/hack/deploy_vars new file mode 100755 index 00000000..1d1c2e6a --- /dev/null +++ b/hack/deploy_vars @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Common deployment variables for Jumpstarter hack scripts +# This file should be sourced after utils + +# Calculate external IP and networking configuration +IP=$(get_external_ip) +BASEDOMAIN="jumpstarter.${IP}.nip.io" +IMG=${IMG:-quay.io/jumpstarter-dev/jumpstarter-controller:latest} +OPERATOR_IMG=${OPERATOR_IMG:-quay.io/jumpstarter-dev/jumpstarter-operator:latest} + +# Determine networking mode and endpoints based on INGRESS_ENABLED +if [ "${INGRESS_ENABLED}" == "true" ]; then + NETWORKING_MODE="ingress" + GRPC_ENDPOINT="grpc.${BASEDOMAIN}:5080" + GRPC_ROUTER_ENDPOINT="router.${BASEDOMAIN}:5080" +else + NETWORKING_MODE="nodeport" + GRPC_ENDPOINT="grpc.${BASEDOMAIN}:8082" + GRPC_ROUTER_ENDPOINT="router.${BASEDOMAIN}:8083" +fi + +# Extract image repository and tag from IMG variable +IMAGE_REPO=$(echo "${IMG}" | cut -d: -f1) +IMAGE_TAG=$(echo "${IMG}" | cut -d: -f2) + +# Export all variables for use in scripts +export IP +export BASEDOMAIN +export NETWORKING_MODE +export GRPC_ENDPOINT +export GRPC_ROUTER_ENDPOINT +export IMAGE_REPO +export IMAGE_TAG +export IMG +export OPERATOR_IMG + diff --git a/hack/deploy_with_helm.sh b/hack/deploy_with_helm.sh index db54bc67..5fec8f26 100755 --- a/hack/deploy_with_helm.sh +++ b/hack/deploy_with_helm.sh @@ -2,86 +2,37 @@ set -eo pipefail SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" -KIND=${KIND:-bin/kind} -GRPCURL=${GRPCURL:-bin/grpcurl} -IMG=${IMG:-quay.io/jumpstarter-dev/jumpstarter-controller:latest} -INGRESS_ENABLED=${INGRESS_ENABLED:-false} +# Source common utilities +source "${SCRIPT_DIR}/utils" -GREEN='\033[0;32m' -NC='\033[0m' # No Color +# Source common deployment variables +source "${SCRIPT_DIR}/deploy_vars" METHOD=install -IP=$("${SCRIPT_DIR}"/get_ext_ip.sh) - kubectl config use-context kind-jumpstarter -HELM_SETS="" +# Install nginx ingress if in ingress mode if [ "${INGRESS_ENABLED}" == "true" ]; then - echo -e "${GREEN}Deploying nginx ingress in kind ...${NC}" - - lsmod | grep ip_tables || \ - (echo "ip_tables module not loaded needed by nginx ingress, please run 'sudo modprobe ip_tables'" && exit 1) - - # before our helm installs, we make sure that kind has an ingress installed - kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml - - echo -e "${GREEN}Waiting for nginx to be ready ...${NC}" - - while ! kubectl get pods --namespace ingress-nginx --selector=app.kubernetes.io/component=controller > /dev/null; do - sleep 1 - done - - kubectl wait --namespace ingress-nginx \ - --for=condition=ready pod \ - --selector=app.kubernetes.io/component=controller \ - --timeout=90s - - HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.ingress.enabled=true" - BASEDOMAIN="jumpstarter.${IP}.nip.io" - GRPC_ENDPOINT="grpc.${BASEDOMAIN}:5080" - GRPC_ROUTER_ENDPOINT="router.${BASEDOMAIN}:5080" + install_nginx_ingress else echo -e "${GREEN}Deploying with nodeport ...${NC}" - HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.nodeport.enabled=true" - BASEDOMAIN="jumpstarter.${IP}.nip.io" - GRPC_ENDPOINT="grpc.${BASEDOMAIN}:8082" - GRPC_ROUTER_ENDPOINT="router.${BASEDOMAIN}:8083" fi +# Build Helm sets based on configuration +HELM_SETS="" HELM_SETS="${HELM_SETS} --set global.baseDomain=${BASEDOMAIN}" HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.endpoint=${GRPC_ENDPOINT}" HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.routerEndpoint=${GRPC_ROUTER_ENDPOINT}" - - -IMAGE_REPO=$(echo ${IMG} | cut -d: -f1) -IMAGE_TAG=$(echo ${IMG} | cut -d: -f2) HELM_SETS="${HELM_SETS} --set jumpstarter-controller.image=${IMAGE_REPO}" HELM_SETS="${HELM_SETS} --set jumpstarter-controller.tag=${IMAGE_TAG}" -# Function to save images to kind, with workaround for github CI and other environment issues -# In github CI, kind gets confused and tries to pull the image from docker instead -# of podman, so if regular docker-image fails we need to: -# * save it to OCI image format -# * then load it into kind -kind_load_image() { - local image=$1 - - # First, try to load the image directly - if ${KIND} load docker-image "${image}" --name jumpstarter; then - echo "Image ${image} loaded successfully." - return - fi - - # Save to tar file - podman save "${image}" | ${KIND} load image-archive /dev/stdin --name jumpstarter - if [ $? -eq 0 ]; then - echo "Image loaded successfully." - else - echo "Error loading image ${image}." - exit 1 - fi -} +# Enable appropriate networking mode +if [ "${NETWORKING_MODE}" == "ingress" ]; then + HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.ingress.enabled=true" +else + HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.nodeport.enabled=true" +fi echo -e "${GREEN}Loading the ${IMG} in kind ...${NC}" # load the docker image into the kind cluster @@ -105,21 +56,8 @@ helm ${METHOD} --namespace jumpstarter-lab \ kubectl config set-context --current --namespace=jumpstarter-lab -echo -e "${GREEN}Waiting for grpc endpoints to be ready:${NC}" -for ep in ${GRPC_ENDPOINT} ${GRPC_ROUTER_ENDPOINT}; do - RETRIES=60 - echo -e "${GREEN} * Checking ${ep} ... ${NC}" - while ! ${GRPCURL} -insecure ${ep} list; do - sleep 2 - RETRIES=$((RETRIES-1)) - if [ ${RETRIES} -eq 0 ]; then - echo -e "${GREEN} * ${ep} not ready after 120s, exiting ... ${NC}" - exit 1 - fi - done -done - +# Check gRPC endpoints are ready +check_grpc_endpoints -echo -e "${GREEN}Jumpstarter controller deployed successfully!${NC}" -echo -e " gRPC endpoint: ${GRPC_ENDPOINT}" -echo -e " gRPC router endpoint: ${GRPC_ROUTER_ENDPOINT}" +# Print success banner +print_deployment_success "Helm" diff --git a/hack/deploy_with_operator.sh b/hack/deploy_with_operator.sh new file mode 100755 index 00000000..d6dd82b9 --- /dev/null +++ b/hack/deploy_with_operator.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +set -eo pipefail +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" + +# Source common utilities +source "${SCRIPT_DIR}/utils" + +# Source common deployment variables +source "${SCRIPT_DIR}/deploy_vars" + +kubectl config use-context kind-jumpstarter + +# Install nginx ingress if in ingress mode +if [ "${INGRESS_ENABLED}" == "true" ]; then + install_nginx_ingress +else + echo -e "${GREEN}Deploying with nodeport ...${NC}" +fi + +echo -e "${GREEN}Loading the ${IMG} in kind ...${NC}" +# load the docker image into the kind cluster +kind_load_image "${IMG}" +kind_load_image "${OPERATOR_IMG}" + +# Deploy the operator +echo -e "${GREEN}Deploying Jumpstarter operator ...${NC}" +kubectl apply -f deploy/operator/dist/install.yaml + +# If operator deployment already exists, restart it to pick up the new image +if kubectl get deployment jumpstarter-operator-controller-manager -n jumpstarter-operator-system > /dev/null 2>&1; then + echo -e "${GREEN}Restarting operator deployment to pick up new image ...${NC}" + kubectl scale deployment jumpstarter-operator-controller-manager -n jumpstarter-operator-system --replicas=0 + kubectl wait --namespace jumpstarter-operator-system \ + --for=delete pod \ + --selector=control-plane=controller-manager \ + --timeout=60s 2>/dev/null || true + kubectl scale deployment jumpstarter-operator-controller-manager -n jumpstarter-operator-system --replicas=1 +fi + +# Wait for operator to be ready +echo -e "${GREEN}Waiting for operator to be ready ...${NC}" +kubectl wait --namespace jumpstarter-operator-system \ + --for=condition=available deployment/jumpstarter-operator-controller-manager \ + --timeout=120s + +# Create namespace for Jumpstarter deployment +echo -e "${GREEN}Creating jumpstarter-lab namespace ...${NC}" +kubectl create namespace jumpstarter-lab --dry-run=client -o yaml | kubectl apply -f - + +# Generate Jumpstarter CR based on networking mode +echo -e "${GREEN}Creating Jumpstarter custom resource ...${NC}" + +# Generate endpoint configuration based on networking mode +if [ "${NETWORKING_MODE}" == "ingress" ]; then + CONTROLLER_ENDPOINT_CONFIG=$(cat <<-END + - hostname: grpc.${BASEDOMAIN} + ingress: + enabled: true + class: "" +END +) + ROUTER_ENDPOINT_CONFIG=$(cat <<-END + - hostname: router.${BASEDOMAIN} + ingress: + enabled: true + class: "" +END +) +else + CONTROLLER_ENDPOINT_CONFIG=$(cat <<-END + - hostname: grpc.${BASEDOMAIN} + nodeport: + enabled: true + port: 8082 +END +) + ROUTER_ENDPOINT_CONFIG=$(cat <<-END + - hostname: router.${BASEDOMAIN} + nodeport: + enabled: true + port: 8083 +END +) +fi + +# Apply the Jumpstarter CR with the appropriate endpoint configuration +cat </dev/null 1>/dev/null; then - ip route get 1.1.1.1 | grep -oP 'src \K\S+' -else - # MacOS does not have ip, so we use route and ifconfig instead - INTERFACE=$(route get 1.1.1.1 | grep interface | awk '{print $2}') - ifconfig | grep $INTERFACE -A 10 | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}' | head -n 1 -fi diff --git a/hack/utils b/hack/utils new file mode 100755 index 00000000..d776e5e9 --- /dev/null +++ b/hack/utils @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +# Common utilities for Jumpstarter hack scripts + +set -eo pipefail + +# Script directory calculation helper +# Usage: SCRIPT_DIR="$(get_script_dir)" +get_script_dir() { + dirname "$(readlink -f "$0")" +} + +# Environment variable defaults +export KIND=${KIND:-bin/kind} +export GRPCURL=${GRPCURL:-bin/grpcurl} +export INGRESS_ENABLED=${INGRESS_ENABLED:-false} + +# Color codes for terminal output +export GREEN='\033[0;32m' +export NC='\033[0m' # No Color + +# Get external IP address +# Returns the IP address used for outbound connections +get_external_ip() { + if which ip 2>/dev/null 1>/dev/null; then + ip route get 1.1.1.1 | grep -oP 'src \K\S+' + else + # MacOS does not have ip, so we use route and ifconfig instead + INTERFACE=$(route get 1.1.1.1 | grep interface | awk '{print $2}') + ifconfig | grep "$INTERFACE" -A 10 | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}' | head -n 1 + fi +} + +# Load Docker/Podman image into kind cluster +# Function to save images to kind, with workaround for github CI and other environment issues +# In github CI, kind gets confused and tries to pull the image from docker instead +# of podman, so if regular docker-image fails we need to: +# * save it to OCI image format +# * then load it into kind +# Args: +# $1: image name (e.g., quay.io/jumpstarter-dev/jumpstarter-controller:latest) +# $2: kind cluster name (default: jumpstarter) +kind_load_image() { + local image=$1 + local cluster_name=${2:-jumpstarter} + + # First, try to load the image directly + if ${KIND} load docker-image "${image}" --name "${cluster_name}" 2>/dev/null; then + echo "Image ${image} loaded successfully." + return + fi + + # Save to tar file + if podman save "${image}" | ${KIND} load image-archive /dev/stdin --name "${cluster_name}"; then + echo "Image loaded successfully." + else + echo "Error loading image ${image}." + exit 1 + fi +} + +# Install nginx ingress in kind cluster +# This function deploys nginx ingress and waits for it to be ready +install_nginx_ingress() { + echo -e "${GREEN}Deploying nginx ingress in kind ...${NC}" + + lsmod | grep ip_tables || \ + (echo "ip_tables module not loaded needed by nginx ingress, please run 'sudo modprobe ip_tables'" && exit 1) + + # Deploy nginx ingress for kind + kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml + + echo -e "${GREEN}Waiting for nginx to be ready ...${NC}" + + while ! kubectl get pods --namespace ingress-nginx --selector=app.kubernetes.io/component=controller > /dev/null 2>&1; do + sleep 1 + done + + kubectl wait --namespace ingress-nginx \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/component=controller \ + --timeout=90s + + echo -e "${GREEN}Nginx ingress installed successfully${NC}" +} + +# Wait for Jumpstarter Kubernetes resources to be created and ready +# This is primarily used for operator deployments where resources are created asynchronously +# Args: +# $1: namespace (default: jumpstarter-lab) +wait_for_jumpstarter_resources() { + local namespace=${1:-jumpstarter-lab} + + echo -e "${GREEN}Waiting for Jumpstarter deployments to be ready ...${NC}" + + # Wait for controller deployment to exist + echo -e "${GREEN} * Waiting for controller deployment to be created ...${NC}" + local timeout=60 + while ! kubectl get deployment jumpstarter-controller -n "${namespace}" > /dev/null 2>&1; do + sleep 2 + timeout=$((timeout - 2)) + if [ ${timeout} -le 0 ]; then + echo -e "${GREEN} * Controller deployment not created after 60s, exiting ...${NC}" + exit 1 + fi + done + + # Wait for router statefulset to exist + echo -e "${GREEN} * Waiting for router statefulset to be created ...${NC}" + timeout=60 + while ! kubectl get statefulset jumpstarter-router -n "${namespace}" > /dev/null 2>&1; do + sleep 2 + timeout=$((timeout - 2)) + if [ ${timeout} -le 0 ]; then + echo -e "${GREEN} * Router statefulset not created after 60s, exiting ...${NC}" + exit 1 + fi + done + + # Wait for controller deployment to be ready + echo -e "${GREEN} * Waiting for controller deployment to be ready ...${NC}" + kubectl wait --namespace "${namespace}" \ + --for=condition=available deployment/jumpstarter-controller \ + --timeout=180s + + # Wait for router statefulset to be ready + echo -e "${GREEN} * Waiting for router pods to be ready ...${NC}" + kubectl wait --namespace "${namespace}" \ + --for=condition=ready pod \ + --selector=app=jumpstarter-router \ + --timeout=180s +} + +# Wait for gRPC endpoint to be ready +# Args: +# $1: endpoint (e.g., grpc.jumpstarter.192.168.1.1.nip.io:8082) +# $2: timeout in seconds (default: 120) +wait_for_grpc_endpoint() { + local endpoint=$1 + local timeout=${2:-120} + local retries=$((timeout / 2)) + + echo -e "${GREEN} * Checking ${endpoint} ... ${NC}" + while ! ${GRPCURL} -insecure "${endpoint}" list; do + sleep 2 + retries=$((retries - 1)) + if [ ${retries} -eq 0 ]; then + echo -e "${GREEN} * ${endpoint} not ready after ${timeout}s, exiting ... ${NC}" + exit 1 + fi + done +} + +# Check both gRPC endpoints (controller and router) are ready +check_grpc_endpoints() { + echo -e "${GREEN}Waiting for grpc endpoints to be ready:${NC}" + wait_for_grpc_endpoint "${GRPC_ENDPOINT}" + wait_for_grpc_endpoint "${GRPC_ROUTER_ENDPOINT}" +} + +# Print deployment success banner +# Args: +# $1: deployment method (e.g., "Helm", "operator") - optional +print_deployment_success() { + local method=${1:-""} + local method_text="" + + if [ -n "${method}" ]; then + method_text=" via ${method}" + fi + + echo -e "${GREEN}Jumpstarter controller deployed successfully${method_text}!${NC}" + echo -e " gRPC endpoint: ${GRPC_ENDPOINT}" + echo -e " gRPC router endpoint: ${GRPC_ROUTER_ENDPOINT}" +} + From e42c890f6300faa1a25f72918ee13b798ee9ffcc Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Fri, 17 Oct 2025 14:36:30 +0200 Subject: [PATCH 05/26] operator: make jumpstarter-controller single-namespace --- .github/workflows/e2e.yaml | 6 ++--- cmd/main.go | 53 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 495c94cd..f8fbb5a8 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -19,10 +19,10 @@ jobs: controller-ref: ${{ github.ref }} # use the matching branch on the jumpstarter repo jumpstarter-ref: ${{ github.event.pull_request.base.ref }} - e2e-tests-28d6b1cc3b49ab9ae176918ab9709a2e2522c97e: + e2e-tests-release-0-7: runs-on: ubuntu-latest steps: - - uses: jumpstarter-dev/jumpstarter-e2e@11a5ce6734be9f089ec3ea6ebf55284616f67fe8 + - uses: jumpstarter-dev/jumpstarter-e2e@release-0.7 with: controller-ref: ${{ github.ref }} - jumpstarter-ref: 28d6b1cc3b49ab9ae176918ab9709a2e2522c97e + jumpstarter-ref: release-0.7 diff --git a/cmd/main.go b/cmd/main.go index f49836c1..8b631464 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -33,6 +33,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -57,6 +58,35 @@ var ( setupLog = ctrl.Log.WithName("setup") ) +const ( + // namespaceFile is the path to the namespace file mounted by Kubernetes + namespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" +) + +// getWatchNamespace returns the namespace the controller should watch. +// It tries multiple sources in order: +// 1. NAMESPACE environment variable (explicit configuration takes precedence) +// 2. Namespace file (automatically mounted by Kubernetes in every pod) +// 3. Empty string (will fail, not supported since 0.8.0) +func getWatchNamespace() string { + // First check NAMESPACE environment variable (explicit configuration) + if ns := os.Getenv("NAMESPACE"); ns != "" { + setupLog.Info("Using namespace from NAMESPACE environment variable", "namespace", ns) + return ns + } + + // Fall back to reading from the namespace file mounted by Kubernetes + if ns, err := os.ReadFile(namespaceFile); err == nil { + namespace := string(ns) + if namespace != "" { + setupLog.Info("Auto-detected namespace from service account", "namespace", namespace) + return namespace + } + } + + return "" +} + func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) @@ -110,7 +140,11 @@ func main() { TLSOpts: tlsOpts, }) - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + // Get the namespace to watch. Try to auto-detect from the pod's service account, + // fall back to NAMESPACE environment variable, or watch all namespaces if neither is available + watchNamespace := getWatchNamespace() + + mgrOptions := ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ BindAddress: metricsAddr, @@ -132,7 +166,22 @@ func main() { // if you are doing or is intended to do any operation such as perform cleanups // after the manager stops then its usage might be unsafe. // LeaderElectionReleaseOnCancel: true, - }) + } + + // If a specific namespace is set, configure the cache to only watch that namespace + if watchNamespace != "" { + mgrOptions.LeaderElectionNamespace = watchNamespace + mgrOptions.Cache = cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + watchNamespace: {}, + }, + } + } else { + setupLog.Error(nil, "Jumpstarter controller can only be configured to work on a single namespace since 0.8.0") + os.Exit(1) + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOptions) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) From 80639b5d6f6e3c467f52f332098caf9762e273e8 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Fri, 17 Oct 2025 14:38:08 +0200 Subject: [PATCH 06/26] operator: start deploying services, RBAC and CRDs --- Makefile | 4 +- .../crd/bases/jumpstarter.dev_clients.yaml | 71 +++++ ...umpstarter.dev_exporteraccesspolicies.yaml | 166 ++++++++++ .../crd/bases/jumpstarter.dev_exporters.yaml | 162 ++++++++++ .../crd/bases/jumpstarter.dev_leases.yaml | 235 ++++++++++++++ deploy/operator/config/crd/kustomization.yaml | 5 + deploy/operator/config/manager/manager.yaml | 1 + deploy/operator/config/rbac/role.yaml | 33 ++ .../jumpstarter/endpoints/endpoints.go | 5 +- .../jumpstarter/endpoints/endpoints_test.go | 22 +- .../jumpstarter/jumpstarter_controller.go | 301 ++++++++++++++++-- .../internal/controller/jumpstarter/rbac.go | 145 +++++++++ deploy/operator/internal/log/levels.go | 18 ++ hack/deploy_with_operator.sh | 3 +- hack/utils | 2 + 15 files changed, 1133 insertions(+), 40 deletions(-) create mode 100644 deploy/operator/config/crd/bases/jumpstarter.dev_clients.yaml create mode 100644 deploy/operator/config/crd/bases/jumpstarter.dev_exporteraccesspolicies.yaml create mode 100644 deploy/operator/config/crd/bases/jumpstarter.dev_exporters.yaml create mode 100644 deploy/operator/config/crd/bases/jumpstarter.dev_leases.yaml create mode 100644 deploy/operator/internal/controller/jumpstarter/rbac.go create mode 100644 deploy/operator/internal/log/levels.go diff --git a/Makefile b/Makefile index cb734174..0fe0b6cc 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,8 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust output:crd:artifacts:config=deploy/helm/jumpstarter/crds/ \ output:rbac:artifacts:config=deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/rbac/ + cp deploy/helm/jumpstarter/crds/* deploy/operator/config/crd/bases/ + .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..." paths="./internal/..." @@ -155,7 +157,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified deploy: docker-build cluster grpcurl ./hack/deploy_with_helm.sh -deploy-with-operator: build-operator docker-build cluster grpcurl +deploy-with-operator: docker-build build-operator docker-build cluster grpcurl ./hack/deploy_with_operator.sh .PHONY: deploy-exporters diff --git a/deploy/operator/config/crd/bases/jumpstarter.dev_clients.yaml b/deploy/operator/config/crd/bases/jumpstarter.dev_clients.yaml new file mode 100644 index 00000000..ae94bdfe --- /dev/null +++ b/deploy/operator/config/crd/bases/jumpstarter.dev_clients.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: clients.jumpstarter.dev +spec: + group: jumpstarter.dev + names: + kind: Client + listKind: ClientList + plural: clients + singular: client + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Client is the Schema for the identities API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClientSpec defines the desired state of Identity + properties: + username: + type: string + type: object + status: + description: ClientStatus defines the observed state of Identity + properties: + credential: + description: |- + INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + Important: Run "make" to regenerate code after modifying this file + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + endpoint: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/operator/config/crd/bases/jumpstarter.dev_exporteraccesspolicies.yaml b/deploy/operator/config/crd/bases/jumpstarter.dev_exporteraccesspolicies.yaml new file mode 100644 index 00000000..ec1b7878 --- /dev/null +++ b/deploy/operator/config/crd/bases/jumpstarter.dev_exporteraccesspolicies.yaml @@ -0,0 +1,166 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: exporteraccesspolicies.jumpstarter.dev +spec: + group: jumpstarter.dev + names: + kind: ExporterAccessPolicy + listKind: ExporterAccessPolicyList + plural: exporteraccesspolicies + singular: exporteraccesspolicy + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ExporterAccessPolicy is the Schema for the exporteraccesspolicies + API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ExporterAccessPolicySpec defines the desired state of ExporterAccessPolicy. + properties: + exporterSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + policies: + items: + properties: + from: + items: + properties: + clientSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: array + maximumDuration: + type: string + priority: + type: integer + spotAccess: + type: boolean + type: object + type: array + type: object + status: + description: ExporterAccessPolicyStatus defines the observed state of + ExporterAccessPolicy. + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/operator/config/crd/bases/jumpstarter.dev_exporters.yaml b/deploy/operator/config/crd/bases/jumpstarter.dev_exporters.yaml new file mode 100644 index 00000000..6383bbf9 --- /dev/null +++ b/deploy/operator/config/crd/bases/jumpstarter.dev_exporters.yaml @@ -0,0 +1,162 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: exporters.jumpstarter.dev +spec: + group: jumpstarter.dev + names: + kind: Exporter + listKind: ExporterList + plural: exporters + singular: exporter + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Exporter is the Schema for the exporters API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ExporterSpec defines the desired state of Exporter + properties: + username: + type: string + type: object + status: + description: ExporterStatus defines the observed state of Exporter + properties: + conditions: + description: |- + INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + Important: Run "make" to regenerate code after modifying this file + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + credential: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + devices: + items: + properties: + labels: + additionalProperties: + type: string + type: object + parent_uuid: + type: string + uuid: + type: string + type: object + type: array + endpoint: + type: string + lastSeen: + format: date-time + type: string + leaseRef: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/operator/config/crd/bases/jumpstarter.dev_leases.yaml b/deploy/operator/config/crd/bases/jumpstarter.dev_leases.yaml new file mode 100644 index 00000000..9aafc859 --- /dev/null +++ b/deploy/operator/config/crd/bases/jumpstarter.dev_leases.yaml @@ -0,0 +1,235 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: leases.jumpstarter.dev +spec: + group: jumpstarter.dev + names: + kind: Lease + listKind: LeaseList + plural: leases + singular: lease + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.ended + name: Ended + type: boolean + - jsonPath: .spec.clientRef.name + name: Client + type: string + - jsonPath: .status.exporterRef.name + name: Exporter + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: Lease is the Schema for the exporters API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: LeaseSpec defines the desired state of Lease + properties: + beginTime: + description: |- + Requested start time. If omitted, lease starts when exporter is acquired. + Immutable after lease starts (cannot change the past). + format: date-time + type: string + clientRef: + description: The client that is requesting the lease + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + duration: + description: |- + Duration of the lease. Must be positive when provided. + Can be omitted (nil) when both BeginTime and EndTime are provided, + in which case it's calculated as EndTime - BeginTime. + type: string + endTime: + description: |- + Requested end time. If specified with BeginTime, Duration is calculated. + Can be updated to extend or shorten active leases. + format: date-time + type: string + release: + description: The release flag requests the controller to end the lease + now + type: boolean + selector: + description: The selector for the exporter to be used + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - clientRef + - selector + type: object + status: + description: LeaseStatus defines the observed state of Lease + properties: + beginTime: + description: |- + If the lease has been acquired an exporter name is assigned + and then it can be used, it will be empty while still pending + format: date-time + type: string + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + endTime: + format: date-time + type: string + ended: + type: boolean + exporterRef: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + priority: + type: integer + spotAccess: + type: boolean + required: + - ended + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/operator/config/crd/kustomization.yaml b/deploy/operator/config/crd/kustomization.yaml index 4fff9a5f..70cefba9 100644 --- a/deploy/operator/config/crd/kustomization.yaml +++ b/deploy/operator/config/crd/kustomization.yaml @@ -3,6 +3,11 @@ # It should be run by config/default resources: - bases/operator.jumpstarter.dev_jumpstarters.yaml +- bases/jumpstarter.dev_clients.yaml +- bases/jumpstarter.dev_exporters.yaml +- bases/jumpstarter.dev_leases.yaml +- bases/jumpstarter.dev_exporteraccesspolicies.yaml + # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/deploy/operator/config/manager/manager.yaml b/deploy/operator/config/manager/manager.yaml index 677b37d4..66b270dc 100644 --- a/deploy/operator/config/manager/manager.yaml +++ b/deploy/operator/config/manager/manager.yaml @@ -64,6 +64,7 @@ spec: - --leader-elect - --health-probe-bind-address=:8081 image: quay.io/jumpstarter-dev/jumpstarter-operator:latest + imagePullPolicy: IfNotPresent name: manager ports: [] securityContext: diff --git a/deploy/operator/config/rbac/role.yaml b/deploy/operator/config/rbac/role.yaml index f531186b..af22da23 100644 --- a/deploy/operator/config/rbac/role.yaml +++ b/deploy/operator/config/rbac/role.yaml @@ -66,6 +66,39 @@ rules: - patch - update - watch +- apiGroups: + - jumpstarter.dev + resources: + - clients + - exporteraccesspolicies + - exporters + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jumpstarter.dev + resources: + - clients/finalizers + - exporters/finalizers + - leases/finalizers + verbs: + - update +- apiGroups: + - jumpstarter.dev + resources: + - clients/status + - exporters/status + - leases/status + verbs: + - get + - patch + - update - apiGroups: - monitoring.coreos.com resources: diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go index d95e5d56..e84efb67 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go @@ -50,7 +50,10 @@ func NewReconciler(client client.Client) *Reconciler { } // ReconcileEndpoint creates or updates a service for the given endpoint -func (r *Reconciler) ReconcileEndpoint(ctx context.Context, namespace string, endpoint *operatorv1alpha1.Endpoint, endpointName string, svcPort corev1.ServicePort) error { +func (r *Reconciler) ReconcileEndpoint(ctx context.Context, namespace string, endpoint *operatorv1alpha1.Endpoint, svcPort corev1.ServicePort) error { + // Extract endpoint name from service port name + endpointName := svcPort.Name + details, err := serviceDetailsForEndpoint(*endpoint) if err != nil { return fmt.Errorf("reconcileEndpoint: failed calculate service type for endpoint %q: %w", endpointName, err) diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go index 3486439c..998eb4ec 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go @@ -62,13 +62,13 @@ var _ = Describe("Endpoints Reconciler", func() { } svcPort := corev1.ServicePort{ - Name: "grpc", + Name: endpointName, Port: 9090, TargetPort: intstr.FromInt(9090), Protocol: corev1.ProtocolTCP, } - err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) Expect(err).NotTo(HaveOccurred()) // Verify the service was created @@ -95,13 +95,13 @@ var _ = Describe("Endpoints Reconciler", func() { } svcPort := corev1.ServicePort{ - Name: "grpc", + Name: endpointName, Port: 9090, TargetPort: intstr.FromInt(9090), Protocol: corev1.ProtocolTCP, } - err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) Expect(err).NotTo(HaveOccurred()) // Verify the service was created @@ -130,13 +130,13 @@ var _ = Describe("Endpoints Reconciler", func() { } svcPort := corev1.ServicePort{ - Name: "grpc", + Name: endpointName, Port: 9090, TargetPort: intstr.FromInt(9090), Protocol: corev1.ProtocolTCP, } - err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) Expect(err).NotTo(HaveOccurred()) // Verify the service was created @@ -165,13 +165,13 @@ var _ = Describe("Endpoints Reconciler", func() { } svcPort := corev1.ServicePort{ - Name: "grpc", + Name: endpointName, Port: 9090, TargetPort: intstr.FromInt(9090), Protocol: corev1.ProtocolTCP, } - err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("both LoadBalancer and NodePort are enabled")) }) @@ -190,20 +190,20 @@ var _ = Describe("Endpoints Reconciler", func() { } svcPort := corev1.ServicePort{ - Name: "grpc", + Name: endpointName, Port: 9090, TargetPort: intstr.FromInt(9090), Protocol: corev1.ProtocolTCP, } - err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) Expect(err).NotTo(HaveOccurred()) // Update the endpoint configuration endpoint.LoadBalancer.Annotations["updated"] = "annotation" endpoint.LoadBalancer.Labels["updated"] = "label" - err = reconciler.ReconcileEndpoint(ctx, namespace, endpoint, endpointName, svcPort) + err = reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) Expect(err).NotTo(HaveOccurred()) // Verify the service was updated diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 67249d3a..a476cca4 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -18,11 +18,14 @@ package jumpstarter import ( "context" + "crypto/rand" + "encoding/base64" "fmt" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -34,6 +37,7 @@ import ( operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/jumpstarter/endpoints" + loglevels "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/log" ) // JumpstarterReconciler reconciles a Jumpstarter object @@ -73,6 +77,18 @@ type JumpstarterReconciler struct { // Monitoring resources // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create;update;patch;delete +// Jumpstarter CRD resources (needed to grant permissions to managed controllers) +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=clients,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=clients/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=clients/finalizers,verbs=update +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporters,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporters/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporters/finalizers,verbs=update +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=leases,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=leases/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=leases/finalizers,verbs=update +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporteraccesspolicies,verbs=get;list;watch;create;update;patch;delete + // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile func (r *JumpstarterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -98,6 +114,12 @@ func (r *JumpstarterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + // Reconcile RBAC resources first + if err := r.reconcileRBAC(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile RBAC") + return ctrl.Result{}, err + } + // Reconcile Controller Deployment if err := r.reconcileControllerDeployment(ctx, &jumpstarter); err != nil { log.Error(err, "Failed to reconcile Controller Deployment") @@ -123,7 +145,10 @@ func (r *JumpstarterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // Reconcile Secrets - r.reconcileSecrets(ctx, &jumpstarter) + if err := r.reconcileSecrets(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile Secrets") + return ctrl.Result{}, err + } // Update status if err := r.updateStatus(ctx, &jumpstarter); err != nil { @@ -175,14 +200,26 @@ func (r *JumpstarterReconciler) reconcileRouterDeployment(ctx context.Context, j func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { // Reconcile controller services for _, endpoint := range jumpstarter.Spec.Controller.GRPC.Endpoints { - if err := r.reconcileEndpointService(ctx, jumpstarter, &endpoint, "controller-grpc"); err != nil { + svcPort := corev1.ServicePort{ + Name: "controller-grpc", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + if err := r.reconcileEndpointService(ctx, jumpstarter, &endpoint, svcPort); err != nil { return err } } // Reconcile router services for _, endpoint := range jumpstarter.Spec.Routers.GRPC.Endpoints { - if err := r.reconcileEndpointService(ctx, jumpstarter, &endpoint, "router-grpc"); err != nil { + svcPort := corev1.ServicePort{ + Name: "router-grpc", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + if err := r.reconcileEndpointService(ctx, jumpstarter, &endpoint, svcPort); err != nil { return err } } @@ -209,24 +246,103 @@ func (r *JumpstarterReconciler) reconcileConfigMaps(ctx context.Context, jumpsta } // reconcileSecrets reconciles all secrets -func (r *JumpstarterReconciler) reconcileSecrets(_ context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) { - // Create TLS secrets for endpoints if cert-manager is not used - // This is a placeholder - actual implementation would generate certificates - _ = jumpstarter.Spec.UseCertManager +// Secrets are only created if they don't exist. They are not updated or deleted +// to preserve secret keys across CR updates and deletions. +func (r *JumpstarterReconciler) reconcileSecrets(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + // Create controller secret if it doesn't exist + controllerSecretName := fmt.Sprintf("%s-controller-secret", jumpstarter.Name) + if err := r.ensureSecretExists(ctx, jumpstarter, controllerSecretName); err != nil { + log.Error(err, "Failed to ensure controller secret exists", "secret", controllerSecretName) + return err + } + + // Create router secret if it doesn't exist + routerSecretName := fmt.Sprintf("%s-router-secret", jumpstarter.Name) + if err := r.ensureSecretExists(ctx, jumpstarter, routerSecretName); err != nil { + log.Error(err, "Failed to ensure router secret exists", "secret", routerSecretName) + return err + } + + return nil } -// reconcileEndpointService reconciles a single endpoint service -func (r *JumpstarterReconciler) reconcileEndpointService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, endpoint *operatorv1alpha1.Endpoint, endpointName string) error { - // Create service port - svcPort := corev1.ServicePort{ - Name: "grpc", - Port: 9090, - TargetPort: intstr.FromInt(9090), - Protocol: corev1.ProtocolTCP, +// ensureSecretExists creates a secret only if it doesn't already exist +func (r *JumpstarterReconciler) ensureSecretExists(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, name string) error { + log := logf.FromContext(ctx) + + // Check if secret already exists + existingSecret := &corev1.Secret{} + err := r.Get(ctx, client.ObjectKey{ + Namespace: jumpstarter.Namespace, + Name: name, + }, existingSecret) + + if err == nil { + // Secret already exists, don't update it + log.V(loglevels.LevelTrace).Info("Secret already exists, skipping creation", "secret", name) + return nil + } + + if !errors.IsNotFound(err) { + // Some other error occurred + return err + } + + // Secret doesn't exist, create it with a random key + randomKey, err := generateRandomKey(32) + if err != nil { + return fmt.Errorf("failed to generate random key: %w", err) + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: jumpstarter.Namespace, + Labels: map[string]string{ + "app": jumpstarter.Name, + "app.kubernetes.io/managed-by": "jumpstarter-operator", + }, + Annotations: map[string]string{ + "jumpstarter.dev/orphan": "true", + }, + }, + StringData: map[string]string{ + "key": randomKey, + }, + } + + // Note: We intentionally do NOT set owner reference here so that + // secrets are not deleted when the Jumpstarter CR is deleted. + // This preserves the secret keys across CR deletions and recreations. + + if err := r.Create(ctx, secret); err != nil { + // Handle race condition where secret was created between Get and Create + if errors.IsAlreadyExists(err) { + log.V(loglevels.LevelDebug).Info("Secret was created by another reconciliation", "secret", name) + return nil + } + return fmt.Errorf("failed to create secret: %w", err) + } + + log.Info("Created new secret with random key", "secret", name) + return nil +} + +// generateRandomKey generates a cryptographically secure random key +func generateRandomKey(length int) (string, error) { + bytes := make([]byte, length) + if _, err := rand.Read(bytes); err != nil { + return "", err } + return base64.URLEncoding.EncodeToString(bytes), nil +} +// reconcileEndpointService reconciles a single endpoint service +func (r *JumpstarterReconciler) reconcileEndpointService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { // Use the endpoint reconciler to create/update the service - return r.EndpointReconciler.ReconcileEndpoint(ctx, jumpstarter.Namespace, endpoint, endpointName, svcPort) + return r.EndpointReconciler.ReconcileEndpoint(ctx, jumpstarter.Namespace, endpoint, servicePort) } // updateStatus updates the status of the Jumpstarter resource @@ -245,6 +361,18 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator "controller": jumpstarter.Name, } + // Build GRPC endpoint from first controller endpoint + // Default to port 443 for TLS gRPC endpoints + grpcEndpoint := "" + if len(jumpstarter.Spec.Controller.GRPC.Endpoints) > 0 { + ep := jumpstarter.Spec.Controller.GRPC.Endpoints[0] + if ep.Hostname != "" { + grpcEndpoint = fmt.Sprintf("%s:443", ep.Hostname) + } else { + grpcEndpoint = fmt.Sprintf("grpc.%s:443", jumpstarter.Spec.BaseDomain) + } + } + return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-controller", jumpstarter.Name), @@ -263,8 +391,53 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "controller", + Name: "manager", Image: jumpstarter.Spec.Controller.Image, + Args: []string{ + "--leader-elect", + "--health-probe-bind-address=:8081", + "-metrics-bind-address=:8080", + }, + Env: []corev1.EnvVar{ + { + Name: "GRPC_ENDPOINT", + Value: grpcEndpoint, + }, + { + Name: "CONTROLLER_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf("%s-controller-secret", jumpstarter.Name), + }, + Key: "key", + }, + }, + }, + { + Name: "ROUTER_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf("%s-router-secret", jumpstarter.Name), + }, + Key: "key", + }, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "GIN_MODE", + Value: "release", + }, + }, Ports: []corev1.ContainerPort{ { ContainerPort: 9090, @@ -272,18 +445,59 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator }, { ContainerPort: 8080, - Name: "http", + Name: "metrics", }, + { + ContainerPort: 8081, + Name: "health", + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8081), + }, + }, + InitialDelaySeconds: 15, + PeriodSeconds: 20, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(8081), + }, + }, + InitialDelaySeconds: 5, + PeriodSeconds: 10, }, Resources: jumpstarter.Spec.Controller.Resources, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: boolPtr(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + }, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: boolPtr(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, }, }, + ServiceAccountName: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), }, }, }, } } +func boolPtr(b bool) *bool { + return &b +} + // createRouterDeployment creates a deployment for the router func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1alpha1.Jumpstarter) *appsv1.Deployment { labels := map[string]string{ @@ -311,6 +525,16 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al { Name: "router", Image: jumpstarter.Spec.Routers.Image, + Env: []corev1.EnvVar{ + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + }, Ports: []corev1.ContainerPort{ { ContainerPort: 9090, @@ -320,6 +544,7 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al Resources: jumpstarter.Spec.Routers.Resources, }, }, + ServiceAccountName: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), TopologySpreadConstraints: jumpstarter.Spec.Routers.TopologySpreadConstraints, }, }, @@ -329,19 +554,43 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al // createConfigMap creates a configmap for jumpstarter configuration func (r *JumpstarterReconciler) createConfigMap(jumpstarter *operatorv1alpha1.Jumpstarter) *corev1.ConfigMap { + // Build router configuration + // Default to port 443 for TLS gRPC endpoints + routerConfig := "default:\n" + if len(jumpstarter.Spec.Routers.GRPC.Endpoints) > 0 { + ep := jumpstarter.Spec.Routers.GRPC.Endpoints[0] + if ep.Hostname != "" { + routerConfig += fmt.Sprintf(" endpoint: %s:443\n", ep.Hostname) + } else { + routerConfig += fmt.Sprintf(" endpoint: router.%s:443\n", jumpstarter.Spec.BaseDomain) + } + } + + // Build config YAML + configYAML := `authentication: + internal: + prefix: internal + jwt: [] +provisioning: + enabled: false +grpc: + keepalive: + minTime: "1s" + permitWithoutStream: true +` + return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-config", jumpstarter.Name), + Name: fmt.Sprintf("%s-controller", jumpstarter.Name), Namespace: jumpstarter.Namespace, Labels: map[string]string{ - "app": jumpstarter.Name, + "app": "jumpstarter-controller", + "control-plane": "controller-manager", }, }, Data: map[string]string{ - "baseDomain": jumpstarter.Spec.BaseDomain, - "useCertManager": fmt.Sprintf("%t", jumpstarter.Spec.UseCertManager), - "controllerImage": jumpstarter.Spec.Controller.Image, - "routerImage": jumpstarter.Spec.Routers.Image, + "config": configYAML, + "router": routerConfig, }, } } @@ -353,7 +602,9 @@ func (r *JumpstarterReconciler) SetupWithManager(mgr ctrl.Manager) error { Named("jumpstarter"). Owns(&appsv1.Deployment{}). Owns(&corev1.Service{}). - Owns(&corev1.Secret{}). Owns(&corev1.ConfigMap{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + // Note: Secrets and ServiceAccounts are intentionally NOT owned to prevent deletion Complete(r) } diff --git a/deploy/operator/internal/controller/jumpstarter/rbac.go b/deploy/operator/internal/controller/jumpstarter/rbac.go new file mode 100644 index 00000000..d7697def --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/rbac.go @@ -0,0 +1,145 @@ +package jumpstarter + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" +) + +// reconcileRBAC reconciles all RBAC resources (ServiceAccount, Role, RoleBinding) +func (r *JumpstarterReconciler) reconcileRBAC(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + // Create ServiceAccount + sa := r.createServiceAccount(jumpstarter) + if err := controllerutil.SetControllerReference(jumpstarter, sa, r.Scheme); err != nil { + return err + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sa, func() error { + return nil + }); err != nil { + return err + } + + // Create Role + role := r.createRole(jumpstarter) + if err := controllerutil.SetControllerReference(jumpstarter, role, r.Scheme); err != nil { + return err + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, role, func() error { + return nil + }); err != nil { + return err + } + + // Create RoleBinding + roleBinding := r.createRoleBinding(jumpstarter) + if err := controllerutil.SetControllerReference(jumpstarter, roleBinding, r.Scheme); err != nil { + return err + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, roleBinding, func() error { + return nil + }); err != nil { + return err + } + + return nil +} + +// createServiceAccount creates a service account for the controller +func (r *JumpstarterReconciler) createServiceAccount(jumpstarter *operatorv1alpha1.Jumpstarter) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: map[string]string{ + "app": "jumpstarter-controller", + "app.kubernetes.io/name": "jumpstarter-controller", + "app.kubernetes.io/managed-by": "jumpstarter-operator", + }, + }, + } +} + +// createRole creates a role with necessary permissions for the controller +func (r *JumpstarterReconciler) createRole(jumpstarter *operatorv1alpha1.Jumpstarter) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-controller-role", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: map[string]string{ + "app": "jumpstarter-controller", + "app.kubernetes.io/name": "jumpstarter-controller", + "app.kubernetes.io/managed-by": "jumpstarter-operator", + }, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + { + APIGroups: []string{"jumpstarter.dev"}, + Resources: []string{"clients", "exporters", "leases", "exporteraccesspolicies"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + { + APIGroups: []string{"jumpstarter.dev"}, + Resources: []string{"clients/status", "exporters/status", "leases/status"}, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{"jumpstarter.dev"}, + Resources: []string{"clients/finalizers", "exporters/finalizers", "leases/finalizers"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + }, + } +} + +// createRoleBinding creates a role binding for the controller +func (r *JumpstarterReconciler) createRoleBinding(jumpstarter *operatorv1alpha1.Jumpstarter) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-controller-rolebinding", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: map[string]string{ + "app": "jumpstarter-controller", + "app.kubernetes.io/name": "jumpstarter-controller", + "app.kubernetes.io/managed-by": "jumpstarter-operator", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: fmt.Sprintf("%s-controller-role", jumpstarter.Name), + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + }, + }, + } +} diff --git a/deploy/operator/internal/log/levels.go b/deploy/operator/internal/log/levels.go new file mode 100644 index 00000000..8b25f203 --- /dev/null +++ b/deploy/operator/internal/log/levels.go @@ -0,0 +1,18 @@ +package log + +// Log levels for use with controller-runtime's logr.Logger.V() method. +// Higher numbers mean more verbose logging. +const ( + // LevelInfo is the standard info level (V(0)) + LevelInfo = 0 + + // LevelDebug is for debug-level logging (V(1)) + // Use for detailed operational information that is useful for debugging + // but not needed during normal operation. + LevelDebug = 1 + + // LevelTrace is for trace-level logging (V(2)) + // Use for very detailed information about internal operations, + // useful for troubleshooting complex issues. + LevelTrace = 2 +) diff --git a/hack/deploy_with_operator.sh b/hack/deploy_with_operator.sh index d6dd82b9..c69e9b22 100755 --- a/hack/deploy_with_operator.sh +++ b/hack/deploy_with_operator.sh @@ -17,8 +17,7 @@ else echo -e "${GREEN}Deploying with nodeport ...${NC}" fi -echo -e "${GREEN}Loading the ${IMG} in kind ...${NC}" -# load the docker image into the kind cluster +# load the container images into the kind cluster kind_load_image "${IMG}" kind_load_image "${OPERATOR_IMG}" diff --git a/hack/utils b/hack/utils index d776e5e9..a9341634 100755 --- a/hack/utils +++ b/hack/utils @@ -43,6 +43,8 @@ kind_load_image() { local image=$1 local cluster_name=${2:-jumpstarter} + echo -e "${GREEN}Loading $1 in kind ...${NC}" + # First, try to load the image directly if ${KIND} load docker-image "${image}" --name "${cluster_name}" 2>/dev/null; then echo "Image ${image} loaded successfully." From 07d8c5018f1de8cef0995b028a40b2e3e28b84b2 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Fri, 17 Oct 2025 14:53:50 +0200 Subject: [PATCH 07/26] operator: include and print build details --- .github/workflows/build.yaml | 14 ++++++++++++++ Dockerfile | 8 +++++++- Makefile | 21 +++++++++++++++++++-- cmd/main.go | 12 ++++++++++++ deploy/operator/Dockerfile | 8 +++++++- deploy/operator/Makefile | 24 +++++++++++++++++++++--- deploy/operator/cmd/main.go | 12 ++++++++++++ 7 files changed, 92 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8e36ed5a..81d00900 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,6 +39,16 @@ jobs: echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "VERSION=${VERSION}" + - name: Set build args + id: build-args + run: | + GIT_COMMIT=$(git rev-parse HEAD) + BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo "git_commit=${GIT_COMMIT}" >> $GITHUB_OUTPUT + echo "build_date=${BUILD_DATE}" >> $GITHUB_OUTPUT + echo "GIT_COMMIT=${GIT_COMMIT}" + echo "BUILD_DATE=${BUILD_DATE}" + - name: Set image tags id: set-tags run: | @@ -79,6 +89,10 @@ jobs: platforms: linux/amd64,linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max + build-args: | + GIT_VERSION=${{ env.VERSION }} + GIT_COMMIT=${{ steps.build-args.outputs.git_commit }} + BUILD_DATE=${{ steps.build-args.outputs.build_date }} publish-helm-charts-containers: needs: build-and-push-image diff --git a/Dockerfile b/Dockerfile index c92f87e9..49ea09c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,9 @@ FROM registry.access.redhat.com/ubi9/go-toolset:1.24.6 AS builder ARG TARGETOS ARG TARGETARCH +ARG GIT_VERSION=unknown +ARG GIT_COMMIT=unknown +ARG BUILD_DATE=unknown # Copy the Go Modules manifests COPY go.mod go.mod @@ -20,7 +23,10 @@ COPY internal/ internal/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -a \ + -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ + -o manager cmd/main.go RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o router cmd/router/main.go FROM registry.access.redhat.com/ubi9/ubi-micro:9.5 diff --git a/Makefile b/Makefile index 0fe0b6cc..9764ded6 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,16 @@ DOCKER_TAG = $(shell echo $(IMG) | cut -d: -f2) # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.30.0 +# Version information +GIT_VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "unknown") +GIT_COMMIT := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown") +BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') + +# LDFLAGS for version information +LDFLAGS := -X main.version=$(GIT_VERSION) \ + -X main.gitCommit=$(GIT_COMMIT) \ + -X main.buildDate=$(BUILD_DATE) + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -91,7 +101,7 @@ build-operator: .PHONY: build build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go + go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go go build -o bin/router cmd/router/main.go .PHONY: run @@ -107,7 +117,11 @@ run-router: manifests generate fmt vet ## Run a router from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . + $(CONTAINER_TOOL) build \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + -t ${IMG} . .PHONY: docker-push docker-push: ## Push docker image with the manager. @@ -127,6 +141,9 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform - $(CONTAINER_TOOL) buildx create --name jumpstarter-controller-builder $(CONTAINER_TOOL) buildx use jumpstarter-controller-builder - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ --tag ${DOCKER_REPO}:${DOCKER_TAG} \ --tag ${DOCKER_REPO}:latest \ -f Dockerfile.cross . diff --git a/cmd/main.go b/cmd/main.go index 8b631464..37f3fd27 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -56,6 +56,11 @@ import ( var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") + + // Version information - set via ldflags at build time + version = "dev" + gitCommit = "unknown" + buildDate = "unknown" ) const ( @@ -120,6 +125,13 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + // Print version information + setupLog.Info("Jumpstarter Controller starting", + "version", version, + "gitCommit", gitCommit, + "buildDate", buildDate, + ) + // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will // prevent from being vulnerable to the HTTP/2 Stream Cancellation and diff --git a/deploy/operator/Dockerfile b/deploy/operator/Dockerfile index c180dbc4..4125f761 100644 --- a/deploy/operator/Dockerfile +++ b/deploy/operator/Dockerfile @@ -2,6 +2,9 @@ FROM registry.access.redhat.com/ubi9/go-toolset:1.24.6 AS builder ARG TARGETOS ARG TARGETARCH +ARG GIT_VERSION=unknown +ARG GIT_COMMIT=unknown +ARG BUILD_DATE=unknown # Copy the Go Modules manifests COPY --chown=1001:0 go.mod go.mod @@ -20,7 +23,10 @@ COPY --chown=1001:0 internal/ internal/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -a \ + -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ + -o manager cmd/main.go FROM registry.access.redhat.com/ubi9/ubi-micro:9.5 WORKDIR / diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 0c9fb348..2e61dc97 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -65,6 +65,16 @@ endif # tools. (i.e. podman) CONTAINER_TOOL ?= podman +# Version information +GIT_VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "unknown") +GIT_COMMIT := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown") +BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') + +# LDFLAGS for version information +LDFLAGS := -X main.version=$(GIT_VERSION) \ + -X main.gitCommit=$(GIT_COMMIT) \ + -X main.buildDate=$(BUILD_DATE) + # Setting SHELL to bash allows bash commands to be executed by recipes. # Options are set to exit when a recipe line exits non-zero or a piped command fails. SHELL = /usr/bin/env bash -o pipefail @@ -157,7 +167,7 @@ lint-config: golangci-lint ## Verify golangci-lint linter configuration .PHONY: build build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go + go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. @@ -168,7 +178,11 @@ run: manifests generate fmt vet ## Run a controller from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . + $(CONTAINER_TOOL) build \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + -t ${IMG} . .PHONY: docker-push docker-push: ## Push docker image with the manager. @@ -187,7 +201,11 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - $(CONTAINER_TOOL) buildx create --name jumpstarter-operator-builder $(CONTAINER_TOOL) buildx use jumpstarter-operator-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + --tag ${IMG} -f Dockerfile.cross . - $(CONTAINER_TOOL) buildx rm jumpstarter-operator-builder rm Dockerfile.cross diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index d6dedef6..b63b907e 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -46,6 +46,11 @@ import ( var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") + + // Version information - set via ldflags at build time + version = "dev" + gitCommit = "unknown" + buildDate = "unknown" ) func init() { @@ -90,6 +95,13 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + // Print version information + setupLog.Info("Jumpstarter Operator starting", + "version", version, + "gitCommit", gitCommit, + "buildDate", buildDate, + ) + // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will // prevent from being vulnerable to the HTTP/2 Stream Cancellation and From 302bc90f81743b4a72a8fc62906a76b1003e7c97 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Fri, 17 Oct 2025 13:23:15 +0000 Subject: [PATCH 08/26] operator: minimal config for now in the deploy operator --- hack/deploy_with_operator.sh | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/hack/deploy_with_operator.sh b/hack/deploy_with_operator.sh index c69e9b22..8a8710cf 100755 --- a/hack/deploy_with_operator.sh +++ b/hack/deploy_with_operator.sh @@ -95,17 +95,8 @@ spec: controller: image: ${IMAGE_REPO} imagePullPolicy: IfNotPresent - replicas: 2 - resources: - requests: - cpu: 100m - memory: 100Mi - exporterOptions: - offlineTimeout: 180s + replicas: 1 grpc: - keepalive: - minTime: 3s - permitWithoutStream: true endpoints: ${CONTROLLER_ENDPOINT_CONFIG} authentication: @@ -115,22 +106,12 @@ ${CONTROLLER_ENDPOINT_CONFIG} routers: image: ${IMAGE_REPO} imagePullPolicy: IfNotPresent - replicas: 3 + replicas: 1 resources: requests: cpu: 100m memory: 100Mi - topologySpreadConstraints: - - topologyKey: "kubernetes.io/hostname" - whenUnsatisfiable: ScheduleAnyway - maxSkew: 1 - - topologyKey: "kubernetes.io/zone" - whenUnsatisfiable: ScheduleAnyway - maxSkew: 1 grpc: - keepalive: - minTime: 3s - permitWithoutStream: true endpoints: ${ROUTER_ENDPOINT_CONFIG} EOF From 4843439d9a7ca8496753a6d9ff7dd537a8edb92d Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Fri, 17 Oct 2025 14:16:37 +0000 Subject: [PATCH 09/26] operator: compatible secrets --- .../jumpstarter/jumpstarter_controller.go | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index a476cca4..757478e0 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -252,14 +252,16 @@ func (r *JumpstarterReconciler) reconcileSecrets(ctx context.Context, jumpstarte log := logf.FromContext(ctx) // Create controller secret if it doesn't exist - controllerSecretName := fmt.Sprintf("%s-controller-secret", jumpstarter.Name) + // Use fixed name to match Helm chart for migration compatibility + controllerSecretName := "jumpstarter-controller-secret" if err := r.ensureSecretExists(ctx, jumpstarter, controllerSecretName); err != nil { log.Error(err, "Failed to ensure controller secret exists", "secret", controllerSecretName) return err } // Create router secret if it doesn't exist - routerSecretName := fmt.Sprintf("%s-router-secret", jumpstarter.Name) + // Use fixed name to match Helm chart for migration compatibility + routerSecretName := "jumpstarter-router-secret" if err := r.ensureSecretExists(ctx, jumpstarter, routerSecretName); err != nil { log.Error(err, "Failed to ensure router secret exists", "secret", routerSecretName) return err @@ -391,8 +393,9 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "manager", - Image: jumpstarter.Spec.Controller.Image, + Name: "manager", + Image: jumpstarter.Spec.Controller.Image, + ImagePullPolicy: jumpstarter.Spec.Controller.ImagePullPolicy, Args: []string{ "--leader-elect", "--health-probe-bind-address=:8081", @@ -408,7 +411,7 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-controller-secret", jumpstarter.Name), + Name: "jumpstarter-controller-secret", }, Key: "key", }, @@ -419,7 +422,7 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-router-secret", jumpstarter.Name), + Name: "jumpstarter-router-secret", }, Key: "key", }, @@ -523,8 +526,9 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "router", - Image: jumpstarter.Spec.Routers.Image, + Name: "router", + Image: jumpstarter.Spec.Routers.Image, + ImagePullPolicy: jumpstarter.Spec.Routers.ImagePullPolicy, Env: []corev1.EnvVar{ { Name: "NAMESPACE", From 4014ce7fa2e6e5ae8e08afb5f3c3fde8ae39f05b Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Tue, 21 Oct 2025 10:07:22 +0000 Subject: [PATCH 10/26] operator: add caching to container builds for local devel --- Dockerfile | 13 ++++++++++--- deploy/operator/Dockerfile | 8 ++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 49ea09c3..818ee32c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,10 @@ COPY go.mod go.mod COPY go.sum go.sum # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer -RUN go mod download +# Cache module downloads across builds +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ + --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + go mod download # Copy the go source COPY cmd/ cmd/ @@ -23,11 +26,15 @@ COPY internal/ internal/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ +--mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ go build -a \ -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ -o manager cmd/main.go -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o router cmd/router/main.go +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ + --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o router cmd/router/main.go FROM registry.access.redhat.com/ubi9/ubi-micro:9.5 WORKDIR / diff --git a/deploy/operator/Dockerfile b/deploy/operator/Dockerfile index 4125f761..3ebc0a9e 100644 --- a/deploy/operator/Dockerfile +++ b/deploy/operator/Dockerfile @@ -11,7 +11,9 @@ COPY --chown=1001:0 go.mod go.mod COPY --chown=1001:0 go.sum go.sum # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer -RUN go mod download +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ + --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + go mod download # Copy the go source COPY --chown=1001:0 cmd/ cmd/ @@ -23,7 +25,9 @@ COPY --chown=1001:0 internal/ internal/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ + --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ go build -a \ -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ -o manager cmd/main.go From 58fec832228bc8b0595234ba9621289845003f0d Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Tue, 21 Oct 2025 16:19:58 +0000 Subject: [PATCH 11/26] operator: deploy router replicas and endpoints --- Makefile | 5 +- .../api/v1alpha1/jumpstarter_types.go | 3 +- ...operator.jumpstarter.dev_jumpstarters.yaml | 9 +- .../jumpstarter/endpoints/endpoints.go | 32 ++ .../jumpstarter/jumpstarter_controller.go | 452 ++++++++++++++++-- hack/deploy_with_operator.sh | 9 +- hack/kind_cluster.yaml | 9 +- hack/utils | 10 +- internal/service/controller_service.go | 2 +- internal/service/router_service.go | 2 +- 10 files changed, 478 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index 9764ded6..780d09a7 100644 --- a/Makefile +++ b/Makefile @@ -174,9 +174,12 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified deploy: docker-build cluster grpcurl ./hack/deploy_with_helm.sh -deploy-with-operator: docker-build build-operator docker-build cluster grpcurl +deploy-with-operator: docker-build build-operator cluster grpcurl ./hack/deploy_with_operator.sh +deploy-with-operator-parallel: + make deploy-with-operator -j5 --output-sync=target + .PHONY: deploy-exporters deploy-exporters: ./hack/demoenv/prepare_exporters.sh diff --git a/deploy/operator/api/v1alpha1/jumpstarter_types.go b/deploy/operator/api/v1alpha1/jumpstarter_types.go index 4da5f89c..ccce26e9 100644 --- a/deploy/operator/api/v1alpha1/jumpstarter_types.go +++ b/deploy/operator/api/v1alpha1/jumpstarter_types.go @@ -381,7 +381,8 @@ type Endpoint struct { // Hostname for this endpoint. // Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. // When optional, the hostname is used for certificate generation and DNS resolution. - // +kubebuilder:validation:Pattern=^[a-z0-9]([a-z0-9\-\.]*[a-z0-9])?$ + // Supports templating with $(replica) for replica-specific hostnames. + // +kubebuilder:validation:Pattern=^[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?$ Hostname string `json:"hostname,omitempty"` // Route configuration for OpenShift clusters. diff --git a/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml b/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml index 9e7332ee..3786c305 100644 --- a/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml +++ b/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml @@ -850,7 +850,8 @@ spec: Hostname for this endpoint. Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. When optional, the hostname is used for certificate generation and DNS resolution. - pattern: ^[a-z0-9]([a-z0-9\-\.]*[a-z0-9])?$ + Supports templating with $(replica) for replica-specific hostnames. + pattern: ^[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?$ type: string ingress: description: |- @@ -1159,7 +1160,8 @@ spec: Hostname for this endpoint. Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. When optional, the hostname is used for certificate generation and DNS resolution. - pattern: ^[a-z0-9]([a-z0-9\-\.]*[a-z0-9])?$ + Supports templating with $(replica) for replica-specific hostnames. + pattern: ^[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?$ type: string ingress: description: |- @@ -1335,7 +1337,8 @@ spec: Hostname for this endpoint. Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. When optional, the hostname is used for certificate generation and DNS resolution. - pattern: ^[a-z0-9]([a-z0-9\-\.]*[a-z0-9])?$ + Supports templating with $(replica) for replica-specific hostnames. + pattern: ^[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?$ type: string ingress: description: |- diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go index e84efb67..124b153a 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go @@ -108,6 +108,15 @@ func (r *Reconciler) createOrUpdateService(ctx context.Context, desiredService * // Service exists, check if it needs updating if r.serviceNeedsUpdate(existingService, desiredService) { + // Preserve existing NodePorts to prevent "port already allocated" errors + if existingService.Spec.Type == corev1.ServiceTypeNodePort || existingService.Spec.Type == corev1.ServiceTypeLoadBalancer { + for i := range existingService.Spec.Ports { + if existingService.Spec.Ports[i].NodePort != 0 && i < len(desiredService.Spec.Ports) { + desiredService.Spec.Ports[i].NodePort = existingService.Spec.Ports[i].NodePort + } + } + } + existingService.Spec = desiredService.Spec existingService.Annotations = desiredService.Annotations existingService.Labels = desiredService.Labels @@ -130,6 +139,29 @@ func (r *Reconciler) serviceNeedsUpdate(existing, desired *corev1.Service) bool return true } + // Check if port details are different + for i := range existing.Spec.Ports { + existingPort := existing.Spec.Ports[i] + desiredPort := desired.Spec.Ports[i] + + // Compare port fields (excluding NodePort which is handled separately) + if existingPort.Name != desiredPort.Name || + existingPort.Protocol != desiredPort.Protocol || + existingPort.Port != desiredPort.Port || + existingPort.TargetPort != desiredPort.TargetPort { + return true + } + + // Compare AppProtocol (handle nil cases) + if (existingPort.AppProtocol == nil) != (desiredPort.AppProtocol == nil) { + return true + } + if existingPort.AppProtocol != nil && desiredPort.AppProtocol != nil && + *existingPort.AppProtocol != *desiredPort.AppProtocol { + return true + } + } + // Check if annotations or labels are different if !utils.MapsEqual(existing.Annotations, desired.Annotations) || !utils.MapsEqual(existing.Labels, desired.Labels) { diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 757478e0..afe15528 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -21,6 +21,7 @@ import ( "crypto/rand" "encoding/base64" "fmt" + "strings" "time" appsv1 "k8s.io/api/apps/v1" @@ -178,52 +179,117 @@ func (r *JumpstarterReconciler) reconcileControllerDeployment(ctx context.Contex return err } -// reconcileRouterDeployment reconciles the router deployment +// reconcileRouterDeployment reconciles router deployments (one per replica) func (r *JumpstarterReconciler) reconcileRouterDeployment(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { - deployment := r.createRouterDeployment(jumpstarter) + log := logf.FromContext(ctx) - // Set the owner reference - if err := controllerutil.SetControllerReference(jumpstarter, deployment, r.Scheme); err != nil { - return err + // Create one deployment per replica + for i := int32(0); i < jumpstarter.Spec.Routers.Replicas; i++ { + deployment := r.createRouterDeployment(jumpstarter, i) + + // Set the owner reference + if err := controllerutil.SetControllerReference(jumpstarter, deployment, r.Scheme); err != nil { + return err + } + + // Create or update the deployment + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + // Update deployment spec if needed + return nil + }) + if err != nil { + return err + } } - // Create or update the deployment - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { - // Update deployment spec if needed - return nil - }) + // Clean up deployments for scaled-down replicas + if err := r.cleanupExcessRouterDeployments(ctx, jumpstarter); err != nil { + log.Error(err, "Failed to cleanup excess router deployments") + return err + } - return err + return nil } // reconcileServices reconciles all services func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + // Reconcile controller services for _, endpoint := range jumpstarter.Spec.Controller.GRPC.Endpoints { + appProtocol := "h2c" svcPort := corev1.ServicePort{ - Name: "controller-grpc", - Port: 9090, - TargetPort: intstr.FromInt(9090), - Protocol: corev1.ProtocolTCP, + Name: "controller-grpc", + Port: 8082, + TargetPort: intstr.FromInt(8082), + Protocol: corev1.ProtocolTCP, + AppProtocol: &appProtocol, + } + // Set NodePort if configured + if endpoint.NodePort != nil && endpoint.NodePort.Enabled && endpoint.NodePort.Port > 0 { + svcPort.NodePort = endpoint.NodePort.Port } - if err := r.reconcileEndpointService(ctx, jumpstarter, &endpoint, svcPort); err != nil { + if err := r.reconcileControllerEndpointService(ctx, jumpstarter, &endpoint, svcPort); err != nil { return err } } - // Reconcile router services - for _, endpoint := range jumpstarter.Spec.Routers.GRPC.Endpoints { - svcPort := corev1.ServicePort{ - Name: "router-grpc", - Port: 9090, - TargetPort: intstr.FromInt(9090), - Protocol: corev1.ProtocolTCP, - } - if err := r.reconcileEndpointService(ctx, jumpstarter, &endpoint, svcPort); err != nil { - return err + // Reconcile router services - one per replica, all endpoints per replica + for i := int32(0); i < jumpstarter.Spec.Routers.Replicas; i++ { + if len(jumpstarter.Spec.Routers.GRPC.Endpoints) > 0 { + // Each replica gets ALL configured endpoints with replica substitution + for endpointIdx, baseEndpoint := range jumpstarter.Spec.Routers.GRPC.Endpoints { + endpoint := r.buildEndpointForReplica(jumpstarter, i, endpointIdx, &baseEndpoint) + + // Build unique service name for this replica AND endpoint + // This allows multiple service types (NodePort, LoadBalancer, etc.) per replica + serviceName := r.buildServiceNameForReplicaEndpoint(jumpstarter, i, endpointIdx) + + appProtocol := "h2c" + svcPort := corev1.ServicePort{ + Name: serviceName, // Unique name per replica+endpoint + Port: 8083, + TargetPort: intstr.FromInt(8083), + Protocol: corev1.ProtocolTCP, + AppProtocol: &appProtocol, + } + // Set NodePort if configured + if endpoint.NodePort != nil && endpoint.NodePort.Enabled && endpoint.NodePort.Port > 0 { + // increase nodeport numbers based in replica, not perfect because it needs to be + // consecutive, but this is mostly for E2E testing. + svcPort.NodePort = endpoint.NodePort.Port + int32(i) + } + if err := r.reconcileRouterReplicaEndpoint(ctx, jumpstarter, i, endpointIdx, &endpoint, svcPort); err != nil { + return err + } + } + } else { + // No endpoints configured, create a default service without ingress/route + endpoint := operatorv1alpha1.Endpoint{ + Hostname: fmt.Sprintf("router-%d.%s", i, jumpstarter.Spec.BaseDomain), + } + + serviceName := fmt.Sprintf("%s-router-%d", jumpstarter.Name, i) + appProtocol := "h2c" + svcPort := corev1.ServicePort{ + Name: serviceName, + Port: 8083, + TargetPort: intstr.FromInt(8083), + Protocol: corev1.ProtocolTCP, + AppProtocol: &appProtocol, + } + if err := r.reconcileRouterReplicaEndpoint(ctx, jumpstarter, i, 0, &endpoint, svcPort); err != nil { + return err + } } } + // Clean up services for scaled-down replicas + if err := r.cleanupExcessRouterServices(ctx, jumpstarter); err != nil { + log.Error(err, "Failed to cleanup excess router services") + return err + } + return nil } @@ -347,6 +413,75 @@ func (r *JumpstarterReconciler) reconcileEndpointService(ctx context.Context, ju return r.EndpointReconciler.ReconcileEndpoint(ctx, jumpstarter.Namespace, endpoint, servicePort) } +// reconcileControllerEndpointService reconciles a controller endpoint service with proper pod selector +func (r *JumpstarterReconciler) reconcileControllerEndpointService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { + // Controller pods have fixed labels: app=jumpstarter-controller + // We need to create a service with selector matching those labels + labels := map[string]string{ + "app": "jumpstarter-controller", + "controller": jumpstarter.Name, + } + + // Determine service type and configuration from endpoint + serviceType := corev1.ServiceTypeClusterIP + annotations := make(map[string]string) + serviceLabels := map[string]string{} + + if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { + serviceType = corev1.ServiceTypeLoadBalancer + for k, v := range endpoint.LoadBalancer.Annotations { + annotations[k] = v + } + for k, v := range endpoint.LoadBalancer.Labels { + serviceLabels[k] = v + } + } else if endpoint.NodePort != nil && endpoint.NodePort.Enabled { + serviceType = corev1.ServiceTypeNodePort + for k, v := range endpoint.NodePort.Annotations { + annotations[k] = v + } + for k, v := range endpoint.NodePort.Labels { + serviceLabels[k] = v + } + } + + // Merge service-specific labels with standard labels + for k, v := range labels { + serviceLabels[k] = v + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: servicePort.Name, + Namespace: jumpstarter.Namespace, + Labels: serviceLabels, + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": "jumpstarter-controller", // Match controller pod labels + }, + Ports: []corev1.ServicePort{servicePort}, + Type: serviceType, + }, + } + + if err := controllerutil.SetControllerReference(jumpstarter, service, r.Scheme); err != nil { + return err + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, func() error { + // Update the service spec + service.Spec.Selector = map[string]string{ + "app": "jumpstarter-controller", + } + service.Spec.Ports = []corev1.ServicePort{servicePort} + service.Spec.Type = serviceType + return nil + }) + return err +} + // updateStatus updates the status of the Jumpstarter resource func (r *JumpstarterReconciler) updateStatus(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { // Update status fields based on current state @@ -443,7 +578,7 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator }, Ports: []corev1.ContainerPort{ { - ContainerPort: 9090, + ContainerPort: 8082, Name: "grpc", }, { @@ -501,21 +636,31 @@ func boolPtr(b bool) *bool { return &b } -// createRouterDeployment creates a deployment for the router -func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1alpha1.Jumpstarter) *appsv1.Deployment { +// createRouterDeployment creates a deployment for a specific router replica +func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32) *appsv1.Deployment { + // Base app label that ALL services for this replica will select + // Individual services will be named with endpoint suffixes, but all select the same pods + baseAppLabel := fmt.Sprintf("%s-router-%d", jumpstarter.Name, replicaIndex) + labels := map[string]string{ - "app": "jumpstarter-router", - "router": jumpstarter.Name, + "app": baseAppLabel, // All services for this replica select by this label + "router": jumpstarter.Name, + "router-index": fmt.Sprintf("%d", replicaIndex), } + // Build router endpoint for this specific replica + routerEndpoint := r.buildRouterEndpointForReplica(jumpstarter, replicaIndex) + + replicas := int32(1) // Each deployment has exactly 1 replica + return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-router", jumpstarter.Name), + Name: fmt.Sprintf("%s-router-%d", jumpstarter.Name, replicaIndex), Namespace: jumpstarter.Namespace, Labels: labels, }, Spec: appsv1.DeploymentSpec{ - Replicas: &jumpstarter.Spec.Routers.Replicas, + Replicas: &replicas, Selector: &metav1.LabelSelector{ MatchLabels: labels, }, @@ -529,7 +674,23 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al Name: "router", Image: jumpstarter.Spec.Routers.Image, ImagePullPolicy: jumpstarter.Spec.Routers.ImagePullPolicy, + Command: []string{"/router"}, Env: []corev1.EnvVar{ + { + Name: "GRPC_ROUTER_ENDPOINT", + Value: routerEndpoint, + }, + { + Name: "ROUTER_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "jumpstarter-router-secret", + }, + Key: "key", + }, + }, + }, { Name: "NAMESPACE", ValueFrom: &corev1.EnvVarSource{ @@ -541,15 +702,28 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al }, Ports: []corev1.ContainerPort{ { - ContainerPort: 9090, + ContainerPort: 8083, Name: "grpc", }, }, Resources: jumpstarter.Spec.Routers.Resources, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: boolPtr(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + }, }, }, - ServiceAccountName: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), - TopologySpreadConstraints: jumpstarter.Spec.Routers.TopologySpreadConstraints, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: boolPtr(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + ServiceAccountName: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), + TopologySpreadConstraints: jumpstarter.Spec.Routers.TopologySpreadConstraints, + TerminationGracePeriodSeconds: int64Ptr(10), }, }, }, @@ -599,6 +773,212 @@ grpc: } } +// buildRouterEndpointForReplica builds the GRPC_ROUTER_ENDPOINT for a specific replica +// This is the primary endpoint the router advertises itself as +func (r *JumpstarterReconciler) buildRouterEndpointForReplica(jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32) string { + // If endpoints are specified, use the first one as the primary endpoint + if len(jumpstarter.Spec.Routers.GRPC.Endpoints) > 0 { + ep := jumpstarter.Spec.Routers.GRPC.Endpoints[0] + hostname := ep.Hostname + if hostname != "" { + hostname = r.substituteReplica(hostname, replicaIndex) + return fmt.Sprintf("%s:443", hostname) + } + } + // Default pattern: router-N.baseDomain + return fmt.Sprintf("router-%d.%s:443", replicaIndex, jumpstarter.Spec.BaseDomain) +} + +// substituteReplica replaces $(replica) placeholder with actual replica index +func (r *JumpstarterReconciler) substituteReplica(hostname string, replicaIndex int32) string { + return strings.ReplaceAll(hostname, "$(replica)", fmt.Sprintf("%d", replicaIndex)) +} + +// int64Ptr returns a pointer to an int64 value +func int64Ptr(i int64) *int64 { + return &i +} + +// buildServiceNameForReplicaEndpoint creates a unique service name for a router replica and endpoint +func (r *JumpstarterReconciler) buildServiceNameForReplicaEndpoint(jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32, endpointIdx int) string { + if endpointIdx == 0 { + // First endpoint uses base name for backwards compatibility + return fmt.Sprintf("%s-router-%d", jumpstarter.Name, replicaIndex) + } + // Additional endpoints get a suffix + return fmt.Sprintf("%s-router-%d-%d", jumpstarter.Name, replicaIndex, endpointIdx) +} + +// buildEndpointForReplica creates an Endpoint struct for a specific router replica and endpoint +func (r *JumpstarterReconciler) buildEndpointForReplica(jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32, endpointIdx int, baseEndpoint *operatorv1alpha1.Endpoint) operatorv1alpha1.Endpoint { + // Copy the base endpoint + endpoint := *baseEndpoint + + // Set or substitute hostname + if endpoint.Hostname != "" { + endpoint.Hostname = r.substituteReplica(endpoint.Hostname, replicaIndex) + } else { + // Default hostname pattern when none specified + if endpointIdx == 0 { + endpoint.Hostname = fmt.Sprintf("router-%d.%s", replicaIndex, jumpstarter.Spec.BaseDomain) + } else { + endpoint.Hostname = fmt.Sprintf("router-%d-%d.%s", replicaIndex, endpointIdx, jumpstarter.Spec.BaseDomain) + } + } + + return endpoint +} + +// reconcileRouterReplicaEndpoint reconciles service, ingress, and route for a specific router replica endpoint +func (r *JumpstarterReconciler) reconcileRouterReplicaEndpoint(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32, endpointIdx int, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { + // Create service with proper selector pointing to the deployment pods + // All services for this replica select the same pods using the base app label + baseAppLabel := fmt.Sprintf("%s-router-%d", jumpstarter.Name, replicaIndex) + + labels := map[string]string{ + "app": "jumpstarter-router", + "router": jumpstarter.Name, + "router-index": fmt.Sprintf("%d", replicaIndex), + "endpoint-idx": fmt.Sprintf("%d", endpointIdx), + } + + // Determine service type and configuration from endpoint + serviceType := corev1.ServiceTypeClusterIP + annotations := make(map[string]string) + + if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { + serviceType = corev1.ServiceTypeLoadBalancer + for k, v := range endpoint.LoadBalancer.Annotations { + annotations[k] = v + } + for k, v := range endpoint.LoadBalancer.Labels { + labels[k] = v + } + } else if endpoint.NodePort != nil && endpoint.NodePort.Enabled { + serviceType = corev1.ServiceTypeNodePort + for k, v := range endpoint.NodePort.Annotations { + annotations[k] = v + } + for k, v := range endpoint.NodePort.Labels { + labels[k] = v + } + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: servicePort.Name, + Namespace: jumpstarter.Namespace, + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": baseAppLabel, // Select pods from this replica's deployment + }, + Ports: []corev1.ServicePort{servicePort}, + Type: serviceType, + }, + } + + if err := controllerutil.SetControllerReference(jumpstarter, service, r.Scheme); err != nil { + return err + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, func() error { + // Preserve existing NodePort if the service already exists + // This prevents "port already allocated" errors during updates + if len(service.Spec.Ports) > 0 && service.Spec.Ports[0].NodePort != 0 { + // Service already exists with a NodePort, preserve it + servicePort.NodePort = service.Spec.Ports[0].NodePort + } + + service.Spec.Selector = map[string]string{ + "app": baseAppLabel, + } + service.Spec.Ports = []corev1.ServicePort{servicePort} + service.Spec.Type = serviceType + return nil + }) + if err != nil { + return err + } + + // Now create ingress/route if configured + // TODO: Create ingress/route based on endpoint configuration + // For now, EndpointReconciler will try to create a service (which already exists) + // but that's okay as it will just update it + return r.EndpointReconciler.ReconcileEndpoint(ctx, jumpstarter.Namespace, endpoint, servicePort) +} + +// cleanupExcessRouterDeployments deletes router deployments that exceed the current replica count +func (r *JumpstarterReconciler) cleanupExcessRouterDeployments(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + // List all deployments with our router label + deploymentList := &appsv1.DeploymentList{} + listOpts := []client.ListOption{ + client.InNamespace(jumpstarter.Namespace), + client.MatchingLabels{ + "router": jumpstarter.Name, + }, + } + + if err := r.List(ctx, deploymentList, listOpts...); err != nil { + return fmt.Errorf("failed to list router deployments: %w", err) + } + + // Delete deployments with replica index >= current replica count + for i := range deploymentList.Items { + deployment := &deploymentList.Items[i] + + // Check if this deployment's name indicates it's beyond the current replica count + // We need to check all indices from current replicas onwards + for idx := jumpstarter.Spec.Routers.Replicas; idx < 100; idx++ { // reasonable upper bound + excessName := fmt.Sprintf("%s-router-%d", jumpstarter.Name, idx) + if deployment.Name == excessName { + log.Info("Deleting excess router deployment", "deployment", deployment.Name, "replicaIndex", idx) + if err := r.Delete(ctx, deployment); err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete excess deployment %s: %w", deployment.Name, err) + } + } + break + } + } + } + + return nil +} + +// cleanupExcessRouterServices deletes router services that exceed the current replica count +func (r *JumpstarterReconciler) cleanupExcessRouterServices(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + // Delete services with replica index >= current replica count + // Services are named "{jumpstarter-name}-router-{N}" by our service port naming + for idx := jumpstarter.Spec.Routers.Replicas; idx < 100; idx++ { // reasonable upper bound + serviceName := fmt.Sprintf("%s-router-%d", jumpstarter.Name, idx) + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: jumpstarter.Namespace, + }, + } + + err := r.Delete(ctx, service) + if err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete excess service %s: %w", serviceName, err) + } + // If not found, we've gone past all excess services, can stop + break + } + log.Info("Deleted excess router service", "service", serviceName, "replicaIndex", idx) + } + + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *JumpstarterReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/hack/deploy_with_operator.sh b/hack/deploy_with_operator.sh index 8a8710cf..e33a4435 100755 --- a/hack/deploy_with_operator.sh +++ b/hack/deploy_with_operator.sh @@ -70,14 +70,13 @@ else - hostname: grpc.${BASEDOMAIN} nodeport: enabled: true - port: 8082 + port: 30010 END ) ROUTER_ENDPOINT_CONFIG=$(cat <<-END - - hostname: router.${BASEDOMAIN} - nodeport: + - nodeport: enabled: true - port: 8083 + port: 30011 END ) fi @@ -106,7 +105,7 @@ ${CONTROLLER_ENDPOINT_CONFIG} routers: image: ${IMAGE_REPO} imagePullPolicy: IfNotPresent - replicas: 1 + replicas: 3 resources: requests: cpu: 100m diff --git a/hack/kind_cluster.yaml b/hack/kind_cluster.yaml index e03fe267..7ffd60b1 100644 --- a/hack/kind_cluster.yaml +++ b/hack/kind_cluster.yaml @@ -20,10 +20,15 @@ nodes: - containerPort: 30010 # grpc nodeport hostPort: 8082 protocol: TCP - - containerPort: 30011 # grpc router nodeport + - containerPort: 30011 # grpc router nodeport (replica 0) hostPort: 8083 protocol: TCP - + - containerPort: 30012 # grpc router nodeport (replica 1) + hostPort: 8084 + protocol: TCP + - containerPort: 30013 # grpc router nodeport (replica 2) + hostPort: 8085 + protocol: TCP - containerPort: 443 hostPort: 5443 protocol: TCP diff --git a/hack/utils b/hack/utils index a9341634..50fd0882 100755 --- a/hack/utils +++ b/hack/utils @@ -106,14 +106,14 @@ wait_for_jumpstarter_resources() { fi done - # Wait for router statefulset to exist - echo -e "${GREEN} * Waiting for router statefulset to be created ...${NC}" + # Wait for router deployment to exist + echo -e "${GREEN} * Waiting for router deployment to be created ...${NC}" timeout=60 - while ! kubectl get statefulset jumpstarter-router -n "${namespace}" > /dev/null 2>&1; do + while ! kubectl get deployment jumpstarter-router-0 -n "${namespace}" > /dev/null 2>&1; do sleep 2 timeout=$((timeout - 2)) if [ ${timeout} -le 0 ]; then - echo -e "${GREEN} * Router statefulset not created after 60s, exiting ...${NC}" + echo -e "${GREEN} * Router deployment not created after 60s, exiting ...${NC}" exit 1 fi done @@ -128,7 +128,7 @@ wait_for_jumpstarter_resources() { echo -e "${GREEN} * Waiting for router pods to be ready ...${NC}" kubectl wait --namespace "${namespace}" \ --for=condition=ready pod \ - --selector=app=jumpstarter-router \ + --selector=app=jumpstarter-router-0 \ --timeout=180s } diff --git a/internal/service/controller_service.go b/internal/service/controller_service.go index d7712bac..75ab4c70 100644 --- a/internal/service/controller_service.go +++ b/internal/service/controller_service.go @@ -745,7 +745,7 @@ func (s *ControllerService) Start(ctx context.Context) error { return err } - logger.Info("Starting Controller grpc service") + logger.Info("Starting Controller grpc service on port 8082") go func() { <-ctx.Done() diff --git a/internal/service/router_service.go b/internal/service/router_service.go index e1167c8b..8d801611 100644 --- a/internal/service/router_service.go +++ b/internal/service/router_service.go @@ -135,7 +135,7 @@ func (s *RouterService) Start(ctx context.Context) error { return err } - log.Info("Starting grpc router service") + log.Info("Starting grpc router service on port 8083") go func() { <-ctx.Done() log.Info("Stopping grpc router service") From f65267cf22552de5793c7f0e1f3ce8da05665074 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 22 Oct 2025 09:32:32 +0000 Subject: [PATCH 12/26] operator: support ClusterIP service types and create all types --- .../api/v1alpha1/jumpstarter_types.go | 21 ++ .../api/v1alpha1/zz_generated.deepcopy.go | 34 +++ ...operator.jumpstarter.dev_jumpstarters.yaml | 81 +++++ .../jumpstarter/jumpstarter_controller.go | 283 ++++++++++++++---- 4 files changed, 356 insertions(+), 63 deletions(-) diff --git a/deploy/operator/api/v1alpha1/jumpstarter_types.go b/deploy/operator/api/v1alpha1/jumpstarter_types.go index ccce26e9..3c6d3d6e 100644 --- a/deploy/operator/api/v1alpha1/jumpstarter_types.go +++ b/deploy/operator/api/v1alpha1/jumpstarter_types.go @@ -404,6 +404,12 @@ type Endpoint struct { // Creates a LoadBalancer service for this endpoint. // Requires cloud provider support for LoadBalancer services. LoadBalancer *LoadBalancerConfig `json:"loadBalancer,omitempty"` + + // ClusterIP configuration for internal service access. + // Creates a ClusterIP service for this endpoint. + // Useful for internal service-to-service communication or when + // using a different method to expose the service externally. + ClusterIP *ClusterIPConfig `json:"clusterIP,omitempty"` } // RouteConfig defines OpenShift Route configuration. @@ -488,6 +494,21 @@ type LoadBalancerConfig struct { Labels map[string]string `json:"labels,omitempty"` } +// ClusterIPConfig defines Kubernetes ClusterIP service configuration. +type ClusterIPConfig struct { + // Enable the ClusterIP service for this endpoint. + // When disabled, no ClusterIP service will be created for this endpoint. + Enabled bool `json:"enabled,omitempty"` + + // Annotations to add to the ClusterIP service. + // Useful for configuring service-specific behavior and load balancer options. + Annotations map[string]string `json:"annotations,omitempty"` + + // Labels to add to the ClusterIP service. + // Useful for monitoring, cost allocation, and resource organization. + Labels map[string]string `json:"labels,omitempty"` +} + // JumpstarterStatus defines the observed state of Jumpstarter. // This field is currently empty but can be extended to include status information // such as deployment status, endpoint URLs, and health information. diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 0d6c3739..a6764a38 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -51,6 +51,35 @@ func (in *AuthenticationConfig) DeepCopy() *AuthenticationConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterIPConfig) DeepCopyInto(out *ClusterIPConfig) { + *out = *in + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterIPConfig. +func (in *ClusterIPConfig) DeepCopy() *ClusterIPConfig { + if in == nil { + return nil + } + out := new(ClusterIPConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControllerConfig) DeepCopyInto(out *ControllerConfig) { *out = *in @@ -94,6 +123,11 @@ func (in *Endpoint) DeepCopyInto(out *Endpoint) { *out = new(LoadBalancerConfig) (*in).DeepCopyInto(*out) } + if in.ClusterIP != nil { + in, out := &in.ClusterIP, &out.ClusterIP + *out = new(ClusterIPConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Endpoint. diff --git a/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml b/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml index 3786c305..61f069a3 100644 --- a/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml +++ b/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml @@ -845,6 +845,33 @@ spec: An endpoint can use one or more networking methods: Route, Ingress, NodePort, or LoadBalancer. Multiple methods can be configured simultaneously for the same hostname. properties: + clusterIP: + description: |- + ClusterIP configuration for internal service access. + Creates a ClusterIP service for this endpoint. + Useful for internal service-to-service communication or when + using a different method to expose the service externally. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations to add to the ClusterIP service. + Useful for configuring service-specific behavior and load balancer options. + type: object + enabled: + description: |- + Enable the ClusterIP service for this endpoint. + When disabled, no ClusterIP service will be created for this endpoint. + type: boolean + labels: + additionalProperties: + type: string + description: |- + Labels to add to the ClusterIP service. + Useful for monitoring, cost allocation, and resource organization. + type: object + type: object hostname: description: |- Hostname for this endpoint. @@ -1155,6 +1182,33 @@ spec: An endpoint can use one or more networking methods: Route, Ingress, NodePort, or LoadBalancer. Multiple methods can be configured simultaneously for the same hostname. properties: + clusterIP: + description: |- + ClusterIP configuration for internal service access. + Creates a ClusterIP service for this endpoint. + Useful for internal service-to-service communication or when + using a different method to expose the service externally. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations to add to the ClusterIP service. + Useful for configuring service-specific behavior and load balancer options. + type: object + enabled: + description: |- + Enable the ClusterIP service for this endpoint. + When disabled, no ClusterIP service will be created for this endpoint. + type: boolean + labels: + additionalProperties: + type: string + description: |- + Labels to add to the ClusterIP service. + Useful for monitoring, cost allocation, and resource organization. + type: object + type: object hostname: description: |- Hostname for this endpoint. @@ -1332,6 +1386,33 @@ spec: An endpoint can use one or more networking methods: Route, Ingress, NodePort, or LoadBalancer. Multiple methods can be configured simultaneously for the same hostname. properties: + clusterIP: + description: |- + ClusterIP configuration for internal service access. + Creates a ClusterIP service for this endpoint. + Useful for internal service-to-service communication or when + using a different method to expose the service externally. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations to add to the ClusterIP service. + Useful for configuring service-specific behavior and load balancer options. + type: object + enabled: + description: |- + Enable the ClusterIP service for this endpoint. + When disabled, no ClusterIP service will be created for this endpoint. + type: boolean + labels: + additionalProperties: + type: string + description: |- + Labels to add to the ClusterIP service. + Useful for monitoring, cost allocation, and resource organization. + type: object + type: object hostname: description: |- Hostname for this endpoint. diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index afe15528..a27deb5f 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -414,45 +414,77 @@ func (r *JumpstarterReconciler) reconcileEndpointService(ctx context.Context, ju } // reconcileControllerEndpointService reconciles a controller endpoint service with proper pod selector +// This function creates a separate service for each enabled service type (ClusterIP, NodePort, LoadBalancer) func (r *JumpstarterReconciler) reconcileControllerEndpointService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { // Controller pods have fixed labels: app=jumpstarter-controller // We need to create a service with selector matching those labels - labels := map[string]string{ + baseLabels := map[string]string{ "app": "jumpstarter-controller", "controller": jumpstarter.Name, } - // Determine service type and configuration from endpoint - serviceType := corev1.ServiceTypeClusterIP - annotations := make(map[string]string) - serviceLabels := map[string]string{} + // Create a service for each enabled service type + // This allows multiple service types to coexist for the same endpoint + // Note: ClusterIP uses no suffix (most common for in-cluster communication) + // LoadBalancer uses "-lb" suffix, NodePort uses "-np" suffix + // LoadBalancer service if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { - serviceType = corev1.ServiceTypeLoadBalancer - for k, v := range endpoint.LoadBalancer.Annotations { - annotations[k] = v + if err := r.createControllerService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeLoadBalancer, "-lb", baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { + return err } - for k, v := range endpoint.LoadBalancer.Labels { - serviceLabels[k] = v + } + + // NodePort service + if endpoint.NodePort != nil && endpoint.NodePort.Enabled { + if err := r.createControllerService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeNodePort, "-np", baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { + return err } - } else if endpoint.NodePort != nil && endpoint.NodePort.Enabled { - serviceType = corev1.ServiceTypeNodePort - for k, v := range endpoint.NodePort.Annotations { - annotations[k] = v + } + + // ClusterIP service (no suffix for cleaner in-cluster service names) + if endpoint.ClusterIP != nil && endpoint.ClusterIP.Enabled { + if err := r.createControllerService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { + return err } - for k, v := range endpoint.NodePort.Labels { - serviceLabels[k] = v + } + + // If no service type is explicitly enabled, create a default ClusterIP service + if (endpoint.LoadBalancer == nil || !endpoint.LoadBalancer.Enabled) && + (endpoint.NodePort == nil || !endpoint.NodePort.Enabled) && + (endpoint.ClusterIP == nil || !endpoint.ClusterIP.Enabled) { + + // TODO: Default to Route or Ingress depending of the type of cluster + if err := r.createControllerService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseLabels, nil, nil); err != nil { + return err } } - // Merge service-specific labels with standard labels - for k, v := range labels { + return nil +} + +// createControllerService creates or updates a single controller service with the specified type and suffix +func (r *JumpstarterReconciler) createControllerService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort, serviceType corev1.ServiceType, nameSuffix string, baseLabels map[string]string, annotations map[string]string, extraLabels map[string]string) error { + // Build service name with suffix to avoid conflicts + serviceName := servicePort.Name + nameSuffix + + // Merge labels + serviceLabels := make(map[string]string) + for k, v := range baseLabels { + serviceLabels[k] = v + } + for k, v := range extraLabels { serviceLabels[k] = v } + // Ensure annotations map is initialized + if annotations == nil { + annotations = make(map[string]string) + } + service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: servicePort.Name, + Name: serviceName, Namespace: jumpstarter.Namespace, Labels: serviceLabels, Annotations: annotations, @@ -477,6 +509,8 @@ func (r *JumpstarterReconciler) reconcileControllerEndpointService(ctx context.C } service.Spec.Ports = []corev1.ServicePort{servicePort} service.Spec.Type = serviceType + service.Labels = serviceLabels + service.Annotations = annotations return nil }) return err @@ -830,50 +864,113 @@ func (r *JumpstarterReconciler) buildEndpointForReplica(jumpstarter *operatorv1a } // reconcileRouterReplicaEndpoint reconciles service, ingress, and route for a specific router replica endpoint +// This function creates a separate service for each enabled service type (ClusterIP, NodePort, LoadBalancer) func (r *JumpstarterReconciler) reconcileRouterReplicaEndpoint(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32, endpointIdx int, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { // Create service with proper selector pointing to the deployment pods // All services for this replica select the same pods using the base app label baseAppLabel := fmt.Sprintf("%s-router-%d", jumpstarter.Name, replicaIndex) - labels := map[string]string{ + baseLabels := map[string]string{ "app": "jumpstarter-router", "router": jumpstarter.Name, "router-index": fmt.Sprintf("%d", replicaIndex), "endpoint-idx": fmt.Sprintf("%d", endpointIdx), } - // Determine service type and configuration from endpoint - serviceType := corev1.ServiceTypeClusterIP - annotations := make(map[string]string) + // Create a service for each enabled service type + // This allows multiple service types to coexist for the same endpoint + // Note: ClusterIP uses no suffix (most common for in-cluster communication) + // LoadBalancer uses "-lb" suffix, NodePort uses "-np" suffix + + // Ingress service + if endpoint.Ingress != nil && endpoint.Ingress.Enabled { + if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "-ing", baseAppLabel, baseLabels, endpoint.Ingress.Annotations, endpoint.Ingress.Labels); err != nil { + return err + } + } + + // Route service + if endpoint.Route != nil && endpoint.Route.Enabled { + if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "-route", baseAppLabel, baseLabels, endpoint.Route.Annotations, endpoint.Route.Labels); err != nil { + return err + } + } + // LoadBalancer service if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { - serviceType = corev1.ServiceTypeLoadBalancer - for k, v := range endpoint.LoadBalancer.Annotations { - annotations[k] = v + if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeLoadBalancer, "-lb", baseAppLabel, baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { + return err } - for k, v := range endpoint.LoadBalancer.Labels { - labels[k] = v + } + + // NodePort service + if endpoint.NodePort != nil && endpoint.NodePort.Enabled { + if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeNodePort, "-np", baseAppLabel, baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { + return err } - } else if endpoint.NodePort != nil && endpoint.NodePort.Enabled { - serviceType = corev1.ServiceTypeNodePort - for k, v := range endpoint.NodePort.Annotations { - annotations[k] = v + } + + // ClusterIP service (no suffix for cleaner in-cluster service names) + if endpoint.ClusterIP != nil && endpoint.ClusterIP.Enabled { + if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseAppLabel, baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { + return err } - for k, v := range endpoint.NodePort.Labels { - labels[k] = v + } + + // If no service type is explicitly enabled, create a default ClusterIP service + if (endpoint.LoadBalancer == nil || !endpoint.LoadBalancer.Enabled) && + (endpoint.NodePort == nil || !endpoint.NodePort.Enabled) && + (endpoint.ClusterIP == nil || !endpoint.ClusterIP.Enabled) { + if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseAppLabel, baseLabels, nil, nil); err != nil { + return err } } + // Now create ingress/route if configured + // Use the first service (or default) for ingress/route endpoints + // Priority: LoadBalancer > NodePort > ClusterIP (no suffix) + serviceName := servicePort.Name + if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { + serviceName = servicePort.Name + "-lb" + } else if endpoint.NodePort != nil && endpoint.NodePort.Enabled { + serviceName = servicePort.Name + "-np" + } + // ClusterIP uses base name (no suffix), so no else clause needed + + servicePortForEndpoint := servicePort + servicePortForEndpoint.Name = serviceName + return r.EndpointReconciler.ReconcileEndpoint(ctx, jumpstarter.Namespace, endpoint, servicePortForEndpoint) +} + +// createRouterService creates or updates a single router service with the specified type and suffix +func (r *JumpstarterReconciler) createRouterService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort, serviceType corev1.ServiceType, nameSuffix string, podSelector string, baseLabels map[string]string, annotations map[string]string, extraLabels map[string]string) error { + // Build service name with suffix to avoid conflicts + serviceName := servicePort.Name + nameSuffix + + // Merge labels + serviceLabels := make(map[string]string) + for k, v := range baseLabels { + serviceLabels[k] = v + } + for k, v := range extraLabels { + serviceLabels[k] = v + } + + // Ensure annotations map is initialized + if annotations == nil { + annotations = make(map[string]string) + } + service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: servicePort.Name, + Name: serviceName, Namespace: jumpstarter.Namespace, - Labels: labels, + Labels: serviceLabels, Annotations: annotations, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{ - "app": baseAppLabel, // Select pods from this replica's deployment + "app": podSelector, // Select pods from this replica's deployment }, Ports: []corev1.ServicePort{servicePort}, Type: serviceType, @@ -887,27 +984,21 @@ func (r *JumpstarterReconciler) reconcileRouterReplicaEndpoint(ctx context.Conte _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, func() error { // Preserve existing NodePort if the service already exists // This prevents "port already allocated" errors during updates - if len(service.Spec.Ports) > 0 && service.Spec.Ports[0].NodePort != 0 { + if serviceType == corev1.ServiceTypeNodePort && len(service.Spec.Ports) > 0 && service.Spec.Ports[0].NodePort != 0 { // Service already exists with a NodePort, preserve it servicePort.NodePort = service.Spec.Ports[0].NodePort } service.Spec.Selector = map[string]string{ - "app": baseAppLabel, + "app": podSelector, } service.Spec.Ports = []corev1.ServicePort{servicePort} service.Spec.Type = serviceType + service.Labels = serviceLabels + service.Annotations = annotations return nil }) - if err != nil { - return err - } - - // Now create ingress/route if configured - // TODO: Create ingress/route based on endpoint configuration - // For now, EndpointReconciler will try to create a service (which already exists) - // but that's okay as it will just update it - return r.EndpointReconciler.ReconcileEndpoint(ctx, jumpstarter.Namespace, endpoint, servicePort) + return err } // cleanupExcessRouterDeployments deletes router deployments that exceed the current replica count @@ -951,29 +1042,95 @@ func (r *JumpstarterReconciler) cleanupExcessRouterDeployments(ctx context.Conte } // cleanupExcessRouterServices deletes router services that exceed the current replica count +// or endpoint count. This ensures that when replicas or endpoints are scaled down, the +// corresponding services are removed. func (r *JumpstarterReconciler) cleanupExcessRouterServices(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { log := logf.FromContext(ctx) - // Delete services with replica index >= current replica count - // Services are named "{jumpstarter-name}-router-{N}" by our service port naming + // Services can have suffixes for different service types + // ClusterIP has no suffix, LoadBalancer has "-lb", NodePort has "-np" + suffixes := []string{"", "-lb", "-np"} + + // 1. Delete services for excess replicas (replica index >= current replica count) for idx := jumpstarter.Spec.Routers.Replicas; idx < 100; idx++ { // reasonable upper bound - serviceName := fmt.Sprintf("%s-router-%d", jumpstarter.Name, idx) - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Namespace: jumpstarter.Namespace, - }, - } + foundAny := false + + // Try to delete services for all endpoints and service types for this replica + for endpointIdx := 0; endpointIdx < 10; endpointIdx++ { // reasonable upper bound for endpoints + for _, suffix := range suffixes { + var serviceName string + if endpointIdx == 0 { + serviceName = fmt.Sprintf("%s-router-%d%s", jumpstarter.Name, idx, suffix) + } else { + serviceName = fmt.Sprintf("%s-router-%d-%d%s", jumpstarter.Name, idx, endpointIdx, suffix) + } - err := r.Delete(ctx, service) - if err != nil { - if !errors.IsNotFound(err) { - return fmt.Errorf("failed to delete excess service %s: %w", serviceName, err) + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: jumpstarter.Namespace, + }, + } + + err := r.Delete(ctx, service) + if err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete excess service %s: %w", serviceName, err) + } + } else { + foundAny = true + log.Info("Deleted excess router service", "service", serviceName, "replicaIndex", idx, "endpointIdx", endpointIdx) + } } - // If not found, we've gone past all excess services, can stop + } + + // If we didn't find any services for this replica index, we've gone past all excess services + if !foundAny { break } - log.Info("Deleted excess router service", "service", serviceName, "replicaIndex", idx) + } + + // 2. Delete services for excess endpoints within valid replicas + numEndpoints := len(jumpstarter.Spec.Routers.GRPC.Endpoints) + if numEndpoints == 0 { + numEndpoints = 1 // default endpoint + } + + for replicaIdx := int32(0); replicaIdx < jumpstarter.Spec.Routers.Replicas; replicaIdx++ { + for endpointIdx := numEndpoints; endpointIdx < 10; endpointIdx++ { // reasonable upper bound + foundAny := false + + for _, suffix := range suffixes { + var serviceName string + if endpointIdx == 0 { + serviceName = fmt.Sprintf("%s-router-%d%s", jumpstarter.Name, replicaIdx, suffix) + } else { + serviceName = fmt.Sprintf("%s-router-%d-%d%s", jumpstarter.Name, replicaIdx, endpointIdx, suffix) + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: jumpstarter.Namespace, + }, + } + + err := r.Delete(ctx, service) + if err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete excess endpoint service %s: %w", serviceName, err) + } + } else { + foundAny = true + log.Info("Deleted excess endpoint service", "service", serviceName, "replicaIndex", replicaIdx, "endpointIdx", endpointIdx) + } + } + + // If we didn't find any services for this endpoint index, we've gone past all excess endpoints + if !foundAny { + break + } + } } return nil From 32ff96fa84d4e3aa5d5f7889b71e8c3be13671e3 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 22 Oct 2025 09:56:31 +0000 Subject: [PATCH 13/26] operator: use address for endpoints instead of hostname In some cases, the port exposed for the service does not necessarily match what the external address will use for exposing. for example a NodePort that is later port mapped to a different port, a node port that is exposed outside from your lan via a reverse proxy, etc... --- .../api/v1alpha1/jumpstarter_types.go | 13 +++-- ...operator.jumpstarter.dev_jumpstarters.yaml | 57 ++++++++++--------- .../jumpstarter/endpoints/endpoints_test.go | 10 ++-- .../jumpstarter/jumpstarter_controller.go | 50 ++++++++++------ .../jumpstarter_controller_test.go | 36 ++++++++---- hack/deploy_with_operator.sh | 13 +++-- 6 files changed, 109 insertions(+), 70 deletions(-) diff --git a/deploy/operator/api/v1alpha1/jumpstarter_types.go b/deploy/operator/api/v1alpha1/jumpstarter_types.go index 3c6d3d6e..af9d4271 100644 --- a/deploy/operator/api/v1alpha1/jumpstarter_types.go +++ b/deploy/operator/api/v1alpha1/jumpstarter_types.go @@ -376,14 +376,15 @@ type RestAPIConfig struct { // Endpoint defines a single endpoint configuration. // An endpoint can use one or more networking methods: Route, Ingress, NodePort, or LoadBalancer. -// Multiple methods can be configured simultaneously for the same hostname. +// Multiple methods can be configured simultaneously for the same address. type Endpoint struct { - // Hostname for this endpoint. + // Address for this endpoint in the format "hostname", "hostname:port", "IPv4", "IPv4:port", "[IPv6]", or "[IPv6]:port". // Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. - // When optional, the hostname is used for certificate generation and DNS resolution. - // Supports templating with $(replica) for replica-specific hostnames. - // +kubebuilder:validation:Pattern=^[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?$ - Hostname string `json:"hostname,omitempty"` + // When optional, the address is used for certificate generation and DNS resolution. + // Supports templating with $(replica) for replica-specific addresses. + // Examples: "grpc.example.com", "grpc.example.com:9090", "192.168.1.1:8080", "[2001:db8::1]:8443", "router-$(replica).example.com" + // +kubebuilder:validation:Pattern=`^(\[[0-9a-fA-F:\.]+\]|[0-9]+(\.[0-9]+){3}|[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?)(:[0-9]+)?$` + Address string `json:"address,omitempty"` // Route configuration for OpenShift clusters. // Creates an OpenShift Route resource for this endpoint. diff --git a/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml b/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml index 61f069a3..dbb182c3 100644 --- a/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml +++ b/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml @@ -843,8 +843,17 @@ spec: description: |- Endpoint defines a single endpoint configuration. An endpoint can use one or more networking methods: Route, Ingress, NodePort, or LoadBalancer. - Multiple methods can be configured simultaneously for the same hostname. + Multiple methods can be configured simultaneously for the same address. properties: + address: + description: |- + Address for this endpoint in the format "hostname", "hostname:port", "IPv4", "IPv4:port", "[IPv6]", or "[IPv6]:port". + Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. + When optional, the address is used for certificate generation and DNS resolution. + Supports templating with $(replica) for replica-specific addresses. + Examples: "grpc.example.com", "grpc.example.com:9090", "192.168.1.1:8080", "[2001:db8::1]:8443", "router-$(replica).example.com" + pattern: ^(\[[0-9a-fA-F:\.]+\]|[0-9]+(\.[0-9]+){3}|[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?)(:[0-9]+)?$ + type: string clusterIP: description: |- ClusterIP configuration for internal service access. @@ -872,14 +881,6 @@ spec: Useful for monitoring, cost allocation, and resource organization. type: object type: object - hostname: - description: |- - Hostname for this endpoint. - Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. - When optional, the hostname is used for certificate generation and DNS resolution. - Supports templating with $(replica) for replica-specific hostnames. - pattern: ^[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?$ - type: string ingress: description: |- Ingress configuration for standard Kubernetes clusters. @@ -1180,8 +1181,17 @@ spec: description: |- Endpoint defines a single endpoint configuration. An endpoint can use one or more networking methods: Route, Ingress, NodePort, or LoadBalancer. - Multiple methods can be configured simultaneously for the same hostname. + Multiple methods can be configured simultaneously for the same address. properties: + address: + description: |- + Address for this endpoint in the format "hostname", "hostname:port", "IPv4", "IPv4:port", "[IPv6]", or "[IPv6]:port". + Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. + When optional, the address is used for certificate generation and DNS resolution. + Supports templating with $(replica) for replica-specific addresses. + Examples: "grpc.example.com", "grpc.example.com:9090", "192.168.1.1:8080", "[2001:db8::1]:8443", "router-$(replica).example.com" + pattern: ^(\[[0-9a-fA-F:\.]+\]|[0-9]+(\.[0-9]+){3}|[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?)(:[0-9]+)?$ + type: string clusterIP: description: |- ClusterIP configuration for internal service access. @@ -1209,14 +1219,6 @@ spec: Useful for monitoring, cost allocation, and resource organization. type: object type: object - hostname: - description: |- - Hostname for this endpoint. - Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. - When optional, the hostname is used for certificate generation and DNS resolution. - Supports templating with $(replica) for replica-specific hostnames. - pattern: ^[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?$ - type: string ingress: description: |- Ingress configuration for standard Kubernetes clusters. @@ -1384,8 +1386,17 @@ spec: description: |- Endpoint defines a single endpoint configuration. An endpoint can use one or more networking methods: Route, Ingress, NodePort, or LoadBalancer. - Multiple methods can be configured simultaneously for the same hostname. + Multiple methods can be configured simultaneously for the same address. properties: + address: + description: |- + Address for this endpoint in the format "hostname", "hostname:port", "IPv4", "IPv4:port", "[IPv6]", or "[IPv6]:port". + Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. + When optional, the address is used for certificate generation and DNS resolution. + Supports templating with $(replica) for replica-specific addresses. + Examples: "grpc.example.com", "grpc.example.com:9090", "192.168.1.1:8080", "[2001:db8::1]:8443", "router-$(replica).example.com" + pattern: ^(\[[0-9a-fA-F:\.]+\]|[0-9]+(\.[0-9]+){3}|[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?)(:[0-9]+)?$ + type: string clusterIP: description: |- ClusterIP configuration for internal service access. @@ -1413,14 +1424,6 @@ spec: Useful for monitoring, cost allocation, and resource organization. type: object type: object - hostname: - description: |- - Hostname for this endpoint. - Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. - When optional, the hostname is used for certificate generation and DNS resolution. - Supports templating with $(replica) for replica-specific hostnames. - pattern: ^[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?$ - type: string ingress: description: |- Ingress configuration for standard Kubernetes clusters. diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go index 998eb4ec..350b5412 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go @@ -58,7 +58,7 @@ var _ = Describe("Endpoints Reconciler", func() { Context("with ClusterIP service type", func() { It("should create a ClusterIP service successfully", func() { endpoint := &operatorv1alpha1.Endpoint{ - Hostname: endpointName, + Address: endpointName, } svcPort := corev1.ServicePort{ @@ -86,7 +86,7 @@ var _ = Describe("Endpoints Reconciler", func() { Context("with LoadBalancer service type", func() { It("should create a LoadBalancer service successfully", func() { endpoint := &operatorv1alpha1.Endpoint{ - Hostname: endpointName, + Address: endpointName, LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ Enabled: true, Annotations: map[string]string{"service.beta.kubernetes.io/aws-load-balancer-type": "nlb"}, @@ -120,7 +120,7 @@ var _ = Describe("Endpoints Reconciler", func() { Context("with NodePort service type", func() { It("should create a NodePort service successfully", func() { endpoint := &operatorv1alpha1.Endpoint{ - Hostname: endpointName, + Address: endpointName, NodePort: &operatorv1alpha1.NodePortConfig{ Enabled: true, Port: 30090, @@ -155,7 +155,7 @@ var _ = Describe("Endpoints Reconciler", func() { Context("with invalid configuration", func() { It("should return an error when both LoadBalancer and NodePort are enabled", func() { endpoint := &operatorv1alpha1.Endpoint{ - Hostname: endpointName, + Address: endpointName, LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ Enabled: true, }, @@ -181,7 +181,7 @@ var _ = Describe("Endpoints Reconciler", func() { It("should update the service when configuration changes", func() { // Create initial service endpoint := &operatorv1alpha1.Endpoint{ - Hostname: endpointName, + Address: endpointName, LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ Enabled: true, Annotations: map[string]string{"initial": "annotation"}, diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index a27deb5f..ffc0e13b 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -21,6 +21,7 @@ import ( "crypto/rand" "encoding/base64" "fmt" + "net" "strings" "time" @@ -266,7 +267,7 @@ func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstart } else { // No endpoints configured, create a default service without ingress/route endpoint := operatorv1alpha1.Endpoint{ - Hostname: fmt.Sprintf("router-%d.%s", i, jumpstarter.Spec.BaseDomain), + Address: fmt.Sprintf("router-%d.%s", i, jumpstarter.Spec.BaseDomain), } serviceName := fmt.Sprintf("%s-router-%d", jumpstarter.Name, i) @@ -537,8 +538,8 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator grpcEndpoint := "" if len(jumpstarter.Spec.Controller.GRPC.Endpoints) > 0 { ep := jumpstarter.Spec.Controller.GRPC.Endpoints[0] - if ep.Hostname != "" { - grpcEndpoint = fmt.Sprintf("%s:443", ep.Hostname) + if ep.Address != "" { + grpcEndpoint = ensurePort(ep.Address, "443") } else { grpcEndpoint = fmt.Sprintf("grpc.%s:443", jumpstarter.Spec.BaseDomain) } @@ -771,8 +772,8 @@ func (r *JumpstarterReconciler) createConfigMap(jumpstarter *operatorv1alpha1.Ju routerConfig := "default:\n" if len(jumpstarter.Spec.Routers.GRPC.Endpoints) > 0 { ep := jumpstarter.Spec.Routers.GRPC.Endpoints[0] - if ep.Hostname != "" { - routerConfig += fmt.Sprintf(" endpoint: %s:443\n", ep.Hostname) + if ep.Address != "" { + routerConfig += fmt.Sprintf(" endpoint: %s:443\n", ep.Address) } else { routerConfig += fmt.Sprintf(" endpoint: router.%s:443\n", jumpstarter.Spec.BaseDomain) } @@ -813,10 +814,10 @@ func (r *JumpstarterReconciler) buildRouterEndpointForReplica(jumpstarter *opera // If endpoints are specified, use the first one as the primary endpoint if len(jumpstarter.Spec.Routers.GRPC.Endpoints) > 0 { ep := jumpstarter.Spec.Routers.GRPC.Endpoints[0] - hostname := ep.Hostname - if hostname != "" { - hostname = r.substituteReplica(hostname, replicaIndex) - return fmt.Sprintf("%s:443", hostname) + address := ep.Address + if address != "" { + address = r.substituteReplica(address, replicaIndex) + return ensurePort(address, "443") } } // Default pattern: router-N.baseDomain @@ -824,8 +825,8 @@ func (r *JumpstarterReconciler) buildRouterEndpointForReplica(jumpstarter *opera } // substituteReplica replaces $(replica) placeholder with actual replica index -func (r *JumpstarterReconciler) substituteReplica(hostname string, replicaIndex int32) string { - return strings.ReplaceAll(hostname, "$(replica)", fmt.Sprintf("%d", replicaIndex)) +func (r *JumpstarterReconciler) substituteReplica(address string, replicaIndex int32) string { + return strings.ReplaceAll(address, "$(replica)", fmt.Sprintf("%d", replicaIndex)) } // int64Ptr returns a pointer to an int64 value @@ -833,6 +834,21 @@ func int64Ptr(i int64) *int64 { return &i } +// ensurePort adds a default port to an address if it doesn't already have one +// Handles IPv4, IPv6, and hostnames correctly using net.SplitHostPort +func ensurePort(address, defaultPort string) string { + // Try to split the address into host and port + _, _, err := net.SplitHostPort(address) + if err == nil { + // Address already has a port, return as-is + return address + } + + // No port found, need to add one + // net.JoinHostPort handles IPv6 addresses correctly (adds brackets if needed) + return net.JoinHostPort(address, defaultPort) +} + // buildServiceNameForReplicaEndpoint creates a unique service name for a router replica and endpoint func (r *JumpstarterReconciler) buildServiceNameForReplicaEndpoint(jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32, endpointIdx int) string { if endpointIdx == 0 { @@ -848,15 +864,15 @@ func (r *JumpstarterReconciler) buildEndpointForReplica(jumpstarter *operatorv1a // Copy the base endpoint endpoint := *baseEndpoint - // Set or substitute hostname - if endpoint.Hostname != "" { - endpoint.Hostname = r.substituteReplica(endpoint.Hostname, replicaIndex) + // Set or substitute address + if endpoint.Address != "" { + endpoint.Address = r.substituteReplica(endpoint.Address, replicaIndex) } else { - // Default hostname pattern when none specified + // Default address pattern when none specified if endpointIdx == 0 { - endpoint.Hostname = fmt.Sprintf("router-%d.%s", replicaIndex, jumpstarter.Spec.BaseDomain) + endpoint.Address = fmt.Sprintf("router-%d.%s", replicaIndex, jumpstarter.Spec.BaseDomain) } else { - endpoint.Hostname = fmt.Sprintf("router-%d-%d.%s", replicaIndex, endpointIdx, jumpstarter.Spec.BaseDomain) + endpoint.Address = fmt.Sprintf("router-%d-%d.%s", replicaIndex, endpointIdx, jumpstarter.Spec.BaseDomain) } } diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go index 5e8b4a65..dab2bca6 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go @@ -66,14 +66,14 @@ var _ = Describe("Jumpstarter Controller", func() { corev1.ResourceMemory: resource.MustParse("100Mi"), }, }, - GRPC: operatorv1alpha1.GRPCConfig{ - Endpoints: []operatorv1alpha1.Endpoint{ - { - Hostname: "controller", - }, + GRPC: operatorv1alpha1.GRPCConfig{ + Endpoints: []operatorv1alpha1.Endpoint{ + { + Address: "controller", }, }, }, + }, Routers: operatorv1alpha1.RoutersConfig{ Image: "quay.io/jumpstarter/jumpstarter:latest", ImagePullPolicy: "IfNotPresent", @@ -84,14 +84,14 @@ var _ = Describe("Jumpstarter Controller", func() { corev1.ResourceMemory: resource.MustParse("100Mi"), }, }, - GRPC: operatorv1alpha1.GRPCConfig{ - Endpoints: []operatorv1alpha1.Endpoint{ - { - Hostname: "router", - }, + GRPC: operatorv1alpha1.GRPCConfig{ + Endpoints: []operatorv1alpha1.Endpoint{ + { + Address: "router", }, }, }, + }, }, } Expect(k8sClient.Create(ctx, resource)).To(Succeed()) @@ -124,3 +124,19 @@ var _ = Describe("Jumpstarter Controller", func() { }) }) }) + +var _ = Describe("ensurePort", func() { + DescribeTable("should handle addresses correctly", + func(address, defaultPort, expected string) { + result := ensurePort(address, defaultPort) + Expect(result).To(Equal(expected)) + }, + Entry("hostname without port", "example.com", "443", "example.com:443"), + Entry("hostname with port", "example.com:8083", "443", "example.com:8083"), + Entry("IPv6 without port", "2001:db8::1", "443", "[2001:db8::1]:443"), + Entry("IPv6 with port", "[2001:db8::1]:8083", "443", "[2001:db8::1]:8083"), + Entry("malformed - too many colons", "host:port:extra", "443", "[host:port:extra]:443"), + Entry("malformed - empty string", "", "443", ":443"), + Entry("malformed - just colon", ":", "443", ":"), + ) +}) diff --git a/hack/deploy_with_operator.sh b/hack/deploy_with_operator.sh index e33a4435..ec12c445 100755 --- a/hack/deploy_with_operator.sh +++ b/hack/deploy_with_operator.sh @@ -52,14 +52,14 @@ echo -e "${GREEN}Creating Jumpstarter custom resource ...${NC}" # Generate endpoint configuration based on networking mode if [ "${NETWORKING_MODE}" == "ingress" ]; then CONTROLLER_ENDPOINT_CONFIG=$(cat <<-END - - hostname: grpc.${BASEDOMAIN} + - address: grpc.${BASEDOMAIN}:443 ingress: enabled: true class: "" END ) ROUTER_ENDPOINT_CONFIG=$(cat <<-END - - hostname: router.${BASEDOMAIN} + - address: router.${BASEDOMAIN}:443 ingress: enabled: true class: "" @@ -67,14 +67,17 @@ END ) else CONTROLLER_ENDPOINT_CONFIG=$(cat <<-END - - hostname: grpc.${BASEDOMAIN} + # this is exposed by a nodeport in 30010 but mapped to 8082 on the host + - address: grpc.${BASEDOMAIN}:8082 nodeport: enabled: true port: 30010 END ) ROUTER_ENDPOINT_CONFIG=$(cat <<-END - - nodeport: + # this is exposed by a nodeport in 30011 but mapped to 8083 on the host + - address: router.${BASEDOMAIN}:8083 + nodeport: enabled: true port: 30011 END @@ -105,7 +108,7 @@ ${CONTROLLER_ENDPOINT_CONFIG} routers: image: ${IMAGE_REPO} imagePullPolicy: IfNotPresent - replicas: 3 + replicas: 1 resources: requests: cpu: 100m From 8b2d973767dbee49069f0b0256344d9a8ae6fbbe Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 22 Oct 2025 10:39:36 +0000 Subject: [PATCH 14/26] operator: config map from go structures vs manual yaml --- deploy/operator/go.mod | 1 + .../jumpstarter/jumpstarter_controller.go | 161 +++++++++++--- internal/config/grpc.go | 23 +- internal/config/types.go | 93 ++++++-- internal/config/types_test.go | 206 ++++++++++++++++++ 5 files changed, 437 insertions(+), 47 deletions(-) create mode 100644 internal/config/types_test.go diff --git a/deploy/operator/go.mod b/deploy/operator/go.mod index de7ca633..7d873332 100644 --- a/deploy/operator/go.mod +++ b/deploy/operator/go.mod @@ -45,6 +45,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/jumpstarter-dev/jumpstarter-controller v0.7.1 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index ffc0e13b..91798fa7 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -32,14 +32,17 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + apiserverv1beta1 "k8s.io/apiserver/pkg/apis/apiserver/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/jumpstarter/endpoints" loglevels "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/log" + "github.com/jumpstarter-dev/jumpstarter-controller/internal/config" ) // JumpstarterReconciler reconciles a Jumpstarter object @@ -296,7 +299,10 @@ func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstart // reconcileConfigMaps reconciles all configmaps func (r *JumpstarterReconciler) reconcileConfigMaps(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { - configMap := r.createConfigMap(jumpstarter) + configMap, err := r.createConfigMap(jumpstarter) + if err != nil { + return fmt.Errorf("failed to create configmap: %w", err) + } // Set the owner reference if err := controllerutil.SetControllerReference(jumpstarter, configMap, r.Scheme); err != nil { @@ -304,7 +310,7 @@ func (r *JumpstarterReconciler) reconcileConfigMaps(ctx context.Context, jumpsta } // Create or update the configmap - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, configMap, func() error { + _, err = controllerutil.CreateOrUpdate(ctx, r.Client, configMap, func() error { // Update configmap data if needed return nil }) @@ -766,31 +772,24 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al } // createConfigMap creates a configmap for jumpstarter configuration -func (r *JumpstarterReconciler) createConfigMap(jumpstarter *operatorv1alpha1.Jumpstarter) *corev1.ConfigMap { - // Build router configuration - // Default to port 443 for TLS gRPC endpoints - routerConfig := "default:\n" - if len(jumpstarter.Spec.Routers.GRPC.Endpoints) > 0 { - ep := jumpstarter.Spec.Routers.GRPC.Endpoints[0] - if ep.Address != "" { - routerConfig += fmt.Sprintf(" endpoint: %s:443\n", ep.Address) - } else { - routerConfig += fmt.Sprintf(" endpoint: router.%s:443\n", jumpstarter.Spec.BaseDomain) - } +func (r *JumpstarterReconciler) createConfigMap(jumpstarter *operatorv1alpha1.Jumpstarter) (*corev1.ConfigMap, error) { + // Build config struct from spec + cfg := r.buildConfig(jumpstarter) + + // Marshal to YAML + configYAML, err := yaml.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("failed to marshal config to YAML: %w", err) } - // Build config YAML - configYAML := `authentication: - internal: - prefix: internal - jwt: [] -provisioning: - enabled: false -grpc: - keepalive: - minTime: "1s" - permitWithoutStream: true -` + // Build router configuration for all replicas + router := r.buildRouter(jumpstarter) + + // Marshal router to YAML + routerYAML, err := yaml.Marshal(router) + if err != nil { + return nil, fmt.Errorf("failed to marshal router to YAML: %w", err) + } return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -802,10 +801,118 @@ grpc: }, }, Data: map[string]string{ - "config": configYAML, - "router": routerConfig, + "config": string(configYAML), + "router": string(routerYAML), + }, + }, nil +} + +// buildConfig builds the controller configuration struct from the CR spec +func (r *JumpstarterReconciler) buildConfig(jumpstarter *operatorv1alpha1.Jumpstarter) config.Config { + cfg := config.Config{ + Provisioning: config.Provisioning{ + Enabled: false, + }, + Grpc: config.Grpc{ + Keepalive: config.Keepalive{ + MinTime: "1s", + PermitWithoutStream: true, + }, }, } + + // Authentication configuration + auth := config.Authentication{ + JWT: jumpstarter.Spec.Controller.Authentication.JWT, + } + + // Internal authentication + if jumpstarter.Spec.Controller.Authentication.Internal.Enabled { + prefix := jumpstarter.Spec.Controller.Authentication.Internal.Prefix + if prefix == "" { + prefix = "internal:" + } + auth.Internal.Prefix = prefix + + if jumpstarter.Spec.Controller.Authentication.Internal.TokenLifetime != nil { + auth.Internal.TokenLifetime = jumpstarter.Spec.Controller.Authentication.Internal.TokenLifetime.Duration.String() + } + } + + // Kubernetes authentication + if jumpstarter.Spec.Controller.Authentication.K8s.Enabled { + auth.K8s.Enabled = true + } + + // Ensure JWT is an empty array, not null + if auth.JWT == nil { + auth.JWT = []apiserverv1beta1.JWTAuthenticator{} + } + + cfg.Authentication = auth + + // gRPC keepalive configuration + if jumpstarter.Spec.Controller.GRPC.Keepalive != nil { + ka := &cfg.Grpc.Keepalive + + if jumpstarter.Spec.Controller.GRPC.Keepalive.MinTime != nil { + ka.MinTime = jumpstarter.Spec.Controller.GRPC.Keepalive.MinTime.Duration.String() + } + + ka.PermitWithoutStream = jumpstarter.Spec.Controller.GRPC.Keepalive.PermitWithoutStream + + if jumpstarter.Spec.Controller.GRPC.Keepalive.Timeout != nil { + ka.Timeout = jumpstarter.Spec.Controller.GRPC.Keepalive.Timeout.Duration.String() + } + + if jumpstarter.Spec.Controller.GRPC.Keepalive.IntervalTime != nil { + ka.IntervalTime = jumpstarter.Spec.Controller.GRPC.Keepalive.IntervalTime.Duration.String() + } + + if jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionIdle != nil { + ka.MaxConnectionIdle = jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionIdle.Duration.String() + } + + if jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionAge != nil { + ka.MaxConnectionAge = jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionAge.Duration.String() + } + + if jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionAgeGrace != nil { + ka.MaxConnectionAgeGrace = jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionAgeGrace.Duration.String() + } + } + + return cfg +} + +// buildRouter builds the router configuration with entries for all replicas +func (r *JumpstarterReconciler) buildRouter(jumpstarter *operatorv1alpha1.Jumpstarter) config.Router { + router := make(config.Router) + + // Create router entry for each replica + for i := int32(0); i < jumpstarter.Spec.Routers.Replicas; i++ { + // First replica is named "default" for backwards compatibility + routerName := "default" + if i > 0 { + routerName = fmt.Sprintf("router-%d", i) + } + + entry := config.RouterEntry{ + Endpoint: r.buildRouterEndpointForReplica(jumpstarter, i), + } + + // Add labels if this is not the default router (replica 0) + // Additional routers get labels to distinguish them + if i > 0 { + entry.Labels = map[string]string{ + "router-index": fmt.Sprintf("%d", i), + } + } + + router[routerName] = entry + } + + return router } // buildRouterEndpointForReplica builds the GRPC_ROUTER_ENDPOINT for a specific replica diff --git a/internal/config/grpc.go b/internal/config/grpc.go index dcb40097..be17f2cf 100644 --- a/internal/config/grpc.go +++ b/internal/config/grpc.go @@ -1,20 +1,31 @@ package config import ( - "time" + "fmt" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" ) +// LoadGrpcConfiguration loads the gRPC server configuration from the parsed Config struct. +// It creates a gRPC server option with keepalive enforcement policy configured. func LoadGrpcConfiguration(config Grpc) (grpc.ServerOption, error) { - minTime, err := time.ParseDuration(config.Keepalive.MinTime) + ka := config.Keepalive + + // Parse MinTime with default of 1s + minTime, err := ParseDuration(ka.MinTime) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to parse keepalive minTime: %w", err) + } + if minTime == 0 { + minTime = 1e9 // 1 second default } - return grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + // Create the keepalive enforcement policy + policy := keepalive.EnforcementPolicy{ MinTime: minTime, - PermitWithoutStream: config.Keepalive.PermitWithoutStream, - }), nil + PermitWithoutStream: ka.PermitWithoutStream, + } + + return grpc.KeepaliveEnforcementPolicy(policy), nil } diff --git a/internal/config/types.go b/internal/config/types.go index d3665b0e..805a022e 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -1,40 +1,105 @@ package config import ( + "time" + apiserverv1beta1 "k8s.io/apiserver/pkg/apis/apiserver/v1beta1" ) +// Config represents the main controller configuration structure. +// This matches the YAML structure in the ConfigMap's "config" key. type Config struct { - Authentication Authentication `json:"authentication"` - Provisioning Provisioning `json:"provisioning"` - Grpc Grpc `json:"grpc"` + Authentication Authentication `json:"authentication" yaml:"authentication"` + Provisioning Provisioning `json:"provisioning" yaml:"provisioning"` + Grpc Grpc `json:"grpc" yaml:"grpc"` } +// Authentication defines the authentication configuration for the controller. +// Supports multiple authentication methods: internal tokens, Kubernetes tokens, and JWT. type Authentication struct { - Internal Internal `json:"internal"` - JWT []apiserverv1beta1.JWTAuthenticator `json:"jwt"` + Internal Internal `json:"internal" yaml:"internal"` + K8s K8s `json:"k8s,omitempty" yaml:"k8s,omitempty"` + JWT []apiserverv1beta1.JWTAuthenticator `json:"jwt" yaml:"jwt"` } -type Provisioning struct { - Enabled bool `json:"enabled"` +// Internal defines the internal token authentication configuration. +type Internal struct { + // Prefix to add to the subject claim of issued tokens (e.g., "internal:") + Prefix string `json:"prefix" yaml:"prefix"` + + // TokenLifetime defines how long issued tokens are valid. + // Parsed as a Go duration string (e.g., "43800h", "30d"). + TokenLifetime string `json:"tokenLifetime,omitempty" yaml:"tokenLifetime,omitempty"` } -type Internal struct { - Prefix string `json:"prefix"` +// K8s defines the Kubernetes service account token authentication configuration. +type K8s struct { + // Enabled indicates whether Kubernetes authentication is enabled. + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` } +// Provisioning defines the provisioning configuration. +type Provisioning struct { + Enabled bool `json:"enabled" yaml:"enabled"` +} + +// Grpc defines the gRPC server configuration. type Grpc struct { - Keepalive Keepalive `json:"keepalive"` + Keepalive Keepalive `json:"keepalive" yaml:"keepalive"` } +// Keepalive defines the gRPC keepalive configuration. +// All duration fields are parsed as Go duration strings (e.g., "1s", "10s", "180s"). type Keepalive struct { - MinTime string `json:"minTime"` - PermitWithoutStream bool `json:"permitWithoutStream"` + // MinTime is the minimum time between keepalives that the server will accept. + // Default: "1s" + MinTime string `json:"minTime,omitempty" yaml:"minTime,omitempty"` + + // PermitWithoutStream allows keepalive pings even when there are no active streams. + // Default: true + PermitWithoutStream bool `json:"permitWithoutStream,omitempty" yaml:"permitWithoutStream,omitempty"` + + // Timeout is the duration to wait for a keepalive ping acknowledgment. + // Default: "180s" + Timeout string `json:"timeout,omitempty" yaml:"timeout,omitempty"` + + // IntervalTime is the duration between keepalive pings. + // Default: "10s" + IntervalTime string `json:"intervalTime,omitempty" yaml:"intervalTime,omitempty"` + + // MaxConnectionIdle is the maximum duration a connection can be idle before being closed. + // Default: infinity (not set) + MaxConnectionIdle string `json:"maxConnectionIdle,omitempty" yaml:"maxConnectionIdle,omitempty"` + + // MaxConnectionAge is the maximum age of a connection before it is closed. + // Default: infinity (not set) + MaxConnectionAge string `json:"maxConnectionAge,omitempty" yaml:"maxConnectionAge,omitempty"` + + // MaxConnectionAgeGrace is the grace period for closing connections that exceed MaxConnectionAge. + // Default: infinity (not set) + MaxConnectionAgeGrace string `json:"maxConnectionAgeGrace,omitempty" yaml:"maxConnectionAgeGrace,omitempty"` } +// Router represents the router configuration mapping. +// This is a map where keys are router names (e.g., "default", "router-1", "router-2") +// and values are RouterEntry structs containing endpoint and label information. +// This matches the YAML structure in the ConfigMap's "router" key. type Router map[string]RouterEntry +// RouterEntry defines a single router endpoint configuration. type RouterEntry struct { - Endpoint string `json:"endpoint"` - Labels map[string]string `json:"labels"` + // Endpoint is the router's gRPC endpoint address (e.g., "router-0.example.com:443") + Endpoint string `json:"endpoint" yaml:"endpoint"` + + // Labels are optional labels to associate with this router entry. + // Used to distinguish between different router instances. + Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` +} + +// ParseDuration is a helper to parse duration strings with better error messages. +func ParseDuration(s string) (time.Duration, error) { + if s == "" { + return 0, nil + } + return time.ParseDuration(s) } diff --git a/internal/config/types_test.go b/internal/config/types_test.go new file mode 100644 index 00000000..a36e8ce3 --- /dev/null +++ b/internal/config/types_test.go @@ -0,0 +1,206 @@ +package config + +import ( + "testing" + + apiserverv1beta1 "k8s.io/apiserver/pkg/apis/apiserver/v1beta1" + "sigs.k8s.io/yaml" +) + +func TestConfigRoundTrip(t *testing.T) { + // Create a config struct + original := Config{ + Authentication: Authentication{ + Internal: Internal{ + Prefix: "internal:", + TokenLifetime: "43800h", + }, + K8s: K8s{ + Enabled: true, + }, + JWT: []apiserverv1beta1.JWTAuthenticator{}, // Empty array + }, + Provisioning: Provisioning{ + Enabled: false, + }, + Grpc: Grpc{ + Keepalive: Keepalive{ + MinTime: "1s", + PermitWithoutStream: true, + Timeout: "180s", + IntervalTime: "10s", + }, + }, + } + + // Marshal to YAML + yamlData, err := yaml.Marshal(original) + if err != nil { + t.Fatalf("Failed to marshal config: %v", err) + } + + // Unmarshal back to struct + var parsed Config + err = yaml.Unmarshal(yamlData, &parsed) + if err != nil { + t.Fatalf("Failed to unmarshal config: %v", err) + } + + // Verify key fields + if parsed.Authentication.Internal.Prefix != original.Authentication.Internal.Prefix { + t.Errorf("Internal prefix mismatch: got %s, want %s", + parsed.Authentication.Internal.Prefix, original.Authentication.Internal.Prefix) + } + + if parsed.Grpc.Keepalive.MinTime != original.Grpc.Keepalive.MinTime { + t.Errorf("Keepalive minTime mismatch: got %s, want %s", + parsed.Grpc.Keepalive.MinTime, original.Grpc.Keepalive.MinTime) + } + + if parsed.Grpc.Keepalive.PermitWithoutStream != original.Grpc.Keepalive.PermitWithoutStream { + t.Errorf("Keepalive permitWithoutStream mismatch: got %v, want %v", + parsed.Grpc.Keepalive.PermitWithoutStream, original.Grpc.Keepalive.PermitWithoutStream) + } +} + +func TestRouterRoundTrip(t *testing.T) { + // Create a router config + original := Router{ + "default": RouterEntry{ + Endpoint: "router-0.example.com:443", + }, + "router-1": RouterEntry{ + Endpoint: "router-1.example.com:443", + Labels: map[string]string{ + "router-index": "1", + }, + }, + "router-2": RouterEntry{ + Endpoint: "router-2.example.com:443", + Labels: map[string]string{ + "router-index": "2", + "zone": "us-east", + }, + }, + } + + // Marshal to YAML + yamlData, err := yaml.Marshal(original) + if err != nil { + t.Fatalf("Failed to marshal router: %v", err) + } + + t.Logf("Generated YAML:\n%s", string(yamlData)) + + // Unmarshal back to struct + var parsed Router + err = yaml.Unmarshal(yamlData, &parsed) + if err != nil { + t.Fatalf("Failed to unmarshal router: %v", err) + } + + // Verify all routers exist + if len(parsed) != len(original) { + t.Errorf("Router count mismatch: got %d, want %d", len(parsed), len(original)) + } + + // Verify default router + if entry, exists := parsed["default"]; !exists { + t.Error("Missing 'default' router") + } else if entry.Endpoint != original["default"].Endpoint { + t.Errorf("Default router endpoint mismatch: got %s, want %s", + entry.Endpoint, original["default"].Endpoint) + } + + // Verify router-1 + if entry, exists := parsed["router-1"]; !exists { + t.Error("Missing 'router-1' router") + } else { + if entry.Endpoint != original["router-1"].Endpoint { + t.Errorf("Router-1 endpoint mismatch: got %s, want %s", + entry.Endpoint, original["router-1"].Endpoint) + } + if entry.Labels["router-index"] != "1" { + t.Errorf("Router-1 index label mismatch: got %s, want 1", + entry.Labels["router-index"]) + } + } + + // Verify router-2 labels + if entry, exists := parsed["router-2"]; !exists { + t.Error("Missing 'router-2' router") + } else { + if len(entry.Labels) != 2 { + t.Errorf("Router-2 label count mismatch: got %d, want 2", len(entry.Labels)) + } + } +} + +func TestParseYAMLToRouter(t *testing.T) { + // Test parsing actual YAML string (like from ConfigMap) + yamlInput := ` +default: + endpoint: router.example.com:443 +router-1: + endpoint: router-1.example.com:443 + labels: + router-index: "1" +router-2: + endpoint: router-2.example.com:443 + labels: + router-index: "2" +` + + var router Router + err := yaml.Unmarshal([]byte(yamlInput), &router) + if err != nil { + t.Fatalf("Failed to unmarshal YAML: %v", err) + } + + // Verify structure + if len(router) != 3 { + t.Errorf("Expected 3 routers, got %d", len(router)) + } + + // Verify default has no labels + if defaultEntry, exists := router["default"]; exists { + if len(defaultEntry.Labels) != 0 { + t.Errorf("Default router should have no labels, got %d", len(defaultEntry.Labels)) + } + } + + // Verify router-1 has labels + if router1, exists := router["router-1"]; exists { + if len(router1.Labels) == 0 { + t.Error("Router-1 should have labels") + } + } +} + +func TestParseDuration(t *testing.T) { + tests := []struct { + input string + wantErr bool + expected string + }{ + {"1s", false, "1s"}, + {"10s", false, "10s"}, + {"1m", false, "1m0s"}, + {"1h", false, "1h0m0s"}, + {"", false, "0s"}, // empty string returns 0 + {"invalid", true, ""}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + duration, err := ParseDuration(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("ParseDuration(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) + return + } + if !tt.wantErr && duration.String() != tt.expected { + t.Errorf("ParseDuration(%q) = %v, want %v", tt.input, duration, tt.expected) + } + }) + } +} From 94520d4a4fd39996e310fc7bc3ea426ee1465fae Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 22 Oct 2025 11:22:50 +0000 Subject: [PATCH 15/26] operator: move build docker context to toplevel the operator needs access to some of the config structures defined in the service itself, and eventually may need access to the APIs defined in the sercice, so we need access to those source files in the build context. --- .../Dockerfile => Dockerfile.operator | 21 ++++++++++++------- deploy/operator/Makefile | 8 +++---- 2 files changed, 18 insertions(+), 11 deletions(-) rename deploy/operator/Dockerfile => Dockerfile.operator (74%) diff --git a/deploy/operator/Dockerfile b/Dockerfile.operator similarity index 74% rename from deploy/operator/Dockerfile rename to Dockerfile.operator index 3ebc0a9e..ccbd42a8 100644 --- a/deploy/operator/Dockerfile +++ b/Dockerfile.operator @@ -7,18 +7,25 @@ ARG GIT_COMMIT=unknown ARG BUILD_DATE=unknown # Copy the Go Modules manifests +COPY --chown=1001:0 deploy/operator/go.mod deploy/operator/go.mod +COPY --chown=1001:0 deploy/operator/go.sum deploy/operator/go.sum COPY --chown=1001:0 go.mod go.mod COPY --chown=1001:0 go.sum go.sum + + # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ - go mod download + cd deploy/operator && go mod download -# Copy the go source -COPY --chown=1001:0 cmd/ cmd/ -COPY --chown=1001:0 api/ api/ +# Copy the base jumpstarter-controller internal/config parts COPY --chown=1001:0 internal/ internal/ +COPY --chown=1001:0 api/ api/ +# Copy the go source +COPY --chown=1001:0 deploy/operator/cmd/ deploy/operator/cmd/ +COPY --chown=1001:0 deploy/operator/api/ deploy/operator/api/ +COPY --chown=1001:0 deploy/operator/internal/ deploy/operator/internal/ # Build # the GOARCH has not a default value to allow the binary be built according to the host where the command @@ -28,13 +35,13 @@ COPY --chown=1001:0 internal/ internal/ RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ - go build -a \ + cd deploy/operator && go build -a \ -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ -o manager cmd/main.go FROM registry.access.redhat.com/ubi9/ubi-micro:9.5 WORKDIR / -COPY --from=builder /opt/app-root/src/manager . +COPY --from=builder /opt/app-root/src/deploy/operator/manager . USER 65532:65532 -ENTRYPOINT ["/manager"] +ENTRYPOINT ["/manager"] \ No newline at end of file diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 2e61dc97..3ab01b2f 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -182,7 +182,7 @@ docker-build: ## Build docker image with the manager. --build-arg GIT_VERSION=$(GIT_VERSION) \ --build-arg GIT_COMMIT=$(GIT_COMMIT) \ --build-arg BUILD_DATE=$(BUILD_DATE) \ - -t ${IMG} . + -t ${IMG} ../../ -f ../../Dockerfile.operator .PHONY: docker-push docker-push: ## Push docker image with the manager. @@ -198,16 +198,16 @@ PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le .PHONY: docker-buildx docker-buildx: ## Build and push docker image for the manager for cross-platform support # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' ../../Dockerfile.operator > ../../Dockerfile.operator.cross - $(CONTAINER_TOOL) buildx create --name jumpstarter-operator-builder $(CONTAINER_TOOL) buildx use jumpstarter-operator-builder - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) \ --build-arg GIT_VERSION=$(GIT_VERSION) \ --build-arg GIT_COMMIT=$(GIT_COMMIT) \ --build-arg BUILD_DATE=$(BUILD_DATE) \ - --tag ${IMG} -f Dockerfile.cross . + --tag ${IMG} -f ../../Dockerfile.operator.cross ../../ - $(CONTAINER_TOOL) buildx rm jumpstarter-operator-builder - rm Dockerfile.cross + rm ../../Dockerfile.operator.cross .PHONY: build-installer build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. From 824d7437a223f58f61f424011ff7d50f4ef1ee45 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 22 Oct 2025 10:58:27 +0000 Subject: [PATCH 16/26] operator: move endpoint handling to endpoint reconciler --- deploy/operator/cmd/main.go | 2 +- deploy/operator/go.mod | 40 ++- deploy/operator/go.sum | 93 +++++++ .../jumpstarter/endpoints/endpoints.go | 198 +++++++++++++- .../jumpstarter/endpoints/endpoints_test.go | 2 +- .../jumpstarter/jumpstarter_controller.go | 253 +----------------- .../jumpstarter_controller_test.go | 22 +- 7 files changed, 342 insertions(+), 268 deletions(-) diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index b63b907e..f1f615db 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -218,7 +218,7 @@ func main() { if err := (&jumpstarter.JumpstarterReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - EndpointReconciler: endpoints.NewReconciler(mgr.GetClient()), + EndpointReconciler: endpoints.NewReconciler(mgr.GetClient(), mgr.GetScheme()), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Jumpstarter") os.Exit(1) diff --git a/deploy/operator/go.mod b/deploy/operator/go.mod index 7d873332..0aa25f51 100644 --- a/deploy/operator/go.mod +++ b/deploy/operator/go.mod @@ -3,6 +3,7 @@ module github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator go 1.24.0 require ( + github.com/jumpstarter-dev/jumpstarter-controller v0.7.1 github.com/onsi/ginkgo/v2 v2.22.2 github.com/onsi/gomega v1.36.2 k8s.io/api v0.33.0 @@ -10,55 +11,88 @@ require ( k8s.io/apiserver v0.33.0 k8s.io/client-go v0.33.0 sigs.k8s.io/controller-runtime v0.21.0 + sigs.k8s.io/yaml v1.4.0 ) replace github.com/jumpstarter-dev/jumpstarter-controller => ../../ require ( cel.dev/expr v0.19.1 // indirect + filippo.io/bigmod v0.0.3 // indirect + filippo.io/keygen v0.0.0-20240718133620-7f162efbbd87 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/bmatcuk/doublestar/v4 v4.8.0 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/coreos/go-oidc v2.3.0+incompatible // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.10.0 // indirect + github.com/go-chi/chi/v5 v5.2.0 // indirect + github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/cel-go v0.23.2 // indirect github.com/google/gnostic-models v0.6.9 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/securecookie v1.1.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/jumpstarter-dev/jumpstarter-controller v0.7.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/muhlemmer/gu v0.3.1 // indirect + github.com/muhlemmer/httpforwarded v0.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pquerna/cachecontrol v0.1.0 // indirect github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/rs/cors v1.11.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/zitadel/logging v0.6.1 // indirect + github.com/zitadel/oidc/v3 v3.34.1 // indirect + github.com/zitadel/schema v1.3.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect go.opentelemetry.io/otel v1.33.0 // indirect @@ -70,6 +104,8 @@ require ( go.opentelemetry.io/proto/otlp v1.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect @@ -85,6 +121,7 @@ require ( google.golang.org/grpc v1.70.0 // indirect google.golang.org/protobuf v1.36.5 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.33.0 // indirect @@ -96,5 +133,4 @@ require ( sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/deploy/operator/go.sum b/deploy/operator/go.sum index 8a062a99..c64b8234 100644 --- a/deploy/operator/go.sum +++ b/deploy/operator/go.sum @@ -1,15 +1,33 @@ cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +filippo.io/bigmod v0.0.3 h1:qmdCFHmEMS+PRwzrW6eUrgA4Q3T8D6bRcjsypDMtWHM= +filippo.io/bigmod v0.0.3/go.mod h1:WxGvOYE0OUaBC2N112Dflb3CjOnMBuNRA2UWZc2UbPE= +filippo.io/keygen v0.0.0-20240718133620-7f162efbbd87 h1:HlcHAMbI9Xvw3aWnhPngghMl5AKE2GOvjmvSGOKzCcI= +filippo.io/keygen v0.0.0-20240718133620-7f162efbbd87/go.mod h1:nAs0+DyACEQGudhkTwlPC9atyqDYC7ZotgZR7D8OwXM= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmatcuk/doublestar/v4 v4.8.0 h1:DSXtrypQddoug1459viM9X9D3dp1Z7993fw36I2kNcQ= +github.com/bmatcuk/doublestar/v4 v4.8.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3 h1:oe6fCvaEpkhyW3qAicT0TnGtyht/UrgvOwMcEgLb7Aw= +github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3/go.mod h1:qdP0gaj0QtgX2RUZhnlVrceJ+Qln8aSlDyJwelLLFeM= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/coreos/go-oidc v2.3.0+incompatible h1:+5vEsrgprdLjjQ9FzIKAzQz1wwPD+83hQRfUIPh7rO0= +github.com/coreos/go-oidc v2.3.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -28,6 +46,16 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-chi/chi/v5 v5.2.0 h1:Aj1EtB0qR2Rdo2dG4O94RIU35w2lvQSj6BRA4+qwFL0= +github.com/go-chi/chi/v5 v5.2.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= +github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -43,10 +71,24 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= @@ -65,6 +107,8 @@ github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/Z github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= +github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -77,6 +121,10 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -86,23 +134,35 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/muhlemmer/gu v0.3.1 h1:7EAqmFrW7n3hETvuAdmFmn4hS8W+z3LgKtrnow+YzNM= +github.com/muhlemmer/gu v0.3.1/go.mod h1:YHtHR+gxM+bKEIIs7Hmi9sPT3ZDUvTN/i88wQpZkrdM= +github.com/muhlemmer/httpforwarded v0.1.0 h1:x4DLrzXdliq8mprgUMR0olDvHGkou5BJsK/vWUetyzY= +github.com/muhlemmer/httpforwarded v0.1.0/go.mod h1:yo9czKedo2pdZhoXe+yDkGVbU0TJ0q9oQ90BVoDEtw0= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc= +github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= @@ -113,7 +173,11 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -126,15 +190,29 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zitadel/logging v0.6.1 h1:Vyzk1rl9Kq9RCevcpX6ujUaTYFX43aa4LkvV1TvUk+Y= +github.com/zitadel/logging v0.6.1/go.mod h1:Y4CyAXHpl3Mig6JOszcV5Rqqsojj+3n7y2F591Mp/ow= +github.com/zitadel/oidc/v3 v3.34.1 h1:/rxx2HxEowd8Sdb8sxcRxTu9pLy3/TXBLrewKOUMTHA= +github.com/zitadel/oidc/v3 v3.34.1/go.mod h1:lhAdAP1iWAnpfWF8CWNiO6yKvGFtPMuAubPwP5JC7Ec= +github.com/zitadel/schema v1.3.0 h1:kQ9W9tvIwZICCKWcMvCEweXET1OcOyGEuFbHs4o5kg0= +github.com/zitadel/schema v1.3.0/go.mod h1:NptN6mkBDFvERUCvZHlvWmmME+gmZ44xzwRXwhzsbtc= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= @@ -161,9 +239,14 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -184,6 +267,9 @@ golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= @@ -203,6 +289,7 @@ golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= @@ -219,8 +306,12 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= +gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -242,6 +333,8 @@ k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUy k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go index 124b153a..881a9ebc 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go @@ -21,17 +21,20 @@ import ( "errors" "fmt" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/utils" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // Reconciler provides endpoint reconciliation functionality type Reconciler struct { Client client.Client + Scheme *runtime.Scheme } // serviceDetails contains the configuration details for creating a service @@ -43,9 +46,10 @@ type serviceDetails struct { } // NewReconciler creates a new endpoint reconciler -func NewReconciler(client client.Client) *Reconciler { +func NewReconciler(client client.Client, scheme *runtime.Scheme) *Reconciler { return &Reconciler{ Client: client, + Scheme: scheme, } } @@ -202,3 +206,191 @@ func serviceDetailsForEndpoint(endpoint operatorv1alpha1.Endpoint) (*serviceDeta Suffix: "", }, nil } + +// ReconcileControllerEndpoint reconciles a controller endpoint service with proper pod selector +// This function creates a separate service for each enabled service type (ClusterIP, NodePort, LoadBalancer) +func (r *Reconciler) ReconcileControllerEndpoint(ctx context.Context, owner metav1.Object, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { + // Controller pods have fixed labels: app=jumpstarter-controller + // We need to create a service with selector matching those labels + baseLabels := map[string]string{ + "app": "jumpstarter-controller", + "controller": owner.GetName(), + } + + // Create a service for each enabled service type + // This allows multiple service types to coexist for the same endpoint + // Note: ClusterIP uses no suffix (most common for in-cluster communication) + // LoadBalancer uses "-lb" suffix, NodePort uses "-np" suffix + + // LoadBalancer service + if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { + if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeLoadBalancer, "-lb", "jumpstarter-controller", baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { + return err + } + } + + // NodePort service + if endpoint.NodePort != nil && endpoint.NodePort.Enabled { + if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeNodePort, "-np", "jumpstarter-controller", baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { + return err + } + } + + // ClusterIP service (no suffix for cleaner in-cluster service names) + if endpoint.ClusterIP != nil && endpoint.ClusterIP.Enabled { + if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", "jumpstarter-controller", baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { + return err + } + } + + // If no service type is explicitly enabled, create a default ClusterIP service + if (endpoint.LoadBalancer == nil || !endpoint.LoadBalancer.Enabled) && + (endpoint.NodePort == nil || !endpoint.NodePort.Enabled) && + (endpoint.ClusterIP == nil || !endpoint.ClusterIP.Enabled) { + + // TODO: Default to Route or Ingress depending of the type of cluster + if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", "jumpstarter-controller", baseLabels, nil, nil); err != nil { + return err + } + } + + return nil +} + +// ReconcileRouterReplicaEndpoint reconciles service, ingress, and route for a specific router replica endpoint +// This function creates a separate service for each enabled service type (ClusterIP, NodePort, LoadBalancer) +func (r *Reconciler) ReconcileRouterReplicaEndpoint(ctx context.Context, owner metav1.Object, replicaIndex int32, endpointIdx int, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { + // Create service with proper selector pointing to the deployment pods + // All services for this replica select the same pods using the base app label + baseAppLabel := fmt.Sprintf("%s-router-%d", owner.GetName(), replicaIndex) + + baseLabels := map[string]string{ + "app": "jumpstarter-router", + "router": owner.GetName(), + "router-index": fmt.Sprintf("%d", replicaIndex), + "endpoint-idx": fmt.Sprintf("%d", endpointIdx), + } + + // Create a service for each enabled service type + // This allows multiple service types to coexist for the same endpoint + // Note: ClusterIP uses no suffix (most common for in-cluster communication) + // LoadBalancer uses "-lb" suffix, NodePort uses "-np" suffix + + // Ingress service + if endpoint.Ingress != nil && endpoint.Ingress.Enabled { + if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "-ing", baseAppLabel, baseLabels, endpoint.Ingress.Annotations, endpoint.Ingress.Labels); err != nil { + return err + } + } + + // Route service + if endpoint.Route != nil && endpoint.Route.Enabled { + if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "-route", baseAppLabel, baseLabels, endpoint.Route.Annotations, endpoint.Route.Labels); err != nil { + return err + } + } + + // LoadBalancer service + if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { + if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeLoadBalancer, "-lb", baseAppLabel, baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { + return err + } + } + + // NodePort service + if endpoint.NodePort != nil && endpoint.NodePort.Enabled { + if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeNodePort, "-np", baseAppLabel, baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { + return err + } + } + + // ClusterIP service (no suffix for cleaner in-cluster service names) + if endpoint.ClusterIP != nil && endpoint.ClusterIP.Enabled { + if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseAppLabel, baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { + return err + } + } + + // If no service type is explicitly enabled, create a default ClusterIP service + if (endpoint.LoadBalancer == nil || !endpoint.LoadBalancer.Enabled) && + (endpoint.NodePort == nil || !endpoint.NodePort.Enabled) && + (endpoint.ClusterIP == nil || !endpoint.ClusterIP.Enabled) { + if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseAppLabel, baseLabels, nil, nil); err != nil { + return err + } + } + + // Now create ingress/route if configured + // Use the first service (or default) for ingress/route endpoints + // Priority: LoadBalancer > NodePort > ClusterIP (no suffix) + serviceName := servicePort.Name + if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { + serviceName = servicePort.Name + "-lb" + } else if endpoint.NodePort != nil && endpoint.NodePort.Enabled { + serviceName = servicePort.Name + "-np" + } + // ClusterIP uses base name (no suffix), so no else clause needed + + servicePortForEndpoint := servicePort + servicePortForEndpoint.Name = serviceName + return r.ReconcileEndpoint(ctx, owner.GetNamespace(), endpoint, servicePortForEndpoint) +} + +// createService creates or updates a single service with the specified type and suffix +func (r *Reconciler) createService(ctx context.Context, owner metav1.Object, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort, serviceType corev1.ServiceType, nameSuffix string, podSelector string, baseLabels map[string]string, annotations map[string]string, extraLabels map[string]string) error { + // Build service name with suffix to avoid conflicts + serviceName := servicePort.Name + nameSuffix + + // Merge labels + serviceLabels := make(map[string]string) + for k, v := range baseLabels { + serviceLabels[k] = v + } + for k, v := range extraLabels { + serviceLabels[k] = v + } + + // Ensure annotations map is initialized + if annotations == nil { + annotations = make(map[string]string) + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: owner.GetNamespace(), + Labels: serviceLabels, + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": podSelector, // Select pods with the specified selector + }, + Ports: []corev1.ServicePort{servicePort}, + Type: serviceType, + }, + } + + if err := controllerutil.SetControllerReference(owner, service, r.Scheme); err != nil { + return err + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, func() error { + // Preserve existing NodePort if the service already exists + // This prevents "port already allocated" errors during updates + if serviceType == corev1.ServiceTypeNodePort && len(service.Spec.Ports) > 0 && service.Spec.Ports[0].NodePort != 0 { + // Service already exists with a NodePort, preserve it + servicePort.NodePort = service.Spec.Ports[0].NodePort + } + + service.Spec.Selector = map[string]string{ + "app": podSelector, + } + service.Spec.Ports = []corev1.ServicePort{servicePort} + service.Spec.Type = serviceType + service.Labels = serviceLabels + service.Annotations = annotations + return nil + }) + return err +} diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go index 350b5412..6394e38f 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go @@ -41,7 +41,7 @@ var _ = Describe("Endpoints Reconciler", func() { var reconciler *Reconciler BeforeEach(func() { - reconciler = NewReconciler(k8sClient) + reconciler = NewReconciler(k8sClient, k8sClient.Scheme()) // Create the test namespace ns := &corev1.Namespace{ diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 91798fa7..811593d3 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -233,7 +233,7 @@ func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstart if endpoint.NodePort != nil && endpoint.NodePort.Enabled && endpoint.NodePort.Port > 0 { svcPort.NodePort = endpoint.NodePort.Port } - if err := r.reconcileControllerEndpointService(ctx, jumpstarter, &endpoint, svcPort); err != nil { + if err := r.EndpointReconciler.ReconcileControllerEndpoint(ctx, jumpstarter, &endpoint, svcPort); err != nil { return err } } @@ -263,7 +263,7 @@ func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstart // consecutive, but this is mostly for E2E testing. svcPort.NodePort = endpoint.NodePort.Port + int32(i) } - if err := r.reconcileRouterReplicaEndpoint(ctx, jumpstarter, i, endpointIdx, &endpoint, svcPort); err != nil { + if err := r.EndpointReconciler.ReconcileRouterReplicaEndpoint(ctx, jumpstarter, i, endpointIdx, &endpoint, svcPort); err != nil { return err } } @@ -282,7 +282,7 @@ func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstart Protocol: corev1.ProtocolTCP, AppProtocol: &appProtocol, } - if err := r.reconcileRouterReplicaEndpoint(ctx, jumpstarter, i, 0, &endpoint, svcPort); err != nil { + if err := r.EndpointReconciler.ReconcileRouterReplicaEndpoint(ctx, jumpstarter, i, 0, &endpoint, svcPort); err != nil { return err } } @@ -414,115 +414,6 @@ func generateRandomKey(length int) (string, error) { return base64.URLEncoding.EncodeToString(bytes), nil } -// reconcileEndpointService reconciles a single endpoint service -func (r *JumpstarterReconciler) reconcileEndpointService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { - // Use the endpoint reconciler to create/update the service - return r.EndpointReconciler.ReconcileEndpoint(ctx, jumpstarter.Namespace, endpoint, servicePort) -} - -// reconcileControllerEndpointService reconciles a controller endpoint service with proper pod selector -// This function creates a separate service for each enabled service type (ClusterIP, NodePort, LoadBalancer) -func (r *JumpstarterReconciler) reconcileControllerEndpointService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { - // Controller pods have fixed labels: app=jumpstarter-controller - // We need to create a service with selector matching those labels - baseLabels := map[string]string{ - "app": "jumpstarter-controller", - "controller": jumpstarter.Name, - } - - // Create a service for each enabled service type - // This allows multiple service types to coexist for the same endpoint - // Note: ClusterIP uses no suffix (most common for in-cluster communication) - // LoadBalancer uses "-lb" suffix, NodePort uses "-np" suffix - - // LoadBalancer service - if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { - if err := r.createControllerService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeLoadBalancer, "-lb", baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { - return err - } - } - - // NodePort service - if endpoint.NodePort != nil && endpoint.NodePort.Enabled { - if err := r.createControllerService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeNodePort, "-np", baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { - return err - } - } - - // ClusterIP service (no suffix for cleaner in-cluster service names) - if endpoint.ClusterIP != nil && endpoint.ClusterIP.Enabled { - if err := r.createControllerService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { - return err - } - } - - // If no service type is explicitly enabled, create a default ClusterIP service - if (endpoint.LoadBalancer == nil || !endpoint.LoadBalancer.Enabled) && - (endpoint.NodePort == nil || !endpoint.NodePort.Enabled) && - (endpoint.ClusterIP == nil || !endpoint.ClusterIP.Enabled) { - - // TODO: Default to Route or Ingress depending of the type of cluster - if err := r.createControllerService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseLabels, nil, nil); err != nil { - return err - } - } - - return nil -} - -// createControllerService creates or updates a single controller service with the specified type and suffix -func (r *JumpstarterReconciler) createControllerService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort, serviceType corev1.ServiceType, nameSuffix string, baseLabels map[string]string, annotations map[string]string, extraLabels map[string]string) error { - // Build service name with suffix to avoid conflicts - serviceName := servicePort.Name + nameSuffix - - // Merge labels - serviceLabels := make(map[string]string) - for k, v := range baseLabels { - serviceLabels[k] = v - } - for k, v := range extraLabels { - serviceLabels[k] = v - } - - // Ensure annotations map is initialized - if annotations == nil { - annotations = make(map[string]string) - } - - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Namespace: jumpstarter.Namespace, - Labels: serviceLabels, - Annotations: annotations, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": "jumpstarter-controller", // Match controller pod labels - }, - Ports: []corev1.ServicePort{servicePort}, - Type: serviceType, - }, - } - - if err := controllerutil.SetControllerReference(jumpstarter, service, r.Scheme); err != nil { - return err - } - - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, func() error { - // Update the service spec - service.Spec.Selector = map[string]string{ - "app": "jumpstarter-controller", - } - service.Spec.Ports = []corev1.ServicePort{servicePort} - service.Spec.Type = serviceType - service.Labels = serviceLabels - service.Annotations = annotations - return nil - }) - return err -} - // updateStatus updates the status of the Jumpstarter resource func (r *JumpstarterReconciler) updateStatus(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { // Update status fields based on current state @@ -986,144 +877,6 @@ func (r *JumpstarterReconciler) buildEndpointForReplica(jumpstarter *operatorv1a return endpoint } -// reconcileRouterReplicaEndpoint reconciles service, ingress, and route for a specific router replica endpoint -// This function creates a separate service for each enabled service type (ClusterIP, NodePort, LoadBalancer) -func (r *JumpstarterReconciler) reconcileRouterReplicaEndpoint(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32, endpointIdx int, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { - // Create service with proper selector pointing to the deployment pods - // All services for this replica select the same pods using the base app label - baseAppLabel := fmt.Sprintf("%s-router-%d", jumpstarter.Name, replicaIndex) - - baseLabels := map[string]string{ - "app": "jumpstarter-router", - "router": jumpstarter.Name, - "router-index": fmt.Sprintf("%d", replicaIndex), - "endpoint-idx": fmt.Sprintf("%d", endpointIdx), - } - - // Create a service for each enabled service type - // This allows multiple service types to coexist for the same endpoint - // Note: ClusterIP uses no suffix (most common for in-cluster communication) - // LoadBalancer uses "-lb" suffix, NodePort uses "-np" suffix - - // Ingress service - if endpoint.Ingress != nil && endpoint.Ingress.Enabled { - if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "-ing", baseAppLabel, baseLabels, endpoint.Ingress.Annotations, endpoint.Ingress.Labels); err != nil { - return err - } - } - - // Route service - if endpoint.Route != nil && endpoint.Route.Enabled { - if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "-route", baseAppLabel, baseLabels, endpoint.Route.Annotations, endpoint.Route.Labels); err != nil { - return err - } - } - - // LoadBalancer service - if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { - if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeLoadBalancer, "-lb", baseAppLabel, baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { - return err - } - } - - // NodePort service - if endpoint.NodePort != nil && endpoint.NodePort.Enabled { - if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeNodePort, "-np", baseAppLabel, baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { - return err - } - } - - // ClusterIP service (no suffix for cleaner in-cluster service names) - if endpoint.ClusterIP != nil && endpoint.ClusterIP.Enabled { - if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseAppLabel, baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { - return err - } - } - - // If no service type is explicitly enabled, create a default ClusterIP service - if (endpoint.LoadBalancer == nil || !endpoint.LoadBalancer.Enabled) && - (endpoint.NodePort == nil || !endpoint.NodePort.Enabled) && - (endpoint.ClusterIP == nil || !endpoint.ClusterIP.Enabled) { - if err := r.createRouterService(ctx, jumpstarter, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseAppLabel, baseLabels, nil, nil); err != nil { - return err - } - } - - // Now create ingress/route if configured - // Use the first service (or default) for ingress/route endpoints - // Priority: LoadBalancer > NodePort > ClusterIP (no suffix) - serviceName := servicePort.Name - if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { - serviceName = servicePort.Name + "-lb" - } else if endpoint.NodePort != nil && endpoint.NodePort.Enabled { - serviceName = servicePort.Name + "-np" - } - // ClusterIP uses base name (no suffix), so no else clause needed - - servicePortForEndpoint := servicePort - servicePortForEndpoint.Name = serviceName - return r.EndpointReconciler.ReconcileEndpoint(ctx, jumpstarter.Namespace, endpoint, servicePortForEndpoint) -} - -// createRouterService creates or updates a single router service with the specified type and suffix -func (r *JumpstarterReconciler) createRouterService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort, serviceType corev1.ServiceType, nameSuffix string, podSelector string, baseLabels map[string]string, annotations map[string]string, extraLabels map[string]string) error { - // Build service name with suffix to avoid conflicts - serviceName := servicePort.Name + nameSuffix - - // Merge labels - serviceLabels := make(map[string]string) - for k, v := range baseLabels { - serviceLabels[k] = v - } - for k, v := range extraLabels { - serviceLabels[k] = v - } - - // Ensure annotations map is initialized - if annotations == nil { - annotations = make(map[string]string) - } - - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Namespace: jumpstarter.Namespace, - Labels: serviceLabels, - Annotations: annotations, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": podSelector, // Select pods from this replica's deployment - }, - Ports: []corev1.ServicePort{servicePort}, - Type: serviceType, - }, - } - - if err := controllerutil.SetControllerReference(jumpstarter, service, r.Scheme); err != nil { - return err - } - - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, func() error { - // Preserve existing NodePort if the service already exists - // This prevents "port already allocated" errors during updates - if serviceType == corev1.ServiceTypeNodePort && len(service.Spec.Ports) > 0 && service.Spec.Ports[0].NodePort != 0 { - // Service already exists with a NodePort, preserve it - servicePort.NodePort = service.Spec.Ports[0].NodePort - } - - service.Spec.Selector = map[string]string{ - "app": podSelector, - } - service.Spec.Ports = []corev1.ServicePort{servicePort} - service.Spec.Type = serviceType - service.Labels = serviceLabels - service.Annotations = annotations - return nil - }) - return err -} - // cleanupExcessRouterDeployments deletes router deployments that exceed the current replica count func (r *JumpstarterReconciler) cleanupExcessRouterDeployments(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { log := logf.FromContext(ctx) diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go index dab2bca6..14540354 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go @@ -66,14 +66,14 @@ var _ = Describe("Jumpstarter Controller", func() { corev1.ResourceMemory: resource.MustParse("100Mi"), }, }, - GRPC: operatorv1alpha1.GRPCConfig{ - Endpoints: []operatorv1alpha1.Endpoint{ - { - Address: "controller", + GRPC: operatorv1alpha1.GRPCConfig{ + Endpoints: []operatorv1alpha1.Endpoint{ + { + Address: "controller", + }, }, }, }, - }, Routers: operatorv1alpha1.RoutersConfig{ Image: "quay.io/jumpstarter/jumpstarter:latest", ImagePullPolicy: "IfNotPresent", @@ -84,14 +84,14 @@ var _ = Describe("Jumpstarter Controller", func() { corev1.ResourceMemory: resource.MustParse("100Mi"), }, }, - GRPC: operatorv1alpha1.GRPCConfig{ - Endpoints: []operatorv1alpha1.Endpoint{ - { - Address: "router", + GRPC: operatorv1alpha1.GRPCConfig{ + Endpoints: []operatorv1alpha1.Endpoint{ + { + Address: "router", + }, }, }, }, - }, }, } Expect(k8sClient.Create(ctx, resource)).To(Succeed()) @@ -112,7 +112,7 @@ var _ = Describe("Jumpstarter Controller", func() { controllerReconciler := &JumpstarterReconciler{ Client: k8sClient, Scheme: k8sClient.Scheme(), - EndpointReconciler: endpoints.NewReconciler(k8sClient), + EndpointReconciler: endpoints.NewReconciler(k8sClient, k8sClient.Scheme()), } _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ From 6e84764caff0be648f6d7a2a977fe6d525beaac4 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 22 Oct 2025 11:53:12 +0000 Subject: [PATCH 17/26] operator: add basic job to deploy with operator --- .github/workflows/pr-kind.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/pr-kind.yaml b/.github/workflows/pr-kind.yaml index d27ff2e7..51e83e95 100644 --- a/.github/workflows/pr-kind.yaml +++ b/.github/workflows/pr-kind.yaml @@ -17,3 +17,14 @@ jobs: - name: Run make deploy run: make deploy + + deploy-with-operator: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run make deploy + run: make deploy-with-operator \ No newline at end of file From 6aee77aca1acb43b4d3c73b943a8ae86aec4bc06 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Thu, 23 Oct 2025 10:17:59 +0000 Subject: [PATCH 18/26] operator: preserve immutable service fields --- .../internal/controller/jumpstarter/endpoints/endpoints.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go index 881a9ebc..48ca640b 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go @@ -120,7 +120,13 @@ func (r *Reconciler) createOrUpdateService(ctx context.Context, desiredService * } } } + // Preserve immutable fields + desiredService.Spec.ClusterIP = existingService.Spec.ClusterIP + desiredService.Spec.ClusterIPs = existingService.Spec.ClusterIPs + desiredService.Spec.IPFamilies = existingService.Spec.IPFamilies + desiredService.Spec.IPFamilyPolicy = existingService.Spec.IPFamilyPolicy + // finally update the existing service spec existingService.Spec = desiredService.Spec existingService.Annotations = desiredService.Annotations existingService.Labels = desiredService.Labels From 554c16c0d78d212de24f4bc3f5b130f98d4f3740 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Thu, 23 Oct 2025 12:29:23 +0000 Subject: [PATCH 19/26] operator: simplify endpoint controller, and remove duplication --- .../jumpstarter/endpoints/endpoints.go | 322 +++++++----------- .../jumpstarter/endpoints/endpoints_test.go | 198 +++++++++-- .../jumpstarter/jumpstarter_controller.go | 13 +- 3 files changed, 289 insertions(+), 244 deletions(-) diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go index 48ca640b..30302a82 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go @@ -18,7 +18,6 @@ package endpoints import ( "context" - "errors" "fmt" corev1 "k8s.io/api/core/v1" @@ -26,9 +25,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" - "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/utils" ) // Reconciler provides endpoint reconciliation functionality @@ -37,14 +36,6 @@ type Reconciler struct { Scheme *runtime.Scheme } -// serviceDetails contains the configuration details for creating a service -type serviceDetails struct { - ServiceType corev1.ServiceType - Annotations map[string]string - Labels map[string]string - Suffix string -} - // NewReconciler creates a new endpoint reconciler func NewReconciler(client client.Client, scheme *runtime.Scheme) *Reconciler { return &Reconciler{ @@ -53,164 +44,88 @@ func NewReconciler(client client.Client, scheme *runtime.Scheme) *Reconciler { } } -// ReconcileEndpoint creates or updates a service for the given endpoint -func (r *Reconciler) ReconcileEndpoint(ctx context.Context, namespace string, endpoint *operatorv1alpha1.Endpoint, svcPort corev1.ServicePort) error { - // Extract endpoint name from service port name - endpointName := svcPort.Name - - details, err := serviceDetailsForEndpoint(*endpoint) - if err != nil { - return fmt.Errorf("reconcileEndpoint: failed calculate service type for endpoint %q: %w", endpointName, err) - } - // add app label to the service - if details.Labels == nil { - details.Labels = make(map[string]string) - } - details.Labels["app"] = endpointName - - // ensure annotations is not nil - if details.Annotations == nil { - details.Annotations = make(map[string]string) - } - - // create the service for the endpoint - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: endpointName, - Namespace: namespace, - Annotations: details.Annotations, - Labels: details.Labels, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": endpointName, - }, - Ports: []corev1.ServicePort{svcPort}, - Type: details.ServiceType, - }, - } - - // Create or update the service using controller-runtime's CreateOrUpdate - _, err = r.createOrUpdateService(ctx, service) - return err -} +// createOrUpdateService creates or updates a service with proper handling of immutable fields +// and owner references. This is the unified service creation method. +func (r *Reconciler) createOrUpdateService(ctx context.Context, service *corev1.Service, owner metav1.Object) error { + log := logf.FromContext(ctx) -// createOrUpdateService creates or updates a service using controller-runtime pattern -func (r *Reconciler) createOrUpdateService(ctx context.Context, desiredService *corev1.Service) (bool, error) { - existingService := &corev1.Service{} - err := r.Client.Get(ctx, client.ObjectKeyFromObject(desiredService), existingService) - if err != nil { - if client.IgnoreNotFound(err) != nil { - return false, err - } - // Service doesn't exist, create it - if err := r.Client.Create(ctx, desiredService); err != nil { - return false, err + if owner != nil { + if err := controllerutil.SetControllerReference(owner, service, r.Scheme); err != nil { + return err } - return true, nil } - // Service exists, check if it needs updating - if r.serviceNeedsUpdate(existingService, desiredService) { - // Preserve existing NodePorts to prevent "port already allocated" errors - if existingService.Spec.Type == corev1.ServiceTypeNodePort || existingService.Spec.Type == corev1.ServiceTypeLoadBalancer { - for i := range existingService.Spec.Ports { - if existingService.Spec.Ports[i].NodePort != 0 && i < len(desiredService.Spec.Ports) { - desiredService.Spec.Ports[i].NodePort = existingService.Spec.Ports[i].NodePort + existingService := &corev1.Service{} + existingService.Name = service.Name + existingService.Namespace = service.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingService, func() error { + // Preserve immutable fields if service already exists + if existingService.CreationTimestamp.IsZero() { + // Service is being created, copy all fields from desired service + existingService.Spec = service.Spec + existingService.Labels = service.Labels + existingService.Annotations = service.Annotations + } else { + // Service exists, preserve immutable fields + preservedClusterIP := existingService.Spec.ClusterIP + preservedClusterIPs := existingService.Spec.ClusterIPs + preservedIPFamilies := existingService.Spec.IPFamilies + preservedIPFamilyPolicy := existingService.Spec.IPFamilyPolicy + + // Preserve existing NodePorts to prevent "port already allocated" errors + if service.Spec.Type == corev1.ServiceTypeNodePort || service.Spec.Type == corev1.ServiceTypeLoadBalancer { + for i := range existingService.Spec.Ports { + if existingService.Spec.Ports[i].NodePort != 0 && i < len(service.Spec.Ports) { + service.Spec.Ports[i].NodePort = existingService.Spec.Ports[i].NodePort + } } } - } - // Preserve immutable fields - desiredService.Spec.ClusterIP = existingService.Spec.ClusterIP - desiredService.Spec.ClusterIPs = existingService.Spec.ClusterIPs - desiredService.Spec.IPFamilies = existingService.Spec.IPFamilies - desiredService.Spec.IPFamilyPolicy = existingService.Spec.IPFamilyPolicy - - // finally update the existing service spec - existingService.Spec = desiredService.Spec - existingService.Annotations = desiredService.Annotations - existingService.Labels = desiredService.Labels - - if err := r.Client.Update(ctx, existingService); err != nil { - return false, err - } - return true, nil - } - return false, nil -} + // Update all mutable fields + existingService.Spec = service.Spec + existingService.Labels = service.Labels + existingService.Annotations = service.Annotations -// serviceNeedsUpdate checks if the service needs to be updated -func (r *Reconciler) serviceNeedsUpdate(existing, desired *corev1.Service) bool { - // Check if specs are different - if existing.Spec.Type != desired.Spec.Type || - len(existing.Spec.Ports) != len(desired.Spec.Ports) || - !utils.MapsEqual(existing.Spec.Selector, desired.Spec.Selector) { - return true - } - - // Check if port details are different - for i := range existing.Spec.Ports { - existingPort := existing.Spec.Ports[i] - desiredPort := desired.Spec.Ports[i] - - // Compare port fields (excluding NodePort which is handled separately) - if existingPort.Name != desiredPort.Name || - existingPort.Protocol != desiredPort.Protocol || - existingPort.Port != desiredPort.Port || - existingPort.TargetPort != desiredPort.TargetPort { - return true + // Restore immutable fields + existingService.Spec.ClusterIP = preservedClusterIP + existingService.Spec.ClusterIPs = preservedClusterIPs + existingService.Spec.IPFamilies = preservedIPFamilies + existingService.Spec.IPFamilyPolicy = preservedIPFamilyPolicy } - // Compare AppProtocol (handle nil cases) - if (existingPort.AppProtocol == nil) != (desiredPort.AppProtocol == nil) { - return true - } - if existingPort.AppProtocol != nil && desiredPort.AppProtocol != nil && - *existingPort.AppProtocol != *desiredPort.AppProtocol { - return true - } - } + return nil + }) - // Check if annotations or labels are different - if !utils.MapsEqual(existing.Annotations, desired.Annotations) || - !utils.MapsEqual(existing.Labels, desired.Labels) { - return true + if err != nil { + log.Error(err, "Failed to reconcile service", + "name", service.Name, + "namespace", service.Namespace, + "type", service.Spec.Type) + return err } - return false -} - -// serviceDetailsForEndpoint returns the service configuration details for the endpoint. -// It returns an error if both LoadBalancer and NodePort are enabled for the same endpoint. -func serviceDetailsForEndpoint(endpoint operatorv1alpha1.Endpoint) (*serviceDetails, error) { - if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled && - endpoint.NodePort != nil && endpoint.NodePort.Enabled { - return nil, errors.New("both LoadBalancer and NodePort are enabled for the same endpoint") - } - if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { - return &serviceDetails{ - ServiceType: corev1.ServiceTypeLoadBalancer, - Annotations: endpoint.LoadBalancer.Annotations, - Labels: endpoint.LoadBalancer.Labels, - Suffix: "-lb", - }, nil - } - if endpoint.NodePort != nil && endpoint.NodePort.Enabled { - return &serviceDetails{ - ServiceType: corev1.ServiceTypeNodePort, - Annotations: endpoint.NodePort.Annotations, - Labels: endpoint.NodePort.Labels, - Suffix: "-nodeport", - }, nil + // Log the operation result + switch op { + case controllerutil.OperationResultCreated: + log.Info("Created service", + "name", service.Name, + "namespace", service.Namespace, + "type", service.Spec.Type, + "selector", service.Spec.Selector) + case controllerutil.OperationResultUpdated: + log.Info("Updated service", + "name", service.Name, + "namespace", service.Namespace, + "type", service.Spec.Type, + "selector", service.Spec.Selector) + case controllerutil.OperationResultNone: + log.V(1).Info("Service already up to date", + "name", service.Name, + "namespace", service.Namespace) } - return &serviceDetails{ - ServiceType: corev1.ServiceTypeClusterIP, - Annotations: nil, - Labels: nil, - Suffix: "", - }, nil + return nil } // ReconcileControllerEndpoint reconciles a controller endpoint service with proper pod selector @@ -223,6 +138,11 @@ func (r *Reconciler) ReconcileControllerEndpoint(ctx context.Context, owner meta "controller": owner.GetName(), } + // Pod selector for controller pods + podSelector := map[string]string{ + "app": "jumpstarter-controller", + } + // Create a service for each enabled service type // This allows multiple service types to coexist for the same endpoint // Note: ClusterIP uses no suffix (most common for in-cluster communication) @@ -230,21 +150,24 @@ func (r *Reconciler) ReconcileControllerEndpoint(ctx context.Context, owner meta // LoadBalancer service if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { - if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeLoadBalancer, "-lb", "jumpstarter-controller", baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { + if err := r.createService(ctx, owner, servicePort, "-lb", corev1.ServiceTypeLoadBalancer, + podSelector, baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { return err } } // NodePort service if endpoint.NodePort != nil && endpoint.NodePort.Enabled { - if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeNodePort, "-np", "jumpstarter-controller", baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { + if err := r.createService(ctx, owner, servicePort, "-np", corev1.ServiceTypeNodePort, + podSelector, baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { return err } } // ClusterIP service (no suffix for cleaner in-cluster service names) if endpoint.ClusterIP != nil && endpoint.ClusterIP.Enabled { - if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", "jumpstarter-controller", baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { + if err := r.createService(ctx, owner, servicePort, "", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { return err } } @@ -255,7 +178,8 @@ func (r *Reconciler) ReconcileControllerEndpoint(ctx context.Context, owner meta (endpoint.ClusterIP == nil || !endpoint.ClusterIP.Enabled) { // TODO: Default to Route or Ingress depending of the type of cluster - if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", "jumpstarter-controller", baseLabels, nil, nil); err != nil { + if err := r.createService(ctx, owner, servicePort, "", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, nil, nil); err != nil { return err } } @@ -266,8 +190,8 @@ func (r *Reconciler) ReconcileControllerEndpoint(ctx context.Context, owner meta // ReconcileRouterReplicaEndpoint reconciles service, ingress, and route for a specific router replica endpoint // This function creates a separate service for each enabled service type (ClusterIP, NodePort, LoadBalancer) func (r *Reconciler) ReconcileRouterReplicaEndpoint(ctx context.Context, owner metav1.Object, replicaIndex int32, endpointIdx int, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { - // Create service with proper selector pointing to the deployment pods - // All services for this replica select the same pods using the base app label + // IMPORTANT: The pod selector must match the actual pod labels + // Router pods have label: app: jumpstarter-router-0 (for replica 0) baseAppLabel := fmt.Sprintf("%s-router-%d", owner.GetName(), replicaIndex) baseLabels := map[string]string{ @@ -277,6 +201,11 @@ func (r *Reconciler) ReconcileRouterReplicaEndpoint(ctx context.Context, owner m "endpoint-idx": fmt.Sprintf("%d", endpointIdx), } + // Pod selector - this MUST match the deployment's pod template labels + podSelector := map[string]string{ + "app": baseAppLabel, // e.g., "jumpstarter-router-0" + } + // Create a service for each enabled service type // This allows multiple service types to coexist for the same endpoint // Note: ClusterIP uses no suffix (most common for in-cluster communication) @@ -284,35 +213,40 @@ func (r *Reconciler) ReconcileRouterReplicaEndpoint(ctx context.Context, owner m // Ingress service if endpoint.Ingress != nil && endpoint.Ingress.Enabled { - if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "-ing", baseAppLabel, baseLabels, endpoint.Ingress.Annotations, endpoint.Ingress.Labels); err != nil { + if err := r.createService(ctx, owner, servicePort, "-ing", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, endpoint.Ingress.Annotations, endpoint.Ingress.Labels); err != nil { return err } } // Route service if endpoint.Route != nil && endpoint.Route.Enabled { - if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "-route", baseAppLabel, baseLabels, endpoint.Route.Annotations, endpoint.Route.Labels); err != nil { + if err := r.createService(ctx, owner, servicePort, "-route", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, endpoint.Route.Annotations, endpoint.Route.Labels); err != nil { return err } } // LoadBalancer service if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { - if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeLoadBalancer, "-lb", baseAppLabel, baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { + if err := r.createService(ctx, owner, servicePort, "-lb", corev1.ServiceTypeLoadBalancer, + podSelector, baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { return err } } // NodePort service if endpoint.NodePort != nil && endpoint.NodePort.Enabled { - if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeNodePort, "-np", baseAppLabel, baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { + if err := r.createService(ctx, owner, servicePort, "-np", corev1.ServiceTypeNodePort, + podSelector, baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { return err } } // ClusterIP service (no suffix for cleaner in-cluster service names) if endpoint.ClusterIP != nil && endpoint.ClusterIP.Enabled { - if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseAppLabel, baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { + if err := r.createService(ctx, owner, servicePort, "", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { return err } } @@ -320,30 +254,27 @@ func (r *Reconciler) ReconcileRouterReplicaEndpoint(ctx context.Context, owner m // If no service type is explicitly enabled, create a default ClusterIP service if (endpoint.LoadBalancer == nil || !endpoint.LoadBalancer.Enabled) && (endpoint.NodePort == nil || !endpoint.NodePort.Enabled) && - (endpoint.ClusterIP == nil || !endpoint.ClusterIP.Enabled) { - if err := r.createService(ctx, owner, endpoint, servicePort, corev1.ServiceTypeClusterIP, "", baseAppLabel, baseLabels, nil, nil); err != nil { + (endpoint.ClusterIP == nil || !endpoint.ClusterIP.Enabled) && + (endpoint.Ingress == nil || !endpoint.Ingress.Enabled) && + (endpoint.Route == nil || !endpoint.Route.Enabled) { + if err := r.createService(ctx, owner, servicePort, "", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, nil, nil); err != nil { return err } } - // Now create ingress/route if configured - // Use the first service (or default) for ingress/route endpoints - // Priority: LoadBalancer > NodePort > ClusterIP (no suffix) - serviceName := servicePort.Name - if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { - serviceName = servicePort.Name + "-lb" - } else if endpoint.NodePort != nil && endpoint.NodePort.Enabled { - serviceName = servicePort.Name + "-np" - } - // ClusterIP uses base name (no suffix), so no else clause needed + // TODO: Create ingress/route resources here instead of calling the deprecated ReconcileEndpoint + // For now, ingress and route are handled by creating ClusterIP services above - servicePortForEndpoint := servicePort - servicePortForEndpoint.Name = serviceName - return r.ReconcileEndpoint(ctx, owner.GetNamespace(), endpoint, servicePortForEndpoint) + return nil } // createService creates or updates a single service with the specified type and suffix -func (r *Reconciler) createService(ctx context.Context, owner metav1.Object, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort, serviceType corev1.ServiceType, nameSuffix string, podSelector string, baseLabels map[string]string, annotations map[string]string, extraLabels map[string]string) error { +// This is the unified service creation method that uses createOrUpdateService internally +func (r *Reconciler) createService(ctx context.Context, owner metav1.Object, servicePort corev1.ServicePort, + nameSuffix string, serviceType corev1.ServiceType, podSelector map[string]string, + baseLabels map[string]string, annotations map[string]string, extraLabels map[string]string) error { + // Build service name with suffix to avoid conflicts serviceName := servicePort.Name + nameSuffix @@ -369,34 +300,11 @@ func (r *Reconciler) createService(ctx context.Context, owner metav1.Object, end Annotations: annotations, }, Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": podSelector, // Select pods with the specified selector - }, - Ports: []corev1.ServicePort{servicePort}, - Type: serviceType, + Selector: podSelector, // Use the provided pod selector map + Ports: []corev1.ServicePort{servicePort}, + Type: serviceType, }, } - if err := controllerutil.SetControllerReference(owner, service, r.Scheme); err != nil { - return err - } - - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, func() error { - // Preserve existing NodePort if the service already exists - // This prevents "port already allocated" errors during updates - if serviceType == corev1.ServiceTypeNodePort && len(service.Spec.Ports) > 0 && service.Spec.Ports[0].NodePort != 0 { - // Service already exists with a NodePort, preserve it - servicePort.NodePort = service.Spec.Ports[0].NodePort - } - - service.Spec.Selector = map[string]string{ - "app": podSelector, - } - service.Spec.Ports = []corev1.ServicePort{servicePort} - service.Spec.Type = serviceType - service.Labels = serviceLabels - service.Annotations = annotations - return nil - }) - return err + return r.createOrUpdateService(ctx, service, owner) } diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go index 6394e38f..12f57d1c 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go @@ -31,14 +31,15 @@ import ( ) var _ = Describe("Endpoints Reconciler", func() { - Context("When reconciling an endpoint", func() { + Context("When reconciling controller endpoints", func() { const ( - namespace = "test-namespace" - endpointName = "test-endpoint" + namespace = "test-namespace" + controllerName = "test-controller" ) ctx := context.Background() var reconciler *Reconciler + var owner *corev1.ConfigMap // Use ConfigMap as a simple owner object for testing BeforeEach(func() { reconciler = NewReconciler(k8sClient, k8sClient.Scheme()) @@ -53,40 +54,56 @@ var _ = Describe("Endpoints Reconciler", func() { if err != nil && !errors.IsAlreadyExists(err) { Expect(err).NotTo(HaveOccurred()) } + + // Create an owner object for testing + owner = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllerName, + Namespace: namespace, + }, + } + err = k8sClient.Create(ctx, owner) + if err != nil && !errors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } }) Context("with ClusterIP service type", func() { It("should create a ClusterIP service successfully", func() { endpoint := &operatorv1alpha1.Endpoint{ - Address: endpointName, + Address: "controller", + ClusterIP: &operatorv1alpha1.ClusterIPConfig{ + Enabled: true, + }, } svcPort := corev1.ServicePort{ - Name: endpointName, + Name: "controller", Port: 9090, TargetPort: intstr.FromInt(9090), Protocol: corev1.ProtocolTCP, } - err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) + err := reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) Expect(err).NotTo(HaveOccurred()) // Verify the service was created service := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: endpointName, + Name: "controller", Namespace: namespace, }, service) Expect(err).NotTo(HaveOccurred()) Expect(service.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) - Expect(service.Labels["app"]).To(Equal(endpointName)) + Expect(service.Spec.Selector["app"]).To(Equal("jumpstarter-controller")) + Expect(service.Labels["app"]).To(Equal("jumpstarter-controller")) }) }) Context("with LoadBalancer service type", func() { It("should create a LoadBalancer service successfully", func() { endpoint := &operatorv1alpha1.Endpoint{ - Address: endpointName, + Address: "controller", LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ Enabled: true, Annotations: map[string]string{"service.beta.kubernetes.io/aws-load-balancer-type": "nlb"}, @@ -95,32 +112,33 @@ var _ = Describe("Endpoints Reconciler", func() { } svcPort := corev1.ServicePort{ - Name: endpointName, + Name: "controller", Port: 9090, TargetPort: intstr.FromInt(9090), Protocol: corev1.ProtocolTCP, } - err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) + err := reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) Expect(err).NotTo(HaveOccurred()) - // Verify the service was created + // Verify the service was created with -lb suffix service := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: endpointName, + Name: "controller-lb", Namespace: namespace, }, service) Expect(err).NotTo(HaveOccurred()) Expect(service.Spec.Type).To(Equal(corev1.ServiceTypeLoadBalancer)) Expect(service.Annotations["service.beta.kubernetes.io/aws-load-balancer-type"]).To(Equal("nlb")) Expect(service.Labels["environment"]).To(Equal("production")) + Expect(service.Spec.Selector["app"]).To(Equal("jumpstarter-controller")) }) }) Context("with NodePort service type", func() { It("should create a NodePort service successfully", func() { endpoint := &operatorv1alpha1.Endpoint{ - Address: endpointName, + Address: "controller", NodePort: &operatorv1alpha1.NodePortConfig{ Enabled: true, Port: 30090, @@ -130,32 +148,33 @@ var _ = Describe("Endpoints Reconciler", func() { } svcPort := corev1.ServicePort{ - Name: endpointName, + Name: "controller", Port: 9090, TargetPort: intstr.FromInt(9090), Protocol: corev1.ProtocolTCP, } - err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) + err := reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) Expect(err).NotTo(HaveOccurred()) - // Verify the service was created + // Verify the service was created with -np suffix service := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: endpointName, + Name: "controller-np", Namespace: namespace, }, service) Expect(err).NotTo(HaveOccurred()) Expect(service.Spec.Type).To(Equal(corev1.ServiceTypeNodePort)) Expect(service.Annotations["nodeport.kubernetes.io/port"]).To(Equal("30090")) Expect(service.Labels["nodeport"]).To(Equal("true")) + Expect(service.Spec.Selector["app"]).To(Equal("jumpstarter-controller")) }) }) - Context("with invalid configuration", func() { - It("should return an error when both LoadBalancer and NodePort are enabled", func() { + Context("with multiple service types enabled", func() { + It("should create all enabled service types", func() { endpoint := &operatorv1alpha1.Endpoint{ - Address: endpointName, + Address: "controller", LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ Enabled: true, }, @@ -165,15 +184,32 @@ var _ = Describe("Endpoints Reconciler", func() { } svcPort := corev1.ServicePort{ - Name: endpointName, + Name: "controller", Port: 9090, TargetPort: intstr.FromInt(9090), Protocol: corev1.ProtocolTCP, } - err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("both LoadBalancer and NodePort are enabled")) + err := reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify LoadBalancer service was created + lbService := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: "controller-lb", + Namespace: namespace, + }, lbService) + Expect(err).NotTo(HaveOccurred()) + Expect(lbService.Spec.Type).To(Equal(corev1.ServiceTypeLoadBalancer)) + + // Verify NodePort service was created + npService := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: "controller-np", + Namespace: namespace, + }, npService) + Expect(err).NotTo(HaveOccurred()) + Expect(npService.Spec.Type).To(Equal(corev1.ServiceTypeNodePort)) }) }) @@ -181,7 +217,7 @@ var _ = Describe("Endpoints Reconciler", func() { It("should update the service when configuration changes", func() { // Create initial service endpoint := &operatorv1alpha1.Endpoint{ - Address: endpointName, + Address: "controller", LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ Enabled: true, Annotations: map[string]string{"initial": "annotation"}, @@ -190,26 +226,26 @@ var _ = Describe("Endpoints Reconciler", func() { } svcPort := corev1.ServicePort{ - Name: endpointName, + Name: "controller", Port: 9090, TargetPort: intstr.FromInt(9090), Protocol: corev1.ProtocolTCP, } - err := reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) + err := reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) Expect(err).NotTo(HaveOccurred()) // Update the endpoint configuration endpoint.LoadBalancer.Annotations["updated"] = "annotation" endpoint.LoadBalancer.Labels["updated"] = "label" - err = reconciler.ReconcileEndpoint(ctx, namespace, endpoint, svcPort) + err = reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) Expect(err).NotTo(HaveOccurred()) // Verify the service was updated service := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: endpointName, + Name: "controller-lb", Namespace: namespace, }, service) Expect(err).NotTo(HaveOccurred()) @@ -220,13 +256,109 @@ var _ = Describe("Endpoints Reconciler", func() { AfterEach(func() { // Clean up created services - service := &corev1.Service{ + services := []string{"controller", "controller-lb", "controller-np"} + for _, svcName := range services { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: namespace, + }, + } + _ = k8sClient.Delete(ctx, service) + } + + // Clean up owner + _ = k8sClient.Delete(ctx, owner) + }) + }) + + Context("When reconciling router replica endpoints", func() { + const ( + namespace = "test-namespace" + routerName = "test-router" + replicaIdx = int32(0) + endpointIdx = 0 + ) + + ctx := context.Background() + var reconciler *Reconciler + var owner *corev1.ConfigMap + + BeforeEach(func() { + reconciler = NewReconciler(k8sClient, k8sClient.Scheme()) + + // Create the test namespace + ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: endpointName, + Name: namespace, + }, + } + err := k8sClient.Create(ctx, ns) + if err != nil && !errors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + // Create an owner object for testing + owner = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: routerName, Namespace: namespace, }, } - _ = k8sClient.Delete(ctx, service) + err = k8sClient.Create(ctx, owner) + if err != nil && !errors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + }) + + Context("with proper pod selector", func() { + It("should create a service with correct pod selector matching deployment labels", func() { + endpoint := &operatorv1alpha1.Endpoint{ + Address: "router", + NodePort: &operatorv1alpha1.NodePortConfig{ + Enabled: true, + }, + } + + svcPort := corev1.ServicePort{ + Name: "router", + Port: 8083, + TargetPort: intstr.FromInt(8083), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileRouterReplicaEndpoint(ctx, owner, replicaIdx, endpointIdx, endpoint, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify the service was created with correct pod selector + service := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: "router-np", + Namespace: namespace, + }, service) + Expect(err).NotTo(HaveOccurred()) + // The selector should be "app: test-router-router-0" (owner.Name + "-router-" + replicaIndex) + Expect(service.Spec.Selector["app"]).To(Equal("test-router-router-0")) + Expect(service.Labels["router"]).To(Equal(routerName)) + Expect(service.Labels["router-index"]).To(Equal("0")) + }) + }) + + AfterEach(func() { + // Clean up created services + services := []string{"router", "router-lb", "router-np", "router-ing", "router-route"} + for _, svcName := range services { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: namespace, + }, + } + _ = k8sClient.Delete(ctx, service) + } + + // Clean up owner + _ = k8sClient.Delete(ctx, owner) }) }) diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 811593d3..c08bc4d8 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -45,6 +45,11 @@ import ( "github.com/jumpstarter-dev/jumpstarter-controller/internal/config" ) +const ( + // appProtocolH2C is the application protocol for HTTP/2 Cleartext + appProtocolH2C = "h2c" +) + // JumpstarterReconciler reconciles a Jumpstarter object type JumpstarterReconciler struct { client.Client @@ -221,7 +226,7 @@ func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstart // Reconcile controller services for _, endpoint := range jumpstarter.Spec.Controller.GRPC.Endpoints { - appProtocol := "h2c" + appProtocol := appProtocolH2C svcPort := corev1.ServicePort{ Name: "controller-grpc", Port: 8082, @@ -249,7 +254,7 @@ func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstart // This allows multiple service types (NodePort, LoadBalancer, etc.) per replica serviceName := r.buildServiceNameForReplicaEndpoint(jumpstarter, i, endpointIdx) - appProtocol := "h2c" + appProtocol := appProtocolH2C svcPort := corev1.ServicePort{ Name: serviceName, // Unique name per replica+endpoint Port: 8083, @@ -261,7 +266,7 @@ func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstart if endpoint.NodePort != nil && endpoint.NodePort.Enabled && endpoint.NodePort.Port > 0 { // increase nodeport numbers based in replica, not perfect because it needs to be // consecutive, but this is mostly for E2E testing. - svcPort.NodePort = endpoint.NodePort.Port + int32(i) + svcPort.NodePort = endpoint.NodePort.Port + i } if err := r.EndpointReconciler.ReconcileRouterReplicaEndpoint(ctx, jumpstarter, i, endpointIdx, &endpoint, svcPort); err != nil { return err @@ -274,7 +279,7 @@ func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstart } serviceName := fmt.Sprintf("%s-router-%d", jumpstarter.Name, i) - appProtocol := "h2c" + appProtocol := appProtocolH2C svcPort := corev1.ServicePort{ Name: serviceName, Port: 8083, From a77b4c1e6dfc19e198aa5759f64f34dab2e40e6f Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Thu, 23 Oct 2025 12:37:07 +0000 Subject: [PATCH 20/26] operator: remove envtest duplication --- .../jumpstarter/endpoints/suite_test.go | 29 +--------- .../controller/jumpstarter/suite_test.go | 29 +--------- .../internal/controller/testutils/envtest.go | 57 +++++++++++++++++++ 3 files changed, 63 insertions(+), 52 deletions(-) create mode 100644 deploy/operator/internal/controller/testutils/envtest.go diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/suite_test.go b/deploy/operator/internal/controller/jumpstarter/endpoints/suite_test.go index b36dfed1..6af1fab4 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/suite_test.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/suite_test.go @@ -18,7 +18,6 @@ package endpoints import ( "context" - "os" "path/filepath" "testing" @@ -33,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/testutils" // +kubebuilder:scaffold:imports ) @@ -71,8 +71,8 @@ var _ = BeforeSuite(func() { } // Retrieve the first found binary directory to allow running tests from IDEs - if getFirstFoundEnvTestBinaryDir() != "" { - testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir() + if binaryDir := testutils.GetFirstFoundEnvTestBinaryDir(6); binaryDir != "" { + testEnv.BinaryAssetsDirectory = binaryDir } // cfg is defined in this file globally. @@ -91,26 +91,3 @@ var _ = AfterSuite(func() { err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) - -// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. -// ENVTEST-based tests depend on specific binaries, usually located in paths set by -// controller-runtime. When running tests directly (e.g., via an IDE) without using -// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. -// -// This function streamlines the process by finding the required binaries, similar to -// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are -// properly set up, run 'make setup-envtest' beforehand. -func getFirstFoundEnvTestBinaryDir() string { - basePath := filepath.Join("..", "..", "..", "..", "..", "..", "bin", "k8s") - entries, err := os.ReadDir(basePath) - if err != nil { - logf.Log.Error(err, "Failed to read directory", "path", basePath) - return "" - } - for _, entry := range entries { - if entry.IsDir() { - return filepath.Join(basePath, entry.Name()) - } - } - return "" -} diff --git a/deploy/operator/internal/controller/jumpstarter/suite_test.go b/deploy/operator/internal/controller/jumpstarter/suite_test.go index aaff5d6d..83034fde 100644 --- a/deploy/operator/internal/controller/jumpstarter/suite_test.go +++ b/deploy/operator/internal/controller/jumpstarter/suite_test.go @@ -18,7 +18,6 @@ package jumpstarter import ( "context" - "os" "path/filepath" "testing" @@ -33,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/testutils" // +kubebuilder:scaffold:imports ) @@ -71,8 +71,8 @@ var _ = BeforeSuite(func() { } // Retrieve the first found binary directory to allow running tests from IDEs - if getFirstFoundEnvTestBinaryDir() != "" { - testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir() + if binaryDir := testutils.GetFirstFoundEnvTestBinaryDir(5); binaryDir != "" { + testEnv.BinaryAssetsDirectory = binaryDir } // cfg is defined in this file globally. @@ -91,26 +91,3 @@ var _ = AfterSuite(func() { err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) - -// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. -// ENVTEST-based tests depend on specific binaries, usually located in paths set by -// controller-runtime. When running tests directly (e.g., via an IDE) without using -// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. -// -// This function streamlines the process by finding the required binaries, similar to -// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are -// properly set up, run 'make setup-envtest' beforehand. -func getFirstFoundEnvTestBinaryDir() string { - basePath := filepath.Join("..", "..", "..", "..", "..", "bin", "k8s") - entries, err := os.ReadDir(basePath) - if err != nil { - logf.Log.Error(err, "Failed to read directory", "path", basePath) - return "" - } - for _, entry := range entries { - if entry.IsDir() { - return filepath.Join(basePath, entry.Name()) - } - } - return "" -} diff --git a/deploy/operator/internal/controller/testutils/envtest.go b/deploy/operator/internal/controller/testutils/envtest.go new file mode 100644 index 00000000..eb5f506d --- /dev/null +++ b/deploy/operator/internal/controller/testutils/envtest.go @@ -0,0 +1,57 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testutils + +import ( + "os" + "path/filepath" + + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +// GetFirstFoundEnvTestBinaryDir locates the first binary in the specified path. +// ENVTEST-based tests depend on specific binaries, usually located in paths set by +// controller-runtime. When running tests directly (e.g., via an IDE) without using +// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. +// +// This function streamlines the process by finding the required binaries, similar to +// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are +// properly set up, run 'make setup-envtest' beforehand. +// +// The depth parameter specifies how many directories up to traverse from the test file +// to reach the operator root (where bin/k8s is located). +func GetFirstFoundEnvTestBinaryDir(depth int) string { + // Build the path based on depth + pathComponents := make([]string, 0, depth+2) + for i := 0; i < depth; i++ { + pathComponents = append(pathComponents, "..") + } + pathComponents = append(pathComponents, "bin", "k8s") + + basePath := filepath.Join(pathComponents...) + entries, err := os.ReadDir(basePath) + if err != nil { + logf.Log.Error(err, "Failed to read directory", "path", basePath) + return "" + } + for _, entry := range entries { + if entry.IsDir() { + return filepath.Join(basePath, entry.Name()) + } + } + return "" +} From e5b7e5dc4c75bc6ab8c23d7280fe36fc2919a12a Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Thu, 23 Oct 2025 12:46:49 +0000 Subject: [PATCH 21/26] operator: safer IMAGE_TAG/REPO --- hack/deploy_vars | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hack/deploy_vars b/hack/deploy_vars index 1d1c2e6a..e54ca915 100755 --- a/hack/deploy_vars +++ b/hack/deploy_vars @@ -20,9 +20,13 @@ else fi # Extract image repository and tag from IMG variable -IMAGE_REPO=$(echo "${IMG}" | cut -d: -f1) -IMAGE_TAG=$(echo "${IMG}" | cut -d: -f2) - +if [[ "${IMG}" == *:* ]]; then + IMAGE_TAG="${IMG##*:}" # everything after the last colon + IMAGE_REPO="${IMG%:*}" # everything before the last colon +else # no tag specified + IMAGE_TAG="latest" + IMAGE_REPO="${IMG}" +fi # Export all variables for use in scripts export IP export BASEDOMAIN From 77df0f41a20873c3d114b323c6bb635cc1e6fd3f Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Thu, 23 Oct 2025 16:07:50 +0000 Subject: [PATCH 22/26] operator: fix endless reconciliation loops some objects stay on a endless reconciliation loops because we don't configure all the fields and they will differ later when k8s updates them to defaults. this commit fixes such problem. --- Makefile | 5 + deploy/operator/go.mod | 3 +- .../controller/jumpstarter/compare.go | 158 ++++++++++ .../jumpstarter/endpoints/endpoints.go | 62 ++-- .../jumpstarter/jumpstarter_controller.go | 274 +++++++++++++++--- .../internal/controller/jumpstarter/rbac.go | 154 ++++++++-- 6 files changed, 548 insertions(+), 108 deletions(-) create mode 100644 deploy/operator/internal/controller/jumpstarter/compare.go diff --git a/Makefile b/Makefile index 780d09a7..f94ac13b 100644 --- a/Makefile +++ b/Makefile @@ -174,9 +174,14 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified deploy: docker-build cluster grpcurl ./hack/deploy_with_helm.sh +.PHONY: deploy-with-operator deploy-with-operator: docker-build build-operator cluster grpcurl ./hack/deploy_with_operator.sh +.PHONY: operator-logs +operator-logs: + kubectl logs -n jumpstarter-operator-system -l app.kubernetes.io/name=jumpstarter-operator -f + deploy-with-operator-parallel: make deploy-with-operator -j5 --output-sync=target diff --git a/deploy/operator/go.mod b/deploy/operator/go.mod index 0aa25f51..cfa4abff 100644 --- a/deploy/operator/go.mod +++ b/deploy/operator/go.mod @@ -3,9 +3,11 @@ module github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator go 1.24.0 require ( + github.com/go-logr/logr v1.4.2 github.com/jumpstarter-dev/jumpstarter-controller v0.7.1 github.com/onsi/ginkgo/v2 v2.22.2 github.com/onsi/gomega v1.36.2 + github.com/pmezard/go-difflib v1.0.0 k8s.io/api v0.33.0 k8s.io/apimachinery v0.33.0 k8s.io/apiserver v0.33.0 @@ -42,7 +44,6 @@ require ( github.com/gin-gonic/gin v1.10.0 // indirect github.com/go-chi/chi/v5 v5.2.0 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect - github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect diff --git a/deploy/operator/internal/controller/jumpstarter/compare.go b/deploy/operator/internal/controller/jumpstarter/compare.go new file mode 100644 index 00000000..d22d2872 --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/compare.go @@ -0,0 +1,158 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jumpstarter + +import ( + "fmt" + + "github.com/go-logr/logr" + "github.com/pmezard/go-difflib/difflib" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/equality" + "sigs.k8s.io/yaml" +) + +// deploymentNeedsUpdate checks if a deployment needs to be updated using K8s semantic equality. +func deploymentNeedsUpdate(existing, desired *appsv1.Deployment) bool { + // Compare labels (only if desired.Labels is non-nil) + if desired.Labels != nil && !equality.Semantic.DeepEqual(existing.Labels, desired.Labels) { + return true + } + + // Compare annotations (only if desired.Annotations is non-nil) + if desired.Annotations != nil && !equality.Semantic.DeepEqual(existing.Annotations, desired.Annotations) { + return true + } + + // Compare the entire Spec using K8s semantic equality (handles nil vs empty automatically) + return !equality.Semantic.DeepEqual(existing.Spec, desired.Spec) +} + +// configMapNeedsUpdate checks if a configmap needs to be updated using K8s semantic equality. +func configMapNeedsUpdate(existing, desired *corev1.ConfigMap, log logr.Logger) bool { + // Compare labels (only if desired.Labels is non-nil) + if desired.Labels != nil && !equality.Semantic.DeepEqual(existing.Labels, desired.Labels) { + return true + } + + // Compare annotations (only if desired.Annotations is non-nil) + if desired.Annotations != nil && !equality.Semantic.DeepEqual(existing.Annotations, desired.Annotations) { + return true + } + + // Compare data (only if desired.Data is non-nil) + if desired.Data != nil && !equality.Semantic.DeepEqual(existing.Data, desired.Data) { + return true + } + + // Compare binary data (only if desired.BinaryData is non-nil) + if desired.BinaryData != nil && !equality.Semantic.DeepEqual(existing.BinaryData, desired.BinaryData) { + return true + } + + return false +} + +// serviceAccountNeedsUpdate checks if a service account needs to be updated using K8s semantic equality. +func serviceAccountNeedsUpdate(existing, desired *corev1.ServiceAccount) bool { + // Compare labels (only if desired.Labels is non-nil) + if desired.Labels != nil && !equality.Semantic.DeepEqual(existing.Labels, desired.Labels) { + return true + } + + // Compare annotations (only if desired.Annotations is non-nil) + if desired.Annotations != nil && !equality.Semantic.DeepEqual(existing.Annotations, desired.Annotations) { + return true + } + + return false +} + +// roleNeedsUpdate checks if a role needs to be updated using K8s semantic equality. +func roleNeedsUpdate(existing, desired *rbacv1.Role) bool { + // Compare labels (only if desired.Labels is non-nil) + if desired.Labels != nil && !equality.Semantic.DeepEqual(existing.Labels, desired.Labels) { + return true + } + + // Compare annotations (only if desired.Annotations is non-nil) + if desired.Annotations != nil && !equality.Semantic.DeepEqual(existing.Annotations, desired.Annotations) { + return true + } + + // Compare rules (only if non-nil in desired) + if desired.Rules != nil && !equality.Semantic.DeepEqual(existing.Rules, desired.Rules) { + return true + } + + return false +} + +// roleBindingNeedsUpdate checks if a role binding needs to be updated using K8s semantic equality. +func roleBindingNeedsUpdate(existing, desired *rbacv1.RoleBinding) bool { + // Compare labels (only if desired.Labels is non-nil) + if desired.Labels != nil && !equality.Semantic.DeepEqual(existing.Labels, desired.Labels) { + return true + } + + // Compare annotations (only if desired.Annotations is non-nil) + if desired.Annotations != nil && !equality.Semantic.DeepEqual(existing.Annotations, desired.Annotations) { + return true + } + + // Compare subjects (only if non-nil in desired) + if desired.Subjects != nil && !equality.Semantic.DeepEqual(existing.Subjects, desired.Subjects) { + return true + } + + // Compare role ref (only if non-zero in desired) + if desired.RoleRef.Name != "" && !equality.Semantic.DeepEqual(existing.RoleRef, desired.RoleRef) { + return true + } + + return false +} + +// generateDiff creates a unified diff between existing and desired resources. +// It works with any Kubernetes resource type. +// Returns the diff string and any error encountered during serialization. +func generateDiff[T any](existing, desired *T) (string, error) { + // Serialize existing resource to YAML + existingYAML, err := yaml.Marshal(existing) + if err != nil { + return "", fmt.Errorf("failed to marshal existing resource: %w", err) + } + + // Serialize desired resource to YAML + desiredYAML, err := yaml.Marshal(desired) + if err != nil { + return "", fmt.Errorf("failed to marshal desired resource: %w", err) + } + + // Generate unified diff + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines(string(existingYAML)), + B: difflib.SplitLines(string(desiredYAML)), + FromFile: "Existing", + ToFile: "Desired", + Context: 3, + } + + return difflib.GetUnifiedDiffString(diff) +} diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go index 30302a82..fce9d71f 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go @@ -49,16 +49,14 @@ func NewReconciler(client client.Client, scheme *runtime.Scheme) *Reconciler { func (r *Reconciler) createOrUpdateService(ctx context.Context, service *corev1.Service, owner metav1.Object) error { log := logf.FromContext(ctx) - if owner != nil { - if err := controllerutil.SetControllerReference(owner, service, r.Scheme); err != nil { - return err - } - } - existingService := &corev1.Service{} existingService.Name = service.Name existingService.Namespace = service.Namespace + if err := controllerutil.SetControllerReference(owner, service, r.Scheme); err != nil { + return err + } + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingService, func() error { // Preserve immutable fields if service already exists if existingService.CreationTimestamp.IsZero() { @@ -66,13 +64,11 @@ func (r *Reconciler) createOrUpdateService(ctx context.Context, service *corev1. existingService.Spec = service.Spec existingService.Labels = service.Labels existingService.Annotations = service.Annotations - } else { - // Service exists, preserve immutable fields - preservedClusterIP := existingService.Spec.ClusterIP - preservedClusterIPs := existingService.Spec.ClusterIPs - preservedIPFamilies := existingService.Spec.IPFamilies - preservedIPFamilyPolicy := existingService.Spec.IPFamilyPolicy + if err := controllerutil.SetControllerReference(owner, service, r.Scheme); err != nil { + return err + } + } else { // Preserve existing NodePorts to prevent "port already allocated" errors if service.Spec.Type == corev1.ServiceTypeNodePort || service.Spec.Type == corev1.ServiceTypeLoadBalancer { for i := range existingService.Spec.Ports { @@ -83,15 +79,18 @@ func (r *Reconciler) createOrUpdateService(ctx context.Context, service *corev1. } // Update all mutable fields - existingService.Spec = service.Spec + if service.Spec.LoadBalancerClass != nil && *service.Spec.LoadBalancerClass != "" { + existingService.Spec.LoadBalancerClass = service.Spec.LoadBalancerClass + } + if service.Spec.ExternalTrafficPolicy != "" { + existingService.Spec.ExternalTrafficPolicy = service.Spec.ExternalTrafficPolicy + } + + existingService.Spec.Ports = service.Spec.Ports + existingService.Spec.Selector = service.Spec.Selector + existingService.Spec.Type = service.Spec.Type existingService.Labels = service.Labels existingService.Annotations = service.Annotations - - // Restore immutable fields - existingService.Spec.ClusterIP = preservedClusterIP - existingService.Spec.ClusterIPs = preservedClusterIPs - existingService.Spec.IPFamilies = preservedIPFamilies - existingService.Spec.IPFamilyPolicy = preservedIPFamilyPolicy } return nil @@ -105,25 +104,12 @@ func (r *Reconciler) createOrUpdateService(ctx context.Context, service *corev1. return err } - // Log the operation result - switch op { - case controllerutil.OperationResultCreated: - log.Info("Created service", - "name", service.Name, - "namespace", service.Namespace, - "type", service.Spec.Type, - "selector", service.Spec.Selector) - case controllerutil.OperationResultUpdated: - log.Info("Updated service", - "name", service.Name, - "namespace", service.Namespace, - "type", service.Spec.Type, - "selector", service.Spec.Selector) - case controllerutil.OperationResultNone: - log.V(1).Info("Service already up to date", - "name", service.Name, - "namespace", service.Namespace) - } + log.Info("Service reconciled", + "name", service.Name, + "namespace", service.Namespace, + "type", service.Spec.Type, + "selector", service.Spec.Selector, + "operation", op) return nil } diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index c08bc4d8..82400691 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -166,26 +166,76 @@ func (r *JumpstarterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - // Requeue after 10 seconds to check for changes - return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + // Requeue after 30 minutes to check for changes + return ctrl.Result{RequeueAfter: 30 * time.Minute}, nil } // reconcileControllerDeployment reconciles the controller deployment func (r *JumpstarterReconciler) reconcileControllerDeployment(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { - deployment := r.createControllerDeployment(jumpstarter) + log := logf.FromContext(ctx) + desiredDeployment := r.createControllerDeployment(jumpstarter) + + controllerutil.SetControllerReference(jumpstarter, desiredDeployment, r.Scheme) + + existingDeployment := &appsv1.Deployment{} + existingDeployment.Name = desiredDeployment.Name + existingDeployment.Namespace = desiredDeployment.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingDeployment, func() error { + // Check if this is a new deployment or an existing one + if existingDeployment.CreationTimestamp.IsZero() { + // Deployment is being created, copy all fields from desired + existingDeployment.Labels = desiredDeployment.Labels + existingDeployment.Annotations = desiredDeployment.Annotations + existingDeployment.Spec = desiredDeployment.Spec + return nil + } + + desiredDeployment.Spec.Template.Spec.DeprecatedServiceAccount = existingDeployment.Spec.Template.Spec.DeprecatedServiceAccount + desiredDeployment.Spec.Template.Spec.SchedulerName = existingDeployment.Spec.Template.Spec.SchedulerName + + // Check if deployment needs update using compare function + if !deploymentNeedsUpdate(existingDeployment, desiredDeployment) { + log.V(1).Info("Controller deployment specs are equal, skipping update", + "name", existingDeployment.Name, + "namespace", existingDeployment.Namespace) + return nil + } + + // Deployment exists, generate and log diff + diff, err := generateDiff(existingDeployment, desiredDeployment) + if err != nil { + log.V(1).Info("Failed to generate deployment diff", "error", err) + } else if diff != "" { + fmt.Printf("\n=== Controller deployment differences detected ===\n") + fmt.Printf("Name: %s\n", existingDeployment.Name) + fmt.Printf("Namespace: %s\n", existingDeployment.Namespace) + fmt.Printf("\n%s\n", diff) + fmt.Printf("========================================\n\n") + } + + // Apply changes + existingDeployment.Labels = desiredDeployment.Labels + existingDeployment.Annotations = desiredDeployment.Annotations + existingDeployment.Spec.Replicas = desiredDeployment.Spec.Replicas + existingDeployment.Spec.Selector = desiredDeployment.Spec.Selector + existingDeployment.Spec.Template = desiredDeployment.Spec.Template + return controllerutil.SetControllerReference(jumpstarter, existingDeployment, r.Scheme) + }) - // Set the owner reference - if err := controllerutil.SetControllerReference(jumpstarter, deployment, r.Scheme); err != nil { + if err != nil { + log.Error(err, "Failed to reconcile controller deployment", + "name", desiredDeployment.Name, + "namespace", desiredDeployment.Namespace) return err } - // Create or update the deployment - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { - // Update deployment spec if needed - return nil - }) + log.Info("Controller deployment reconciled", + "name", existingDeployment.Name, + "namespace", existingDeployment.Namespace, + "operation", op) - return err + return nil } // reconcileRouterDeployment reconciles router deployments (one per replica) @@ -194,21 +244,68 @@ func (r *JumpstarterReconciler) reconcileRouterDeployment(ctx context.Context, j // Create one deployment per replica for i := int32(0); i < jumpstarter.Spec.Routers.Replicas; i++ { - deployment := r.createRouterDeployment(jumpstarter, i) - - // Set the owner reference - if err := controllerutil.SetControllerReference(jumpstarter, deployment, r.Scheme); err != nil { - return err - } + desiredDeployment := r.createRouterDeployment(jumpstarter, i) + + controllerutil.SetControllerReference(jumpstarter, desiredDeployment, r.Scheme) + + existingDeployment := &appsv1.Deployment{} + existingDeployment.Name = desiredDeployment.Name + existingDeployment.Namespace = desiredDeployment.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingDeployment, func() error { + // Check if this is a new deployment or an existing one + if existingDeployment.CreationTimestamp.IsZero() { + // Deployment is being created, copy all fields from desired + existingDeployment.Labels = desiredDeployment.Labels + existingDeployment.Annotations = desiredDeployment.Annotations + existingDeployment.Spec = desiredDeployment.Spec + return nil + } + desiredDeployment.Spec.Template.Spec.SchedulerName = existingDeployment.Spec.Template.Spec.SchedulerName + desiredDeployment.Spec.Template.Spec.DeprecatedServiceAccount = existingDeployment.Spec.Template.Spec.DeprecatedServiceAccount + + if !deploymentNeedsUpdate(existingDeployment, desiredDeployment) { + log.V(1).Info("Router deployment specs are equal, skipping update", + "name", existingDeployment.Name, + "namespace", existingDeployment.Namespace, + "replica", i) + return nil + } + // Deployment exists, generate and log diff + diff, err := generateDiff(existingDeployment, desiredDeployment) + if err != nil { + log.V(1).Info("Failed to generate deployment diff", "error", err) + } else if diff != "" { + fmt.Printf("\n=== Router deployment differences detected ===\n") + fmt.Printf("Name: %s\n", existingDeployment.Name) + fmt.Printf("Namespace: %s\n", existingDeployment.Namespace) + fmt.Printf("Replica: %d\n", i) + fmt.Printf("\n%s\n", diff) + fmt.Printf("==============================================\n\n") + } - // Create or update the deployment - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { - // Update deployment spec if needed - return nil + // Apply changes + existingDeployment.Labels = desiredDeployment.Labels + existingDeployment.Annotations = desiredDeployment.Annotations + existingDeployment.Spec.Replicas = desiredDeployment.Spec.Replicas + existingDeployment.Spec.Selector = desiredDeployment.Spec.Selector + existingDeployment.Spec.Template = desiredDeployment.Spec.Template + return controllerutil.SetControllerReference(jumpstarter, existingDeployment, r.Scheme) }) + if err != nil { + log.Error(err, "Failed to reconcile router deployment", + "name", desiredDeployment.Name, + "namespace", desiredDeployment.Namespace, + "replica", i) return err } + + log.Info("Router deployment reconciled", + "name", existingDeployment.Name, + "namespace", existingDeployment.Namespace, + "replica", i, + "operation", op) } // Clean up deployments for scaled-down replicas @@ -304,23 +401,56 @@ func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstart // reconcileConfigMaps reconciles all configmaps func (r *JumpstarterReconciler) reconcileConfigMaps(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { - configMap, err := r.createConfigMap(jumpstarter) + log := logf.FromContext(ctx) + desiredConfigMap, err := r.createConfigMap(jumpstarter) if err != nil { return fmt.Errorf("failed to create configmap: %w", err) } - // Set the owner reference - if err := controllerutil.SetControllerReference(jumpstarter, configMap, r.Scheme); err != nil { + existingConfigMap := &corev1.ConfigMap{} + existingConfigMap.Name = desiredConfigMap.Name + existingConfigMap.Namespace = desiredConfigMap.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingConfigMap, func() error { + // Check if this is a new configmap or an existing one + if existingConfigMap.CreationTimestamp.IsZero() { + // ConfigMap is being created, copy all fields from desired + existingConfigMap.Labels = desiredConfigMap.Labels + existingConfigMap.Annotations = desiredConfigMap.Annotations + existingConfigMap.Data = desiredConfigMap.Data + existingConfigMap.BinaryData = desiredConfigMap.BinaryData + return controllerutil.SetControllerReference(jumpstarter, existingConfigMap, r.Scheme) + } + + // ConfigMap exists, check if update is needed + if !configMapNeedsUpdate(existingConfigMap, desiredConfigMap, log) { + log.V(1).Info("ConfigMap is up to date, skipping update", + "name", existingConfigMap.Name, + "namespace", existingConfigMap.Namespace) + return nil + } + + // Update needed - apply changes + existingConfigMap.Labels = desiredConfigMap.Labels + existingConfigMap.Annotations = desiredConfigMap.Annotations + existingConfigMap.Data = desiredConfigMap.Data + existingConfigMap.BinaryData = desiredConfigMap.BinaryData + return controllerutil.SetControllerReference(jumpstarter, existingConfigMap, r.Scheme) + }) + + if err != nil { + log.Error(err, "Failed to reconcile configmap", + "name", desiredConfigMap.Name, + "namespace", desiredConfigMap.Namespace) return err } - // Create or update the configmap - _, err = controllerutil.CreateOrUpdate(ctx, r.Client, configMap, func() error { - // Update configmap data if needed - return nil - }) + log.Info("ConfigMap reconciled", + "name", existingConfigMap.Name, + "namespace", existingConfigMap.Namespace, + "operation", op) - return err + return nil } // reconcileSecrets reconciles all secrets @@ -454,7 +584,16 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator Labels: labels, }, Spec: appsv1.DeploymentSpec{ - Replicas: &jumpstarter.Spec.Controller.Replicas, + Replicas: &jumpstarter.Spec.Controller.Replicas, + ProgressDeadlineSeconds: int32Ptr(600), + RevisionHistoryLimit: int32Ptr(10), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &intstr.IntOrString{Type: intstr.String, StrVal: "25%"}, + MaxUnavailable: &intstr.IntOrString{Type: intstr.String, StrVal: "25%"}, + }, + }, Selector: &metav1.LabelSelector{ MatchLabels: labels, }, @@ -463,6 +602,9 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator Labels: labels, }, Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + DNSPolicy: corev1.DNSClusterFirst, + TerminationGracePeriodSeconds: int64Ptr(30), Containers: []corev1.Container{ { Name: "manager", @@ -504,7 +646,8 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator Name: "NAMESPACE", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", + FieldPath: "metadata.namespace", + APIVersion: "v1", }, }, }, @@ -517,37 +660,50 @@ func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operator { ContainerPort: 8082, Name: "grpc", + Protocol: corev1.ProtocolTCP, }, { ContainerPort: 8080, Name: "metrics", + Protocol: corev1.ProtocolTCP, }, { ContainerPort: 8081, Name: "health", + Protocol: corev1.ProtocolTCP, }, }, LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/healthz", - Port: intstr.FromInt(8081), + Path: "/healthz", + Port: intstr.FromInt(8081), + Scheme: corev1.URISchemeHTTP, }, }, InitialDelaySeconds: 15, PeriodSeconds: 20, + TimeoutSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 3, }, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/readyz", - Port: intstr.FromInt(8081), + Path: "/readyz", + Port: intstr.FromInt(8081), + Scheme: corev1.URISchemeHTTP, }, }, InitialDelaySeconds: 5, PeriodSeconds: 10, + TimeoutSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 3, }, - Resources: jumpstarter.Spec.Controller.Resources, + Resources: jumpstarter.Spec.Controller.Resources, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: corev1.TerminationMessageReadFile, SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: boolPtr(false), Capabilities: &corev1.Capabilities{ @@ -588,8 +744,6 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al // Build router endpoint for this specific replica routerEndpoint := r.buildRouterEndpointForReplica(jumpstarter, replicaIndex) - replicas := int32(1) // Each deployment has exactly 1 replica - return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-router-%d", jumpstarter.Name, replicaIndex), @@ -597,7 +751,16 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al Labels: labels, }, Spec: appsv1.DeploymentSpec{ - Replicas: &replicas, + Replicas: int32Ptr(1), // Each deployment for the router needs to have exactly 1 replica + ProgressDeadlineSeconds: int32Ptr(600), + RevisionHistoryLimit: int32Ptr(10), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &intstr.IntOrString{Type: intstr.String, StrVal: "25%"}, + MaxUnavailable: &intstr.IntOrString{Type: intstr.String, StrVal: "25%"}, + }, + }, Selector: &metav1.LabelSelector{ MatchLabels: labels, }, @@ -606,6 +769,9 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al Labels: labels, }, Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + DNSPolicy: corev1.DNSClusterFirst, + TerminationGracePeriodSeconds: int64Ptr(30), Containers: []corev1.Container{ { Name: "router", @@ -632,7 +798,8 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al Name: "NAMESPACE", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", + FieldPath: "metadata.namespace", + APIVersion: "v1", }, }, }, @@ -641,9 +808,22 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al { ContainerPort: 8083, Name: "grpc", + Protocol: corev1.ProtocolTCP, + }, + { + ContainerPort: 8080, + Name: "metrics", + Protocol: corev1.ProtocolTCP, + }, + { + ContainerPort: 8081, + Name: "health", + Protocol: corev1.ProtocolTCP, }, }, - Resources: jumpstarter.Spec.Routers.Resources, + Resources: jumpstarter.Spec.Routers.Resources, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: corev1.TerminationMessageReadFile, SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: boolPtr(false), Capabilities: &corev1.Capabilities{ @@ -658,9 +838,8 @@ func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1al Type: corev1.SeccompProfileTypeRuntimeDefault, }, }, - ServiceAccountName: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), - TopologySpreadConstraints: jumpstarter.Spec.Routers.TopologySpreadConstraints, - TerminationGracePeriodSeconds: int64Ptr(10), + ServiceAccountName: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), + TopologySpreadConstraints: jumpstarter.Spec.Routers.TopologySpreadConstraints, }, }, }, @@ -832,6 +1011,11 @@ func (r *JumpstarterReconciler) substituteReplica(address string, replicaIndex i return strings.ReplaceAll(address, "$(replica)", fmt.Sprintf("%d", replicaIndex)) } +// int32Ptr returns a pointer to an int32 value +func int32Ptr(i int32) *int32 { + return &i +} + // int64Ptr returns a pointer to an int64 value func int64Ptr(i int64) *int64 { return &i diff --git a/deploy/operator/internal/controller/jumpstarter/rbac.go b/deploy/operator/internal/controller/jumpstarter/rbac.go index d7697def..c02d6c4e 100644 --- a/deploy/operator/internal/controller/jumpstarter/rbac.go +++ b/deploy/operator/internal/controller/jumpstarter/rbac.go @@ -8,45 +8,151 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" ) // reconcileRBAC reconciles all RBAC resources (ServiceAccount, Role, RoleBinding) func (r *JumpstarterReconciler) reconcileRBAC(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { - // Create ServiceAccount - sa := r.createServiceAccount(jumpstarter) - if err := controllerutil.SetControllerReference(jumpstarter, sa, r.Scheme); err != nil { - return err - } - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sa, func() error { - return nil - }); err != nil { - return err - } + log := logf.FromContext(ctx) - // Create Role - role := r.createRole(jumpstarter) - if err := controllerutil.SetControllerReference(jumpstarter, role, r.Scheme); err != nil { - return err - } - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, role, func() error { - return nil - }); err != nil { + // ServiceAccount + desiredSA := r.createServiceAccount(jumpstarter) + controllerutil.SetControllerReference(jumpstarter, desiredSA, r.Scheme) + + existingSA := &corev1.ServiceAccount{} + existingSA.Name = desiredSA.Name + existingSA.Namespace = desiredSA.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingSA, func() error { + // Check if this is a new service account or an existing one + if existingSA.CreationTimestamp.IsZero() { + // ServiceAccount is being created, copy all fields from desired + existingSA.Labels = desiredSA.Labels + existingSA.Annotations = desiredSA.Annotations + + return nil + } + + // ServiceAccount exists, check if update is needed + if !serviceAccountNeedsUpdate(existingSA, desiredSA) { + log.V(1).Info("ServiceAccount is up to date, skipping update", + "name", existingSA.Name, + "namespace", existingSA.Namespace) + return nil + } + + // Update needed - apply changes + existingSA.Labels = desiredSA.Labels + existingSA.Annotations = desiredSA.Annotations + return controllerutil.SetControllerReference(jumpstarter, existingSA, r.Scheme) + }) + + if err != nil { + log.Error(err, "Failed to reconcile ServiceAccount", + "name", desiredSA.Name, + "namespace", desiredSA.Namespace) return err } - // Create RoleBinding - roleBinding := r.createRoleBinding(jumpstarter) - if err := controllerutil.SetControllerReference(jumpstarter, roleBinding, r.Scheme); err != nil { + log.Info("ServiceAccount reconciled", + "name", existingSA.Name, + "namespace", existingSA.Namespace, + "operation", op) + + // Role + desiredRole := r.createRole(jumpstarter) + controllerutil.SetControllerReference(jumpstarter, desiredRole, r.Scheme) + + existingRole := &rbacv1.Role{} + existingRole.Name = desiredRole.Name + existingRole.Namespace = desiredRole.Namespace + + op, err = controllerutil.CreateOrUpdate(ctx, r.Client, existingRole, func() error { + // Check if this is a new role or an existing one + if existingRole.CreationTimestamp.IsZero() { + // Role is being created, copy all fields from desired + existingRole.Labels = desiredRole.Labels + existingRole.Annotations = desiredRole.Annotations + existingRole.Rules = desiredRole.Rules + return nil + } + + // Role exists, check if update is needed + if !roleNeedsUpdate(existingRole, desiredRole) { + log.V(1).Info("Role is up to date, skipping update", + "name", existingRole.Name, + "namespace", existingRole.Namespace) + return nil + } + + // Update needed - apply changes + existingRole.Labels = desiredRole.Labels + existingRole.Annotations = desiredRole.Annotations + existingRole.Rules = desiredRole.Rules + return controllerutil.SetControllerReference(jumpstarter, existingRole, r.Scheme) + }) + + if err != nil { + log.Error(err, "Failed to reconcile Role", + "name", desiredRole.Name, + "namespace", desiredRole.Namespace) return err } - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, roleBinding, func() error { - return nil - }); err != nil { + + log.Info("Role reconciled", + "name", existingRole.Name, + "namespace", existingRole.Namespace, + "operation", op) + + // RoleBinding + desiredRoleBinding := r.createRoleBinding(jumpstarter) + controllerutil.SetControllerReference(jumpstarter, desiredRoleBinding, r.Scheme) + + existingRoleBinding := &rbacv1.RoleBinding{} + existingRoleBinding.Name = desiredRoleBinding.Name + existingRoleBinding.Namespace = desiredRoleBinding.Namespace + + op, err = controllerutil.CreateOrUpdate(ctx, r.Client, existingRoleBinding, func() error { + // Check if this is a new role binding or an existing one + if existingRoleBinding.CreationTimestamp.IsZero() { + // RoleBinding is being created, copy all fields from desired + existingRoleBinding.Labels = desiredRoleBinding.Labels + existingRoleBinding.Annotations = desiredRoleBinding.Annotations + existingRoleBinding.Subjects = desiredRoleBinding.Subjects + existingRoleBinding.RoleRef = desiredRoleBinding.RoleRef + return nil + } + + // RoleBinding exists, check if update is needed + if !roleBindingNeedsUpdate(existingRoleBinding, desiredRoleBinding) { + log.V(1).Info("RoleBinding is up to date, skipping update", + "name", existingRoleBinding.Name, + "namespace", existingRoleBinding.Namespace) + return nil + } + + // Update needed - apply changes + existingRoleBinding.Labels = desiredRoleBinding.Labels + existingRoleBinding.Annotations = desiredRoleBinding.Annotations + existingRoleBinding.Subjects = desiredRoleBinding.Subjects + existingRoleBinding.RoleRef = desiredRoleBinding.RoleRef + return controllerutil.SetControllerReference(jumpstarter, existingRoleBinding, r.Scheme) + }) + + if err != nil { + log.Error(err, "Failed to reconcile RoleBinding", + "name", desiredRoleBinding.Name, + "namespace", desiredRoleBinding.Namespace) return err } + log.Info("RoleBinding reconciled", + "name", existingRoleBinding.Name, + "namespace", existingRoleBinding.Namespace, + "operation", op) + return nil } From 096b27a93a441b0541b14d7a9c00de856011fc63 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Fri, 24 Oct 2025 11:43:26 +0000 Subject: [PATCH 23/26] operator: do not own the service account to avoid garbage collection --- deploy/operator/internal/controller/jumpstarter/rbac.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/operator/internal/controller/jumpstarter/rbac.go b/deploy/operator/internal/controller/jumpstarter/rbac.go index c02d6c4e..bfdc3a78 100644 --- a/deploy/operator/internal/controller/jumpstarter/rbac.go +++ b/deploy/operator/internal/controller/jumpstarter/rbac.go @@ -18,8 +18,9 @@ func (r *JumpstarterReconciler) reconcileRBAC(ctx context.Context, jumpstarter * log := logf.FromContext(ctx) // ServiceAccount + // Note: We intentionally do NOT set controller reference on ServiceAccount to prevent + // it from being garbage collected when the Jumpstarter CR is deleted desiredSA := r.createServiceAccount(jumpstarter) - controllerutil.SetControllerReference(jumpstarter, desiredSA, r.Scheme) existingSA := &corev1.ServiceAccount{} existingSA.Name = desiredSA.Name @@ -31,7 +32,6 @@ func (r *JumpstarterReconciler) reconcileRBAC(ctx context.Context, jumpstarter * // ServiceAccount is being created, copy all fields from desired existingSA.Labels = desiredSA.Labels existingSA.Annotations = desiredSA.Annotations - return nil } @@ -46,7 +46,7 @@ func (r *JumpstarterReconciler) reconcileRBAC(ctx context.Context, jumpstarter * // Update needed - apply changes existingSA.Labels = desiredSA.Labels existingSA.Annotations = desiredSA.Annotations - return controllerutil.SetControllerReference(jumpstarter, existingSA, r.Scheme) + return nil }) if err != nil { From 9392eb9ab8c1f617504bff0476c71ddbb061a4ee Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Fri, 24 Oct 2025 11:52:46 +0000 Subject: [PATCH 24/26] operator: include version information in the router too --- Dockerfile | 5 ++++- Makefile | 2 +- cmd/router/main.go | 14 ++++++++++++ deploy/operator/internal/utils/maps.go | 30 -------------------------- hack/deploy_with_operator.sh | 2 +- 5 files changed, 20 insertions(+), 33 deletions(-) delete mode 100644 deploy/operator/internal/utils/maps.go diff --git a/Dockerfile b/Dockerfile index 818ee32c..106652d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,10 @@ RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=10 -o manager cmd/main.go RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ - CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o router cmd/router/main.go + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -a \ + -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ + -o router cmd/router/main.go FROM registry.access.redhat.com/ubi9/ubi-micro:9.5 WORKDIR / diff --git a/Makefile b/Makefile index f94ac13b..4d7ee6c6 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ build-operator: .PHONY: build build: manifests generate fmt vet ## Build manager binary. go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go - go build -o bin/router cmd/router/main.go + go build -ldflags "$(LDFLAGS)" -o bin/router cmd/router/main.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. diff --git a/cmd/router/main.go b/cmd/router/main.go index beb1570b..04faaf10 100644 --- a/cmd/router/main.go +++ b/cmd/router/main.go @@ -34,6 +34,13 @@ import ( _ "google.golang.org/grpc/encoding/gzip" ) +var ( + // Version information - set via ldflags at build time + version = "dev" + gitCommit = "unknown" + buildDate = "unknown" +) + func main() { opts := zap.Options{} opts.BindFlags(flag.CommandLine) @@ -44,6 +51,13 @@ func main() { logger := ctrl.Log.WithName("router") ctx := logr.NewContext(context.Background(), logger) + // Print version information + logger.Info("Jumpstarter Router starting", + "version", version, + "gitCommit", gitCommit, + "buildDate", buildDate, + ) + cfg := ctrl.GetConfigOrDie() client, err := kclient.New(cfg, kclient.Options{}) if err != nil { diff --git a/deploy/operator/internal/utils/maps.go b/deploy/operator/internal/utils/maps.go deleted file mode 100644 index aadc58f9..00000000 --- a/deploy/operator/internal/utils/maps.go +++ /dev/null @@ -1,30 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -// MapsEqual compares two string maps for equality -func MapsEqual(a, b map[string]string) bool { - if len(a) != len(b) { - return false - } - for k, v := range a { - if b[k] != v { - return false - } - } - return true -} diff --git a/hack/deploy_with_operator.sh b/hack/deploy_with_operator.sh index ec12c445..ccd0d0c6 100755 --- a/hack/deploy_with_operator.sh +++ b/hack/deploy_with_operator.sh @@ -11,7 +11,7 @@ source "${SCRIPT_DIR}/deploy_vars" kubectl config use-context kind-jumpstarter # Install nginx ingress if in ingress mode -if [ "${INGRESS_ENABLED}" == "true" ]; then +if [ "${NETWORKING_MODE}" = "ingress" ]; then install_nginx_ingress else echo -e "${GREEN}Deploying with nodeport ...${NC}" From f1668b07dfa3ee5b4eb9c2b3eb65132cb389e25c Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 27 Oct 2025 10:14:41 +0000 Subject: [PATCH 25/26] operator: set controller references correctly --- .../controller/jumpstarter/endpoints/endpoints.go | 11 ++--------- .../controller/jumpstarter/jumpstarter_controller.go | 8 ++------ .../operator/internal/controller/jumpstarter/rbac.go | 6 ++---- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go index fce9d71f..aed06ee3 100644 --- a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go @@ -53,10 +53,6 @@ func (r *Reconciler) createOrUpdateService(ctx context.Context, service *corev1. existingService.Name = service.Name existingService.Namespace = service.Namespace - if err := controllerutil.SetControllerReference(owner, service, r.Scheme); err != nil { - return err - } - op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingService, func() error { // Preserve immutable fields if service already exists if existingService.CreationTimestamp.IsZero() { @@ -64,9 +60,7 @@ func (r *Reconciler) createOrUpdateService(ctx context.Context, service *corev1. existingService.Spec = service.Spec existingService.Labels = service.Labels existingService.Annotations = service.Annotations - if err := controllerutil.SetControllerReference(owner, service, r.Scheme); err != nil { - return err - } + return controllerutil.SetControllerReference(owner, existingService, r.Scheme) } else { // Preserve existing NodePorts to prevent "port already allocated" errors @@ -91,9 +85,8 @@ func (r *Reconciler) createOrUpdateService(ctx context.Context, service *corev1. existingService.Spec.Type = service.Spec.Type existingService.Labels = service.Labels existingService.Annotations = service.Annotations + return controllerutil.SetControllerReference(owner, existingService, r.Scheme) } - - return nil }) if err != nil { diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 82400691..5009999d 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -175,8 +175,6 @@ func (r *JumpstarterReconciler) reconcileControllerDeployment(ctx context.Contex log := logf.FromContext(ctx) desiredDeployment := r.createControllerDeployment(jumpstarter) - controllerutil.SetControllerReference(jumpstarter, desiredDeployment, r.Scheme) - existingDeployment := &appsv1.Deployment{} existingDeployment.Name = desiredDeployment.Name existingDeployment.Namespace = desiredDeployment.Namespace @@ -188,7 +186,7 @@ func (r *JumpstarterReconciler) reconcileControllerDeployment(ctx context.Contex existingDeployment.Labels = desiredDeployment.Labels existingDeployment.Annotations = desiredDeployment.Annotations existingDeployment.Spec = desiredDeployment.Spec - return nil + return controllerutil.SetControllerReference(jumpstarter, existingDeployment, r.Scheme) } desiredDeployment.Spec.Template.Spec.DeprecatedServiceAccount = existingDeployment.Spec.Template.Spec.DeprecatedServiceAccount @@ -246,8 +244,6 @@ func (r *JumpstarterReconciler) reconcileRouterDeployment(ctx context.Context, j for i := int32(0); i < jumpstarter.Spec.Routers.Replicas; i++ { desiredDeployment := r.createRouterDeployment(jumpstarter, i) - controllerutil.SetControllerReference(jumpstarter, desiredDeployment, r.Scheme) - existingDeployment := &appsv1.Deployment{} existingDeployment.Name = desiredDeployment.Name existingDeployment.Namespace = desiredDeployment.Namespace @@ -259,7 +255,7 @@ func (r *JumpstarterReconciler) reconcileRouterDeployment(ctx context.Context, j existingDeployment.Labels = desiredDeployment.Labels existingDeployment.Annotations = desiredDeployment.Annotations existingDeployment.Spec = desiredDeployment.Spec - return nil + return controllerutil.SetControllerReference(jumpstarter, existingDeployment, r.Scheme) } desiredDeployment.Spec.Template.Spec.SchedulerName = existingDeployment.Spec.Template.Spec.SchedulerName desiredDeployment.Spec.Template.Spec.DeprecatedServiceAccount = existingDeployment.Spec.Template.Spec.DeprecatedServiceAccount diff --git a/deploy/operator/internal/controller/jumpstarter/rbac.go b/deploy/operator/internal/controller/jumpstarter/rbac.go index bfdc3a78..a0e6486b 100644 --- a/deploy/operator/internal/controller/jumpstarter/rbac.go +++ b/deploy/operator/internal/controller/jumpstarter/rbac.go @@ -63,7 +63,6 @@ func (r *JumpstarterReconciler) reconcileRBAC(ctx context.Context, jumpstarter * // Role desiredRole := r.createRole(jumpstarter) - controllerutil.SetControllerReference(jumpstarter, desiredRole, r.Scheme) existingRole := &rbacv1.Role{} existingRole.Name = desiredRole.Name @@ -76,7 +75,7 @@ func (r *JumpstarterReconciler) reconcileRBAC(ctx context.Context, jumpstarter * existingRole.Labels = desiredRole.Labels existingRole.Annotations = desiredRole.Annotations existingRole.Rules = desiredRole.Rules - return nil + return controllerutil.SetControllerReference(jumpstarter, existingRole, r.Scheme) } // Role exists, check if update is needed @@ -108,7 +107,6 @@ func (r *JumpstarterReconciler) reconcileRBAC(ctx context.Context, jumpstarter * // RoleBinding desiredRoleBinding := r.createRoleBinding(jumpstarter) - controllerutil.SetControllerReference(jumpstarter, desiredRoleBinding, r.Scheme) existingRoleBinding := &rbacv1.RoleBinding{} existingRoleBinding.Name = desiredRoleBinding.Name @@ -122,7 +120,7 @@ func (r *JumpstarterReconciler) reconcileRBAC(ctx context.Context, jumpstarter * existingRoleBinding.Annotations = desiredRoleBinding.Annotations existingRoleBinding.Subjects = desiredRoleBinding.Subjects existingRoleBinding.RoleRef = desiredRoleBinding.RoleRef - return nil + return controllerutil.SetControllerReference(jumpstarter, existingRoleBinding, r.Scheme) } // RoleBinding exists, check if update is needed From 4d5c98d760649f1005f41dcd9c8f0231c84ae7d0 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 27 Oct 2025 10:37:19 +0000 Subject: [PATCH 26/26] operator: complete rbac for accesspolicies --- deploy/operator/config/rbac/role.yaml | 2 ++ .../internal/controller/jumpstarter/jumpstarter_controller.go | 2 ++ deploy/operator/internal/controller/jumpstarter/rbac.go | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/deploy/operator/config/rbac/role.yaml b/deploy/operator/config/rbac/role.yaml index af22da23..d9a8394d 100644 --- a/deploy/operator/config/rbac/role.yaml +++ b/deploy/operator/config/rbac/role.yaml @@ -85,6 +85,7 @@ rules: - jumpstarter.dev resources: - clients/finalizers + - exporteraccesspolicies/finalizers - exporters/finalizers - leases/finalizers verbs: @@ -93,6 +94,7 @@ rules: - jumpstarter.dev resources: - clients/status + - exporteraccesspolicies/status - exporters/status - leases/status verbs: diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 5009999d..c6d7542b 100644 --- a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -98,6 +98,8 @@ type JumpstarterReconciler struct { // +kubebuilder:rbac:groups=jumpstarter.dev,resources=leases/status,verbs=get;update;patch // +kubebuilder:rbac:groups=jumpstarter.dev,resources=leases/finalizers,verbs=update // +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporteraccesspolicies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporteraccesspolicies/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporteraccesspolicies/finalizers,verbs=update // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile diff --git a/deploy/operator/internal/controller/jumpstarter/rbac.go b/deploy/operator/internal/controller/jumpstarter/rbac.go index a0e6486b..e0333618 100644 --- a/deploy/operator/internal/controller/jumpstarter/rbac.go +++ b/deploy/operator/internal/controller/jumpstarter/rbac.go @@ -199,12 +199,12 @@ func (r *JumpstarterReconciler) createRole(jumpstarter *operatorv1alpha1.Jumpsta }, { APIGroups: []string{"jumpstarter.dev"}, - Resources: []string{"clients/status", "exporters/status", "leases/status"}, + Resources: []string{"clients/status", "exporters/status", "leases/status", "exporteraccesspolicies/status"}, Verbs: []string{"get", "update", "patch"}, }, { APIGroups: []string{"jumpstarter.dev"}, - Resources: []string{"clients/finalizers", "exporters/finalizers", "leases/finalizers"}, + Resources: []string{"clients/finalizers", "exporters/finalizers", "leases/finalizers", "exporteraccesspolicies/finalizers"}, Verbs: []string{"update"}, }, {