diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8e36ed5a..81d00900 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,6 +39,16 @@ jobs: echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "VERSION=${VERSION}" + - name: Set build args + id: build-args + run: | + GIT_COMMIT=$(git rev-parse HEAD) + BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo "git_commit=${GIT_COMMIT}" >> $GITHUB_OUTPUT + echo "build_date=${BUILD_DATE}" >> $GITHUB_OUTPUT + echo "GIT_COMMIT=${GIT_COMMIT}" + echo "BUILD_DATE=${BUILD_DATE}" + - name: Set image tags id: set-tags run: | @@ -79,6 +89,10 @@ jobs: platforms: linux/amd64,linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max + build-args: | + GIT_VERSION=${{ env.VERSION }} + GIT_COMMIT=${{ steps.build-args.outputs.git_commit }} + BUILD_DATE=${{ steps.build-args.outputs.build_date }} publish-helm-charts-containers: needs: build-and-push-image diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 495c94cd..f8fbb5a8 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -19,10 +19,10 @@ jobs: controller-ref: ${{ github.ref }} # use the matching branch on the jumpstarter repo jumpstarter-ref: ${{ github.event.pull_request.base.ref }} - e2e-tests-28d6b1cc3b49ab9ae176918ab9709a2e2522c97e: + e2e-tests-release-0-7: runs-on: ubuntu-latest steps: - - uses: jumpstarter-dev/jumpstarter-e2e@11a5ce6734be9f089ec3ea6ebf55284616f67fe8 + - uses: jumpstarter-dev/jumpstarter-e2e@release-0.7 with: controller-ref: ${{ github.ref }} - jumpstarter-ref: 28d6b1cc3b49ab9ae176918ab9709a2e2522c97e + jumpstarter-ref: release-0.7 diff --git a/.github/workflows/pr-kind.yaml b/.github/workflows/pr-kind.yaml index d27ff2e7..51e83e95 100644 --- a/.github/workflows/pr-kind.yaml +++ b/.github/workflows/pr-kind.yaml @@ -17,3 +17,14 @@ jobs: - name: Run make deploy run: make deploy + + deploy-with-operator: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run make deploy + run: make deploy-with-operator \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c92f87e9..106652d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,13 +2,19 @@ FROM registry.access.redhat.com/ubi9/go-toolset:1.24.6 AS builder ARG TARGETOS ARG TARGETARCH +ARG GIT_VERSION=unknown +ARG GIT_COMMIT=unknown +ARG BUILD_DATE=unknown # Copy the Go Modules manifests COPY go.mod go.mod COPY go.sum go.sum # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer -RUN go mod download +# Cache module downloads across builds +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ + --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + go mod download # Copy the go source COPY cmd/ cmd/ @@ -20,8 +26,18 @@ COPY internal/ internal/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o router cmd/router/main.go +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ +--mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -a \ + -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ + -o manager cmd/main.go +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ + --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -a \ + -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ + -o router cmd/router/main.go FROM registry.access.redhat.com/ubi9/ubi-micro:9.5 WORKDIR / diff --git a/Dockerfile.operator b/Dockerfile.operator new file mode 100644 index 00000000..ccbd42a8 --- /dev/null +++ b/Dockerfile.operator @@ -0,0 +1,47 @@ +# Build the manager binary +FROM registry.access.redhat.com/ubi9/go-toolset:1.24.6 AS builder +ARG TARGETOS +ARG TARGETARCH +ARG GIT_VERSION=unknown +ARG GIT_COMMIT=unknown +ARG BUILD_DATE=unknown + +# Copy the Go Modules manifests +COPY --chown=1001:0 deploy/operator/go.mod deploy/operator/go.mod +COPY --chown=1001:0 deploy/operator/go.sum deploy/operator/go.sum +COPY --chown=1001:0 go.mod go.mod +COPY --chown=1001:0 go.sum go.sum + + +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ + --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + cd deploy/operator && go mod download + +# Copy the base jumpstarter-controller internal/config parts +COPY --chown=1001:0 internal/ internal/ +COPY --chown=1001:0 api/ api/ +# Copy the go source +COPY --chown=1001:0 deploy/operator/cmd/ deploy/operator/cmd/ +COPY --chown=1001:0 deploy/operator/api/ deploy/operator/api/ +COPY --chown=1001:0 deploy/operator/internal/ deploy/operator/internal/ + +# Build +# the GOARCH has not a default value to allow the binary be built according to the host where the command +# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO +# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, +# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ + --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + cd deploy/operator && go build -a \ + -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ + -o manager cmd/main.go + +FROM registry.access.redhat.com/ubi9/ubi-micro:9.5 +WORKDIR / +COPY --from=builder /opt/app-root/src/deploy/operator/manager . +USER 65532:65532 + +ENTRYPOINT ["/manager"] \ No newline at end of file diff --git a/Makefile b/Makefile index 184e6ae6..4d7ee6c6 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,16 @@ DOCKER_TAG = $(shell echo $(IMG) | cut -d: -f2) # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.30.0 +# Version information +GIT_VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "unknown") +GIT_COMMIT := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown") +BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') + +# LDFLAGS for version information +LDFLAGS := -X main.version=$(GIT_VERSION) \ + -X main.gitCommit=$(GIT_COMMIT) \ + -X main.buildDate=$(BUILD_DATE) + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -53,6 +63,8 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust output:crd:artifacts:config=deploy/helm/jumpstarter/crds/ \ output:rbac:artifacts:config=deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/rbac/ + cp deploy/helm/jumpstarter/crds/* deploy/operator/config/crd/bases/ + .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..." paths="./internal/..." @@ -83,11 +95,14 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes $(GOLANGCI_LINT) run --fix ##@ Build +.PHONY: build-operator +build-operator: + make -C deploy/operator build-installer docker-build .PHONY: build build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go - go build -o bin/router cmd/router/main.go + go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go + go build -ldflags "$(LDFLAGS)" -o bin/router cmd/router/main.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. @@ -102,7 +117,11 @@ run-router: manifests generate fmt vet ## Run a router from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . + $(CONTAINER_TOOL) build \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + -t ${IMG} . .PHONY: docker-push docker-push: ## Push docker image with the manager. @@ -122,6 +141,9 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform - $(CONTAINER_TOOL) buildx create --name jumpstarter-controller-builder $(CONTAINER_TOOL) buildx use jumpstarter-controller-builder - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ --tag ${DOCKER_REPO}:${DOCKER_TAG} \ --tag ${DOCKER_REPO}:latest \ -f Dockerfile.cross . @@ -152,6 +174,17 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified deploy: docker-build cluster grpcurl ./hack/deploy_with_helm.sh +.PHONY: deploy-with-operator +deploy-with-operator: docker-build build-operator cluster grpcurl + ./hack/deploy_with_operator.sh + +.PHONY: operator-logs +operator-logs: + kubectl logs -n jumpstarter-operator-system -l app.kubernetes.io/name=jumpstarter-operator -f + +deploy-with-operator-parallel: + make deploy-with-operator -j5 --output-sync=target + .PHONY: deploy-exporters deploy-exporters: ./hack/demoenv/prepare_exporters.sh @@ -185,7 +218,7 @@ GRPCURL = $(LOCALBIN)/grpcurl KUSTOMIZE_VERSION ?= v5.4.1 CONTROLLER_TOOLS_VERSION ?= v0.16.3 ENVTEST_VERSION ?= release-0.18 -GOLANGCI_LINT_VERSION ?= v2.1.2 +GOLANGCI_LINT_VERSION ?= v2.5.0 KIND_VERSION ?= v0.27.0 GRPCURL_VERSION ?= v1.9.2 diff --git a/cmd/main.go b/cmd/main.go index f49836c1..37f3fd27 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -33,6 +33,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -55,8 +56,42 @@ import ( var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") + + // Version information - set via ldflags at build time + version = "dev" + gitCommit = "unknown" + buildDate = "unknown" +) + +const ( + // namespaceFile is the path to the namespace file mounted by Kubernetes + namespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" ) +// getWatchNamespace returns the namespace the controller should watch. +// It tries multiple sources in order: +// 1. NAMESPACE environment variable (explicit configuration takes precedence) +// 2. Namespace file (automatically mounted by Kubernetes in every pod) +// 3. Empty string (will fail, not supported since 0.8.0) +func getWatchNamespace() string { + // First check NAMESPACE environment variable (explicit configuration) + if ns := os.Getenv("NAMESPACE"); ns != "" { + setupLog.Info("Using namespace from NAMESPACE environment variable", "namespace", ns) + return ns + } + + // Fall back to reading from the namespace file mounted by Kubernetes + if ns, err := os.ReadFile(namespaceFile); err == nil { + namespace := string(ns) + if namespace != "" { + setupLog.Info("Auto-detected namespace from service account", "namespace", namespace) + return namespace + } + } + + return "" +} + func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) @@ -90,6 +125,13 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + // Print version information + setupLog.Info("Jumpstarter Controller starting", + "version", version, + "gitCommit", gitCommit, + "buildDate", buildDate, + ) + // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will // prevent from being vulnerable to the HTTP/2 Stream Cancellation and @@ -110,7 +152,11 @@ func main() { TLSOpts: tlsOpts, }) - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + // Get the namespace to watch. Try to auto-detect from the pod's service account, + // fall back to NAMESPACE environment variable, or watch all namespaces if neither is available + watchNamespace := getWatchNamespace() + + mgrOptions := ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ BindAddress: metricsAddr, @@ -132,7 +178,22 @@ func main() { // if you are doing or is intended to do any operation such as perform cleanups // after the manager stops then its usage might be unsafe. // LeaderElectionReleaseOnCancel: true, - }) + } + + // If a specific namespace is set, configure the cache to only watch that namespace + if watchNamespace != "" { + mgrOptions.LeaderElectionNamespace = watchNamespace + mgrOptions.Cache = cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + watchNamespace: {}, + }, + } + } else { + setupLog.Error(nil, "Jumpstarter controller can only be configured to work on a single namespace since 0.8.0") + os.Exit(1) + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOptions) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) diff --git a/cmd/router/main.go b/cmd/router/main.go index beb1570b..04faaf10 100644 --- a/cmd/router/main.go +++ b/cmd/router/main.go @@ -34,6 +34,13 @@ import ( _ "google.golang.org/grpc/encoding/gzip" ) +var ( + // Version information - set via ldflags at build time + version = "dev" + gitCommit = "unknown" + buildDate = "unknown" +) + func main() { opts := zap.Options{} opts.BindFlags(flag.CommandLine) @@ -44,6 +51,13 @@ func main() { logger := ctrl.Log.WithName("router") ctx := logr.NewContext(context.Background(), logger) + // Print version information + logger.Info("Jumpstarter Router starting", + "version", version, + "gitCommit", gitCommit, + "buildDate", buildDate, + ) + cfg := ctrl.GetConfigOrDie() client, err := kclient.New(cfg, kclient.Options{}) if err != nil { diff --git a/deploy/operator/Dockerfile b/deploy/operator/Dockerfile deleted file mode 100644 index c180dbc4..00000000 --- a/deploy/operator/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# Build the manager binary -FROM registry.access.redhat.com/ubi9/go-toolset:1.24.6 AS builder -ARG TARGETOS -ARG TARGETARCH - -# Copy the Go Modules manifests -COPY --chown=1001:0 go.mod go.mod -COPY --chown=1001:0 go.sum go.sum -# cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN go mod download - -# Copy the go source -COPY --chown=1001:0 cmd/ cmd/ -COPY --chown=1001:0 api/ api/ -COPY --chown=1001:0 internal/ internal/ - -# Build -# the GOARCH has not a default value to allow the binary be built according to the host where the command -# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO -# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, -# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go - -FROM registry.access.redhat.com/ubi9/ubi-micro:9.5 -WORKDIR / -COPY --from=builder /opt/app-root/src/manager . -USER 65532:65532 - -ENTRYPOINT ["/manager"] diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 5c5efa97..3ab01b2f 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -29,7 +29,7 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) # # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both # jumpstarter.dev/jumpstarter-operator-bundle:$VERSION and jumpstarter.dev/jumpstarter-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= jumpstarter.dev/jumpstarter-operator +IMAGE_TAG_BASE ?= quay.io/jumpstarter-dev/jumpstarter-operator # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) @@ -50,7 +50,7 @@ endif # This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. OPERATOR_SDK_VERSION ?= v1.41.1 # Image URL to use all building/pushing image targets -IMG ?= controller:latest +IMG ?= quay.io/jumpstarter-dev/jumpstarter-operator:latest # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -65,6 +65,16 @@ endif # tools. (i.e. podman) CONTAINER_TOOL ?= podman +# Version information +GIT_VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "unknown") +GIT_COMMIT := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown") +BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') + +# LDFLAGS for version information +LDFLAGS := -X main.version=$(GIT_VERSION) \ + -X main.gitCommit=$(GIT_COMMIT) \ + -X main.buildDate=$(BUILD_DATE) + # Setting SHELL to bash allows bash commands to be executed by recipes. # Options are set to exit when a recipe line exits non-zero or a piped command fails. SHELL = /usr/bin/env bash -o pipefail @@ -157,7 +167,7 @@ lint-config: golangci-lint ## Verify golangci-lint linter configuration .PHONY: build build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go + go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. @@ -168,7 +178,11 @@ run: manifests generate fmt vet ## Run a controller from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . + $(CONTAINER_TOOL) build \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + -t ${IMG} ../../ -f ../../Dockerfile.operator .PHONY: docker-push docker-push: ## Push docker image with the manager. @@ -184,12 +198,16 @@ PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le .PHONY: docker-buildx docker-buildx: ## Build and push docker image for the manager for cross-platform support # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' ../../Dockerfile.operator > ../../Dockerfile.operator.cross - $(CONTAINER_TOOL) buildx create --name jumpstarter-operator-builder $(CONTAINER_TOOL) buildx use jumpstarter-operator-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + --tag ${IMG} -f ../../Dockerfile.operator.cross ../../ - $(CONTAINER_TOOL) buildx rm jumpstarter-operator-builder - rm Dockerfile.cross + rm ../../Dockerfile.operator.cross .PHONY: build-installer build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. @@ -242,7 +260,7 @@ CONTROLLER_TOOLS_VERSION ?= v0.18.0 ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') #ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}') -GOLANGCI_LINT_VERSION ?= v2.1.0 +GOLANGCI_LINT_VERSION ?= v2.5.0 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. diff --git a/deploy/operator/api/v1alpha1/jumpstarter_types.go b/deploy/operator/api/v1alpha1/jumpstarter_types.go index 4da5f89c..af9d4271 100644 --- a/deploy/operator/api/v1alpha1/jumpstarter_types.go +++ b/deploy/operator/api/v1alpha1/jumpstarter_types.go @@ -376,13 +376,15 @@ type RestAPIConfig struct { // Endpoint defines a single endpoint configuration. // An endpoint can use one or more networking methods: Route, Ingress, NodePort, or LoadBalancer. -// Multiple methods can be configured simultaneously for the same hostname. +// Multiple methods can be configured simultaneously for the same address. type Endpoint struct { - // Hostname for this endpoint. + // Address for this endpoint in the format "hostname", "hostname:port", "IPv4", "IPv4:port", "[IPv6]", or "[IPv6]:port". // Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. - // When optional, the hostname is used for certificate generation and DNS resolution. - // +kubebuilder:validation:Pattern=^[a-z0-9]([a-z0-9\-\.]*[a-z0-9])?$ - Hostname string `json:"hostname,omitempty"` + // When optional, the address is used for certificate generation and DNS resolution. + // Supports templating with $(replica) for replica-specific addresses. + // Examples: "grpc.example.com", "grpc.example.com:9090", "192.168.1.1:8080", "[2001:db8::1]:8443", "router-$(replica).example.com" + // +kubebuilder:validation:Pattern=`^(\[[0-9a-fA-F:\.]+\]|[0-9]+(\.[0-9]+){3}|[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?)(:[0-9]+)?$` + Address string `json:"address,omitempty"` // Route configuration for OpenShift clusters. // Creates an OpenShift Route resource for this endpoint. @@ -403,6 +405,12 @@ type Endpoint struct { // Creates a LoadBalancer service for this endpoint. // Requires cloud provider support for LoadBalancer services. LoadBalancer *LoadBalancerConfig `json:"loadBalancer,omitempty"` + + // ClusterIP configuration for internal service access. + // Creates a ClusterIP service for this endpoint. + // Useful for internal service-to-service communication or when + // using a different method to expose the service externally. + ClusterIP *ClusterIPConfig `json:"clusterIP,omitempty"` } // RouteConfig defines OpenShift Route configuration. @@ -487,6 +495,21 @@ type LoadBalancerConfig struct { Labels map[string]string `json:"labels,omitempty"` } +// ClusterIPConfig defines Kubernetes ClusterIP service configuration. +type ClusterIPConfig struct { + // Enable the ClusterIP service for this endpoint. + // When disabled, no ClusterIP service will be created for this endpoint. + Enabled bool `json:"enabled,omitempty"` + + // Annotations to add to the ClusterIP service. + // Useful for configuring service-specific behavior and load balancer options. + Annotations map[string]string `json:"annotations,omitempty"` + + // Labels to add to the ClusterIP service. + // Useful for monitoring, cost allocation, and resource organization. + Labels map[string]string `json:"labels,omitempty"` +} + // JumpstarterStatus defines the observed state of Jumpstarter. // This field is currently empty but can be extended to include status information // such as deployment status, endpoint URLs, and health information. diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 1be37fc2..a6764a38 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ //go:build !ignore_autogenerated /* -Copyright 2024. +Copyright 2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -51,6 +51,35 @@ func (in *AuthenticationConfig) DeepCopy() *AuthenticationConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterIPConfig) DeepCopyInto(out *ClusterIPConfig) { + *out = *in + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterIPConfig. +func (in *ClusterIPConfig) DeepCopy() *ClusterIPConfig { + if in == nil { + return nil + } + out := new(ClusterIPConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControllerConfig) DeepCopyInto(out *ControllerConfig) { *out = *in @@ -94,6 +123,11 @@ func (in *Endpoint) DeepCopyInto(out *Endpoint) { *out = new(LoadBalancerConfig) (*in).DeepCopyInto(*out) } + if in.ClusterIP != nil { + in, out := &in.ClusterIP, &out.ClusterIP + *out = new(ClusterIPConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Endpoint. diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index c404f2e2..f1f615db 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -38,13 +38,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" - "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/jumpstarter" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/jumpstarter/endpoints" // +kubebuilder:scaffold:imports ) var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") + + // Version information - set via ldflags at build time + version = "dev" + gitCommit = "unknown" + buildDate = "unknown" ) func init() { @@ -89,6 +95,13 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + // Print version information + setupLog.Info("Jumpstarter Operator starting", + "version", version, + "gitCommit", gitCommit, + "buildDate", buildDate, + ) + // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will // prevent from being vulnerable to the HTTP/2 Stream Cancellation and @@ -202,9 +215,10 @@ func main() { os.Exit(1) } - if err := (&controller.JumpstarterReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + if err := (&jumpstarter.JumpstarterReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + EndpointReconciler: endpoints.NewReconciler(mgr.GetClient(), mgr.GetScheme()), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Jumpstarter") os.Exit(1) diff --git a/deploy/operator/config/crd/bases/jumpstarter.dev_clients.yaml b/deploy/operator/config/crd/bases/jumpstarter.dev_clients.yaml new file mode 100644 index 00000000..ae94bdfe --- /dev/null +++ b/deploy/operator/config/crd/bases/jumpstarter.dev_clients.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: clients.jumpstarter.dev +spec: + group: jumpstarter.dev + names: + kind: Client + listKind: ClientList + plural: clients + singular: client + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Client is the Schema for the identities API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClientSpec defines the desired state of Identity + properties: + username: + type: string + type: object + status: + description: ClientStatus defines the observed state of Identity + properties: + credential: + description: |- + INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + Important: Run "make" to regenerate code after modifying this file + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + endpoint: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/operator/config/crd/bases/jumpstarter.dev_exporteraccesspolicies.yaml b/deploy/operator/config/crd/bases/jumpstarter.dev_exporteraccesspolicies.yaml new file mode 100644 index 00000000..ec1b7878 --- /dev/null +++ b/deploy/operator/config/crd/bases/jumpstarter.dev_exporteraccesspolicies.yaml @@ -0,0 +1,166 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: exporteraccesspolicies.jumpstarter.dev +spec: + group: jumpstarter.dev + names: + kind: ExporterAccessPolicy + listKind: ExporterAccessPolicyList + plural: exporteraccesspolicies + singular: exporteraccesspolicy + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ExporterAccessPolicy is the Schema for the exporteraccesspolicies + API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ExporterAccessPolicySpec defines the desired state of ExporterAccessPolicy. + properties: + exporterSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + policies: + items: + properties: + from: + items: + properties: + clientSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: array + maximumDuration: + type: string + priority: + type: integer + spotAccess: + type: boolean + type: object + type: array + type: object + status: + description: ExporterAccessPolicyStatus defines the observed state of + ExporterAccessPolicy. + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/operator/config/crd/bases/jumpstarter.dev_exporters.yaml b/deploy/operator/config/crd/bases/jumpstarter.dev_exporters.yaml new file mode 100644 index 00000000..6383bbf9 --- /dev/null +++ b/deploy/operator/config/crd/bases/jumpstarter.dev_exporters.yaml @@ -0,0 +1,162 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: exporters.jumpstarter.dev +spec: + group: jumpstarter.dev + names: + kind: Exporter + listKind: ExporterList + plural: exporters + singular: exporter + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Exporter is the Schema for the exporters API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ExporterSpec defines the desired state of Exporter + properties: + username: + type: string + type: object + status: + description: ExporterStatus defines the observed state of Exporter + properties: + conditions: + description: |- + INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + Important: Run "make" to regenerate code after modifying this file + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + credential: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + devices: + items: + properties: + labels: + additionalProperties: + type: string + type: object + parent_uuid: + type: string + uuid: + type: string + type: object + type: array + endpoint: + type: string + lastSeen: + format: date-time + type: string + leaseRef: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/operator/config/crd/bases/jumpstarter.dev_leases.yaml b/deploy/operator/config/crd/bases/jumpstarter.dev_leases.yaml new file mode 100644 index 00000000..9aafc859 --- /dev/null +++ b/deploy/operator/config/crd/bases/jumpstarter.dev_leases.yaml @@ -0,0 +1,235 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: leases.jumpstarter.dev +spec: + group: jumpstarter.dev + names: + kind: Lease + listKind: LeaseList + plural: leases + singular: lease + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.ended + name: Ended + type: boolean + - jsonPath: .spec.clientRef.name + name: Client + type: string + - jsonPath: .status.exporterRef.name + name: Exporter + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: Lease is the Schema for the exporters API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: LeaseSpec defines the desired state of Lease + properties: + beginTime: + description: |- + Requested start time. If omitted, lease starts when exporter is acquired. + Immutable after lease starts (cannot change the past). + format: date-time + type: string + clientRef: + description: The client that is requesting the lease + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + duration: + description: |- + Duration of the lease. Must be positive when provided. + Can be omitted (nil) when both BeginTime and EndTime are provided, + in which case it's calculated as EndTime - BeginTime. + type: string + endTime: + description: |- + Requested end time. If specified with BeginTime, Duration is calculated. + Can be updated to extend or shorten active leases. + format: date-time + type: string + release: + description: The release flag requests the controller to end the lease + now + type: boolean + selector: + description: The selector for the exporter to be used + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - clientRef + - selector + type: object + status: + description: LeaseStatus defines the observed state of Lease + properties: + beginTime: + description: |- + If the lease has been acquired an exporter name is assigned + and then it can be used, it will be empty while still pending + format: date-time + type: string + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + endTime: + format: date-time + type: string + ended: + type: boolean + exporterRef: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + priority: + type: integer + spotAccess: + type: boolean + required: + - ended + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml b/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml index 9e7332ee..dbb182c3 100644 --- a/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml +++ b/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml @@ -843,15 +843,44 @@ spec: description: |- Endpoint defines a single endpoint configuration. An endpoint can use one or more networking methods: Route, Ingress, NodePort, or LoadBalancer. - Multiple methods can be configured simultaneously for the same hostname. + Multiple methods can be configured simultaneously for the same address. properties: - hostname: + address: description: |- - Hostname for this endpoint. + Address for this endpoint in the format "hostname", "hostname:port", "IPv4", "IPv4:port", "[IPv6]", or "[IPv6]:port". Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. - When optional, the hostname is used for certificate generation and DNS resolution. - pattern: ^[a-z0-9]([a-z0-9\-\.]*[a-z0-9])?$ + When optional, the address is used for certificate generation and DNS resolution. + Supports templating with $(replica) for replica-specific addresses. + Examples: "grpc.example.com", "grpc.example.com:9090", "192.168.1.1:8080", "[2001:db8::1]:8443", "router-$(replica).example.com" + pattern: ^(\[[0-9a-fA-F:\.]+\]|[0-9]+(\.[0-9]+){3}|[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?)(:[0-9]+)?$ type: string + clusterIP: + description: |- + ClusterIP configuration for internal service access. + Creates a ClusterIP service for this endpoint. + Useful for internal service-to-service communication or when + using a different method to expose the service externally. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations to add to the ClusterIP service. + Useful for configuring service-specific behavior and load balancer options. + type: object + enabled: + description: |- + Enable the ClusterIP service for this endpoint. + When disabled, no ClusterIP service will be created for this endpoint. + type: boolean + labels: + additionalProperties: + type: string + description: |- + Labels to add to the ClusterIP service. + Useful for monitoring, cost allocation, and resource organization. + type: object + type: object ingress: description: |- Ingress configuration for standard Kubernetes clusters. @@ -1152,15 +1181,44 @@ spec: description: |- Endpoint defines a single endpoint configuration. An endpoint can use one or more networking methods: Route, Ingress, NodePort, or LoadBalancer. - Multiple methods can be configured simultaneously for the same hostname. + Multiple methods can be configured simultaneously for the same address. properties: - hostname: + address: description: |- - Hostname for this endpoint. + Address for this endpoint in the format "hostname", "hostname:port", "IPv4", "IPv4:port", "[IPv6]", or "[IPv6]:port". Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. - When optional, the hostname is used for certificate generation and DNS resolution. - pattern: ^[a-z0-9]([a-z0-9\-\.]*[a-z0-9])?$ + When optional, the address is used for certificate generation and DNS resolution. + Supports templating with $(replica) for replica-specific addresses. + Examples: "grpc.example.com", "grpc.example.com:9090", "192.168.1.1:8080", "[2001:db8::1]:8443", "router-$(replica).example.com" + pattern: ^(\[[0-9a-fA-F:\.]+\]|[0-9]+(\.[0-9]+){3}|[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?)(:[0-9]+)?$ type: string + clusterIP: + description: |- + ClusterIP configuration for internal service access. + Creates a ClusterIP service for this endpoint. + Useful for internal service-to-service communication or when + using a different method to expose the service externally. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations to add to the ClusterIP service. + Useful for configuring service-specific behavior and load balancer options. + type: object + enabled: + description: |- + Enable the ClusterIP service for this endpoint. + When disabled, no ClusterIP service will be created for this endpoint. + type: boolean + labels: + additionalProperties: + type: string + description: |- + Labels to add to the ClusterIP service. + Useful for monitoring, cost allocation, and resource organization. + type: object + type: object ingress: description: |- Ingress configuration for standard Kubernetes clusters. @@ -1328,15 +1386,44 @@ spec: description: |- Endpoint defines a single endpoint configuration. An endpoint can use one or more networking methods: Route, Ingress, NodePort, or LoadBalancer. - Multiple methods can be configured simultaneously for the same hostname. + Multiple methods can be configured simultaneously for the same address. properties: - hostname: + address: description: |- - Hostname for this endpoint. + Address for this endpoint in the format "hostname", "hostname:port", "IPv4", "IPv4:port", "[IPv6]", or "[IPv6]:port". Required for Route and Ingress endpoints. Optional for NodePort and LoadBalancer endpoints. - When optional, the hostname is used for certificate generation and DNS resolution. - pattern: ^[a-z0-9]([a-z0-9\-\.]*[a-z0-9])?$ + When optional, the address is used for certificate generation and DNS resolution. + Supports templating with $(replica) for replica-specific addresses. + Examples: "grpc.example.com", "grpc.example.com:9090", "192.168.1.1:8080", "[2001:db8::1]:8443", "router-$(replica).example.com" + pattern: ^(\[[0-9a-fA-F:\.]+\]|[0-9]+(\.[0-9]+){3}|[a-z0-9$]([a-z0-9\-\.\$\(\)]*[a-z0-9\)])?)(:[0-9]+)?$ type: string + clusterIP: + description: |- + ClusterIP configuration for internal service access. + Creates a ClusterIP service for this endpoint. + Useful for internal service-to-service communication or when + using a different method to expose the service externally. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations to add to the ClusterIP service. + Useful for configuring service-specific behavior and load balancer options. + type: object + enabled: + description: |- + Enable the ClusterIP service for this endpoint. + When disabled, no ClusterIP service will be created for this endpoint. + type: boolean + labels: + additionalProperties: + type: string + description: |- + Labels to add to the ClusterIP service. + Useful for monitoring, cost allocation, and resource organization. + type: object + type: object ingress: description: |- Ingress configuration for standard Kubernetes clusters. diff --git a/deploy/operator/config/crd/kustomization.yaml b/deploy/operator/config/crd/kustomization.yaml index 4fff9a5f..70cefba9 100644 --- a/deploy/operator/config/crd/kustomization.yaml +++ b/deploy/operator/config/crd/kustomization.yaml @@ -3,6 +3,11 @@ # It should be run by config/default resources: - bases/operator.jumpstarter.dev_jumpstarters.yaml +- bases/jumpstarter.dev_clients.yaml +- bases/jumpstarter.dev_exporters.yaml +- bases/jumpstarter.dev_leases.yaml +- bases/jumpstarter.dev_exporteraccesspolicies.yaml + # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/deploy/operator/config/manager/kustomization.yaml b/deploy/operator/config/manager/kustomization.yaml index 5c5f0b84..9c94df03 100644 --- a/deploy/operator/config/manager/kustomization.yaml +++ b/deploy/operator/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/jumpstarter-dev/jumpstarter-operator + newTag: latest diff --git a/deploy/operator/config/manager/manager.yaml b/deploy/operator/config/manager/manager.yaml index 9c429c08..66b270dc 100644 --- a/deploy/operator/config/manager/manager.yaml +++ b/deploy/operator/config/manager/manager.yaml @@ -63,7 +63,8 @@ spec: args: - --leader-elect - --health-probe-bind-address=:8081 - image: controller:latest + image: quay.io/jumpstarter-dev/jumpstarter-operator:latest + imagePullPolicy: IfNotPresent name: manager ports: [] securityContext: diff --git a/deploy/operator/config/rbac/role.yaml b/deploy/operator/config/rbac/role.yaml index a3a3c3ea..d9a8394d 100644 --- a/deploy/operator/config/rbac/role.yaml +++ b/deploy/operator/config/rbac/role.yaml @@ -4,6 +4,135 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + - serviceaccounts + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - services/status + verbs: + - get + - patch + - update +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments/status + verbs: + - get + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jumpstarter.dev + resources: + - clients + - exporteraccesspolicies + - exporters + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jumpstarter.dev + resources: + - clients/finalizers + - exporteraccesspolicies/finalizers + - exporters/finalizers + - leases/finalizers + verbs: + - update +- apiGroups: + - jumpstarter.dev + resources: + - clients/status + - exporteraccesspolicies/status + - exporters/status + - leases/status + verbs: + - get + - patch + - update +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - get + - patch + - update - apiGroups: - operator.jumpstarter.dev resources: @@ -30,3 +159,36 @@ rules: - get - patch - update +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + - roles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - route.openshift.io + resources: + - routes/status + verbs: + - get + - patch + - update diff --git a/deploy/operator/go.mod b/deploy/operator/go.mod index de7ca633..cfa4abff 100644 --- a/deploy/operator/go.mod +++ b/deploy/operator/go.mod @@ -3,61 +3,97 @@ module github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator go 1.24.0 require ( + github.com/go-logr/logr v1.4.2 + github.com/jumpstarter-dev/jumpstarter-controller v0.7.1 github.com/onsi/ginkgo/v2 v2.22.2 github.com/onsi/gomega v1.36.2 + github.com/pmezard/go-difflib v1.0.0 k8s.io/api v0.33.0 k8s.io/apimachinery v0.33.0 k8s.io/apiserver v0.33.0 k8s.io/client-go v0.33.0 sigs.k8s.io/controller-runtime v0.21.0 + sigs.k8s.io/yaml v1.4.0 ) replace github.com/jumpstarter-dev/jumpstarter-controller => ../../ require ( cel.dev/expr v0.19.1 // indirect + filippo.io/bigmod v0.0.3 // indirect + filippo.io/keygen v0.0.0-20240718133620-7f162efbbd87 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/bmatcuk/doublestar/v4 v4.8.0 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/coreos/go-oidc v2.3.0+incompatible // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.10.0 // indirect + github.com/go-chi/chi/v5 v5.2.0 // indirect + github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/cel-go v0.23.2 // indirect github.com/google/gnostic-models v0.6.9 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/securecookie v1.1.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/muhlemmer/gu v0.3.1 // indirect + github.com/muhlemmer/httpforwarded v0.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pquerna/cachecontrol v0.1.0 // indirect github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/rs/cors v1.11.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/zitadel/logging v0.6.1 // indirect + github.com/zitadel/oidc/v3 v3.34.1 // indirect + github.com/zitadel/schema v1.3.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect go.opentelemetry.io/otel v1.33.0 // indirect @@ -69,6 +105,8 @@ require ( go.opentelemetry.io/proto/otlp v1.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect @@ -84,6 +122,7 @@ require ( google.golang.org/grpc v1.70.0 // indirect google.golang.org/protobuf v1.36.5 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.33.0 // indirect @@ -95,5 +134,4 @@ require ( sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/deploy/operator/go.sum b/deploy/operator/go.sum index 8a062a99..c64b8234 100644 --- a/deploy/operator/go.sum +++ b/deploy/operator/go.sum @@ -1,15 +1,33 @@ cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +filippo.io/bigmod v0.0.3 h1:qmdCFHmEMS+PRwzrW6eUrgA4Q3T8D6bRcjsypDMtWHM= +filippo.io/bigmod v0.0.3/go.mod h1:WxGvOYE0OUaBC2N112Dflb3CjOnMBuNRA2UWZc2UbPE= +filippo.io/keygen v0.0.0-20240718133620-7f162efbbd87 h1:HlcHAMbI9Xvw3aWnhPngghMl5AKE2GOvjmvSGOKzCcI= +filippo.io/keygen v0.0.0-20240718133620-7f162efbbd87/go.mod h1:nAs0+DyACEQGudhkTwlPC9atyqDYC7ZotgZR7D8OwXM= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmatcuk/doublestar/v4 v4.8.0 h1:DSXtrypQddoug1459viM9X9D3dp1Z7993fw36I2kNcQ= +github.com/bmatcuk/doublestar/v4 v4.8.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3 h1:oe6fCvaEpkhyW3qAicT0TnGtyht/UrgvOwMcEgLb7Aw= +github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3/go.mod h1:qdP0gaj0QtgX2RUZhnlVrceJ+Qln8aSlDyJwelLLFeM= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/coreos/go-oidc v2.3.0+incompatible h1:+5vEsrgprdLjjQ9FzIKAzQz1wwPD+83hQRfUIPh7rO0= +github.com/coreos/go-oidc v2.3.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -28,6 +46,16 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-chi/chi/v5 v5.2.0 h1:Aj1EtB0qR2Rdo2dG4O94RIU35w2lvQSj6BRA4+qwFL0= +github.com/go-chi/chi/v5 v5.2.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= +github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -43,10 +71,24 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= @@ -65,6 +107,8 @@ github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/Z github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= +github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -77,6 +121,10 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -86,23 +134,35 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/muhlemmer/gu v0.3.1 h1:7EAqmFrW7n3hETvuAdmFmn4hS8W+z3LgKtrnow+YzNM= +github.com/muhlemmer/gu v0.3.1/go.mod h1:YHtHR+gxM+bKEIIs7Hmi9sPT3ZDUvTN/i88wQpZkrdM= +github.com/muhlemmer/httpforwarded v0.1.0 h1:x4DLrzXdliq8mprgUMR0olDvHGkou5BJsK/vWUetyzY= +github.com/muhlemmer/httpforwarded v0.1.0/go.mod h1:yo9czKedo2pdZhoXe+yDkGVbU0TJ0q9oQ90BVoDEtw0= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc= +github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= @@ -113,7 +173,11 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -126,15 +190,29 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zitadel/logging v0.6.1 h1:Vyzk1rl9Kq9RCevcpX6ujUaTYFX43aa4LkvV1TvUk+Y= +github.com/zitadel/logging v0.6.1/go.mod h1:Y4CyAXHpl3Mig6JOszcV5Rqqsojj+3n7y2F591Mp/ow= +github.com/zitadel/oidc/v3 v3.34.1 h1:/rxx2HxEowd8Sdb8sxcRxTu9pLy3/TXBLrewKOUMTHA= +github.com/zitadel/oidc/v3 v3.34.1/go.mod h1:lhAdAP1iWAnpfWF8CWNiO6yKvGFtPMuAubPwP5JC7Ec= +github.com/zitadel/schema v1.3.0 h1:kQ9W9tvIwZICCKWcMvCEweXET1OcOyGEuFbHs4o5kg0= +github.com/zitadel/schema v1.3.0/go.mod h1:NptN6mkBDFvERUCvZHlvWmmME+gmZ44xzwRXwhzsbtc= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= @@ -161,9 +239,14 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -184,6 +267,9 @@ golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= @@ -203,6 +289,7 @@ golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= @@ -219,8 +306,12 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= +gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -242,6 +333,8 @@ k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUy k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= diff --git a/deploy/operator/internal/controller/jumpstarter/compare.go b/deploy/operator/internal/controller/jumpstarter/compare.go new file mode 100644 index 00000000..d22d2872 --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/compare.go @@ -0,0 +1,158 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jumpstarter + +import ( + "fmt" + + "github.com/go-logr/logr" + "github.com/pmezard/go-difflib/difflib" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/equality" + "sigs.k8s.io/yaml" +) + +// deploymentNeedsUpdate checks if a deployment needs to be updated using K8s semantic equality. +func deploymentNeedsUpdate(existing, desired *appsv1.Deployment) bool { + // Compare labels (only if desired.Labels is non-nil) + if desired.Labels != nil && !equality.Semantic.DeepEqual(existing.Labels, desired.Labels) { + return true + } + + // Compare annotations (only if desired.Annotations is non-nil) + if desired.Annotations != nil && !equality.Semantic.DeepEqual(existing.Annotations, desired.Annotations) { + return true + } + + // Compare the entire Spec using K8s semantic equality (handles nil vs empty automatically) + return !equality.Semantic.DeepEqual(existing.Spec, desired.Spec) +} + +// configMapNeedsUpdate checks if a configmap needs to be updated using K8s semantic equality. +func configMapNeedsUpdate(existing, desired *corev1.ConfigMap, log logr.Logger) bool { + // Compare labels (only if desired.Labels is non-nil) + if desired.Labels != nil && !equality.Semantic.DeepEqual(existing.Labels, desired.Labels) { + return true + } + + // Compare annotations (only if desired.Annotations is non-nil) + if desired.Annotations != nil && !equality.Semantic.DeepEqual(existing.Annotations, desired.Annotations) { + return true + } + + // Compare data (only if desired.Data is non-nil) + if desired.Data != nil && !equality.Semantic.DeepEqual(existing.Data, desired.Data) { + return true + } + + // Compare binary data (only if desired.BinaryData is non-nil) + if desired.BinaryData != nil && !equality.Semantic.DeepEqual(existing.BinaryData, desired.BinaryData) { + return true + } + + return false +} + +// serviceAccountNeedsUpdate checks if a service account needs to be updated using K8s semantic equality. +func serviceAccountNeedsUpdate(existing, desired *corev1.ServiceAccount) bool { + // Compare labels (only if desired.Labels is non-nil) + if desired.Labels != nil && !equality.Semantic.DeepEqual(existing.Labels, desired.Labels) { + return true + } + + // Compare annotations (only if desired.Annotations is non-nil) + if desired.Annotations != nil && !equality.Semantic.DeepEqual(existing.Annotations, desired.Annotations) { + return true + } + + return false +} + +// roleNeedsUpdate checks if a role needs to be updated using K8s semantic equality. +func roleNeedsUpdate(existing, desired *rbacv1.Role) bool { + // Compare labels (only if desired.Labels is non-nil) + if desired.Labels != nil && !equality.Semantic.DeepEqual(existing.Labels, desired.Labels) { + return true + } + + // Compare annotations (only if desired.Annotations is non-nil) + if desired.Annotations != nil && !equality.Semantic.DeepEqual(existing.Annotations, desired.Annotations) { + return true + } + + // Compare rules (only if non-nil in desired) + if desired.Rules != nil && !equality.Semantic.DeepEqual(existing.Rules, desired.Rules) { + return true + } + + return false +} + +// roleBindingNeedsUpdate checks if a role binding needs to be updated using K8s semantic equality. +func roleBindingNeedsUpdate(existing, desired *rbacv1.RoleBinding) bool { + // Compare labels (only if desired.Labels is non-nil) + if desired.Labels != nil && !equality.Semantic.DeepEqual(existing.Labels, desired.Labels) { + return true + } + + // Compare annotations (only if desired.Annotations is non-nil) + if desired.Annotations != nil && !equality.Semantic.DeepEqual(existing.Annotations, desired.Annotations) { + return true + } + + // Compare subjects (only if non-nil in desired) + if desired.Subjects != nil && !equality.Semantic.DeepEqual(existing.Subjects, desired.Subjects) { + return true + } + + // Compare role ref (only if non-zero in desired) + if desired.RoleRef.Name != "" && !equality.Semantic.DeepEqual(existing.RoleRef, desired.RoleRef) { + return true + } + + return false +} + +// generateDiff creates a unified diff between existing and desired resources. +// It works with any Kubernetes resource type. +// Returns the diff string and any error encountered during serialization. +func generateDiff[T any](existing, desired *T) (string, error) { + // Serialize existing resource to YAML + existingYAML, err := yaml.Marshal(existing) + if err != nil { + return "", fmt.Errorf("failed to marshal existing resource: %w", err) + } + + // Serialize desired resource to YAML + desiredYAML, err := yaml.Marshal(desired) + if err != nil { + return "", fmt.Errorf("failed to marshal desired resource: %w", err) + } + + // Generate unified diff + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines(string(existingYAML)), + B: difflib.SplitLines(string(desiredYAML)), + FromFile: "Existing", + ToFile: "Desired", + Context: 3, + } + + return difflib.GetUnifiedDiffString(diff) +} diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go new file mode 100644 index 00000000..aed06ee3 --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints.go @@ -0,0 +1,289 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endpoints + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" +) + +// Reconciler provides endpoint reconciliation functionality +type Reconciler struct { + Client client.Client + Scheme *runtime.Scheme +} + +// NewReconciler creates a new endpoint reconciler +func NewReconciler(client client.Client, scheme *runtime.Scheme) *Reconciler { + return &Reconciler{ + Client: client, + Scheme: scheme, + } +} + +// createOrUpdateService creates or updates a service with proper handling of immutable fields +// and owner references. This is the unified service creation method. +func (r *Reconciler) createOrUpdateService(ctx context.Context, service *corev1.Service, owner metav1.Object) error { + log := logf.FromContext(ctx) + + existingService := &corev1.Service{} + existingService.Name = service.Name + existingService.Namespace = service.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingService, func() error { + // Preserve immutable fields if service already exists + if existingService.CreationTimestamp.IsZero() { + // Service is being created, copy all fields from desired service + existingService.Spec = service.Spec + existingService.Labels = service.Labels + existingService.Annotations = service.Annotations + return controllerutil.SetControllerReference(owner, existingService, r.Scheme) + + } else { + // Preserve existing NodePorts to prevent "port already allocated" errors + if service.Spec.Type == corev1.ServiceTypeNodePort || service.Spec.Type == corev1.ServiceTypeLoadBalancer { + for i := range existingService.Spec.Ports { + if existingService.Spec.Ports[i].NodePort != 0 && i < len(service.Spec.Ports) { + service.Spec.Ports[i].NodePort = existingService.Spec.Ports[i].NodePort + } + } + } + + // Update all mutable fields + if service.Spec.LoadBalancerClass != nil && *service.Spec.LoadBalancerClass != "" { + existingService.Spec.LoadBalancerClass = service.Spec.LoadBalancerClass + } + if service.Spec.ExternalTrafficPolicy != "" { + existingService.Spec.ExternalTrafficPolicy = service.Spec.ExternalTrafficPolicy + } + + existingService.Spec.Ports = service.Spec.Ports + existingService.Spec.Selector = service.Spec.Selector + existingService.Spec.Type = service.Spec.Type + existingService.Labels = service.Labels + existingService.Annotations = service.Annotations + return controllerutil.SetControllerReference(owner, existingService, r.Scheme) + } + }) + + if err != nil { + log.Error(err, "Failed to reconcile service", + "name", service.Name, + "namespace", service.Namespace, + "type", service.Spec.Type) + return err + } + + log.Info("Service reconciled", + "name", service.Name, + "namespace", service.Namespace, + "type", service.Spec.Type, + "selector", service.Spec.Selector, + "operation", op) + + return nil +} + +// ReconcileControllerEndpoint reconciles a controller endpoint service with proper pod selector +// This function creates a separate service for each enabled service type (ClusterIP, NodePort, LoadBalancer) +func (r *Reconciler) ReconcileControllerEndpoint(ctx context.Context, owner metav1.Object, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { + // Controller pods have fixed labels: app=jumpstarter-controller + // We need to create a service with selector matching those labels + baseLabels := map[string]string{ + "app": "jumpstarter-controller", + "controller": owner.GetName(), + } + + // Pod selector for controller pods + podSelector := map[string]string{ + "app": "jumpstarter-controller", + } + + // Create a service for each enabled service type + // This allows multiple service types to coexist for the same endpoint + // Note: ClusterIP uses no suffix (most common for in-cluster communication) + // LoadBalancer uses "-lb" suffix, NodePort uses "-np" suffix + + // LoadBalancer service + if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { + if err := r.createService(ctx, owner, servicePort, "-lb", corev1.ServiceTypeLoadBalancer, + podSelector, baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { + return err + } + } + + // NodePort service + if endpoint.NodePort != nil && endpoint.NodePort.Enabled { + if err := r.createService(ctx, owner, servicePort, "-np", corev1.ServiceTypeNodePort, + podSelector, baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { + return err + } + } + + // ClusterIP service (no suffix for cleaner in-cluster service names) + if endpoint.ClusterIP != nil && endpoint.ClusterIP.Enabled { + if err := r.createService(ctx, owner, servicePort, "", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { + return err + } + } + + // If no service type is explicitly enabled, create a default ClusterIP service + if (endpoint.LoadBalancer == nil || !endpoint.LoadBalancer.Enabled) && + (endpoint.NodePort == nil || !endpoint.NodePort.Enabled) && + (endpoint.ClusterIP == nil || !endpoint.ClusterIP.Enabled) { + + // TODO: Default to Route or Ingress depending of the type of cluster + if err := r.createService(ctx, owner, servicePort, "", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, nil, nil); err != nil { + return err + } + } + + return nil +} + +// ReconcileRouterReplicaEndpoint reconciles service, ingress, and route for a specific router replica endpoint +// This function creates a separate service for each enabled service type (ClusterIP, NodePort, LoadBalancer) +func (r *Reconciler) ReconcileRouterReplicaEndpoint(ctx context.Context, owner metav1.Object, replicaIndex int32, endpointIdx int, endpoint *operatorv1alpha1.Endpoint, servicePort corev1.ServicePort) error { + // IMPORTANT: The pod selector must match the actual pod labels + // Router pods have label: app: jumpstarter-router-0 (for replica 0) + baseAppLabel := fmt.Sprintf("%s-router-%d", owner.GetName(), replicaIndex) + + baseLabels := map[string]string{ + "app": "jumpstarter-router", + "router": owner.GetName(), + "router-index": fmt.Sprintf("%d", replicaIndex), + "endpoint-idx": fmt.Sprintf("%d", endpointIdx), + } + + // Pod selector - this MUST match the deployment's pod template labels + podSelector := map[string]string{ + "app": baseAppLabel, // e.g., "jumpstarter-router-0" + } + + // Create a service for each enabled service type + // This allows multiple service types to coexist for the same endpoint + // Note: ClusterIP uses no suffix (most common for in-cluster communication) + // LoadBalancer uses "-lb" suffix, NodePort uses "-np" suffix + + // Ingress service + if endpoint.Ingress != nil && endpoint.Ingress.Enabled { + if err := r.createService(ctx, owner, servicePort, "-ing", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, endpoint.Ingress.Annotations, endpoint.Ingress.Labels); err != nil { + return err + } + } + + // Route service + if endpoint.Route != nil && endpoint.Route.Enabled { + if err := r.createService(ctx, owner, servicePort, "-route", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, endpoint.Route.Annotations, endpoint.Route.Labels); err != nil { + return err + } + } + + // LoadBalancer service + if endpoint.LoadBalancer != nil && endpoint.LoadBalancer.Enabled { + if err := r.createService(ctx, owner, servicePort, "-lb", corev1.ServiceTypeLoadBalancer, + podSelector, baseLabels, endpoint.LoadBalancer.Annotations, endpoint.LoadBalancer.Labels); err != nil { + return err + } + } + + // NodePort service + if endpoint.NodePort != nil && endpoint.NodePort.Enabled { + if err := r.createService(ctx, owner, servicePort, "-np", corev1.ServiceTypeNodePort, + podSelector, baseLabels, endpoint.NodePort.Annotations, endpoint.NodePort.Labels); err != nil { + return err + } + } + + // ClusterIP service (no suffix for cleaner in-cluster service names) + if endpoint.ClusterIP != nil && endpoint.ClusterIP.Enabled { + if err := r.createService(ctx, owner, servicePort, "", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, endpoint.ClusterIP.Annotations, endpoint.ClusterIP.Labels); err != nil { + return err + } + } + + // If no service type is explicitly enabled, create a default ClusterIP service + if (endpoint.LoadBalancer == nil || !endpoint.LoadBalancer.Enabled) && + (endpoint.NodePort == nil || !endpoint.NodePort.Enabled) && + (endpoint.ClusterIP == nil || !endpoint.ClusterIP.Enabled) && + (endpoint.Ingress == nil || !endpoint.Ingress.Enabled) && + (endpoint.Route == nil || !endpoint.Route.Enabled) { + if err := r.createService(ctx, owner, servicePort, "", corev1.ServiceTypeClusterIP, + podSelector, baseLabels, nil, nil); err != nil { + return err + } + } + + // TODO: Create ingress/route resources here instead of calling the deprecated ReconcileEndpoint + // For now, ingress and route are handled by creating ClusterIP services above + + return nil +} + +// createService creates or updates a single service with the specified type and suffix +// This is the unified service creation method that uses createOrUpdateService internally +func (r *Reconciler) createService(ctx context.Context, owner metav1.Object, servicePort corev1.ServicePort, + nameSuffix string, serviceType corev1.ServiceType, podSelector map[string]string, + baseLabels map[string]string, annotations map[string]string, extraLabels map[string]string) error { + + // Build service name with suffix to avoid conflicts + serviceName := servicePort.Name + nameSuffix + + // Merge labels + serviceLabels := make(map[string]string) + for k, v := range baseLabels { + serviceLabels[k] = v + } + for k, v := range extraLabels { + serviceLabels[k] = v + } + + // Ensure annotations map is initialized + if annotations == nil { + annotations = make(map[string]string) + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: owner.GetNamespace(), + Labels: serviceLabels, + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Selector: podSelector, // Use the provided pod selector map + Ports: []corev1.ServicePort{servicePort}, + Type: serviceType, + }, + } + + return r.createOrUpdateService(ctx, service, owner) +} diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go new file mode 100644 index 00000000..12f57d1c --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/endpoints_test.go @@ -0,0 +1,365 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endpoints + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" +) + +var _ = Describe("Endpoints Reconciler", func() { + Context("When reconciling controller endpoints", func() { + const ( + namespace = "test-namespace" + controllerName = "test-controller" + ) + + ctx := context.Background() + var reconciler *Reconciler + var owner *corev1.ConfigMap // Use ConfigMap as a simple owner object for testing + + BeforeEach(func() { + reconciler = NewReconciler(k8sClient, k8sClient.Scheme()) + + // Create the test namespace + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + err := k8sClient.Create(ctx, ns) + if err != nil && !errors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + // Create an owner object for testing + owner = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllerName, + Namespace: namespace, + }, + } + err = k8sClient.Create(ctx, owner) + if err != nil && !errors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + }) + + Context("with ClusterIP service type", func() { + It("should create a ClusterIP service successfully", func() { + endpoint := &operatorv1alpha1.Endpoint{ + Address: "controller", + ClusterIP: &operatorv1alpha1.ClusterIPConfig{ + Enabled: true, + }, + } + + svcPort := corev1.ServicePort{ + Name: "controller", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify the service was created + service := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: "controller", + Namespace: namespace, + }, service) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) + Expect(service.Spec.Selector["app"]).To(Equal("jumpstarter-controller")) + Expect(service.Labels["app"]).To(Equal("jumpstarter-controller")) + }) + }) + + Context("with LoadBalancer service type", func() { + It("should create a LoadBalancer service successfully", func() { + endpoint := &operatorv1alpha1.Endpoint{ + Address: "controller", + LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ + Enabled: true, + Annotations: map[string]string{"service.beta.kubernetes.io/aws-load-balancer-type": "nlb"}, + Labels: map[string]string{"environment": "production"}, + }, + } + + svcPort := corev1.ServicePort{ + Name: "controller", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify the service was created with -lb suffix + service := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: "controller-lb", + Namespace: namespace, + }, service) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Spec.Type).To(Equal(corev1.ServiceTypeLoadBalancer)) + Expect(service.Annotations["service.beta.kubernetes.io/aws-load-balancer-type"]).To(Equal("nlb")) + Expect(service.Labels["environment"]).To(Equal("production")) + Expect(service.Spec.Selector["app"]).To(Equal("jumpstarter-controller")) + }) + }) + + Context("with NodePort service type", func() { + It("should create a NodePort service successfully", func() { + endpoint := &operatorv1alpha1.Endpoint{ + Address: "controller", + NodePort: &operatorv1alpha1.NodePortConfig{ + Enabled: true, + Port: 30090, + Annotations: map[string]string{"nodeport.kubernetes.io/port": "30090"}, + Labels: map[string]string{"nodeport": "true"}, + }, + } + + svcPort := corev1.ServicePort{ + Name: "controller", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify the service was created with -np suffix + service := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: "controller-np", + Namespace: namespace, + }, service) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Spec.Type).To(Equal(corev1.ServiceTypeNodePort)) + Expect(service.Annotations["nodeport.kubernetes.io/port"]).To(Equal("30090")) + Expect(service.Labels["nodeport"]).To(Equal("true")) + Expect(service.Spec.Selector["app"]).To(Equal("jumpstarter-controller")) + }) + }) + + Context("with multiple service types enabled", func() { + It("should create all enabled service types", func() { + endpoint := &operatorv1alpha1.Endpoint{ + Address: "controller", + LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ + Enabled: true, + }, + NodePort: &operatorv1alpha1.NodePortConfig{ + Enabled: true, + }, + } + + svcPort := corev1.ServicePort{ + Name: "controller", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify LoadBalancer service was created + lbService := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: "controller-lb", + Namespace: namespace, + }, lbService) + Expect(err).NotTo(HaveOccurred()) + Expect(lbService.Spec.Type).To(Equal(corev1.ServiceTypeLoadBalancer)) + + // Verify NodePort service was created + npService := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: "controller-np", + Namespace: namespace, + }, npService) + Expect(err).NotTo(HaveOccurred()) + Expect(npService.Spec.Type).To(Equal(corev1.ServiceTypeNodePort)) + }) + }) + + Context("when updating an existing service", func() { + It("should update the service when configuration changes", func() { + // Create initial service + endpoint := &operatorv1alpha1.Endpoint{ + Address: "controller", + LoadBalancer: &operatorv1alpha1.LoadBalancerConfig{ + Enabled: true, + Annotations: map[string]string{"initial": "annotation"}, + Labels: map[string]string{"initial": "label"}, + }, + } + + svcPort := corev1.ServicePort{ + Name: "controller", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Update the endpoint configuration + endpoint.LoadBalancer.Annotations["updated"] = "annotation" + endpoint.LoadBalancer.Labels["updated"] = "label" + + err = reconciler.ReconcileControllerEndpoint(ctx, owner, endpoint, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify the service was updated + service := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: "controller-lb", + Namespace: namespace, + }, service) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Annotations["updated"]).To(Equal("annotation")) + Expect(service.Labels["updated"]).To(Equal("label")) + }) + }) + + AfterEach(func() { + // Clean up created services + services := []string{"controller", "controller-lb", "controller-np"} + for _, svcName := range services { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: namespace, + }, + } + _ = k8sClient.Delete(ctx, service) + } + + // Clean up owner + _ = k8sClient.Delete(ctx, owner) + }) + }) + + Context("When reconciling router replica endpoints", func() { + const ( + namespace = "test-namespace" + routerName = "test-router" + replicaIdx = int32(0) + endpointIdx = 0 + ) + + ctx := context.Background() + var reconciler *Reconciler + var owner *corev1.ConfigMap + + BeforeEach(func() { + reconciler = NewReconciler(k8sClient, k8sClient.Scheme()) + + // Create the test namespace + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + err := k8sClient.Create(ctx, ns) + if err != nil && !errors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + // Create an owner object for testing + owner = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: routerName, + Namespace: namespace, + }, + } + err = k8sClient.Create(ctx, owner) + if err != nil && !errors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + }) + + Context("with proper pod selector", func() { + It("should create a service with correct pod selector matching deployment labels", func() { + endpoint := &operatorv1alpha1.Endpoint{ + Address: "router", + NodePort: &operatorv1alpha1.NodePortConfig{ + Enabled: true, + }, + } + + svcPort := corev1.ServicePort{ + Name: "router", + Port: 8083, + TargetPort: intstr.FromInt(8083), + Protocol: corev1.ProtocolTCP, + } + + err := reconciler.ReconcileRouterReplicaEndpoint(ctx, owner, replicaIdx, endpointIdx, endpoint, svcPort) + Expect(err).NotTo(HaveOccurred()) + + // Verify the service was created with correct pod selector + service := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: "router-np", + Namespace: namespace, + }, service) + Expect(err).NotTo(HaveOccurred()) + // The selector should be "app: test-router-router-0" (owner.Name + "-router-" + replicaIndex) + Expect(service.Spec.Selector["app"]).To(Equal("test-router-router-0")) + Expect(service.Labels["router"]).To(Equal(routerName)) + Expect(service.Labels["router-index"]).To(Equal("0")) + }) + }) + + AfterEach(func() { + // Clean up created services + services := []string{"router", "router-lb", "router-np", "router-ing", "router-route"} + for _, svcName := range services { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: namespace, + }, + } + _ = k8sClient.Delete(ctx, service) + } + + // Clean up owner + _ = k8sClient.Delete(ctx, owner) + }) + }) + +}) diff --git a/deploy/operator/internal/controller/jumpstarter/endpoints/suite_test.go b/deploy/operator/internal/controller/jumpstarter/endpoints/suite_test.go new file mode 100644 index 00000000..6af1fab4 --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/endpoints/suite_test.go @@ -0,0 +1,93 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endpoints + +import ( + "context" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/testutils" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + ctx context.Context + cancel context.CancelFunc + testEnv *envtest.Environment + cfg *rest.Config + k8sClient client.Client +) + +func TestEndpoints(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Endpoints Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + var err error + err = operatorv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + // Retrieve the first found binary directory to allow running tests from IDEs + if binaryDir := testutils.GetFirstFoundEnvTestBinaryDir(6); binaryDir != "" { + testEnv.BinaryAssetsDirectory = binaryDir + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go new file mode 100644 index 00000000..c6d7542b --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -0,0 +1,1214 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jumpstarter + +import ( + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "net" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + apiserverv1beta1 "k8s.io/apiserver/pkg/apis/apiserver/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/jumpstarter/endpoints" + loglevels "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/log" + "github.com/jumpstarter-dev/jumpstarter-controller/internal/config" +) + +const ( + // appProtocolH2C is the application protocol for HTTP/2 Cleartext + appProtocolH2C = "h2c" +) + +// JumpstarterReconciler reconciles a Jumpstarter object +type JumpstarterReconciler struct { + client.Client + Scheme *runtime.Scheme + EndpointReconciler *endpoints.Reconciler +} + +// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters/finalizers,verbs=update + +// Core Kubernetes resources +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=services/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch + +// RBAC resources +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete + +// Leader election +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete + +// Networking resources +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes/status,verbs=get;update;patch + +// Monitoring resources +// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create;update;patch;delete + +// Jumpstarter CRD resources (needed to grant permissions to managed controllers) +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=clients,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=clients/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=clients/finalizers,verbs=update +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporters,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporters/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporters/finalizers,verbs=update +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=leases,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=leases/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=leases/finalizers,verbs=update +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporteraccesspolicies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporteraccesspolicies/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporteraccesspolicies/finalizers,verbs=update + +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile +func (r *JumpstarterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + // Fetch the Jumpstarter instance + var jumpstarter operatorv1alpha1.Jumpstarter + if err := r.Get(ctx, req.NamespacedName, &jumpstarter); err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + log.Info("Jumpstarter resource not found. Ignoring since object must be deleted.") + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + log.Error(err, "Failed to get Jumpstarter") + return ctrl.Result{}, err + } + + // Check if the instance is marked to be deleted + if jumpstarter.GetDeletionTimestamp() != nil { + // Handle finalizer logic here if needed + return ctrl.Result{}, nil + } + + // Reconcile RBAC resources first + if err := r.reconcileRBAC(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile RBAC") + return ctrl.Result{}, err + } + + // Reconcile Controller Deployment + if err := r.reconcileControllerDeployment(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile Controller Deployment") + return ctrl.Result{}, err + } + + // Reconcile Router Deployment + if err := r.reconcileRouterDeployment(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile Router Deployment") + return ctrl.Result{}, err + } + + // Reconcile Services + if err := r.reconcileServices(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile Services") + return ctrl.Result{}, err + } + + // Reconcile ConfigMaps + if err := r.reconcileConfigMaps(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile ConfigMaps") + return ctrl.Result{}, err + } + + // Reconcile Secrets + if err := r.reconcileSecrets(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile Secrets") + return ctrl.Result{}, err + } + + // Update status + if err := r.updateStatus(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to update status") + return ctrl.Result{}, err + } + + // Requeue after 30 minutes to check for changes + return ctrl.Result{RequeueAfter: 30 * time.Minute}, nil +} + +// reconcileControllerDeployment reconciles the controller deployment +func (r *JumpstarterReconciler) reconcileControllerDeployment(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + desiredDeployment := r.createControllerDeployment(jumpstarter) + + existingDeployment := &appsv1.Deployment{} + existingDeployment.Name = desiredDeployment.Name + existingDeployment.Namespace = desiredDeployment.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingDeployment, func() error { + // Check if this is a new deployment or an existing one + if existingDeployment.CreationTimestamp.IsZero() { + // Deployment is being created, copy all fields from desired + existingDeployment.Labels = desiredDeployment.Labels + existingDeployment.Annotations = desiredDeployment.Annotations + existingDeployment.Spec = desiredDeployment.Spec + return controllerutil.SetControllerReference(jumpstarter, existingDeployment, r.Scheme) + } + + desiredDeployment.Spec.Template.Spec.DeprecatedServiceAccount = existingDeployment.Spec.Template.Spec.DeprecatedServiceAccount + desiredDeployment.Spec.Template.Spec.SchedulerName = existingDeployment.Spec.Template.Spec.SchedulerName + + // Check if deployment needs update using compare function + if !deploymentNeedsUpdate(existingDeployment, desiredDeployment) { + log.V(1).Info("Controller deployment specs are equal, skipping update", + "name", existingDeployment.Name, + "namespace", existingDeployment.Namespace) + return nil + } + + // Deployment exists, generate and log diff + diff, err := generateDiff(existingDeployment, desiredDeployment) + if err != nil { + log.V(1).Info("Failed to generate deployment diff", "error", err) + } else if diff != "" { + fmt.Printf("\n=== Controller deployment differences detected ===\n") + fmt.Printf("Name: %s\n", existingDeployment.Name) + fmt.Printf("Namespace: %s\n", existingDeployment.Namespace) + fmt.Printf("\n%s\n", diff) + fmt.Printf("========================================\n\n") + } + + // Apply changes + existingDeployment.Labels = desiredDeployment.Labels + existingDeployment.Annotations = desiredDeployment.Annotations + existingDeployment.Spec.Replicas = desiredDeployment.Spec.Replicas + existingDeployment.Spec.Selector = desiredDeployment.Spec.Selector + existingDeployment.Spec.Template = desiredDeployment.Spec.Template + return controllerutil.SetControllerReference(jumpstarter, existingDeployment, r.Scheme) + }) + + if err != nil { + log.Error(err, "Failed to reconcile controller deployment", + "name", desiredDeployment.Name, + "namespace", desiredDeployment.Namespace) + return err + } + + log.Info("Controller deployment reconciled", + "name", existingDeployment.Name, + "namespace", existingDeployment.Namespace, + "operation", op) + + return nil +} + +// reconcileRouterDeployment reconciles router deployments (one per replica) +func (r *JumpstarterReconciler) reconcileRouterDeployment(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + // Create one deployment per replica + for i := int32(0); i < jumpstarter.Spec.Routers.Replicas; i++ { + desiredDeployment := r.createRouterDeployment(jumpstarter, i) + + existingDeployment := &appsv1.Deployment{} + existingDeployment.Name = desiredDeployment.Name + existingDeployment.Namespace = desiredDeployment.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingDeployment, func() error { + // Check if this is a new deployment or an existing one + if existingDeployment.CreationTimestamp.IsZero() { + // Deployment is being created, copy all fields from desired + existingDeployment.Labels = desiredDeployment.Labels + existingDeployment.Annotations = desiredDeployment.Annotations + existingDeployment.Spec = desiredDeployment.Spec + return controllerutil.SetControllerReference(jumpstarter, existingDeployment, r.Scheme) + } + desiredDeployment.Spec.Template.Spec.SchedulerName = existingDeployment.Spec.Template.Spec.SchedulerName + desiredDeployment.Spec.Template.Spec.DeprecatedServiceAccount = existingDeployment.Spec.Template.Spec.DeprecatedServiceAccount + + if !deploymentNeedsUpdate(existingDeployment, desiredDeployment) { + log.V(1).Info("Router deployment specs are equal, skipping update", + "name", existingDeployment.Name, + "namespace", existingDeployment.Namespace, + "replica", i) + return nil + } + // Deployment exists, generate and log diff + diff, err := generateDiff(existingDeployment, desiredDeployment) + if err != nil { + log.V(1).Info("Failed to generate deployment diff", "error", err) + } else if diff != "" { + fmt.Printf("\n=== Router deployment differences detected ===\n") + fmt.Printf("Name: %s\n", existingDeployment.Name) + fmt.Printf("Namespace: %s\n", existingDeployment.Namespace) + fmt.Printf("Replica: %d\n", i) + fmt.Printf("\n%s\n", diff) + fmt.Printf("==============================================\n\n") + } + + // Apply changes + existingDeployment.Labels = desiredDeployment.Labels + existingDeployment.Annotations = desiredDeployment.Annotations + existingDeployment.Spec.Replicas = desiredDeployment.Spec.Replicas + existingDeployment.Spec.Selector = desiredDeployment.Spec.Selector + existingDeployment.Spec.Template = desiredDeployment.Spec.Template + return controllerutil.SetControllerReference(jumpstarter, existingDeployment, r.Scheme) + }) + + if err != nil { + log.Error(err, "Failed to reconcile router deployment", + "name", desiredDeployment.Name, + "namespace", desiredDeployment.Namespace, + "replica", i) + return err + } + + log.Info("Router deployment reconciled", + "name", existingDeployment.Name, + "namespace", existingDeployment.Namespace, + "replica", i, + "operation", op) + } + + // Clean up deployments for scaled-down replicas + if err := r.cleanupExcessRouterDeployments(ctx, jumpstarter); err != nil { + log.Error(err, "Failed to cleanup excess router deployments") + return err + } + + return nil +} + +// reconcileServices reconciles all services +func (r *JumpstarterReconciler) reconcileServices(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + // Reconcile controller services + for _, endpoint := range jumpstarter.Spec.Controller.GRPC.Endpoints { + appProtocol := appProtocolH2C + svcPort := corev1.ServicePort{ + Name: "controller-grpc", + Port: 8082, + TargetPort: intstr.FromInt(8082), + Protocol: corev1.ProtocolTCP, + AppProtocol: &appProtocol, + } + // Set NodePort if configured + if endpoint.NodePort != nil && endpoint.NodePort.Enabled && endpoint.NodePort.Port > 0 { + svcPort.NodePort = endpoint.NodePort.Port + } + if err := r.EndpointReconciler.ReconcileControllerEndpoint(ctx, jumpstarter, &endpoint, svcPort); err != nil { + return err + } + } + + // Reconcile router services - one per replica, all endpoints per replica + for i := int32(0); i < jumpstarter.Spec.Routers.Replicas; i++ { + if len(jumpstarter.Spec.Routers.GRPC.Endpoints) > 0 { + // Each replica gets ALL configured endpoints with replica substitution + for endpointIdx, baseEndpoint := range jumpstarter.Spec.Routers.GRPC.Endpoints { + endpoint := r.buildEndpointForReplica(jumpstarter, i, endpointIdx, &baseEndpoint) + + // Build unique service name for this replica AND endpoint + // This allows multiple service types (NodePort, LoadBalancer, etc.) per replica + serviceName := r.buildServiceNameForReplicaEndpoint(jumpstarter, i, endpointIdx) + + appProtocol := appProtocolH2C + svcPort := corev1.ServicePort{ + Name: serviceName, // Unique name per replica+endpoint + Port: 8083, + TargetPort: intstr.FromInt(8083), + Protocol: corev1.ProtocolTCP, + AppProtocol: &appProtocol, + } + // Set NodePort if configured + if endpoint.NodePort != nil && endpoint.NodePort.Enabled && endpoint.NodePort.Port > 0 { + // increase nodeport numbers based in replica, not perfect because it needs to be + // consecutive, but this is mostly for E2E testing. + svcPort.NodePort = endpoint.NodePort.Port + i + } + if err := r.EndpointReconciler.ReconcileRouterReplicaEndpoint(ctx, jumpstarter, i, endpointIdx, &endpoint, svcPort); err != nil { + return err + } + } + } else { + // No endpoints configured, create a default service without ingress/route + endpoint := operatorv1alpha1.Endpoint{ + Address: fmt.Sprintf("router-%d.%s", i, jumpstarter.Spec.BaseDomain), + } + + serviceName := fmt.Sprintf("%s-router-%d", jumpstarter.Name, i) + appProtocol := appProtocolH2C + svcPort := corev1.ServicePort{ + Name: serviceName, + Port: 8083, + TargetPort: intstr.FromInt(8083), + Protocol: corev1.ProtocolTCP, + AppProtocol: &appProtocol, + } + if err := r.EndpointReconciler.ReconcileRouterReplicaEndpoint(ctx, jumpstarter, i, 0, &endpoint, svcPort); err != nil { + return err + } + } + } + + // Clean up services for scaled-down replicas + if err := r.cleanupExcessRouterServices(ctx, jumpstarter); err != nil { + log.Error(err, "Failed to cleanup excess router services") + return err + } + + return nil +} + +// reconcileConfigMaps reconciles all configmaps +func (r *JumpstarterReconciler) reconcileConfigMaps(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + desiredConfigMap, err := r.createConfigMap(jumpstarter) + if err != nil { + return fmt.Errorf("failed to create configmap: %w", err) + } + + existingConfigMap := &corev1.ConfigMap{} + existingConfigMap.Name = desiredConfigMap.Name + existingConfigMap.Namespace = desiredConfigMap.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingConfigMap, func() error { + // Check if this is a new configmap or an existing one + if existingConfigMap.CreationTimestamp.IsZero() { + // ConfigMap is being created, copy all fields from desired + existingConfigMap.Labels = desiredConfigMap.Labels + existingConfigMap.Annotations = desiredConfigMap.Annotations + existingConfigMap.Data = desiredConfigMap.Data + existingConfigMap.BinaryData = desiredConfigMap.BinaryData + return controllerutil.SetControllerReference(jumpstarter, existingConfigMap, r.Scheme) + } + + // ConfigMap exists, check if update is needed + if !configMapNeedsUpdate(existingConfigMap, desiredConfigMap, log) { + log.V(1).Info("ConfigMap is up to date, skipping update", + "name", existingConfigMap.Name, + "namespace", existingConfigMap.Namespace) + return nil + } + + // Update needed - apply changes + existingConfigMap.Labels = desiredConfigMap.Labels + existingConfigMap.Annotations = desiredConfigMap.Annotations + existingConfigMap.Data = desiredConfigMap.Data + existingConfigMap.BinaryData = desiredConfigMap.BinaryData + return controllerutil.SetControllerReference(jumpstarter, existingConfigMap, r.Scheme) + }) + + if err != nil { + log.Error(err, "Failed to reconcile configmap", + "name", desiredConfigMap.Name, + "namespace", desiredConfigMap.Namespace) + return err + } + + log.Info("ConfigMap reconciled", + "name", existingConfigMap.Name, + "namespace", existingConfigMap.Namespace, + "operation", op) + + return nil +} + +// reconcileSecrets reconciles all secrets +// Secrets are only created if they don't exist. They are not updated or deleted +// to preserve secret keys across CR updates and deletions. +func (r *JumpstarterReconciler) reconcileSecrets(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + // Create controller secret if it doesn't exist + // Use fixed name to match Helm chart for migration compatibility + controllerSecretName := "jumpstarter-controller-secret" + if err := r.ensureSecretExists(ctx, jumpstarter, controllerSecretName); err != nil { + log.Error(err, "Failed to ensure controller secret exists", "secret", controllerSecretName) + return err + } + + // Create router secret if it doesn't exist + // Use fixed name to match Helm chart for migration compatibility + routerSecretName := "jumpstarter-router-secret" + if err := r.ensureSecretExists(ctx, jumpstarter, routerSecretName); err != nil { + log.Error(err, "Failed to ensure router secret exists", "secret", routerSecretName) + return err + } + + return nil +} + +// ensureSecretExists creates a secret only if it doesn't already exist +func (r *JumpstarterReconciler) ensureSecretExists(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter, name string) error { + log := logf.FromContext(ctx) + + // Check if secret already exists + existingSecret := &corev1.Secret{} + err := r.Get(ctx, client.ObjectKey{ + Namespace: jumpstarter.Namespace, + Name: name, + }, existingSecret) + + if err == nil { + // Secret already exists, don't update it + log.V(loglevels.LevelTrace).Info("Secret already exists, skipping creation", "secret", name) + return nil + } + + if !errors.IsNotFound(err) { + // Some other error occurred + return err + } + + // Secret doesn't exist, create it with a random key + randomKey, err := generateRandomKey(32) + if err != nil { + return fmt.Errorf("failed to generate random key: %w", err) + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: jumpstarter.Namespace, + Labels: map[string]string{ + "app": jumpstarter.Name, + "app.kubernetes.io/managed-by": "jumpstarter-operator", + }, + Annotations: map[string]string{ + "jumpstarter.dev/orphan": "true", + }, + }, + StringData: map[string]string{ + "key": randomKey, + }, + } + + // Note: We intentionally do NOT set owner reference here so that + // secrets are not deleted when the Jumpstarter CR is deleted. + // This preserves the secret keys across CR deletions and recreations. + + if err := r.Create(ctx, secret); err != nil { + // Handle race condition where secret was created between Get and Create + if errors.IsAlreadyExists(err) { + log.V(loglevels.LevelDebug).Info("Secret was created by another reconciliation", "secret", name) + return nil + } + return fmt.Errorf("failed to create secret: %w", err) + } + + log.Info("Created new secret with random key", "secret", name) + return nil +} + +// generateRandomKey generates a cryptographically secure random key +func generateRandomKey(length int) (string, error) { + bytes := make([]byte, length) + if _, err := rand.Read(bytes); err != nil { + return "", err + } + return base64.URLEncoding.EncodeToString(bytes), nil +} + +// updateStatus updates the status of the Jumpstarter resource +func (r *JumpstarterReconciler) updateStatus(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + // Update status fields based on current state + // This is a placeholder - actual implementation would check deployment status, etc. + // TODO: Add status fields to JumpstarterStatus in the API types + + return nil +} + +// createControllerDeployment creates a deployment for the controller +func (r *JumpstarterReconciler) createControllerDeployment(jumpstarter *operatorv1alpha1.Jumpstarter) *appsv1.Deployment { + labels := map[string]string{ + "app": "jumpstarter-controller", + "controller": jumpstarter.Name, + } + + // Build GRPC endpoint from first controller endpoint + // Default to port 443 for TLS gRPC endpoints + grpcEndpoint := "" + if len(jumpstarter.Spec.Controller.GRPC.Endpoints) > 0 { + ep := jumpstarter.Spec.Controller.GRPC.Endpoints[0] + if ep.Address != "" { + grpcEndpoint = ensurePort(ep.Address, "443") + } else { + grpcEndpoint = fmt.Sprintf("grpc.%s:443", jumpstarter.Spec.BaseDomain) + } + } + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-controller", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &jumpstarter.Spec.Controller.Replicas, + ProgressDeadlineSeconds: int32Ptr(600), + RevisionHistoryLimit: int32Ptr(10), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &intstr.IntOrString{Type: intstr.String, StrVal: "25%"}, + MaxUnavailable: &intstr.IntOrString{Type: intstr.String, StrVal: "25%"}, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + DNSPolicy: corev1.DNSClusterFirst, + TerminationGracePeriodSeconds: int64Ptr(30), + Containers: []corev1.Container{ + { + Name: "manager", + Image: jumpstarter.Spec.Controller.Image, + ImagePullPolicy: jumpstarter.Spec.Controller.ImagePullPolicy, + Args: []string{ + "--leader-elect", + "--health-probe-bind-address=:8081", + "-metrics-bind-address=:8080", + }, + Env: []corev1.EnvVar{ + { + Name: "GRPC_ENDPOINT", + Value: grpcEndpoint, + }, + { + Name: "CONTROLLER_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "jumpstarter-controller-secret", + }, + Key: "key", + }, + }, + }, + { + Name: "ROUTER_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "jumpstarter-router-secret", + }, + Key: "key", + }, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + APIVersion: "v1", + }, + }, + }, + { + Name: "GIN_MODE", + Value: "release", + }, + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 8082, + Name: "grpc", + Protocol: corev1.ProtocolTCP, + }, + { + ContainerPort: 8080, + Name: "metrics", + Protocol: corev1.ProtocolTCP, + }, + { + ContainerPort: 8081, + Name: "health", + Protocol: corev1.ProtocolTCP, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8081), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 15, + PeriodSeconds: 20, + TimeoutSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(8081), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 5, + PeriodSeconds: 10, + TimeoutSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + Resources: jumpstarter.Spec.Controller.Resources, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: boolPtr(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + }, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: boolPtr(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + ServiceAccountName: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), + }, + }, + }, + } +} + +func boolPtr(b bool) *bool { + return &b +} + +// createRouterDeployment creates a deployment for a specific router replica +func (r *JumpstarterReconciler) createRouterDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32) *appsv1.Deployment { + // Base app label that ALL services for this replica will select + // Individual services will be named with endpoint suffixes, but all select the same pods + baseAppLabel := fmt.Sprintf("%s-router-%d", jumpstarter.Name, replicaIndex) + + labels := map[string]string{ + "app": baseAppLabel, // All services for this replica select by this label + "router": jumpstarter.Name, + "router-index": fmt.Sprintf("%d", replicaIndex), + } + + // Build router endpoint for this specific replica + routerEndpoint := r.buildRouterEndpointForReplica(jumpstarter, replicaIndex) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-router-%d", jumpstarter.Name, replicaIndex), + Namespace: jumpstarter.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(1), // Each deployment for the router needs to have exactly 1 replica + ProgressDeadlineSeconds: int32Ptr(600), + RevisionHistoryLimit: int32Ptr(10), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &intstr.IntOrString{Type: intstr.String, StrVal: "25%"}, + MaxUnavailable: &intstr.IntOrString{Type: intstr.String, StrVal: "25%"}, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + DNSPolicy: corev1.DNSClusterFirst, + TerminationGracePeriodSeconds: int64Ptr(30), + Containers: []corev1.Container{ + { + Name: "router", + Image: jumpstarter.Spec.Routers.Image, + ImagePullPolicy: jumpstarter.Spec.Routers.ImagePullPolicy, + Command: []string{"/router"}, + Env: []corev1.EnvVar{ + { + Name: "GRPC_ROUTER_ENDPOINT", + Value: routerEndpoint, + }, + { + Name: "ROUTER_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "jumpstarter-router-secret", + }, + Key: "key", + }, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + APIVersion: "v1", + }, + }, + }, + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 8083, + Name: "grpc", + Protocol: corev1.ProtocolTCP, + }, + { + ContainerPort: 8080, + Name: "metrics", + Protocol: corev1.ProtocolTCP, + }, + { + ContainerPort: 8081, + Name: "health", + Protocol: corev1.ProtocolTCP, + }, + }, + Resources: jumpstarter.Spec.Routers.Resources, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: boolPtr(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + }, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: boolPtr(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + ServiceAccountName: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), + TopologySpreadConstraints: jumpstarter.Spec.Routers.TopologySpreadConstraints, + }, + }, + }, + } +} + +// createConfigMap creates a configmap for jumpstarter configuration +func (r *JumpstarterReconciler) createConfigMap(jumpstarter *operatorv1alpha1.Jumpstarter) (*corev1.ConfigMap, error) { + // Build config struct from spec + cfg := r.buildConfig(jumpstarter) + + // Marshal to YAML + configYAML, err := yaml.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("failed to marshal config to YAML: %w", err) + } + + // Build router configuration for all replicas + router := r.buildRouter(jumpstarter) + + // Marshal router to YAML + routerYAML, err := yaml.Marshal(router) + if err != nil { + return nil, fmt.Errorf("failed to marshal router to YAML: %w", err) + } + + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-controller", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: map[string]string{ + "app": "jumpstarter-controller", + "control-plane": "controller-manager", + }, + }, + Data: map[string]string{ + "config": string(configYAML), + "router": string(routerYAML), + }, + }, nil +} + +// buildConfig builds the controller configuration struct from the CR spec +func (r *JumpstarterReconciler) buildConfig(jumpstarter *operatorv1alpha1.Jumpstarter) config.Config { + cfg := config.Config{ + Provisioning: config.Provisioning{ + Enabled: false, + }, + Grpc: config.Grpc{ + Keepalive: config.Keepalive{ + MinTime: "1s", + PermitWithoutStream: true, + }, + }, + } + + // Authentication configuration + auth := config.Authentication{ + JWT: jumpstarter.Spec.Controller.Authentication.JWT, + } + + // Internal authentication + if jumpstarter.Spec.Controller.Authentication.Internal.Enabled { + prefix := jumpstarter.Spec.Controller.Authentication.Internal.Prefix + if prefix == "" { + prefix = "internal:" + } + auth.Internal.Prefix = prefix + + if jumpstarter.Spec.Controller.Authentication.Internal.TokenLifetime != nil { + auth.Internal.TokenLifetime = jumpstarter.Spec.Controller.Authentication.Internal.TokenLifetime.Duration.String() + } + } + + // Kubernetes authentication + if jumpstarter.Spec.Controller.Authentication.K8s.Enabled { + auth.K8s.Enabled = true + } + + // Ensure JWT is an empty array, not null + if auth.JWT == nil { + auth.JWT = []apiserverv1beta1.JWTAuthenticator{} + } + + cfg.Authentication = auth + + // gRPC keepalive configuration + if jumpstarter.Spec.Controller.GRPC.Keepalive != nil { + ka := &cfg.Grpc.Keepalive + + if jumpstarter.Spec.Controller.GRPC.Keepalive.MinTime != nil { + ka.MinTime = jumpstarter.Spec.Controller.GRPC.Keepalive.MinTime.Duration.String() + } + + ka.PermitWithoutStream = jumpstarter.Spec.Controller.GRPC.Keepalive.PermitWithoutStream + + if jumpstarter.Spec.Controller.GRPC.Keepalive.Timeout != nil { + ka.Timeout = jumpstarter.Spec.Controller.GRPC.Keepalive.Timeout.Duration.String() + } + + if jumpstarter.Spec.Controller.GRPC.Keepalive.IntervalTime != nil { + ka.IntervalTime = jumpstarter.Spec.Controller.GRPC.Keepalive.IntervalTime.Duration.String() + } + + if jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionIdle != nil { + ka.MaxConnectionIdle = jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionIdle.Duration.String() + } + + if jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionAge != nil { + ka.MaxConnectionAge = jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionAge.Duration.String() + } + + if jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionAgeGrace != nil { + ka.MaxConnectionAgeGrace = jumpstarter.Spec.Controller.GRPC.Keepalive.MaxConnectionAgeGrace.Duration.String() + } + } + + return cfg +} + +// buildRouter builds the router configuration with entries for all replicas +func (r *JumpstarterReconciler) buildRouter(jumpstarter *operatorv1alpha1.Jumpstarter) config.Router { + router := make(config.Router) + + // Create router entry for each replica + for i := int32(0); i < jumpstarter.Spec.Routers.Replicas; i++ { + // First replica is named "default" for backwards compatibility + routerName := "default" + if i > 0 { + routerName = fmt.Sprintf("router-%d", i) + } + + entry := config.RouterEntry{ + Endpoint: r.buildRouterEndpointForReplica(jumpstarter, i), + } + + // Add labels if this is not the default router (replica 0) + // Additional routers get labels to distinguish them + if i > 0 { + entry.Labels = map[string]string{ + "router-index": fmt.Sprintf("%d", i), + } + } + + router[routerName] = entry + } + + return router +} + +// buildRouterEndpointForReplica builds the GRPC_ROUTER_ENDPOINT for a specific replica +// This is the primary endpoint the router advertises itself as +func (r *JumpstarterReconciler) buildRouterEndpointForReplica(jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32) string { + // If endpoints are specified, use the first one as the primary endpoint + if len(jumpstarter.Spec.Routers.GRPC.Endpoints) > 0 { + ep := jumpstarter.Spec.Routers.GRPC.Endpoints[0] + address := ep.Address + if address != "" { + address = r.substituteReplica(address, replicaIndex) + return ensurePort(address, "443") + } + } + // Default pattern: router-N.baseDomain + return fmt.Sprintf("router-%d.%s:443", replicaIndex, jumpstarter.Spec.BaseDomain) +} + +// substituteReplica replaces $(replica) placeholder with actual replica index +func (r *JumpstarterReconciler) substituteReplica(address string, replicaIndex int32) string { + return strings.ReplaceAll(address, "$(replica)", fmt.Sprintf("%d", replicaIndex)) +} + +// int32Ptr returns a pointer to an int32 value +func int32Ptr(i int32) *int32 { + return &i +} + +// int64Ptr returns a pointer to an int64 value +func int64Ptr(i int64) *int64 { + return &i +} + +// ensurePort adds a default port to an address if it doesn't already have one +// Handles IPv4, IPv6, and hostnames correctly using net.SplitHostPort +func ensurePort(address, defaultPort string) string { + // Try to split the address into host and port + _, _, err := net.SplitHostPort(address) + if err == nil { + // Address already has a port, return as-is + return address + } + + // No port found, need to add one + // net.JoinHostPort handles IPv6 addresses correctly (adds brackets if needed) + return net.JoinHostPort(address, defaultPort) +} + +// buildServiceNameForReplicaEndpoint creates a unique service name for a router replica and endpoint +func (r *JumpstarterReconciler) buildServiceNameForReplicaEndpoint(jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32, endpointIdx int) string { + if endpointIdx == 0 { + // First endpoint uses base name for backwards compatibility + return fmt.Sprintf("%s-router-%d", jumpstarter.Name, replicaIndex) + } + // Additional endpoints get a suffix + return fmt.Sprintf("%s-router-%d-%d", jumpstarter.Name, replicaIndex, endpointIdx) +} + +// buildEndpointForReplica creates an Endpoint struct for a specific router replica and endpoint +func (r *JumpstarterReconciler) buildEndpointForReplica(jumpstarter *operatorv1alpha1.Jumpstarter, replicaIndex int32, endpointIdx int, baseEndpoint *operatorv1alpha1.Endpoint) operatorv1alpha1.Endpoint { + // Copy the base endpoint + endpoint := *baseEndpoint + + // Set or substitute address + if endpoint.Address != "" { + endpoint.Address = r.substituteReplica(endpoint.Address, replicaIndex) + } else { + // Default address pattern when none specified + if endpointIdx == 0 { + endpoint.Address = fmt.Sprintf("router-%d.%s", replicaIndex, jumpstarter.Spec.BaseDomain) + } else { + endpoint.Address = fmt.Sprintf("router-%d-%d.%s", replicaIndex, endpointIdx, jumpstarter.Spec.BaseDomain) + } + } + + return endpoint +} + +// cleanupExcessRouterDeployments deletes router deployments that exceed the current replica count +func (r *JumpstarterReconciler) cleanupExcessRouterDeployments(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + // List all deployments with our router label + deploymentList := &appsv1.DeploymentList{} + listOpts := []client.ListOption{ + client.InNamespace(jumpstarter.Namespace), + client.MatchingLabels{ + "router": jumpstarter.Name, + }, + } + + if err := r.List(ctx, deploymentList, listOpts...); err != nil { + return fmt.Errorf("failed to list router deployments: %w", err) + } + + // Delete deployments with replica index >= current replica count + for i := range deploymentList.Items { + deployment := &deploymentList.Items[i] + + // Check if this deployment's name indicates it's beyond the current replica count + // We need to check all indices from current replicas onwards + for idx := jumpstarter.Spec.Routers.Replicas; idx < 100; idx++ { // reasonable upper bound + excessName := fmt.Sprintf("%s-router-%d", jumpstarter.Name, idx) + if deployment.Name == excessName { + log.Info("Deleting excess router deployment", "deployment", deployment.Name, "replicaIndex", idx) + if err := r.Delete(ctx, deployment); err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete excess deployment %s: %w", deployment.Name, err) + } + } + break + } + } + } + + return nil +} + +// cleanupExcessRouterServices deletes router services that exceed the current replica count +// or endpoint count. This ensures that when replicas or endpoints are scaled down, the +// corresponding services are removed. +func (r *JumpstarterReconciler) cleanupExcessRouterServices(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + // Services can have suffixes for different service types + // ClusterIP has no suffix, LoadBalancer has "-lb", NodePort has "-np" + suffixes := []string{"", "-lb", "-np"} + + // 1. Delete services for excess replicas (replica index >= current replica count) + for idx := jumpstarter.Spec.Routers.Replicas; idx < 100; idx++ { // reasonable upper bound + foundAny := false + + // Try to delete services for all endpoints and service types for this replica + for endpointIdx := 0; endpointIdx < 10; endpointIdx++ { // reasonable upper bound for endpoints + for _, suffix := range suffixes { + var serviceName string + if endpointIdx == 0 { + serviceName = fmt.Sprintf("%s-router-%d%s", jumpstarter.Name, idx, suffix) + } else { + serviceName = fmt.Sprintf("%s-router-%d-%d%s", jumpstarter.Name, idx, endpointIdx, suffix) + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: jumpstarter.Namespace, + }, + } + + err := r.Delete(ctx, service) + if err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete excess service %s: %w", serviceName, err) + } + } else { + foundAny = true + log.Info("Deleted excess router service", "service", serviceName, "replicaIndex", idx, "endpointIdx", endpointIdx) + } + } + } + + // If we didn't find any services for this replica index, we've gone past all excess services + if !foundAny { + break + } + } + + // 2. Delete services for excess endpoints within valid replicas + numEndpoints := len(jumpstarter.Spec.Routers.GRPC.Endpoints) + if numEndpoints == 0 { + numEndpoints = 1 // default endpoint + } + + for replicaIdx := int32(0); replicaIdx < jumpstarter.Spec.Routers.Replicas; replicaIdx++ { + for endpointIdx := numEndpoints; endpointIdx < 10; endpointIdx++ { // reasonable upper bound + foundAny := false + + for _, suffix := range suffixes { + var serviceName string + if endpointIdx == 0 { + serviceName = fmt.Sprintf("%s-router-%d%s", jumpstarter.Name, replicaIdx, suffix) + } else { + serviceName = fmt.Sprintf("%s-router-%d-%d%s", jumpstarter.Name, replicaIdx, endpointIdx, suffix) + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: jumpstarter.Namespace, + }, + } + + err := r.Delete(ctx, service) + if err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete excess endpoint service %s: %w", serviceName, err) + } + } else { + foundAny = true + log.Info("Deleted excess endpoint service", "service", serviceName, "replicaIndex", replicaIdx, "endpointIdx", endpointIdx) + } + } + + // If we didn't find any services for this endpoint index, we've gone past all excess endpoints + if !foundAny { + break + } + } + } + + return nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *JumpstarterReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&operatorv1alpha1.Jumpstarter{}). + Named("jumpstarter"). + Owns(&appsv1.Deployment{}). + Owns(&corev1.Service{}). + Owns(&corev1.ConfigMap{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + // Note: Secrets and ServiceAccounts are intentionally NOT owned to prevent deletion + Complete(r) +} diff --git a/deploy/operator/internal/controller/jumpstarter_controller_test.go b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go similarity index 51% rename from deploy/operator/internal/controller/jumpstarter_controller_test.go rename to deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go index a66ca3ed..14540354 100644 --- a/deploy/operator/internal/controller/jumpstarter_controller_test.go +++ b/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller_test.go @@ -14,20 +14,22 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package jumpstarter import ( "context" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/jumpstarter/endpoints" ) var _ = Describe("Jumpstarter Controller", func() { @@ -51,7 +53,46 @@ var _ = Describe("Jumpstarter Controller", func() { Name: resourceName, Namespace: "default", }, - // TODO(user): Specify other spec details if needed. + Spec: operatorv1alpha1.JumpstarterSpec{ + BaseDomain: "example.com", + UseCertManager: true, + Controller: operatorv1alpha1.ControllerConfig{ + Image: "quay.io/jumpstarter/jumpstarter:latest", + ImagePullPolicy: "IfNotPresent", + Replicas: 1, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + GRPC: operatorv1alpha1.GRPCConfig{ + Endpoints: []operatorv1alpha1.Endpoint{ + { + Address: "controller", + }, + }, + }, + }, + Routers: operatorv1alpha1.RoutersConfig{ + Image: "quay.io/jumpstarter/jumpstarter:latest", + ImagePullPolicy: "IfNotPresent", + Replicas: 1, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + GRPC: operatorv1alpha1.GRPCConfig{ + Endpoints: []operatorv1alpha1.Endpoint{ + { + Address: "router", + }, + }, + }, + }, + }, } Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } @@ -69,8 +110,9 @@ var _ = Describe("Jumpstarter Controller", func() { It("should successfully reconcile the resource", func() { By("Reconciling the created resource") controllerReconciler := &JumpstarterReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), + Client: k8sClient, + Scheme: k8sClient.Scheme(), + EndpointReconciler: endpoints.NewReconciler(k8sClient, k8sClient.Scheme()), } _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ @@ -82,3 +124,19 @@ var _ = Describe("Jumpstarter Controller", func() { }) }) }) + +var _ = Describe("ensurePort", func() { + DescribeTable("should handle addresses correctly", + func(address, defaultPort, expected string) { + result := ensurePort(address, defaultPort) + Expect(result).To(Equal(expected)) + }, + Entry("hostname without port", "example.com", "443", "example.com:443"), + Entry("hostname with port", "example.com:8083", "443", "example.com:8083"), + Entry("IPv6 without port", "2001:db8::1", "443", "[2001:db8::1]:443"), + Entry("IPv6 with port", "[2001:db8::1]:8083", "443", "[2001:db8::1]:8083"), + Entry("malformed - too many colons", "host:port:extra", "443", "[host:port:extra]:443"), + Entry("malformed - empty string", "", "443", ":443"), + Entry("malformed - just colon", ":", "443", ":"), + ) +}) diff --git a/deploy/operator/internal/controller/jumpstarter/rbac.go b/deploy/operator/internal/controller/jumpstarter/rbac.go new file mode 100644 index 00000000..e0333618 --- /dev/null +++ b/deploy/operator/internal/controller/jumpstarter/rbac.go @@ -0,0 +1,249 @@ +package jumpstarter + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" +) + +// reconcileRBAC reconciles all RBAC resources (ServiceAccount, Role, RoleBinding) +func (r *JumpstarterReconciler) reconcileRBAC(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + // ServiceAccount + // Note: We intentionally do NOT set controller reference on ServiceAccount to prevent + // it from being garbage collected when the Jumpstarter CR is deleted + desiredSA := r.createServiceAccount(jumpstarter) + + existingSA := &corev1.ServiceAccount{} + existingSA.Name = desiredSA.Name + existingSA.Namespace = desiredSA.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingSA, func() error { + // Check if this is a new service account or an existing one + if existingSA.CreationTimestamp.IsZero() { + // ServiceAccount is being created, copy all fields from desired + existingSA.Labels = desiredSA.Labels + existingSA.Annotations = desiredSA.Annotations + return nil + } + + // ServiceAccount exists, check if update is needed + if !serviceAccountNeedsUpdate(existingSA, desiredSA) { + log.V(1).Info("ServiceAccount is up to date, skipping update", + "name", existingSA.Name, + "namespace", existingSA.Namespace) + return nil + } + + // Update needed - apply changes + existingSA.Labels = desiredSA.Labels + existingSA.Annotations = desiredSA.Annotations + return nil + }) + + if err != nil { + log.Error(err, "Failed to reconcile ServiceAccount", + "name", desiredSA.Name, + "namespace", desiredSA.Namespace) + return err + } + + log.Info("ServiceAccount reconciled", + "name", existingSA.Name, + "namespace", existingSA.Namespace, + "operation", op) + + // Role + desiredRole := r.createRole(jumpstarter) + + existingRole := &rbacv1.Role{} + existingRole.Name = desiredRole.Name + existingRole.Namespace = desiredRole.Namespace + + op, err = controllerutil.CreateOrUpdate(ctx, r.Client, existingRole, func() error { + // Check if this is a new role or an existing one + if existingRole.CreationTimestamp.IsZero() { + // Role is being created, copy all fields from desired + existingRole.Labels = desiredRole.Labels + existingRole.Annotations = desiredRole.Annotations + existingRole.Rules = desiredRole.Rules + return controllerutil.SetControllerReference(jumpstarter, existingRole, r.Scheme) + } + + // Role exists, check if update is needed + if !roleNeedsUpdate(existingRole, desiredRole) { + log.V(1).Info("Role is up to date, skipping update", + "name", existingRole.Name, + "namespace", existingRole.Namespace) + return nil + } + + // Update needed - apply changes + existingRole.Labels = desiredRole.Labels + existingRole.Annotations = desiredRole.Annotations + existingRole.Rules = desiredRole.Rules + return controllerutil.SetControllerReference(jumpstarter, existingRole, r.Scheme) + }) + + if err != nil { + log.Error(err, "Failed to reconcile Role", + "name", desiredRole.Name, + "namespace", desiredRole.Namespace) + return err + } + + log.Info("Role reconciled", + "name", existingRole.Name, + "namespace", existingRole.Namespace, + "operation", op) + + // RoleBinding + desiredRoleBinding := r.createRoleBinding(jumpstarter) + + existingRoleBinding := &rbacv1.RoleBinding{} + existingRoleBinding.Name = desiredRoleBinding.Name + existingRoleBinding.Namespace = desiredRoleBinding.Namespace + + op, err = controllerutil.CreateOrUpdate(ctx, r.Client, existingRoleBinding, func() error { + // Check if this is a new role binding or an existing one + if existingRoleBinding.CreationTimestamp.IsZero() { + // RoleBinding is being created, copy all fields from desired + existingRoleBinding.Labels = desiredRoleBinding.Labels + existingRoleBinding.Annotations = desiredRoleBinding.Annotations + existingRoleBinding.Subjects = desiredRoleBinding.Subjects + existingRoleBinding.RoleRef = desiredRoleBinding.RoleRef + return controllerutil.SetControllerReference(jumpstarter, existingRoleBinding, r.Scheme) + } + + // RoleBinding exists, check if update is needed + if !roleBindingNeedsUpdate(existingRoleBinding, desiredRoleBinding) { + log.V(1).Info("RoleBinding is up to date, skipping update", + "name", existingRoleBinding.Name, + "namespace", existingRoleBinding.Namespace) + return nil + } + + // Update needed - apply changes + existingRoleBinding.Labels = desiredRoleBinding.Labels + existingRoleBinding.Annotations = desiredRoleBinding.Annotations + existingRoleBinding.Subjects = desiredRoleBinding.Subjects + existingRoleBinding.RoleRef = desiredRoleBinding.RoleRef + return controllerutil.SetControllerReference(jumpstarter, existingRoleBinding, r.Scheme) + }) + + if err != nil { + log.Error(err, "Failed to reconcile RoleBinding", + "name", desiredRoleBinding.Name, + "namespace", desiredRoleBinding.Namespace) + return err + } + + log.Info("RoleBinding reconciled", + "name", existingRoleBinding.Name, + "namespace", existingRoleBinding.Namespace, + "operation", op) + + return nil +} + +// createServiceAccount creates a service account for the controller +func (r *JumpstarterReconciler) createServiceAccount(jumpstarter *operatorv1alpha1.Jumpstarter) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: map[string]string{ + "app": "jumpstarter-controller", + "app.kubernetes.io/name": "jumpstarter-controller", + "app.kubernetes.io/managed-by": "jumpstarter-operator", + }, + }, + } +} + +// createRole creates a role with necessary permissions for the controller +func (r *JumpstarterReconciler) createRole(jumpstarter *operatorv1alpha1.Jumpstarter) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-controller-role", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: map[string]string{ + "app": "jumpstarter-controller", + "app.kubernetes.io/name": "jumpstarter-controller", + "app.kubernetes.io/managed-by": "jumpstarter-operator", + }, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + { + APIGroups: []string{"jumpstarter.dev"}, + Resources: []string{"clients", "exporters", "leases", "exporteraccesspolicies"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + { + APIGroups: []string{"jumpstarter.dev"}, + Resources: []string{"clients/status", "exporters/status", "leases/status", "exporteraccesspolicies/status"}, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{"jumpstarter.dev"}, + Resources: []string{"clients/finalizers", "exporters/finalizers", "leases/finalizers", "exporteraccesspolicies/finalizers"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + }, + } +} + +// createRoleBinding creates a role binding for the controller +func (r *JumpstarterReconciler) createRoleBinding(jumpstarter *operatorv1alpha1.Jumpstarter) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-controller-rolebinding", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: map[string]string{ + "app": "jumpstarter-controller", + "app.kubernetes.io/name": "jumpstarter-controller", + "app.kubernetes.io/managed-by": "jumpstarter-operator", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: fmt.Sprintf("%s-controller-role", jumpstarter.Name), + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: fmt.Sprintf("%s-controller-manager", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + }, + }, + } +} diff --git a/deploy/operator/internal/controller/suite_test.go b/deploy/operator/internal/controller/jumpstarter/suite_test.go similarity index 67% rename from deploy/operator/internal/controller/suite_test.go rename to deploy/operator/internal/controller/jumpstarter/suite_test.go index 746a5b30..83034fde 100644 --- a/deploy/operator/internal/controller/suite_test.go +++ b/deploy/operator/internal/controller/jumpstarter/suite_test.go @@ -14,11 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package jumpstarter import ( "context" - "os" "path/filepath" "testing" @@ -33,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/internal/controller/testutils" // +kubebuilder:scaffold:imports ) @@ -66,13 +66,13 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, } // Retrieve the first found binary directory to allow running tests from IDEs - if getFirstFoundEnvTestBinaryDir() != "" { - testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir() + if binaryDir := testutils.GetFirstFoundEnvTestBinaryDir(5); binaryDir != "" { + testEnv.BinaryAssetsDirectory = binaryDir } // cfg is defined in this file globally. @@ -91,26 +91,3 @@ var _ = AfterSuite(func() { err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) - -// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. -// ENVTEST-based tests depend on specific binaries, usually located in paths set by -// controller-runtime. When running tests directly (e.g., via an IDE) without using -// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. -// -// This function streamlines the process by finding the required binaries, similar to -// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are -// properly set up, run 'make setup-envtest' beforehand. -func getFirstFoundEnvTestBinaryDir() string { - basePath := filepath.Join("..", "..", "bin", "k8s") - entries, err := os.ReadDir(basePath) - if err != nil { - logf.Log.Error(err, "Failed to read directory", "path", basePath) - return "" - } - for _, entry := range entries { - if entry.IsDir() { - return filepath.Join(basePath, entry.Name()) - } - } - return "" -} diff --git a/deploy/operator/internal/controller/jumpstarter_controller.go b/deploy/operator/internal/controller/jumpstarter_controller.go deleted file mode 100644 index 95a3d516..00000000 --- a/deploy/operator/internal/controller/jumpstarter_controller.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter-controller/deploy/operator/api/v1alpha1" -) - -// JumpstarterReconciler reconciles a Jumpstarter object -type JumpstarterReconciler struct { - client.Client - Scheme *runtime.Scheme -} - -// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=operator.jumpstarter.dev,resources=jumpstarters/finalizers,verbs=update - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the Jumpstarter object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile -func (r *JumpstarterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = logf.FromContext(ctx) - - // TODO(user): your logic here - - return ctrl.Result{}, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *JumpstarterReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&operatorv1alpha1.Jumpstarter{}). - Named("jumpstarter"). - Complete(r) -} diff --git a/deploy/operator/internal/controller/testutils/envtest.go b/deploy/operator/internal/controller/testutils/envtest.go new file mode 100644 index 00000000..eb5f506d --- /dev/null +++ b/deploy/operator/internal/controller/testutils/envtest.go @@ -0,0 +1,57 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testutils + +import ( + "os" + "path/filepath" + + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +// GetFirstFoundEnvTestBinaryDir locates the first binary in the specified path. +// ENVTEST-based tests depend on specific binaries, usually located in paths set by +// controller-runtime. When running tests directly (e.g., via an IDE) without using +// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. +// +// This function streamlines the process by finding the required binaries, similar to +// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are +// properly set up, run 'make setup-envtest' beforehand. +// +// The depth parameter specifies how many directories up to traverse from the test file +// to reach the operator root (where bin/k8s is located). +func GetFirstFoundEnvTestBinaryDir(depth int) string { + // Build the path based on depth + pathComponents := make([]string, 0, depth+2) + for i := 0; i < depth; i++ { + pathComponents = append(pathComponents, "..") + } + pathComponents = append(pathComponents, "bin", "k8s") + + basePath := filepath.Join(pathComponents...) + entries, err := os.ReadDir(basePath) + if err != nil { + logf.Log.Error(err, "Failed to read directory", "path", basePath) + return "" + } + for _, entry := range entries { + if entry.IsDir() { + return filepath.Join(basePath, entry.Name()) + } + } + return "" +} diff --git a/deploy/operator/internal/log/levels.go b/deploy/operator/internal/log/levels.go new file mode 100644 index 00000000..8b25f203 --- /dev/null +++ b/deploy/operator/internal/log/levels.go @@ -0,0 +1,18 @@ +package log + +// Log levels for use with controller-runtime's logr.Logger.V() method. +// Higher numbers mean more verbose logging. +const ( + // LevelInfo is the standard info level (V(0)) + LevelInfo = 0 + + // LevelDebug is for debug-level logging (V(1)) + // Use for detailed operational information that is useful for debugging + // but not needed during normal operation. + LevelDebug = 1 + + // LevelTrace is for trace-level logging (V(2)) + // Use for very detailed information about internal operations, + // useful for troubleshooting complex issues. + LevelTrace = 2 +) diff --git a/hack/deploy_vars b/hack/deploy_vars new file mode 100755 index 00000000..e54ca915 --- /dev/null +++ b/hack/deploy_vars @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Common deployment variables for Jumpstarter hack scripts +# This file should be sourced after utils + +# Calculate external IP and networking configuration +IP=$(get_external_ip) +BASEDOMAIN="jumpstarter.${IP}.nip.io" +IMG=${IMG:-quay.io/jumpstarter-dev/jumpstarter-controller:latest} +OPERATOR_IMG=${OPERATOR_IMG:-quay.io/jumpstarter-dev/jumpstarter-operator:latest} + +# Determine networking mode and endpoints based on INGRESS_ENABLED +if [ "${INGRESS_ENABLED}" == "true" ]; then + NETWORKING_MODE="ingress" + GRPC_ENDPOINT="grpc.${BASEDOMAIN}:5080" + GRPC_ROUTER_ENDPOINT="router.${BASEDOMAIN}:5080" +else + NETWORKING_MODE="nodeport" + GRPC_ENDPOINT="grpc.${BASEDOMAIN}:8082" + GRPC_ROUTER_ENDPOINT="router.${BASEDOMAIN}:8083" +fi + +# Extract image repository and tag from IMG variable +if [[ "${IMG}" == *:* ]]; then + IMAGE_TAG="${IMG##*:}" # everything after the last colon + IMAGE_REPO="${IMG%:*}" # everything before the last colon +else # no tag specified + IMAGE_TAG="latest" + IMAGE_REPO="${IMG}" +fi +# Export all variables for use in scripts +export IP +export BASEDOMAIN +export NETWORKING_MODE +export GRPC_ENDPOINT +export GRPC_ROUTER_ENDPOINT +export IMAGE_REPO +export IMAGE_TAG +export IMG +export OPERATOR_IMG + diff --git a/hack/deploy_with_helm.sh b/hack/deploy_with_helm.sh index db54bc67..5fec8f26 100755 --- a/hack/deploy_with_helm.sh +++ b/hack/deploy_with_helm.sh @@ -2,86 +2,37 @@ set -eo pipefail SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" -KIND=${KIND:-bin/kind} -GRPCURL=${GRPCURL:-bin/grpcurl} -IMG=${IMG:-quay.io/jumpstarter-dev/jumpstarter-controller:latest} -INGRESS_ENABLED=${INGRESS_ENABLED:-false} +# Source common utilities +source "${SCRIPT_DIR}/utils" -GREEN='\033[0;32m' -NC='\033[0m' # No Color +# Source common deployment variables +source "${SCRIPT_DIR}/deploy_vars" METHOD=install -IP=$("${SCRIPT_DIR}"/get_ext_ip.sh) - kubectl config use-context kind-jumpstarter -HELM_SETS="" +# Install nginx ingress if in ingress mode if [ "${INGRESS_ENABLED}" == "true" ]; then - echo -e "${GREEN}Deploying nginx ingress in kind ...${NC}" - - lsmod | grep ip_tables || \ - (echo "ip_tables module not loaded needed by nginx ingress, please run 'sudo modprobe ip_tables'" && exit 1) - - # before our helm installs, we make sure that kind has an ingress installed - kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml - - echo -e "${GREEN}Waiting for nginx to be ready ...${NC}" - - while ! kubectl get pods --namespace ingress-nginx --selector=app.kubernetes.io/component=controller > /dev/null; do - sleep 1 - done - - kubectl wait --namespace ingress-nginx \ - --for=condition=ready pod \ - --selector=app.kubernetes.io/component=controller \ - --timeout=90s - - HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.ingress.enabled=true" - BASEDOMAIN="jumpstarter.${IP}.nip.io" - GRPC_ENDPOINT="grpc.${BASEDOMAIN}:5080" - GRPC_ROUTER_ENDPOINT="router.${BASEDOMAIN}:5080" + install_nginx_ingress else echo -e "${GREEN}Deploying with nodeport ...${NC}" - HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.nodeport.enabled=true" - BASEDOMAIN="jumpstarter.${IP}.nip.io" - GRPC_ENDPOINT="grpc.${BASEDOMAIN}:8082" - GRPC_ROUTER_ENDPOINT="router.${BASEDOMAIN}:8083" fi +# Build Helm sets based on configuration +HELM_SETS="" HELM_SETS="${HELM_SETS} --set global.baseDomain=${BASEDOMAIN}" HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.endpoint=${GRPC_ENDPOINT}" HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.routerEndpoint=${GRPC_ROUTER_ENDPOINT}" - - -IMAGE_REPO=$(echo ${IMG} | cut -d: -f1) -IMAGE_TAG=$(echo ${IMG} | cut -d: -f2) HELM_SETS="${HELM_SETS} --set jumpstarter-controller.image=${IMAGE_REPO}" HELM_SETS="${HELM_SETS} --set jumpstarter-controller.tag=${IMAGE_TAG}" -# Function to save images to kind, with workaround for github CI and other environment issues -# In github CI, kind gets confused and tries to pull the image from docker instead -# of podman, so if regular docker-image fails we need to: -# * save it to OCI image format -# * then load it into kind -kind_load_image() { - local image=$1 - - # First, try to load the image directly - if ${KIND} load docker-image "${image}" --name jumpstarter; then - echo "Image ${image} loaded successfully." - return - fi - - # Save to tar file - podman save "${image}" | ${KIND} load image-archive /dev/stdin --name jumpstarter - if [ $? -eq 0 ]; then - echo "Image loaded successfully." - else - echo "Error loading image ${image}." - exit 1 - fi -} +# Enable appropriate networking mode +if [ "${NETWORKING_MODE}" == "ingress" ]; then + HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.ingress.enabled=true" +else + HELM_SETS="${HELM_SETS} --set jumpstarter-controller.grpc.nodeport.enabled=true" +fi echo -e "${GREEN}Loading the ${IMG} in kind ...${NC}" # load the docker image into the kind cluster @@ -105,21 +56,8 @@ helm ${METHOD} --namespace jumpstarter-lab \ kubectl config set-context --current --namespace=jumpstarter-lab -echo -e "${GREEN}Waiting for grpc endpoints to be ready:${NC}" -for ep in ${GRPC_ENDPOINT} ${GRPC_ROUTER_ENDPOINT}; do - RETRIES=60 - echo -e "${GREEN} * Checking ${ep} ... ${NC}" - while ! ${GRPCURL} -insecure ${ep} list; do - sleep 2 - RETRIES=$((RETRIES-1)) - if [ ${RETRIES} -eq 0 ]; then - echo -e "${GREEN} * ${ep} not ready after 120s, exiting ... ${NC}" - exit 1 - fi - done -done - +# Check gRPC endpoints are ready +check_grpc_endpoints -echo -e "${GREEN}Jumpstarter controller deployed successfully!${NC}" -echo -e " gRPC endpoint: ${GRPC_ENDPOINT}" -echo -e " gRPC router endpoint: ${GRPC_ROUTER_ENDPOINT}" +# Print success banner +print_deployment_success "Helm" diff --git a/hack/deploy_with_operator.sh b/hack/deploy_with_operator.sh new file mode 100755 index 00000000..ccd0d0c6 --- /dev/null +++ b/hack/deploy_with_operator.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -eo pipefail +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" + +# Source common utilities +source "${SCRIPT_DIR}/utils" + +# Source common deployment variables +source "${SCRIPT_DIR}/deploy_vars" + +kubectl config use-context kind-jumpstarter + +# Install nginx ingress if in ingress mode +if [ "${NETWORKING_MODE}" = "ingress" ]; then + install_nginx_ingress +else + echo -e "${GREEN}Deploying with nodeport ...${NC}" +fi + +# load the container images into the kind cluster +kind_load_image "${IMG}" +kind_load_image "${OPERATOR_IMG}" + +# Deploy the operator +echo -e "${GREEN}Deploying Jumpstarter operator ...${NC}" +kubectl apply -f deploy/operator/dist/install.yaml + +# If operator deployment already exists, restart it to pick up the new image +if kubectl get deployment jumpstarter-operator-controller-manager -n jumpstarter-operator-system > /dev/null 2>&1; then + echo -e "${GREEN}Restarting operator deployment to pick up new image ...${NC}" + kubectl scale deployment jumpstarter-operator-controller-manager -n jumpstarter-operator-system --replicas=0 + kubectl wait --namespace jumpstarter-operator-system \ + --for=delete pod \ + --selector=control-plane=controller-manager \ + --timeout=60s 2>/dev/null || true + kubectl scale deployment jumpstarter-operator-controller-manager -n jumpstarter-operator-system --replicas=1 +fi + +# Wait for operator to be ready +echo -e "${GREEN}Waiting for operator to be ready ...${NC}" +kubectl wait --namespace jumpstarter-operator-system \ + --for=condition=available deployment/jumpstarter-operator-controller-manager \ + --timeout=120s + +# Create namespace for Jumpstarter deployment +echo -e "${GREEN}Creating jumpstarter-lab namespace ...${NC}" +kubectl create namespace jumpstarter-lab --dry-run=client -o yaml | kubectl apply -f - + +# Generate Jumpstarter CR based on networking mode +echo -e "${GREEN}Creating Jumpstarter custom resource ...${NC}" + +# Generate endpoint configuration based on networking mode +if [ "${NETWORKING_MODE}" == "ingress" ]; then + CONTROLLER_ENDPOINT_CONFIG=$(cat <<-END + - address: grpc.${BASEDOMAIN}:443 + ingress: + enabled: true + class: "" +END +) + ROUTER_ENDPOINT_CONFIG=$(cat <<-END + - address: router.${BASEDOMAIN}:443 + ingress: + enabled: true + class: "" +END +) +else + CONTROLLER_ENDPOINT_CONFIG=$(cat <<-END + # this is exposed by a nodeport in 30010 but mapped to 8082 on the host + - address: grpc.${BASEDOMAIN}:8082 + nodeport: + enabled: true + port: 30010 +END +) + ROUTER_ENDPOINT_CONFIG=$(cat <<-END + # this is exposed by a nodeport in 30011 but mapped to 8083 on the host + - address: router.${BASEDOMAIN}:8083 + nodeport: + enabled: true + port: 30011 +END +) +fi + +# Apply the Jumpstarter CR with the appropriate endpoint configuration +cat </dev/null 1>/dev/null; then - ip route get 1.1.1.1 | grep -oP 'src \K\S+' -else - # MacOS does not have ip, so we use route and ifconfig instead - INTERFACE=$(route get 1.1.1.1 | grep interface | awk '{print $2}') - ifconfig | grep $INTERFACE -A 10 | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}' | head -n 1 -fi diff --git a/hack/kind_cluster.yaml b/hack/kind_cluster.yaml index e03fe267..7ffd60b1 100644 --- a/hack/kind_cluster.yaml +++ b/hack/kind_cluster.yaml @@ -20,10 +20,15 @@ nodes: - containerPort: 30010 # grpc nodeport hostPort: 8082 protocol: TCP - - containerPort: 30011 # grpc router nodeport + - containerPort: 30011 # grpc router nodeport (replica 0) hostPort: 8083 protocol: TCP - + - containerPort: 30012 # grpc router nodeport (replica 1) + hostPort: 8084 + protocol: TCP + - containerPort: 30013 # grpc router nodeport (replica 2) + hostPort: 8085 + protocol: TCP - containerPort: 443 hostPort: 5443 protocol: TCP diff --git a/hack/utils b/hack/utils new file mode 100755 index 00000000..50fd0882 --- /dev/null +++ b/hack/utils @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Common utilities for Jumpstarter hack scripts + +set -eo pipefail + +# Script directory calculation helper +# Usage: SCRIPT_DIR="$(get_script_dir)" +get_script_dir() { + dirname "$(readlink -f "$0")" +} + +# Environment variable defaults +export KIND=${KIND:-bin/kind} +export GRPCURL=${GRPCURL:-bin/grpcurl} +export INGRESS_ENABLED=${INGRESS_ENABLED:-false} + +# Color codes for terminal output +export GREEN='\033[0;32m' +export NC='\033[0m' # No Color + +# Get external IP address +# Returns the IP address used for outbound connections +get_external_ip() { + if which ip 2>/dev/null 1>/dev/null; then + ip route get 1.1.1.1 | grep -oP 'src \K\S+' + else + # MacOS does not have ip, so we use route and ifconfig instead + INTERFACE=$(route get 1.1.1.1 | grep interface | awk '{print $2}') + ifconfig | grep "$INTERFACE" -A 10 | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}' | head -n 1 + fi +} + +# Load Docker/Podman image into kind cluster +# Function to save images to kind, with workaround for github CI and other environment issues +# In github CI, kind gets confused and tries to pull the image from docker instead +# of podman, so if regular docker-image fails we need to: +# * save it to OCI image format +# * then load it into kind +# Args: +# $1: image name (e.g., quay.io/jumpstarter-dev/jumpstarter-controller:latest) +# $2: kind cluster name (default: jumpstarter) +kind_load_image() { + local image=$1 + local cluster_name=${2:-jumpstarter} + + echo -e "${GREEN}Loading $1 in kind ...${NC}" + + # First, try to load the image directly + if ${KIND} load docker-image "${image}" --name "${cluster_name}" 2>/dev/null; then + echo "Image ${image} loaded successfully." + return + fi + + # Save to tar file + if podman save "${image}" | ${KIND} load image-archive /dev/stdin --name "${cluster_name}"; then + echo "Image loaded successfully." + else + echo "Error loading image ${image}." + exit 1 + fi +} + +# Install nginx ingress in kind cluster +# This function deploys nginx ingress and waits for it to be ready +install_nginx_ingress() { + echo -e "${GREEN}Deploying nginx ingress in kind ...${NC}" + + lsmod | grep ip_tables || \ + (echo "ip_tables module not loaded needed by nginx ingress, please run 'sudo modprobe ip_tables'" && exit 1) + + # Deploy nginx ingress for kind + kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml + + echo -e "${GREEN}Waiting for nginx to be ready ...${NC}" + + while ! kubectl get pods --namespace ingress-nginx --selector=app.kubernetes.io/component=controller > /dev/null 2>&1; do + sleep 1 + done + + kubectl wait --namespace ingress-nginx \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/component=controller \ + --timeout=90s + + echo -e "${GREEN}Nginx ingress installed successfully${NC}" +} + +# Wait for Jumpstarter Kubernetes resources to be created and ready +# This is primarily used for operator deployments where resources are created asynchronously +# Args: +# $1: namespace (default: jumpstarter-lab) +wait_for_jumpstarter_resources() { + local namespace=${1:-jumpstarter-lab} + + echo -e "${GREEN}Waiting for Jumpstarter deployments to be ready ...${NC}" + + # Wait for controller deployment to exist + echo -e "${GREEN} * Waiting for controller deployment to be created ...${NC}" + local timeout=60 + while ! kubectl get deployment jumpstarter-controller -n "${namespace}" > /dev/null 2>&1; do + sleep 2 + timeout=$((timeout - 2)) + if [ ${timeout} -le 0 ]; then + echo -e "${GREEN} * Controller deployment not created after 60s, exiting ...${NC}" + exit 1 + fi + done + + # Wait for router deployment to exist + echo -e "${GREEN} * Waiting for router deployment to be created ...${NC}" + timeout=60 + while ! kubectl get deployment jumpstarter-router-0 -n "${namespace}" > /dev/null 2>&1; do + sleep 2 + timeout=$((timeout - 2)) + if [ ${timeout} -le 0 ]; then + echo -e "${GREEN} * Router deployment not created after 60s, exiting ...${NC}" + exit 1 + fi + done + + # Wait for controller deployment to be ready + echo -e "${GREEN} * Waiting for controller deployment to be ready ...${NC}" + kubectl wait --namespace "${namespace}" \ + --for=condition=available deployment/jumpstarter-controller \ + --timeout=180s + + # Wait for router statefulset to be ready + echo -e "${GREEN} * Waiting for router pods to be ready ...${NC}" + kubectl wait --namespace "${namespace}" \ + --for=condition=ready pod \ + --selector=app=jumpstarter-router-0 \ + --timeout=180s +} + +# Wait for gRPC endpoint to be ready +# Args: +# $1: endpoint (e.g., grpc.jumpstarter.192.168.1.1.nip.io:8082) +# $2: timeout in seconds (default: 120) +wait_for_grpc_endpoint() { + local endpoint=$1 + local timeout=${2:-120} + local retries=$((timeout / 2)) + + echo -e "${GREEN} * Checking ${endpoint} ... ${NC}" + while ! ${GRPCURL} -insecure "${endpoint}" list; do + sleep 2 + retries=$((retries - 1)) + if [ ${retries} -eq 0 ]; then + echo -e "${GREEN} * ${endpoint} not ready after ${timeout}s, exiting ... ${NC}" + exit 1 + fi + done +} + +# Check both gRPC endpoints (controller and router) are ready +check_grpc_endpoints() { + echo -e "${GREEN}Waiting for grpc endpoints to be ready:${NC}" + wait_for_grpc_endpoint "${GRPC_ENDPOINT}" + wait_for_grpc_endpoint "${GRPC_ROUTER_ENDPOINT}" +} + +# Print deployment success banner +# Args: +# $1: deployment method (e.g., "Helm", "operator") - optional +print_deployment_success() { + local method=${1:-""} + local method_text="" + + if [ -n "${method}" ]; then + method_text=" via ${method}" + fi + + echo -e "${GREEN}Jumpstarter controller deployed successfully${method_text}!${NC}" + echo -e " gRPC endpoint: ${GRPC_ENDPOINT}" + echo -e " gRPC router endpoint: ${GRPC_ROUTER_ENDPOINT}" +} + diff --git a/internal/config/grpc.go b/internal/config/grpc.go index dcb40097..be17f2cf 100644 --- a/internal/config/grpc.go +++ b/internal/config/grpc.go @@ -1,20 +1,31 @@ package config import ( - "time" + "fmt" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" ) +// LoadGrpcConfiguration loads the gRPC server configuration from the parsed Config struct. +// It creates a gRPC server option with keepalive enforcement policy configured. func LoadGrpcConfiguration(config Grpc) (grpc.ServerOption, error) { - minTime, err := time.ParseDuration(config.Keepalive.MinTime) + ka := config.Keepalive + + // Parse MinTime with default of 1s + minTime, err := ParseDuration(ka.MinTime) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to parse keepalive minTime: %w", err) + } + if minTime == 0 { + minTime = 1e9 // 1 second default } - return grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + // Create the keepalive enforcement policy + policy := keepalive.EnforcementPolicy{ MinTime: minTime, - PermitWithoutStream: config.Keepalive.PermitWithoutStream, - }), nil + PermitWithoutStream: ka.PermitWithoutStream, + } + + return grpc.KeepaliveEnforcementPolicy(policy), nil } diff --git a/internal/config/types.go b/internal/config/types.go index d3665b0e..805a022e 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -1,40 +1,105 @@ package config import ( + "time" + apiserverv1beta1 "k8s.io/apiserver/pkg/apis/apiserver/v1beta1" ) +// Config represents the main controller configuration structure. +// This matches the YAML structure in the ConfigMap's "config" key. type Config struct { - Authentication Authentication `json:"authentication"` - Provisioning Provisioning `json:"provisioning"` - Grpc Grpc `json:"grpc"` + Authentication Authentication `json:"authentication" yaml:"authentication"` + Provisioning Provisioning `json:"provisioning" yaml:"provisioning"` + Grpc Grpc `json:"grpc" yaml:"grpc"` } +// Authentication defines the authentication configuration for the controller. +// Supports multiple authentication methods: internal tokens, Kubernetes tokens, and JWT. type Authentication struct { - Internal Internal `json:"internal"` - JWT []apiserverv1beta1.JWTAuthenticator `json:"jwt"` + Internal Internal `json:"internal" yaml:"internal"` + K8s K8s `json:"k8s,omitempty" yaml:"k8s,omitempty"` + JWT []apiserverv1beta1.JWTAuthenticator `json:"jwt" yaml:"jwt"` } -type Provisioning struct { - Enabled bool `json:"enabled"` +// Internal defines the internal token authentication configuration. +type Internal struct { + // Prefix to add to the subject claim of issued tokens (e.g., "internal:") + Prefix string `json:"prefix" yaml:"prefix"` + + // TokenLifetime defines how long issued tokens are valid. + // Parsed as a Go duration string (e.g., "43800h", "30d"). + TokenLifetime string `json:"tokenLifetime,omitempty" yaml:"tokenLifetime,omitempty"` } -type Internal struct { - Prefix string `json:"prefix"` +// K8s defines the Kubernetes service account token authentication configuration. +type K8s struct { + // Enabled indicates whether Kubernetes authentication is enabled. + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` } +// Provisioning defines the provisioning configuration. +type Provisioning struct { + Enabled bool `json:"enabled" yaml:"enabled"` +} + +// Grpc defines the gRPC server configuration. type Grpc struct { - Keepalive Keepalive `json:"keepalive"` + Keepalive Keepalive `json:"keepalive" yaml:"keepalive"` } +// Keepalive defines the gRPC keepalive configuration. +// All duration fields are parsed as Go duration strings (e.g., "1s", "10s", "180s"). type Keepalive struct { - MinTime string `json:"minTime"` - PermitWithoutStream bool `json:"permitWithoutStream"` + // MinTime is the minimum time between keepalives that the server will accept. + // Default: "1s" + MinTime string `json:"minTime,omitempty" yaml:"minTime,omitempty"` + + // PermitWithoutStream allows keepalive pings even when there are no active streams. + // Default: true + PermitWithoutStream bool `json:"permitWithoutStream,omitempty" yaml:"permitWithoutStream,omitempty"` + + // Timeout is the duration to wait for a keepalive ping acknowledgment. + // Default: "180s" + Timeout string `json:"timeout,omitempty" yaml:"timeout,omitempty"` + + // IntervalTime is the duration between keepalive pings. + // Default: "10s" + IntervalTime string `json:"intervalTime,omitempty" yaml:"intervalTime,omitempty"` + + // MaxConnectionIdle is the maximum duration a connection can be idle before being closed. + // Default: infinity (not set) + MaxConnectionIdle string `json:"maxConnectionIdle,omitempty" yaml:"maxConnectionIdle,omitempty"` + + // MaxConnectionAge is the maximum age of a connection before it is closed. + // Default: infinity (not set) + MaxConnectionAge string `json:"maxConnectionAge,omitempty" yaml:"maxConnectionAge,omitempty"` + + // MaxConnectionAgeGrace is the grace period for closing connections that exceed MaxConnectionAge. + // Default: infinity (not set) + MaxConnectionAgeGrace string `json:"maxConnectionAgeGrace,omitempty" yaml:"maxConnectionAgeGrace,omitempty"` } +// Router represents the router configuration mapping. +// This is a map where keys are router names (e.g., "default", "router-1", "router-2") +// and values are RouterEntry structs containing endpoint and label information. +// This matches the YAML structure in the ConfigMap's "router" key. type Router map[string]RouterEntry +// RouterEntry defines a single router endpoint configuration. type RouterEntry struct { - Endpoint string `json:"endpoint"` - Labels map[string]string `json:"labels"` + // Endpoint is the router's gRPC endpoint address (e.g., "router-0.example.com:443") + Endpoint string `json:"endpoint" yaml:"endpoint"` + + // Labels are optional labels to associate with this router entry. + // Used to distinguish between different router instances. + Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` +} + +// ParseDuration is a helper to parse duration strings with better error messages. +func ParseDuration(s string) (time.Duration, error) { + if s == "" { + return 0, nil + } + return time.ParseDuration(s) } diff --git a/internal/config/types_test.go b/internal/config/types_test.go new file mode 100644 index 00000000..a36e8ce3 --- /dev/null +++ b/internal/config/types_test.go @@ -0,0 +1,206 @@ +package config + +import ( + "testing" + + apiserverv1beta1 "k8s.io/apiserver/pkg/apis/apiserver/v1beta1" + "sigs.k8s.io/yaml" +) + +func TestConfigRoundTrip(t *testing.T) { + // Create a config struct + original := Config{ + Authentication: Authentication{ + Internal: Internal{ + Prefix: "internal:", + TokenLifetime: "43800h", + }, + K8s: K8s{ + Enabled: true, + }, + JWT: []apiserverv1beta1.JWTAuthenticator{}, // Empty array + }, + Provisioning: Provisioning{ + Enabled: false, + }, + Grpc: Grpc{ + Keepalive: Keepalive{ + MinTime: "1s", + PermitWithoutStream: true, + Timeout: "180s", + IntervalTime: "10s", + }, + }, + } + + // Marshal to YAML + yamlData, err := yaml.Marshal(original) + if err != nil { + t.Fatalf("Failed to marshal config: %v", err) + } + + // Unmarshal back to struct + var parsed Config + err = yaml.Unmarshal(yamlData, &parsed) + if err != nil { + t.Fatalf("Failed to unmarshal config: %v", err) + } + + // Verify key fields + if parsed.Authentication.Internal.Prefix != original.Authentication.Internal.Prefix { + t.Errorf("Internal prefix mismatch: got %s, want %s", + parsed.Authentication.Internal.Prefix, original.Authentication.Internal.Prefix) + } + + if parsed.Grpc.Keepalive.MinTime != original.Grpc.Keepalive.MinTime { + t.Errorf("Keepalive minTime mismatch: got %s, want %s", + parsed.Grpc.Keepalive.MinTime, original.Grpc.Keepalive.MinTime) + } + + if parsed.Grpc.Keepalive.PermitWithoutStream != original.Grpc.Keepalive.PermitWithoutStream { + t.Errorf("Keepalive permitWithoutStream mismatch: got %v, want %v", + parsed.Grpc.Keepalive.PermitWithoutStream, original.Grpc.Keepalive.PermitWithoutStream) + } +} + +func TestRouterRoundTrip(t *testing.T) { + // Create a router config + original := Router{ + "default": RouterEntry{ + Endpoint: "router-0.example.com:443", + }, + "router-1": RouterEntry{ + Endpoint: "router-1.example.com:443", + Labels: map[string]string{ + "router-index": "1", + }, + }, + "router-2": RouterEntry{ + Endpoint: "router-2.example.com:443", + Labels: map[string]string{ + "router-index": "2", + "zone": "us-east", + }, + }, + } + + // Marshal to YAML + yamlData, err := yaml.Marshal(original) + if err != nil { + t.Fatalf("Failed to marshal router: %v", err) + } + + t.Logf("Generated YAML:\n%s", string(yamlData)) + + // Unmarshal back to struct + var parsed Router + err = yaml.Unmarshal(yamlData, &parsed) + if err != nil { + t.Fatalf("Failed to unmarshal router: %v", err) + } + + // Verify all routers exist + if len(parsed) != len(original) { + t.Errorf("Router count mismatch: got %d, want %d", len(parsed), len(original)) + } + + // Verify default router + if entry, exists := parsed["default"]; !exists { + t.Error("Missing 'default' router") + } else if entry.Endpoint != original["default"].Endpoint { + t.Errorf("Default router endpoint mismatch: got %s, want %s", + entry.Endpoint, original["default"].Endpoint) + } + + // Verify router-1 + if entry, exists := parsed["router-1"]; !exists { + t.Error("Missing 'router-1' router") + } else { + if entry.Endpoint != original["router-1"].Endpoint { + t.Errorf("Router-1 endpoint mismatch: got %s, want %s", + entry.Endpoint, original["router-1"].Endpoint) + } + if entry.Labels["router-index"] != "1" { + t.Errorf("Router-1 index label mismatch: got %s, want 1", + entry.Labels["router-index"]) + } + } + + // Verify router-2 labels + if entry, exists := parsed["router-2"]; !exists { + t.Error("Missing 'router-2' router") + } else { + if len(entry.Labels) != 2 { + t.Errorf("Router-2 label count mismatch: got %d, want 2", len(entry.Labels)) + } + } +} + +func TestParseYAMLToRouter(t *testing.T) { + // Test parsing actual YAML string (like from ConfigMap) + yamlInput := ` +default: + endpoint: router.example.com:443 +router-1: + endpoint: router-1.example.com:443 + labels: + router-index: "1" +router-2: + endpoint: router-2.example.com:443 + labels: + router-index: "2" +` + + var router Router + err := yaml.Unmarshal([]byte(yamlInput), &router) + if err != nil { + t.Fatalf("Failed to unmarshal YAML: %v", err) + } + + // Verify structure + if len(router) != 3 { + t.Errorf("Expected 3 routers, got %d", len(router)) + } + + // Verify default has no labels + if defaultEntry, exists := router["default"]; exists { + if len(defaultEntry.Labels) != 0 { + t.Errorf("Default router should have no labels, got %d", len(defaultEntry.Labels)) + } + } + + // Verify router-1 has labels + if router1, exists := router["router-1"]; exists { + if len(router1.Labels) == 0 { + t.Error("Router-1 should have labels") + } + } +} + +func TestParseDuration(t *testing.T) { + tests := []struct { + input string + wantErr bool + expected string + }{ + {"1s", false, "1s"}, + {"10s", false, "10s"}, + {"1m", false, "1m0s"}, + {"1h", false, "1h0m0s"}, + {"", false, "0s"}, // empty string returns 0 + {"invalid", true, ""}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + duration, err := ParseDuration(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("ParseDuration(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) + return + } + if !tt.wantErr && duration.String() != tt.expected { + t.Errorf("ParseDuration(%q) = %v, want %v", tt.input, duration, tt.expected) + } + }) + } +} diff --git a/internal/service/controller_service.go b/internal/service/controller_service.go index d7712bac..75ab4c70 100644 --- a/internal/service/controller_service.go +++ b/internal/service/controller_service.go @@ -745,7 +745,7 @@ func (s *ControllerService) Start(ctx context.Context) error { return err } - logger.Info("Starting Controller grpc service") + logger.Info("Starting Controller grpc service on port 8082") go func() { <-ctx.Done() diff --git a/internal/service/router_service.go b/internal/service/router_service.go index e1167c8b..8d801611 100644 --- a/internal/service/router_service.go +++ b/internal/service/router_service.go @@ -135,7 +135,7 @@ func (s *RouterService) Start(ctx context.Context) error { return err } - log.Info("Starting grpc router service") + log.Info("Starting grpc router service on port 8083") go func() { <-ctx.Done() log.Info("Stopping grpc router service")