diff --git a/Makefile b/Makefile index 5b5e4b967..8860e115b 100644 --- a/Makefile +++ b/Makefile @@ -161,12 +161,14 @@ fmt: goimports ## Format the code using goimports .PHONY: test test: manifests generate fmt vet envtest $(LOCALBIN) ## Run tests. We need LOCALBIN=$(LOCALBIN) to get correct default-config path - mkdir -p $(LOCALBIN)/default-config && rm -fr $(LOCALBIN)/default-config/* && cp config/profile/$(PROFILE)/default-config/* $(LOCALBIN)/default-config + mkdir -p $(LOCALBIN)/default-config && rm -fr $(LOCALBIN)/default-config/* && cp -r config/profile/$(PROFILE)/default-config/* $(LOCALBIN)/default-config + mkdir -p $(LOCALBIN)/plugin-deps && rm -fr $(LOCALBIN)/plugin-deps/* && cp -r config/profile/$(PROFILE)/plugin-deps/* $(LOCALBIN)/plugin-deps 2>/dev/null || : LOCALBIN=$(LOCALBIN) KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(PKGS) -coverprofile cover.out .PHONY: integration-test integration-test: ginkgo manifests generate fmt vet envtest $(LOCALBIN) ## Run integration_tests. We need LOCALBIN=$(LOCALBIN) to get correct default-config path - mkdir -p $(LOCALBIN)/default-config && rm -fr $(LOCALBIN)/default-config/* && cp config/profile/$(PROFILE)/default-config/* $(LOCALBIN)/default-config + mkdir -p $(LOCALBIN)/default-config && rm -fr $(LOCALBIN)/default-config/* && cp -r config/profile/$(PROFILE)/default-config/* $(LOCALBIN)/default-config + mkdir -p $(LOCALBIN)/plugin-deps && rm -fr $(LOCALBIN)/plugin-deps/* && cp -r config/profile/$(PROFILE)/plugin-deps/* $(LOCALBIN)/plugin-deps 2>/dev/null || : LOCALBIN=$(LOCALBIN) KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -v -r $(ARGS) integration_tests # After this time, Ginkgo will emit progress reports, so we can get visibility into long-running tests. @@ -218,9 +220,8 @@ build: manifests generate fmt vet ## Build manager binary. .PHONY: run run: manifests generate fmt vet $(LOCALBIN) ## Run a controller from your host. - mkdir -p $(LOCALBIN)/default-config/ - rm -fr $(LOCALBIN)/default-config/* - cp config/profile/$(PROFILE)/default-config/* $(LOCALBIN)/default-config/ + mkdir -p $(LOCALBIN)/default-config/ && rm -fr $(LOCALBIN)/default-config/* && cp -r config/profile/$(PROFILE)/default-config/* $(LOCALBIN)/default-config/ + mkdir -p $(LOCALBIN)/plugin-deps/ && rm -fr $(LOCALBIN)/plugin-deps/* && cp -r config/profile/$(PROFILE)/plugin-deps/* $(LOCALBIN)/plugin-deps/ 2>/dev/null || : go run -C $(LOCALBIN) ../cmd/main.go # by default images expire from quay registry after 14 days @@ -363,6 +364,22 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(KUSTOMIZE) build config/profile/$(PROFILE) | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - +.PHONY: plugin-infra +plugin-infra: + @if [ -d "config/profile/$(PROFILE)/plugin-infra" ]; then \ + $(KUSTOMIZE) build config/profile/$(PROFILE)/plugin-infra | $(KUBECTL) apply -f -; \ + else \ + echo "Directory config/profile/$(PROFILE)/plugin-infra does not exist."; \ + fi + +.PHONY: plugin-infra-undeploy +plugin-infra-undeploy: + @if [ -d "config/profile/$(PROFILE)/plugin-infra" ]; then \ + $(KUSTOMIZE) build config/profile/$(PROFILE)/plugin-infra | $(KUBECTL) delete -f -; \ + else \ + echo "Directory config/profile/$(PROFILE)/plugin-infra does not exist."; \ + fi + ##@ OLM Deployment # It has to be the same namespace as ./config/default/kustomization.yaml -> namespace diff --git a/bundle/backstage.io/manifests/backstage-default-config_v1_configmap.yaml b/bundle/backstage.io/manifests/backstage-default-config_v1_configmap.yaml index 8cdbc58f1..9dbc745b3 100644 --- a/bundle/backstage.io/manifests/backstage-default-config_v1_configmap.yaml +++ b/bundle/backstage.io/manifests/backstage-default-config_v1_configmap.yaml @@ -177,7 +177,7 @@ data: spec: containers: - name: backstage-backend - image: ghcr.io/backstage/backstage:1.28.4 + image: ghcr.io/backstage/backstage:1.38.1 imagePullPolicy: IfNotPresent command: - "node" diff --git a/bundle/backstage.io/manifests/backstage-operator.clusterserviceversion.yaml b/bundle/backstage.io/manifests/backstage-operator.clusterserviceversion.yaml index 9fbdad6ec..b822ba80d 100644 --- a/bundle/backstage.io/manifests/backstage-operator.clusterserviceversion.yaml +++ b/bundle/backstage.io/manifests/backstage-operator.clusterserviceversion.yaml @@ -35,7 +35,7 @@ metadata: } } ] - createdAt: "2025-04-25T07:39:51Z" + createdAt: "2025-05-12T12:54:39Z" description: Backstage Operator operators.operatorframework.io/builder: operator-sdk-v1.37.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 @@ -250,6 +250,8 @@ spec: volumeMounts: - mountPath: /default-config name: default-config + - mountPath: /plugin-deps + name: plugin-deps securityContext: runAsNonRoot: true serviceAccountName: backstage-controller-manager @@ -258,6 +260,10 @@ spec: - configMap: name: backstage-default-config name: default-config + - configMap: + name: plugin-deps + optional: true + name: plugin-deps permissions: - rules: - apiGroups: diff --git a/bundle/rhdh/manifests/backstage-operator.clusterserviceversion.yaml b/bundle/rhdh/manifests/backstage-operator.clusterserviceversion.yaml index eb65a71ca..cde2ec2d9 100644 --- a/bundle/rhdh/manifests/backstage-operator.clusterserviceversion.yaml +++ b/bundle/rhdh/manifests/backstage-operator.clusterserviceversion.yaml @@ -39,7 +39,7 @@ metadata: categories: Developer Tools certified: "true" containerImage: registry.redhat.io/rhdh/rhdh-rhel9-operator:1.7 - createdAt: "2025-04-25T07:39:52Z" + createdAt: "2025-05-12T12:54:40Z" description: Red Hat Developer Hub is a Red Hat supported version of Backstage. It comes with pre-built plug-ins and configuration settings, supports use of an external database, and can help streamline the process of setting up a self-managed @@ -302,6 +302,8 @@ spec: volumeMounts: - mountPath: /default-config name: default-config + - mountPath: /plugin-deps + name: plugin-deps securityContext: runAsNonRoot: true serviceAccountName: rhdh-controller-manager @@ -310,6 +312,10 @@ spec: - configMap: name: rhdh-default-config name: default-config + - configMap: + name: rhdh-plugin-deps + optional: true + name: plugin-deps permissions: - rules: - apiGroups: diff --git a/bundle/rhdh/manifests/rhdh-default-config_v1_configmap.yaml b/bundle/rhdh/manifests/rhdh-default-config_v1_configmap.yaml index 0a720c066..0abbddac9 100644 --- a/bundle/rhdh/manifests/rhdh-default-config_v1_configmap.yaml +++ b/bundle/rhdh/manifests/rhdh-default-config_v1_configmap.yaml @@ -238,6 +238,8 @@ data: env: - name: NPM_CONFIG_USERCONFIG value: /opt/app-root/src/.npmrc.dynamic-plugins + - name: MAX_ENTRY_SIZE + value: "30000000" volumeMounts: - mountPath: /dynamic-plugins-root name: dynamic-plugins-root @@ -348,28 +350,72 @@ data: securityContext: # any group id fsGroup: 1001 - dynamic-plugins.yaml: |- + dynamic-plugins.yaml: | + #apiVersion: v1 + #kind: ConfigMap + #metadata: + # name: default-dynamic-plugins # must be the same as (deployment.yaml).spec.template.spec.volumes.name.dynamic-plugins-conf.configMap.name + #data: + # "dynamic-plugins.yaml": | + # ########################################################################################################### + # # /!\ WARNING + # # + # # This is the default dynamic plugins configuration file created and managed by the Operator for your CR. + # # Do NOT edit this manually in the Cluster, as your changes will be overridden by the Operator upon the + # # next reconciliation. + # # If you want to customize the dynamic plugins, you should create your own dynamic-plugins ConfigMap + # # and reference it in your CR. + # # See https://docs.redhat.com/en/documentation/red_hat_developer_hub/1.4/html/installing_and_viewing_plugins_in_red_hat_developer_hub/rhdh-installing-rhdh-plugins_title-plugins-rhdh-about#proc-config-dynamic-plugins-rhdh-operator_rhdh-installing-rhdh-plugins + # # for more details or https://github.com/redhat-developer/rhdh-operator/blob/main/examples/rhdh-cr.yaml + # # for an example. + # ########################################################################################################### + # includes: + # - dynamic-plugins.default.yaml + # plugins: [] + #--- apiVersion: v1 kind: ConfigMap metadata: - name: default-dynamic-plugins # must be the same as (deployment.yaml).spec.template.spec.volumes.name.dynamic-plugins-conf.configMap.name + name: default-dynamic-plugins data: - "dynamic-plugins.yaml": | - ########################################################################################################### - # /!\ WARNING - # - # This is the default dynamic plugins configuration file created and managed by the Operator for your CR. - # Do NOT edit this manually in the Cluster, as your changes will be overridden by the Operator upon the - # next reconciliation. - # If you want to customize the dynamic plugins, you should create your own dynamic-plugins ConfigMap - # and reference it in your CR. - # See https://docs.redhat.com/en/documentation/red_hat_developer_hub/1.4/html/installing_and_viewing_plugins_in_red_hat_developer_hub/rhdh-installing-rhdh-plugins_title-plugins-rhdh-about#proc-config-dynamic-plugins-rhdh-operator_rhdh-installing-rhdh-plugins - # for more details or https://github.com/redhat-developer/rhdh-operator/blob/main/examples/rhdh-cr.yaml - # for an example. - ########################################################################################################### + dynamic-plugins.yaml: | includes: - dynamic-plugins.default.yaml - plugins: [] + plugins: + - disabled: true + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-orchestrator-1.5.1.tgz" + integrity: sha512-7VOe+XGTUzrdO/av0DNHbydOjB3Lo+XdCs6fj3JVODLP7Ypd3GXHf/nssYxG5ZYC9F1t9MNeguE2bZOB6ckqTA== + pluginConfig: + dynamicPlugins: + frontend: + red-hat-developer-hub.backstage-plugin-orchestrator: + appIcons: + - importName: OrchestratorIcon + module: OrchestratorPlugin + name: orchestratorIcon + dynamicRoutes: + - importName: OrchestratorPage + menuItem: + icon: orchestratorIcon + text: Orchestrator + module: OrchestratorPlugin + path: /orchestrator + - disabled: true + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-orchestrator-backend-dynamic-1.5.1.tgz" + integrity: sha512-VIenFStdq9QvvmgmEMG8O7b2wqIebvEcqNeJ9SWZ8jen9t+efTK6D3Rde74LQ1no1QaHLx8RoxNCOuTUEF8O/g== + pluginConfig: + orchestrator: + dataIndexService: + url: http://sonataflow-platform-data-index-service + dependencies: + - ref: sonataflow + - disabled: true + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-scaffolder-backend-module-orchestrator-dynamic-1.5.1.tgz" + integrity: sha512-bnVQjVsUZ470Vgm2kd5Lo/bVa2fF0q4GufBDc/8oTQsnP3zZJQqKFvFElBTCjY76RqkECydlvZ1UFybSzvockQ== + pluginConfig: + orchestrator: + dataIndexService: + url: http://sonataflow-platform-data-index-service route.yaml: |- apiVersion: route.openshift.io/v1 kind: Route diff --git a/bundle/rhdh/manifests/rhdh-plugin-deps_v1_configmap.yaml b/bundle/rhdh/manifests/rhdh-plugin-deps_v1_configmap.yaml new file mode 100644 index 000000000..5ec78accf --- /dev/null +++ b/bundle/rhdh/manifests/rhdh-plugin-deps_v1_configmap.yaml @@ -0,0 +1,121 @@ +apiVersion: v1 +data: + sonataflow.yaml: | + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: allow-knative-to-sonataflow-and-workflows # hardcoded + spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + # Allow knative events to be delivered to workflows. + kubernetes.io/metadata.name: knative-eventing + - namespaceSelector: + matchLabels: + # Allow auxiliary knative function for workflow (such as m2k-save-transformation) + kubernetes.io/metadata.name: knative-serving + --- + # NetworkPolicy to unblock incoming traffic to the namespace + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: allow-external-communication # hardcoded + spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + # Allow knative events to be delivered to workflows. + policy-group.network.openshift.io/ingress: "" + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: allow-intra-network # hardcoded + spec: + # Apply this policy to all pods in the namespace + podSelector: {} + # Specify policy type as 'Ingress' to control incoming traffic rules + policyTypes: + - Ingress + ingress: + - from: + # Allow ingress from any pod within the same namespace + - podSelector: {} + --- + # NetworkPolicy to allow openshift-user-workload-monitoring pods to access all pods within the workflow's namespace + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: allow-monitoring-to-sonataflow-and-workflows # hardcoded + spec: + # Apply this policy to all pods in the namespace + podSelector: {} + # Specify policy type as 'Ingress' to control incoming traffic rules + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + # Allow openshift-user-workload-monitoring pods to access the workflow. + kubernetes.io/metadata.name: openshift-user-workload-monitoring + --- + apiVersion: operator.knative.dev/v1beta1 + kind: KnativeEventing + metadata: + name: knative-eventing + namespace: knative-eventing + spec: + Registry: {} + --- + apiVersion: operator.knative.dev/v1beta1 + kind: KnativeServing + metadata: + name: knative-serving + namespace: knative-serving + spec: + controller-custom-certs: + name: "" + type: "" + registry: {} + --- + apiVersion: sonataflow.org/v1alpha08 + kind: SonataFlowPlatform + metadata: + name: sonataflow-platform + spec: + monitoring: + enabled: true + services: + dataIndex: + enabled: true + persistence: + postgresql: + secretRef: + name: backstage-psql-secret-{{backstage-name}} + userKey: POSTGRES_USER + passwordKey: POSTGRES_PASSWORD + serviceRef: + name: backstage-psql-{{backstage-name}} + namespace: {{backstage-ns}} + databaseName: backstage_plugin_orchestrator + jobService: + enabled: true + persistence: + postgresql: + secretRef: + name: backstage-psql-secret-{{backstage-name}} + userKey: POSTGRES_USER + passwordKey: POSTGRES_PASSWORD + serviceRef: + name: backstage-psql-{{backstage-name}} + namespace: {{backstage-ns}} + databaseName: backstage_plugin_orchestrator +kind: ConfigMap +metadata: + name: rhdh-plugin-deps diff --git a/config/manager/deployment.yaml b/config/manager/deployment.yaml index 03d47efb7..f273f3bdc 100644 --- a/config/manager/deployment.yaml +++ b/config/manager/deployment.yaml @@ -95,9 +95,16 @@ spec: volumeMounts: - mountPath: /default-config name: default-config + - mountPath: /plugin-deps + name: plugin-deps + serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 volumes: - name: default-config configMap: name: default-config + - name: plugin-deps + configMap: + name: plugin-deps + optional: true \ No newline at end of file diff --git a/config/profile/backstage.io/default-config/deployment.yaml b/config/profile/backstage.io/default-config/deployment.yaml index 16609f6f5..eac3a90dc 100644 --- a/config/profile/backstage.io/default-config/deployment.yaml +++ b/config/profile/backstage.io/default-config/deployment.yaml @@ -14,7 +14,7 @@ spec: spec: containers: - name: backstage-backend - image: ghcr.io/backstage/backstage:1.28.4 + image: ghcr.io/backstage/backstage:1.38.1 imagePullPolicy: IfNotPresent command: - "node" diff --git a/config/profile/backstage.io/kustomization.yaml b/config/profile/backstage.io/kustomization.yaml index 357bf01d6..5f6ab1283 100644 --- a/config/profile/backstage.io/kustomization.yaml +++ b/config/profile/backstage.io/kustomization.yaml @@ -1,14 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -# Adds namespace to all resources. namespace: backstage-system -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. namePrefix: backstage- resources: diff --git a/config/profile/rhdh/default-config/deployment.yaml b/config/profile/rhdh/default-config/deployment.yaml index e726cd663..fa21dcfca 100644 --- a/config/profile/rhdh/default-config/deployment.yaml +++ b/config/profile/rhdh/default-config/deployment.yaml @@ -62,6 +62,8 @@ spec: env: - name: NPM_CONFIG_USERCONFIG value: /opt/app-root/src/.npmrc.dynamic-plugins + - name: MAX_ENTRY_SIZE + value: "30000000" volumeMounts: - mountPath: /dynamic-plugins-root name: dynamic-plugins-root diff --git a/config/profile/rhdh/default-config/dynamic-plugins.yaml b/config/profile/rhdh/default-config/dynamic-plugins.yaml index 5c0c99a1a..87770acb9 100644 --- a/config/profile/rhdh/default-config/dynamic-plugins.yaml +++ b/config/profile/rhdh/default-config/dynamic-plugins.yaml @@ -1,21 +1,65 @@ +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: default-dynamic-plugins # must be the same as (deployment.yaml).spec.template.spec.volumes.name.dynamic-plugins-conf.configMap.name +#data: +# "dynamic-plugins.yaml": | +# ########################################################################################################### +# # /!\ WARNING +# # +# # This is the default dynamic plugins configuration file created and managed by the Operator for your CR. +# # Do NOT edit this manually in the Cluster, as your changes will be overridden by the Operator upon the +# # next reconciliation. +# # If you want to customize the dynamic plugins, you should create your own dynamic-plugins ConfigMap +# # and reference it in your CR. +# # See https://docs.redhat.com/en/documentation/red_hat_developer_hub/1.4/html/installing_and_viewing_plugins_in_red_hat_developer_hub/rhdh-installing-rhdh-plugins_title-plugins-rhdh-about#proc-config-dynamic-plugins-rhdh-operator_rhdh-installing-rhdh-plugins +# # for more details or https://github.com/redhat-developer/rhdh-operator/blob/main/examples/rhdh-cr.yaml +# # for an example. +# ########################################################################################################### +# includes: +# - dynamic-plugins.default.yaml +# plugins: [] +#--- apiVersion: v1 kind: ConfigMap metadata: - name: default-dynamic-plugins # must be the same as (deployment.yaml).spec.template.spec.volumes.name.dynamic-plugins-conf.configMap.name + name: default-dynamic-plugins data: - "dynamic-plugins.yaml": | - ########################################################################################################### - # /!\ WARNING - # - # This is the default dynamic plugins configuration file created and managed by the Operator for your CR. - # Do NOT edit this manually in the Cluster, as your changes will be overridden by the Operator upon the - # next reconciliation. - # If you want to customize the dynamic plugins, you should create your own dynamic-plugins ConfigMap - # and reference it in your CR. - # See https://docs.redhat.com/en/documentation/red_hat_developer_hub/1.4/html/installing_and_viewing_plugins_in_red_hat_developer_hub/rhdh-installing-rhdh-plugins_title-plugins-rhdh-about#proc-config-dynamic-plugins-rhdh-operator_rhdh-installing-rhdh-plugins - # for more details or https://github.com/redhat-developer/rhdh-operator/blob/main/examples/rhdh-cr.yaml - # for an example. - ########################################################################################################### + dynamic-plugins.yaml: | includes: - dynamic-plugins.default.yaml - plugins: [] \ No newline at end of file + plugins: + - disabled: true + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-orchestrator-1.5.1.tgz" + integrity: sha512-7VOe+XGTUzrdO/av0DNHbydOjB3Lo+XdCs6fj3JVODLP7Ypd3GXHf/nssYxG5ZYC9F1t9MNeguE2bZOB6ckqTA== + pluginConfig: + dynamicPlugins: + frontend: + red-hat-developer-hub.backstage-plugin-orchestrator: + appIcons: + - importName: OrchestratorIcon + module: OrchestratorPlugin + name: orchestratorIcon + dynamicRoutes: + - importName: OrchestratorPage + menuItem: + icon: orchestratorIcon + text: Orchestrator + module: OrchestratorPlugin + path: /orchestrator + - disabled: true + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-orchestrator-backend-dynamic-1.5.1.tgz" + integrity: sha512-VIenFStdq9QvvmgmEMG8O7b2wqIebvEcqNeJ9SWZ8jen9t+efTK6D3Rde74LQ1no1QaHLx8RoxNCOuTUEF8O/g== + pluginConfig: + orchestrator: + dataIndexService: + url: http://sonataflow-platform-data-index-service + dependencies: + - ref: sonataflow + - disabled: true + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-scaffolder-backend-module-orchestrator-dynamic-1.5.1.tgz" + integrity: sha512-bnVQjVsUZ470Vgm2kd5Lo/bVa2fF0q4GufBDc/8oTQsnP3zZJQqKFvFElBTCjY76RqkECydlvZ1UFybSzvockQ== + pluginConfig: + orchestrator: + dataIndexService: + url: http://sonataflow-platform-data-index-service diff --git a/config/profile/rhdh/kustomization.yaml b/config/profile/rhdh/kustomization.yaml index abdc79c59..c24c144cc 100644 --- a/config/profile/rhdh/kustomization.yaml +++ b/config/profile/rhdh/kustomization.yaml @@ -1,14 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -# Adds namespace to all resources. namespace: rhdh-operator -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. namePrefix: rhdh- resources: @@ -17,6 +11,7 @@ resources: - ../../manager - namespace.yaml + images: - name: controller newName: quay.io/rhdh/rhdh-rhel9-operator @@ -46,3 +41,6 @@ configMapGenerator: - default-config/db-statefulset.yaml.k8s - default-config/service.yaml.k8s name: default-config +- files: + - plugin-deps/sonataflow.yaml + name: plugin-deps diff --git a/config/profile/rhdh/namespace.yaml b/config/profile/rhdh/namespace.yaml index db987089f..2bbc54ecd 100644 --- a/config/profile/rhdh/namespace.yaml +++ b/config/profile/rhdh/namespace.yaml @@ -9,4 +9,4 @@ metadata: app.kubernetes.io/created-by: backstage-operator app.kubernetes.io/part-of: backstage-operator app.kubernetes.io/managed-by: kustomize - name: operator + name: rhdh-operator diff --git a/config/profile/rhdh/patches/deployment-patch.yaml b/config/profile/rhdh/patches/deployment-patch.yaml index a4097939c..404a5248a 100644 --- a/config/profile/rhdh/patches/deployment-patch.yaml +++ b/config/profile/rhdh/patches/deployment-patch.yaml @@ -27,3 +27,4 @@ spec: value: quay.io/fedora/postgresql-15:latest - name: RELATED_IMAGE_backstage value: quay.io/rhdh/rhdh-hub-rhel9:next + diff --git a/config/profile/rhdh/plugin-deps/sonataflow.yaml b/config/profile/rhdh/plugin-deps/sonataflow.yaml new file mode 100644 index 000000000..b020fcd6f --- /dev/null +++ b/config/profile/rhdh/plugin-deps/sonataflow.yaml @@ -0,0 +1,115 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-knative-to-sonataflow-and-workflows # hardcoded +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + # Allow knative events to be delivered to workflows. + kubernetes.io/metadata.name: knative-eventing + - namespaceSelector: + matchLabels: + # Allow auxiliary knative function for workflow (such as m2k-save-transformation) + kubernetes.io/metadata.name: knative-serving +--- +# NetworkPolicy to unblock incoming traffic to the namespace +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-external-communication # hardcoded +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + # Allow knative events to be delivered to workflows. + policy-group.network.openshift.io/ingress: "" +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-intra-network # hardcoded +spec: + # Apply this policy to all pods in the namespace + podSelector: {} + # Specify policy type as 'Ingress' to control incoming traffic rules + policyTypes: + - Ingress + ingress: + - from: + # Allow ingress from any pod within the same namespace + - podSelector: {} +--- +# NetworkPolicy to allow openshift-user-workload-monitoring pods to access all pods within the workflow's namespace +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-monitoring-to-sonataflow-and-workflows # hardcoded +spec: + # Apply this policy to all pods in the namespace + podSelector: {} + # Specify policy type as 'Ingress' to control incoming traffic rules + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + # Allow openshift-user-workload-monitoring pods to access the workflow. + kubernetes.io/metadata.name: openshift-user-workload-monitoring +--- +apiVersion: operator.knative.dev/v1beta1 +kind: KnativeEventing +metadata: + name: knative-eventing + namespace: knative-eventing +spec: + Registry: {} +--- +apiVersion: operator.knative.dev/v1beta1 +kind: KnativeServing +metadata: + name: knative-serving + namespace: knative-serving +spec: + controller-custom-certs: + name: "" + type: "" + registry: {} +--- +apiVersion: sonataflow.org/v1alpha08 +kind: SonataFlowPlatform +metadata: + name: sonataflow-platform +spec: + monitoring: + enabled: true + services: + dataIndex: + enabled: true + persistence: + postgresql: + secretRef: + name: backstage-psql-secret-{{backstage-name}} + userKey: POSTGRES_USER + passwordKey: POSTGRES_PASSWORD + serviceRef: + name: backstage-psql-{{backstage-name}} + namespace: {{backstage-ns}} + databaseName: backstage_plugin_orchestrator + jobService: + enabled: true + persistence: + postgresql: + secretRef: + name: backstage-psql-secret-{{backstage-name}} + userKey: POSTGRES_USER + passwordKey: POSTGRES_PASSWORD + serviceRef: + name: backstage-psql-{{backstage-name}} + namespace: {{backstage-ns}} + databaseName: backstage_plugin_orchestrator diff --git a/config/profile/rhdh/plugin-infra/kustomization.yaml b/config/profile/rhdh/plugin-infra/kustomization.yaml new file mode 100644 index 000000000..34b1b9cb8 --- /dev/null +++ b/config/profile/rhdh/plugin-infra/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- orchestrator/rbac-sonataflow.yaml +- orchestrator/infra-sonataflow.yaml +- orchestrator/infra-serverless.yaml + diff --git a/config/profile/rhdh/plugin-infra/orchestrator/infra-serverless.yaml b/config/profile/rhdh/plugin-infra/orchestrator/infra-serverless.yaml new file mode 100644 index 000000000..085fe8036 --- /dev/null +++ b/config/profile/rhdh/plugin-infra/orchestrator/infra-serverless.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-serverless +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: serverless-operator-group + namespace: openshift-serverless +spec: +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: serverless-operator + namespace: openshift-serverless +spec: + channel: stable # channel of an operator package to subscribe to + installPlanApproval: Automatic # whether the update should be installed automatically + name: serverless-operator # name of the operator package + source: redhat-operators # name of the catalog source + sourceNamespace: openshift-marketplace +--- +apiVersion: v1 +kind: Namespace +metadata: + name: knative-serving +--- +apiVersion: v1 +kind: Namespace +metadata: + name: knative-eventing + diff --git a/config/profile/rhdh/plugin-infra/orchestrator/infra-sonataflow.yaml b/config/profile/rhdh/plugin-infra/orchestrator/infra-sonataflow.yaml new file mode 100644 index 000000000..ed686bd8d --- /dev/null +++ b/config/profile/rhdh/plugin-infra/orchestrator/infra-sonataflow.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-serverless-logic +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-serverless-logic + namespace: openshift-serverless-logic +spec: +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: logic-operator-rhel8 + namespace: openshift-serverless-logic +spec: + channel: alpha # channel of an operator package to subscribe to + installPlanApproval: Automatic # whether the update should be installed automatically + name: logic-operator-rhel8 # name of the operator package + source: redhat-operators # name of the catalog source + sourceNamespace: openshift-marketplace + startingCSV: logic-operator-rhel8.v1.35.0 # The initial version of the operator \ No newline at end of file diff --git a/config/profile/rhdh/plugin-infra/orchestrator/rbac-sonataflow.yaml b/config/profile/rhdh/plugin-infra/orchestrator/rbac-sonataflow.yaml new file mode 100644 index 000000000..997af3e8c --- /dev/null +++ b/config/profile/rhdh/plugin-infra/orchestrator/rbac-sonataflow.yaml @@ -0,0 +1,62 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rhdh-manager-sonataflow-role +rules: + - apiGroups: + - "sonataflow.org" + resources: + - sonataflowplatforms + - sonataflows + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - "operator.knative.dev" + resources: + - knativeeventings + - knativeservings + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/instance: manager-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/part-of: backstage-operator + name: rhdh-manager-sonataflow-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rhdh-manager-sonataflow-role +subjects: + - kind: ServiceAccount + name: rhdh-controller-manager + namespace: rhdh-operator diff --git a/dist/backstage.io/install.yaml b/dist/backstage.io/install.yaml index 12a1f4aff..5b5bcc9ce 100644 --- a/dist/backstage.io/install.yaml +++ b/dist/backstage.io/install.yaml @@ -1688,7 +1688,7 @@ data: spec: containers: - name: backstage-backend - image: ghcr.io/backstage/backstage:1.28.4 + image: ghcr.io/backstage/backstage:1.38.1 imagePullPolicy: IfNotPresent command: - "node" @@ -1832,6 +1832,8 @@ spec: volumeMounts: - mountPath: /default-config name: default-config + - mountPath: /plugin-deps + name: plugin-deps securityContext: runAsNonRoot: true serviceAccountName: backstage-controller-manager @@ -1840,3 +1842,7 @@ spec: - configMap: name: backstage-default-config name: default-config + - configMap: + name: plugin-deps + optional: true + name: plugin-deps diff --git a/dist/rhdh/install.yaml b/dist/rhdh/install.yaml index a6a664463..b58b24499 100644 --- a/dist/rhdh/install.yaml +++ b/dist/rhdh/install.yaml @@ -1749,6 +1749,8 @@ data: env: - name: NPM_CONFIG_USERCONFIG value: /opt/app-root/src/.npmrc.dynamic-plugins + - name: MAX_ENTRY_SIZE + value: "30000000" volumeMounts: - mountPath: /dynamic-plugins-root name: dynamic-plugins-root @@ -1859,28 +1861,72 @@ data: securityContext: # any group id fsGroup: 1001 - dynamic-plugins.yaml: |- + dynamic-plugins.yaml: | + #apiVersion: v1 + #kind: ConfigMap + #metadata: + # name: default-dynamic-plugins # must be the same as (deployment.yaml).spec.template.spec.volumes.name.dynamic-plugins-conf.configMap.name + #data: + # "dynamic-plugins.yaml": | + # ########################################################################################################### + # # /!\ WARNING + # # + # # This is the default dynamic plugins configuration file created and managed by the Operator for your CR. + # # Do NOT edit this manually in the Cluster, as your changes will be overridden by the Operator upon the + # # next reconciliation. + # # If you want to customize the dynamic plugins, you should create your own dynamic-plugins ConfigMap + # # and reference it in your CR. + # # See https://docs.redhat.com/en/documentation/red_hat_developer_hub/1.4/html/installing_and_viewing_plugins_in_red_hat_developer_hub/rhdh-installing-rhdh-plugins_title-plugins-rhdh-about#proc-config-dynamic-plugins-rhdh-operator_rhdh-installing-rhdh-plugins + # # for more details or https://github.com/redhat-developer/rhdh-operator/blob/main/examples/rhdh-cr.yaml + # # for an example. + # ########################################################################################################### + # includes: + # - dynamic-plugins.default.yaml + # plugins: [] + #--- apiVersion: v1 kind: ConfigMap metadata: - name: default-dynamic-plugins # must be the same as (deployment.yaml).spec.template.spec.volumes.name.dynamic-plugins-conf.configMap.name + name: default-dynamic-plugins data: - "dynamic-plugins.yaml": | - ########################################################################################################### - # /!\ WARNING - # - # This is the default dynamic plugins configuration file created and managed by the Operator for your CR. - # Do NOT edit this manually in the Cluster, as your changes will be overridden by the Operator upon the - # next reconciliation. - # If you want to customize the dynamic plugins, you should create your own dynamic-plugins ConfigMap - # and reference it in your CR. - # See https://docs.redhat.com/en/documentation/red_hat_developer_hub/1.4/html/installing_and_viewing_plugins_in_red_hat_developer_hub/rhdh-installing-rhdh-plugins_title-plugins-rhdh-about#proc-config-dynamic-plugins-rhdh-operator_rhdh-installing-rhdh-plugins - # for more details or https://github.com/redhat-developer/rhdh-operator/blob/main/examples/rhdh-cr.yaml - # for an example. - ########################################################################################################### + dynamic-plugins.yaml: | includes: - dynamic-plugins.default.yaml - plugins: [] + plugins: + - disabled: true + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-orchestrator-1.5.1.tgz" + integrity: sha512-7VOe+XGTUzrdO/av0DNHbydOjB3Lo+XdCs6fj3JVODLP7Ypd3GXHf/nssYxG5ZYC9F1t9MNeguE2bZOB6ckqTA== + pluginConfig: + dynamicPlugins: + frontend: + red-hat-developer-hub.backstage-plugin-orchestrator: + appIcons: + - importName: OrchestratorIcon + module: OrchestratorPlugin + name: orchestratorIcon + dynamicRoutes: + - importName: OrchestratorPage + menuItem: + icon: orchestratorIcon + text: Orchestrator + module: OrchestratorPlugin + path: /orchestrator + - disabled: true + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-orchestrator-backend-dynamic-1.5.1.tgz" + integrity: sha512-VIenFStdq9QvvmgmEMG8O7b2wqIebvEcqNeJ9SWZ8jen9t+efTK6D3Rde74LQ1no1QaHLx8RoxNCOuTUEF8O/g== + pluginConfig: + orchestrator: + dataIndexService: + url: http://sonataflow-platform-data-index-service + dependencies: + - ref: sonataflow + - disabled: true + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-scaffolder-backend-module-orchestrator-dynamic-1.5.1.tgz" + integrity: sha512-bnVQjVsUZ470Vgm2kd5Lo/bVa2fF0q4GufBDc/8oTQsnP3zZJQqKFvFElBTCjY76RqkECydlvZ1UFybSzvockQ== + pluginConfig: + orchestrator: + dataIndexService: + url: http://sonataflow-platform-data-index-service route.yaml: |- apiVersion: route.openshift.io/v1 kind: Route @@ -1931,6 +1977,129 @@ metadata: namespace: rhdh-operator --- apiVersion: v1 +data: + sonataflow.yaml: | + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: allow-knative-to-sonataflow-and-workflows # hardcoded + spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + # Allow knative events to be delivered to workflows. + kubernetes.io/metadata.name: knative-eventing + - namespaceSelector: + matchLabels: + # Allow auxiliary knative function for workflow (such as m2k-save-transformation) + kubernetes.io/metadata.name: knative-serving + --- + # NetworkPolicy to unblock incoming traffic to the namespace + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: allow-external-communication # hardcoded + spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + # Allow knative events to be delivered to workflows. + policy-group.network.openshift.io/ingress: "" + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: allow-intra-network # hardcoded + spec: + # Apply this policy to all pods in the namespace + podSelector: {} + # Specify policy type as 'Ingress' to control incoming traffic rules + policyTypes: + - Ingress + ingress: + - from: + # Allow ingress from any pod within the same namespace + - podSelector: {} + --- + # NetworkPolicy to allow openshift-user-workload-monitoring pods to access all pods within the workflow's namespace + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: allow-monitoring-to-sonataflow-and-workflows # hardcoded + spec: + # Apply this policy to all pods in the namespace + podSelector: {} + # Specify policy type as 'Ingress' to control incoming traffic rules + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + # Allow openshift-user-workload-monitoring pods to access the workflow. + kubernetes.io/metadata.name: openshift-user-workload-monitoring + --- + apiVersion: operator.knative.dev/v1beta1 + kind: KnativeEventing + metadata: + name: knative-eventing + namespace: knative-eventing + spec: + Registry: {} + --- + apiVersion: operator.knative.dev/v1beta1 + kind: KnativeServing + metadata: + name: knative-serving + namespace: knative-serving + spec: + controller-custom-certs: + name: "" + type: "" + registry: {} + --- + apiVersion: sonataflow.org/v1alpha08 + kind: SonataFlowPlatform + metadata: + name: sonataflow-platform + spec: + monitoring: + enabled: true + services: + dataIndex: + enabled: true + persistence: + postgresql: + secretRef: + name: backstage-psql-secret-{{backstage-name}} + userKey: POSTGRES_USER + passwordKey: POSTGRES_PASSWORD + serviceRef: + name: backstage-psql-{{backstage-name}} + namespace: {{backstage-ns}} + databaseName: backstage_plugin_orchestrator + jobService: + enabled: true + persistence: + postgresql: + secretRef: + name: backstage-psql-secret-{{backstage-name}} + userKey: POSTGRES_USER + passwordKey: POSTGRES_PASSWORD + serviceRef: + name: backstage-psql-{{backstage-name}} + namespace: {{backstage-ns}} + databaseName: backstage_plugin_orchestrator +kind: ConfigMap +metadata: + name: rhdh-plugin-deps + namespace: rhdh-operator +--- +apiVersion: v1 kind: Service metadata: labels: @@ -2045,6 +2214,8 @@ spec: volumeMounts: - mountPath: /default-config name: default-config + - mountPath: /plugin-deps + name: plugin-deps securityContext: runAsNonRoot: true serviceAccountName: rhdh-controller-manager @@ -2053,3 +2224,7 @@ spec: - configMap: name: rhdh-default-config name: default-config + - configMap: + name: rhdh-plugin-deps + optional: true + name: plugin-deps diff --git a/docs/configuration.md b/docs/configuration.md index d799b563c..cb75d09f2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -12,21 +12,21 @@ The Default Configuration defines the structure of all Backstage instances withi ### Default Configuration Files -| Key/File Name | Object Kind | Object Name | Mandatory | Multi | Version | Notes | -|----------------------|--------------------------------|--------------------------------------------|--------------|-------|---------|-------------------------------------------------| -| deployment.yaml | appsv1.Deployment | backstage- | Yes | No | >=0.1.x | Backstage deployment | -| service.yaml | corev1.Service | backstage- | Yes | No | >=0.1.x | Backstage Service | -| db-statefulset.yaml | appsv1.StatefulSet | backstage-psql- | For local DB | No | >=0.1.x | PostgreSQL StatefulSet | -| db-service.yaml | corev1.Service | backstage-psql- | For local DB | No | >=0.1.x | PostgreSQL Service | -| db-secret.yaml | corev1.Secret | backstage-psql-secret- | For local DB | No | >=0.1.x | Secret to connect Backstage to PGSQL | -| route.yaml | openshift.Route | backstage- | No (for OCP) | No | >=0.1.x | Route exposing Backstage service | -| app-config.yaml | corev1.ConfigMap | backstage-appconfig- | No | No | >=0.2.x | Backstage app-config.yaml | -| configmap-files.yaml | corev1.ConfigMap | backstage-files- | No | No | >=0.2.x | Backstage config file inclusions from configMap | -| configmap-envs.yaml | corev1.ConfigMap | backstage-envs- | No | No | >=0.2.x | Backstage environment variables from ConfigMap | -| secret-files.yaml | []corev1.Secret | backstage-files- | No | Yes | >=0.2.x | Backstage config file inclusions from Secret | -| secret-envs.yaml | []corev1.Secret | backstage-envs- | No | Yes | >=0.2.x | Backstage environment variables from Secret | -| dynamic-plugins.yaml | corev1.ConfigMap | backstage-dynamic-plugins- | No | No | >=0.2.x | Dynamic plugins configuration | -| pvcs.yaml | []corev1.PersistentVolumeClaim | backstage-<cr-name>-<pvc-name> | No | Yes | >=0.4.x | List of PVC objects to be mounted to containers | +| Key/File Name | Object Kind | Object Name | Mandatory | Multi | Version | Notes | +|------------------------------------------|--------------------------------|--------------------------------------------|--------------|-------|---------|-------------------------------------------------| +| deployment.yaml | appsv1.Deployment | backstage- | Yes | No | >=0.1.x | Backstage deployment | +| service.yaml | corev1.Service | backstage- | Yes | No | >=0.1.x | Backstage Service | +| db-statefulset.yaml | appsv1.StatefulSet | backstage-psql- | For local DB | No | >=0.1.x | PostgreSQL StatefulSet | +| db-service.yaml | corev1.Service | backstage-psql- | For local DB | No | >=0.1.x | PostgreSQL Service | +| db-secret.yaml | corev1.Secret | backstage-psql-secret- | For local DB | No | >=0.1.x | Secret to connect Backstage to PGSQL | +| route.yaml | openshift.Route | backstage- | No (for OCP) | No | >=0.1.x | Route exposing Backstage service | +| app-config.yaml | corev1.ConfigMap | backstage-appconfig- | No | No | >=0.2.x | Backstage app-config.yaml | +| configmap-files.yaml | corev1.ConfigMap | backstage-files- | No | No | >=0.2.x | Backstage config file inclusions from configMap | +| configmap-envs.yaml | corev1.ConfigMap | backstage-envs- | No | No | >=0.2.x | Backstage environment variables from ConfigMap | +| secret-files.yaml | []corev1.Secret | backstage-files- | No | Yes | >=0.2.x | Backstage config file inclusions from Secret | +| secret-envs.yaml | []corev1.Secret | backstage-envs- | No | Yes | >=0.2.x | Backstage environment variables from Secret | +| [dynamic-plugins.yaml](#dynamic-plugins) | corev1.ConfigMap | backstage-dynamic-plugins- | No | No | >=0.2.x | Dynamic plugins configuration | +| pvcs.yaml | []corev1.PersistentVolumeClaim | backstage-<cr-name>-<pvc-name> | No | Yes | >=0.4.x | List of PVC objects to be mounted to containers | **Meanings of "Mandatory" Column:** - **Yes** - Must be configured; deployment will fail otherwise. @@ -507,6 +507,9 @@ spec: application: dynamicPluginsConfigMapName: "dynamic-plugins-config" ``` +Starting from version **0.7.0**, the Operator supports dynamic plugins dependencies. For more details, refer to [Dynamic Plugins Dependencies](dynamic-plugins.md). + +```yaml #### Deployment Parameters diff --git a/docs/developer.md b/docs/developer.md index 5e57016fd..b6a535a79 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -90,6 +90,11 @@ To undeploy the controller from the cluster: make undeploy ``` +In a case if Profile contain plugin infrastructure manifests `/config/profile//plugin-infra` it can be deployed by: +```sh +make plugin-infra [PROFILE=] +``` + **UnDeploy the controller from the cluster:** ```sh diff --git a/docs/dynamic-plugins.md b/docs/dynamic-plugins.md new file mode 100644 index 000000000..1098bfcb7 --- /dev/null +++ b/docs/dynamic-plugins.md @@ -0,0 +1,115 @@ +## Dynamic plugins dependency management + +### Overview +Dynamic plugins configured for the Backstage CR may require certain Kubernetes resources to be configured to make the plugin work. These are referred to as 'plugin dependencies'. Starting from version 1.7, it is possible to automatically create these resources when the Backstage CR is applied to the cluster. + +### Profile Configuration +Plugin dependency configuration for a specific profile is done via the `/config/profile/{PROFILE}/plugin-deps` directory. To enable this, the administrator should place the required resources as Kubernetes manifests in YAML format within **plugin-deps** directory. + +**Example Directory Structure**: +```txt +config/ + profile/ + rhdh/ + kustomization.yaml + plugin-deps/ + example-dep1.yaml + example-dep2.yaml +``` +Here, **example-dep1.yaml** and **example-dep2.yaml** are the plugin dependencies for the example plugin. + +**Notes:** + +* If a resource manifest does not specify a namespace, it will be created in the namespace of the Backstage CR. +* Resources may contain **{{backstage-name}}** and **{{backstage-ns}}** placeholders, which will be replaced with the name and namespace of the Backstage CR, respectively. + +The `kustomization.yaml` file should contain the following lines: +```yaml + +configMapGenerator: + - files: + - plugin-deps/example-dep1.yaml + - plugin-deps/example-dep2.yaml + name: plugin-deps +``` + +### Plugin dependencies infrastructure + +If plugin dependencies require infrastructural resources (e.g., RoleBindings, CustomResources) and if the User (Administrator) wants it to be deployed (see Note below), they can be specified in the /config/profile/{PROFILE}/plugin-infra directory. To create these resources (along with the operator deployment), use the `make plugin-infra` command. + +**Note**: Be cautious when running this command on a production cluster, as it may reconfigure cluster-scoped resources. + +### Plugin configuration + +To create the plugin dependencies when the Backstage CR is applied, they must be referenced in the **dependencies** field of the plugin configuration. The operator will look for the **plugin-deps** directory and create the resources described in the files within this directory. + +Plugin dependencies can be referenced in the dynamic-plugins' ConfigMap. This can either be part of the profile's [default configuration](configuration.md/#default-configuration-files) for all Backstage CRs or part of the [ConfigMap referenced in the Backstage CR](configuration.md/#dynamic-plugins). Starting from version 1.7, plugin dependencies can be included in the dynamic plugin configuration. Each `dependencies.ref` value can either match the full file name or serve as a prefix for the file name. The operator will look for files in the `plugin-deps` directory whose names either start with the specified `ref` value or exactly match it. These files will be used to create the resources described within them. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: default-dynamic-plugins +data: + dynamic-plugins.yaml: | + includes: + - dynamic-plugins.default.yaml + plugins: + - disabled: false + package: "path-or-url-to-example-plugin" + dependencies: + - ref: example-dep +``` + +In this example, both example-dep1.yaml and example-dep1.yaml will be picked and operator create the resources described in the files. + +### Example: Orchestrator plugin dependencies +The orchestrator plugin (as of v1.5.1) consists of three dynamic plugins: +- orchestrator-backend +- orchestrator-frontend +- scaffolder-backend-module +See [example](/examples/orchestrator.yaml) for a complete configuration of the orchestrator plugin. + + +The orchestrator plugin has the following dependencies: +- A Sonataflowplatform custom resource - created in the namespace of the Backstage CR. +- Knativeeventing and Knativeserving custom resources to be created in the knative-eventing and knative-serving namespaces respectively. +- A set of NetworkPolicies to allow traffic between Knative resources created in the namespace of Backstage CR, traffic for monitoring, and intra-namespace traffic. +See [profile/rhdh/plugin-deps](/config/profile/rhdh/plugin-deps) + +The orchestrator-backend plugin uses the service **sonataflow-platform-data-index-service**, which is created by the SonataFlowPlatform CR. This service is used to communicate with the SonataFlow platform. + +**Important:** The sonataflowplatform CR contains dataIndex service that requires PostgreSQL database. + +```yaml + persistence: + postgresql: + secretRef: + name: backstage-psql-secret-{{backstage-name}} + userKey: POSTGRES_USER + passwordKey: POSTGRES_PASSWORD + serviceRef: + name: backstage-psql-{{backstage-name}} + namespace: {{backstage-ns}} + databaseName: backstage_plugin_orchestrator +``` + +Current **default** implementation of the orchestrator plugin dependencies uses: +- the PostgreSQL database created by Backstage for Orchestrator plugin, named **backstage_plugin_orchestrator** +- the Secret created by Backstage operator for the PostgreSQL with **POSTGRES_USER** and **POSTGRES_PASSWORD** keys as the database credentials in the Backstage CR namespace. +- the Service created by Backstage operator for the PostgreSQL database with the name **backstage-psql-{{backstage-name}}** in the Backstage CR namespace. + +**Note:** This default implementation is not recommended for production. For production environments, you should configure an external database connection instead (**TODO: Provide an example with an external database**). + +**Known issue:** +Since DB secret with credentials is automatically generated by the Operator every time new Backstage CR created and sonataflowplatform is not recreated and so uses the "old" secret, the orchestrator plugin will not work properly. To resolve this, the user must manually recreate the SonataFlowPlatform CR or its pods. + +The SonataFlowPlatform CR requires the SonataFlow operator to be installed. In an OpenShift environment, it is done by installing the OpenShift Serverless Operator and the OpenShift Serverless Logic Operator by installing respective OLM Subscriptions (see [infra-serverless.yaml](/config/profile/rhdh/plugin-infra/orchestrator/infra-serverless.yaml) and [infra-sonataflow.yaml](/config/profile/rhdh/plugin-infra/orchestrator/infra-sonataflow.yaml)). + +**Note:** Current Subscriptions configuration uses **Automatic** install plan (**spec.installPlanApproval: Automatic**), consider to change it to **Manual** if you want to control the installation of the operators (see [Operator Installation with OLM](https://olm.operatorframework.io/docs/tasks/install-operator-with-olm) for more details). + +Additionally, to enable the Backstage operator to work with the SonataFlow platform, its ServiceAccount must be granted the appropriate permissions. + +These manifests are defined as plugin infrastructure in the profile/rhdh/operator/plugin-infra/orchestrator directory. + +**Note:** Using plugin infrastructure (plugin-infra) can be risky in production, as it modifies cluster-scoped resources. \ No newline at end of file diff --git a/examples/orchestrator.yaml b/examples/orchestrator.yaml new file mode 100644 index 000000000..e354dcf70 --- /dev/null +++ b/examples/orchestrator.yaml @@ -0,0 +1,67 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: orchestrator-plugin +data: + dynamic-plugins.yaml: | + includes: + - dynamic-plugins.default.yaml + plugins: + - disabled: false + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-orchestrator-1.5.1.tgz" + integrity: sha512-7VOe+XGTUzrdO/av0DNHbydOjB3Lo+XdCs6fj3JVODLP7Ypd3GXHf/nssYxG5ZYC9F1t9MNeguE2bZOB6ckqTA== + pluginConfig: + dynamicPlugins: + frontend: + red-hat-developer-hub.backstage-plugin-orchestrator: + appIcons: + - importName: OrchestratorIcon + module: OrchestratorPlugin + name: orchestratorIcon + dynamicRoutes: + - importName: OrchestratorPage + menuItem: + icon: orchestratorIcon + text: Orchestrator + module: OrchestratorPlugin + path: /orchestrator + - disabled: false + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-orchestrator-backend-dynamic-1.5.1.tgz" + integrity: sha512-VIenFStdq9QvvmgmEMG8O7b2wqIebvEcqNeJ9SWZ8jen9t+efTK6D3Rde74LQ1no1QaHLx8RoxNCOuTUEF8O/g== + pluginConfig: + orchestrator: + dataIndexService: + url: http://sonataflow-platform-data-index-service + dependencies: + - ref: sonataflow + - disabled: false + package: "https://github.com/rhdhorchestrator/orchestrator-plugins-internal-release/releases/download/v1.5.1/backstage-plugin-scaffolder-backend-module-orchestrator-dynamic-1.5.1.tgz" + integrity: sha512-bnVQjVsUZ470Vgm2kd5Lo/bVa2fF0q4GufBDc/8oTQsnP3zZJQqKFvFElBTCjY76RqkECydlvZ1UFybSzvockQ== + pluginConfig: + orchestrator: + dataIndexService: + url: http://sonataflow-platform-data-index-service +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config-rhdh +data: + app-config-rhdh.yaml: |- + auth: + environment: development + providers: + guest: + # using the guest user to query the '/api/dynamic-plugins-info/loaded-plugins' endpoint. + dangerouslyAllowOutsideDevelopment: true +--- +apiVersion: rhdh.redhat.com/v1alpha3 +kind: Backstage +metadata: + name: orchestrator +spec: + application: + appConfig: + configMaps: + - name: app-config-rhdh + dynamicPluginsConfigMapName: orchestrator-plugin diff --git a/internal/controller/backstage_controller.go b/internal/controller/backstage_controller.go index f06f53b1e..8b4ccb869 100644 --- a/internal/controller/backstage_controller.go +++ b/internal/controller/backstage_controller.go @@ -108,6 +108,12 @@ func (r *BackstageReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, errorAndStatus(&backstage, "failed to initialize backstage model", err) } + // Apply the plugin dependencies + if err := r.applyPluginDeps(ctx, req.NamespacedName, bsModel.DynamicPlugins); err != nil { + return ctrl.Result{}, errorAndStatus(&backstage, "failed to apply plugin dependencies", err) + } + + // Apply the runtime objects err = r.applyObjects(ctx, bsModel.RuntimeObjects) if err != nil { return ctrl.Result{}, errorAndStatus(&backstage, "failed to apply backstage objects", err) @@ -123,7 +129,7 @@ func (r *BackstageReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( func errorAndStatus(backstage *bs.Backstage, msg string, err error) error { setStatusCondition(backstage, bs.BackstageConditionTypeDeployed, metav1.ConditionFalse, bs.BackstageConditionReasonFailed, fmt.Sprintf("%s %s", msg, err)) - return fmt.Errorf("%s %w", msg, err) + return fmt.Errorf("%s: %w", msg, err) } func (r *BackstageReconciler) applyObjects(ctx context.Context, objects []model.RuntimeObject) error { diff --git a/internal/controller/plugin-deps.go b/internal/controller/plugin-deps.go new file mode 100644 index 000000000..fe0198f95 --- /dev/null +++ b/internal/controller/plugin-deps.go @@ -0,0 +1,56 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/redhat-developer/rhdh-operator/pkg/model" + "k8s.io/apimachinery/pkg/types" + + "sigs.k8s.io/controller-runtime/pkg/log" + + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (r *BackstageReconciler) applyPluginDeps(ctx context.Context, nsName types.NamespacedName, plugins model.DynamicPlugins) error { + + lg := log.FromContext(ctx) + + objects, err := model.GetPluginDeps(nsName.Name, nsName.Namespace, plugins) + if err != nil { + return fmt.Errorf("failed to read YAML files: %w", err) + } + + // Process the objects as needed + var errs []error + for _, obj := range objects { + // Apply the unstructured object + lg.V(1).Info("apply plugin dependency: ", "name", obj.GetName(), "kind", obj.GetKind(), "namespace", obj.GetNamespace()) + + // Set the namespace if not set + if obj.GetNamespace() == "" { + obj.SetNamespace(nsName.Namespace) + } + + if err := r.Patch(ctx, obj, client.Apply, &client.PatchOptions{FieldManager: BackstageFieldManager, Force: ptr.To(true)}); err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return combineErrors(errs) + } + + return nil +} + +func combineErrors(errs []error) error { + var sb strings.Builder + for _, err := range errs { + sb.WriteString(err.Error() + "\n") + } + return errors.New(sb.String()) +} diff --git a/pkg/model/dynamic-plugins.go b/pkg/model/dynamic-plugins.go index d40766b6b..78c7236f9 100644 --- a/pkg/model/dynamic-plugins.go +++ b/pkg/model/dynamic-plugins.go @@ -4,6 +4,8 @@ import ( "fmt" "os" + "gopkg.in/yaml.v2" + "golang.org/x/exp/maps" "sigs.k8s.io/controller-runtime/pkg/client" @@ -40,6 +42,25 @@ type DynamicPlugins struct { ConfigMap *corev1.ConfigMap } +type DynaPluginsConfig struct { + // we do not really support Includes here, that's what is processed by the installation script + // in the dynamic-plugins container. Keeping it here for the sake of completeness + Includes []string `yaml:"includes"` + Plugins []DynaPlugin `yaml:"plugins"` +} + +type DynaPlugin struct { + Package string `yaml:"package"` + Integrity string `yaml:"integrity"` + Disabled bool `yaml:"disabled"` + PluginConfig map[string]interface{} `yaml:"pluginConfig"` + Dependencies []PluginDependency `yaml:"dependencies"` +} + +type PluginDependency struct { + Ref string `yaml:"ref"` +} + func init() { registerConfig("dynamic-plugins.yaml", DynamicPluginsFactory{}, false) } @@ -60,15 +81,20 @@ func addDynamicPluginsFromSpec(spec bsv1.BackstageSpec, model *BackstageModel) e } dp := model.ExternalConfig.DynamicPlugins - if dp.Data == nil || len(dp.Data) != 1 || dp.Data[DynamicPluginsFile] == "" { - return fmt.Errorf("dynamic plugin configMap expects exactly one Data key named '%s' ", DynamicPluginsFile) + + if dp.Data == nil || dp.Data[DynamicPluginsFile] == "" { + return fmt.Errorf("dynamic plugin configMap expects '%s' Data key", DynamicPluginsFile) } - model.backstageDeployment.mountFilesFrom([]string{dynamicPluginInitContainerName}, ConfigMapObjectKind, - dp.Name, ic.WorkingDir, DynamicPluginsFile, true, maps.Keys(dp.Data)) + if dp.Data[DynamicPluginsFile] != "" { + model.backstageDeployment.mountFilesFrom([]string{dynamicPluginInitContainerName}, ConfigMapObjectKind, + dp.Name, ic.WorkingDir, DynamicPluginsFile, true, maps.Keys(dp.Data)) + } - return nil + // if default dynamic-plugins set, the value from model.DynamicPlugins will differ from the one in model.RuntimeObjects + model.DynamicPlugins.ConfigMap = &dp + return nil } // implementation of RuntimeObject interface @@ -92,16 +118,25 @@ func (p *DynamicPlugins) EmptyObject() client.Object { // implementation of RuntimeObject interface func (p *DynamicPlugins) addToModel(model *BackstageModel, backstage bsv1.Backstage) (bool, error) { - if p.ConfigMap == nil || (backstage.Spec.Application != nil && backstage.Spec.Application.DynamicPluginsConfigMapName != "") { - return false, nil + if p.ConfigMap == nil { + if backstage.Spec.Application != nil && backstage.Spec.Application.DynamicPluginsConfigMapName != "" { + p.ConfigMap = &corev1.ConfigMap{} + } else { + return false, nil + } } model.setRuntimeObject(p) + model.DynamicPlugins = *p return true, nil } // implementation of RuntimeObject interface // ConfigMap name must be the same as (deployment.yaml).spec.template.spec.volumes.name.dynamic-plugins-conf.ConfigMap.name -func (p *DynamicPlugins) updateAndValidate(model *BackstageModel, _ bsv1.Backstage) error { +func (p *DynamicPlugins) updateAndValidate(model *BackstageModel, backstage bsv1.Backstage) error { + + if backstage.Spec.Application != nil && backstage.Spec.Application.DynamicPluginsConfigMapName != "" { + return nil + } _, initContainer := DynamicPluginsInitContainer(model.backstageDeployment.deployment.Spec.Template.Spec.InitContainers) if initContainer == nil { @@ -128,6 +163,46 @@ func (p *DynamicPlugins) setMetaInfo(backstage bsv1.Backstage, scheme *runtime.S setMetaInfo(p.ConfigMap, backstage, scheme) } +// Dependencies returns a list of plugin dependencies +func (p *DynamicPlugins) Dependencies() ([]PluginDependency, error) { + ps, err := p.pluginsFromConfigMap() + if err != nil { + return nil, err + } + + result := make([]PluginDependency, 0) + + for _, pp := range ps { + if pp.Disabled { + continue + } + + result = append(result, pp.Dependencies...) + } + + return result, nil +} + +// returns a list of plugins from the configMap +func (p *DynamicPlugins) pluginsFromConfigMap() ([]DynaPlugin, error) { + if p.ConfigMap == nil { + return []DynaPlugin{}, nil + } + + data := p.ConfigMap.Data[DynamicPluginsFile] + if data == "" { + return []DynaPlugin{}, nil + } + + var pluginsConfig DynaPluginsConfig + err := yaml.Unmarshal([]byte(data), &pluginsConfig) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal dynamic plugins data: %w", err) + } + + return pluginsConfig.Plugins, nil +} + // returns initContainer supposed to initialize DynamicPlugins // TODO consider to use a label to identify instead func DynamicPluginsInitContainer(initContainers []corev1.Container) (int, *corev1.Container) { diff --git a/pkg/model/dynamic-plugins_test.go b/pkg/model/dynamic-plugins_test.go index 924d8df6d..5c92544c8 100644 --- a/pkg/model/dynamic-plugins_test.go +++ b/pkg/model/dynamic-plugins_test.go @@ -6,6 +6,8 @@ import ( "github.com/redhat-developer/rhdh-operator/pkg/platform" + "gopkg.in/yaml.v2" + "github.com/redhat-developer/rhdh-operator/pkg/utils" "k8s.io/utils/ptr" @@ -62,7 +64,8 @@ func TestDynamicPluginsInvalidKeyName(t *testing.T) { _, err := InitObjects(context.TODO(), *bs, testObj.externalConfig, platform.Default, testObj.scheme) assert.Error(t, err) - assert.Contains(t, err.Error(), "expects exactly one Data key named 'dynamic-plugins.yaml'") + //assert.Contains(t, err.Error(), "expects exactly one Data key named 'dynamic-plugins.yaml'") + assert.Contains(t, err.Error(), "dynamic plugin configMap expects 'dynamic-plugins.yaml' Data key") } @@ -93,6 +96,10 @@ func TestDefaultDynamicPlugins(t *testing.T) { //vol-default-dynamic-plugins assert.Equal(t, 4, len(ic.VolumeMounts)) + deps, err := model.DynamicPlugins.Dependencies() + assert.NoError(t, err) + assert.Equal(t, 0, len(deps)) + } func TestDefaultAndSpecifiedDynamicPlugins(t *testing.T) { @@ -106,7 +113,7 @@ func TestDefaultAndSpecifiedDynamicPlugins(t *testing.T) { testObj.externalConfig.DynamicPlugins = corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: "dplugin"}, - Data: map[string]string{DynamicPluginsFile: "tt"}, + Data: map[string]string{DynamicPluginsFile: "dynamic-plugins.yaml: | \n plugins: []"}, } model, err := InitObjects(context.TODO(), *bs, testObj.externalConfig, platform.Default, testObj.scheme) @@ -122,6 +129,10 @@ func TestDefaultAndSpecifiedDynamicPlugins(t *testing.T) { //vol-dplugin assert.Equal(t, 4, len(ic.VolumeMounts)) assert.Equal(t, utils.GenerateVolumeNameFromCmOrSecret("dplugin"), ic.VolumeMounts[3].Name) + + deps, err := model.DynamicPlugins.Dependencies() + assert.NoError(t, err) + assert.Equal(t, 0, len(deps)) } func TestDynamicPluginsFailOnArbitraryDepl(t *testing.T) { @@ -154,6 +165,48 @@ func TestNotConfiguredDPsNotInTheModel(t *testing.T) { } } +func TestWithDynamicPluginsDeps(t *testing.T) { + + bs := testDynamicPluginsBackstage.DeepCopy() + bs.Spec.Application.DynamicPluginsConfigMapName = "dplugin" + + testObj := createBackstageTest(*bs).withDefaultConfig(true). + addToDefaultConfig("dynamic-plugins.yaml", "raw-dynamic-plugins.yaml"). + addToDefaultConfig("deployment.yaml", "janus-deployment.yaml") + + yamlData := `"dynamic-plugins.yaml": | +plugins: + - package: "plugin-a" + disabled: false + dependencies: + - ref: "dependency-1" + - ref: "dependency-2" +` + + testObj.externalConfig.DynamicPlugins = corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "dplugin"}, + Data: map[string]string{DynamicPluginsFile: yamlData}, + } + + model, err := InitObjects(context.TODO(), *bs, testObj.externalConfig, platform.Default, testObj.scheme) + + assert.NoError(t, err) + assert.NotNil(t, model) + + // dependencies from external config + // - ref: "dependency-1" + // - ref: "dependency-2" + deps, err := model.DynamicPlugins.Dependencies() + assert.NoError(t, err) + assert.Equal(t, 2, len(deps)) + + // dependencies from default config = [] + depends, err := (model.getRuntimeObjectByType(&DynamicPlugins{})).(*DynamicPlugins).Dependencies() + assert.NoError(t, err) + assert.Equal(t, 0, len(depends)) + +} + func initContainer(model *BackstageModel) *corev1.Container { for _, v := range model.backstageDeployment.deployment.Spec.Template.Spec.InitContainers { if v.Name == dynamicPluginInitContainerName { @@ -162,3 +215,107 @@ func initContainer(model *BackstageModel) *corev1.Container { } return nil } + +func TestUnmarshalDynaPluginsConfig(t *testing.T) { + yamlData := ` +plugins: + - package: "plugin-a" + integrity: "sha256-abc123" + disabled: false + pluginConfig: + key1: "value1" + key2: "value2" + dependencies: + - ref: "dependency-1" + - ref: "dependency-2" + - package: "plugin-b" + integrity: "sha256-def456" + disabled: true + pluginConfig: + key3: "value3" + dependencies: [] +` + + var config DynaPluginsConfig + err := yaml.Unmarshal([]byte(yamlData), &config) + assert.NoError(t, err) + + // Validate plugins + assert.Equal(t, 2, len(config.Plugins)) + + // Validate first plugin + pluginA := config.Plugins[0] + assert.Equal(t, "plugin-a", pluginA.Package) + assert.Equal(t, "sha256-abc123", pluginA.Integrity) + assert.False(t, pluginA.Disabled) + assert.Equal(t, "value1", pluginA.PluginConfig["key1"]) + assert.Equal(t, "value2", pluginA.PluginConfig["key2"]) + assert.Equal(t, 2, len(pluginA.Dependencies)) + assert.Equal(t, "dependency-1", pluginA.Dependencies[0].Ref) + assert.Equal(t, "dependency-2", pluginA.Dependencies[1].Ref) + + // Validate second plugin + pluginB := config.Plugins[1] + assert.Equal(t, "plugin-b", pluginB.Package) + assert.Equal(t, "sha256-def456", pluginB.Integrity) + assert.True(t, pluginB.Disabled) + assert.Equal(t, "value3", pluginB.PluginConfig["key3"]) + assert.Empty(t, pluginB.Dependencies) +} + +func TestDynamicPluginsDependencies(t *testing.T) { + // Case 1: Plugins with dependencies + yamlDataWithDeps := ` +plugins: + - package: "plugin-a" + disabled: false + dependencies: + - ref: "dependency-1" + - ref: "dependency-2" + - package: "plugin-b" + disabled: false + dependencies: + - ref: "dependency-3" + - package: "plugin-disabled" + disabled: true + dependencies: + - ref: "dependency-4" +` + + dpWithDeps := &DynamicPlugins{ + ConfigMap: &corev1.ConfigMap{ + Data: map[string]string{ + DynamicPluginsFile: yamlDataWithDeps, + }, + }, + } + + deps, err := dpWithDeps.Dependencies() + assert.NoError(t, err) + assert.Equal(t, 3, len(deps)) + assert.Equal(t, "dependency-1", deps[0].Ref) + assert.Equal(t, "dependency-2", deps[1].Ref) + assert.Equal(t, "dependency-3", deps[2].Ref) + + // Case 2: Plugins without dependencies + yamlDataWithoutDeps := ` +plugins: + - package: "plugin-c" + disabled: false + - package: "plugin-d" + disabled: false +` + + dpWithoutDeps := &DynamicPlugins{ + ConfigMap: &corev1.ConfigMap{ + Data: map[string]string{ + DynamicPluginsFile: yamlDataWithoutDeps, + }, + }, + } + + deps, err = dpWithoutDeps.Dependencies() + assert.NoError(t, err) + assert.NotNil(t, deps) + assert.Equal(t, 0, len(deps)) // Ensure it returns an empty slice, not nil +} diff --git a/pkg/model/plugin_deps.go b/pkg/model/plugin_deps.go new file mode 100644 index 000000000..c0fd31967 --- /dev/null +++ b/pkg/model/plugin_deps.go @@ -0,0 +1,102 @@ +package model + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/redhat-developer/rhdh-operator/pkg/utils" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func GetPluginDeps(bsName, bsNamespace string, plugins DynamicPlugins) ([]*unstructured.Unstructured, error) { + dir := filepath.Join(os.Getenv("LOCALBIN"), "plugin-deps") + pdeps, err := plugins.Dependencies() + if err != nil { + return nil, fmt.Errorf("failed to get plugin dependencies: %w", err) + } + + //get refs from enabled + var refs []string + for _, dep := range pdeps { + if dep.Ref != "" { + refs = append(refs, dep.Ref) + } + } + + return ReadPluginDeps(dir, bsName, bsNamespace, refs) + +} + +// ReadPluginDeps reads the plugin dependencies from the specified directory +// and returns a slice of unstructured.Unstructured objects. +func ReadPluginDeps(rootDir, bsName, bsNamespace string, enabled []string) ([]*unstructured.Unstructured, error) { + + if !utils.DirectoryExists(rootDir) { + return []*unstructured.Unstructured{}, nil + } + + var objects []*unstructured.Unstructured + + // Read the directory tree + files, err := getDepsFiles(rootDir, enabled) + + if err != nil { + return nil, err + } + + for _, file := range files { + if !utils.IsYamlFile(file) { + continue + } + + // Read file content + content, err := os.ReadFile(filepath.Clean(file)) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %w", file, err) + } + + // Perform substitutions + modifiedContent := strings.ReplaceAll(string(content), "{{backstage-name}}", bsName) + modifiedContent = strings.ReplaceAll(modifiedContent, "{{backstage-ns}}", bsNamespace) + + // Parse the modified content + objs, err := utils.ReadYamlContent(modifiedContent) + + if err != nil { + return nil, fmt.Errorf("failed to read YAML file %s: %w", file, err) + } + objects = append(objects, objs...) + } + + return objects, nil +} + +func getDepsFiles(root string, enabledPrefixes []string) ([]string, error) { + var files []string + + // Read the directory contents + entries, err := os.ReadDir(root) + if err != nil { + return nil, fmt.Errorf("failed to read directory %s: %w", root, err) + } + + // Iterate over the entries and filter by prefixes + for _, entry := range entries { + if entry.IsDir() { + continue // Skip directories + } + + // Check if the file name starts with any of the enabled prefixes + for _, prefix := range enabledPrefixes { + if strings.HasPrefix(entry.Name(), prefix) { + files = append(files, filepath.Join(root, entry.Name())) + break + } + } + } + + return files, nil +} diff --git a/pkg/model/plugin_deps_test.go b/pkg/model/plugin_deps_test.go new file mode 100644 index 000000000..4090dbe87 --- /dev/null +++ b/pkg/model/plugin_deps_test.go @@ -0,0 +1,63 @@ +package model + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReadPluginDeps(t *testing.T) { + dir := t.TempDir() + + // Create files in the root directory + file1 := filepath.Join(dir, "sonata.yaml") + file2 := filepath.Join(dir, "otherplugin.yaml") + file3 := filepath.Join(dir, "sonata-config.yaml") + file4 := filepath.Join(dir, "unrelated.txt") + + err := os.WriteFile(file1, []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: sonata"), 0644) + assert.NoError(t, err) + err = os.WriteFile(file2, []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: test2"), 0644) + assert.NoError(t, err) + err = os.WriteFile(file3, []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: sonata"), 0644) + assert.NoError(t, err) + err = os.WriteFile(file4, []byte("some unrelated content"), 0644) + assert.NoError(t, err) + + objects, err := ReadPluginDeps(dir, "", "", []string{"sonata"}) + assert.NoError(t, err) + assert.Len(t, objects, 2) + + // Verify the names of the objects + assert.Equal(t, "sonata", objects[0].GetName()) + assert.Equal(t, "sonata", objects[1].GetName()) +} + +func TestReadPluginDepsSubstitutions(t *testing.T) { + + dir := t.TempDir() + + file1 := filepath.Join(dir, "file1.yaml") + yamlContent := ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{backstage-name}} + namespace: {{backstage-ns}} +` + err := os.WriteFile(file1, []byte(yamlContent), 0644) + assert.NoError(t, err) + + // Call ReadPluginDeps with substitution values + bsName := "test-name" + bsNamespace := "test-namespace" + objects, err := ReadPluginDeps(dir, bsName, bsNamespace, []string{"file1"}) + assert.NoError(t, err) + assert.Len(t, objects, 1) + + // Verify the substitutions + assert.Equal(t, "test-name", objects[0].GetName()) + assert.Equal(t, "test-namespace", objects[0].GetNamespace()) +} diff --git a/pkg/model/runtime.go b/pkg/model/runtime.go index 72515041a..fc97e1e85 100644 --- a/pkg/model/runtime.go +++ b/pkg/model/runtime.go @@ -45,8 +45,9 @@ type BackstageModel struct { LocalDbService *DbService LocalDbSecret *DbSecret - route *BackstageRoute - appConfig *AppConfig + route *BackstageRoute + appConfig *AppConfig + DynamicPlugins DynamicPlugins RuntimeObjects []RuntimeObject @@ -103,7 +104,7 @@ func InitObjects(ctx context.Context, backstage bsv1.Backstage, externalConfig E lg := log.FromContext(ctx) lg.V(1) - model := &BackstageModel{RuntimeObjects: make([]RuntimeObject, 0), ExternalConfig: externalConfig, localDbEnabled: backstage.Spec.IsLocalDbEnabled(), isOpenshift: platform.IsOpenshift()} + model := &BackstageModel{RuntimeObjects: make([]RuntimeObject, 0), ExternalConfig: externalConfig, localDbEnabled: backstage.Spec.IsLocalDbEnabled(), isOpenshift: platform.IsOpenshift(), DynamicPlugins: DynamicPlugins{}} // looping through the registered runtimeConfig objects initializing the model for _, conf := range runtimeConfig { diff --git a/pkg/model/testdata/dynamic-plugins-deps.yaml b/pkg/model/testdata/dynamic-plugins-deps.yaml new file mode 100644 index 000000000..3dd06ab15 --- /dev/null +++ b/pkg/model/testdata/dynamic-plugins-deps.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: default-dynamic-plugins # must be the same as (deployment.yaml).spec.template.spec.volumes.name.dynamic-plugins-conf.configMap.name +data: + "enabled-plugins-deps.txt": | + orchestrator/* + "dynamic-plugins.yaml": | + includes: + - dynamic-plugins.default.yaml + plugins: [] \ No newline at end of file diff --git a/pkg/utils/yaml.go b/pkg/utils/yaml.go new file mode 100644 index 000000000..524b5f0ca --- /dev/null +++ b/pkg/utils/yaml.go @@ -0,0 +1,104 @@ +package utils + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/yaml" +) + +func ReadYamlFilesFromDir(dir string) ([]*unstructured.Unstructured, error) { + + if !DirectoryExists(dir) { + return []*unstructured.Unstructured{}, nil + } + + var objects []*unstructured.Unstructured + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !IsYamlFile(d.Name()) { + return nil + } + + objs, err := ReadYamlFile(path) + if err != nil { + return fmt.Errorf("failed to read YAML file %s: %w", path, err) + } + objects = append(objects, objs...) + return nil + }) + + if err != nil { + return nil, err + } + + return objects, nil +} + +func ReadYamlFile(path string) ([]*unstructured.Unstructured, error) { + fpath := filepath.Clean(path) + if _, err := os.Stat(fpath); err != nil { + return nil, err + } + conf, err := os.ReadFile(fpath) + if err != nil { + return nil, fmt.Errorf("failed to read YAML file: %w", err) + } + + dec := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(conf), 1000) + var objects []*unstructured.Unstructured + for { + obj := &unstructured.Unstructured{} + err := dec.Decode(obj) + if err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("failed to decode YAML: %w", err) + } + objects = append(objects, obj) + } + + return objects, nil +} + +func ReadYamlContent(content string) ([]*unstructured.Unstructured, error) { + // Create a YAML decoder from the content + dec := yaml.NewYAMLOrJSONDecoder(bytes.NewReader([]byte(content)), 1000) + var objects []*unstructured.Unstructured + + // Decode the content into unstructured objects + for { + obj := &unstructured.Unstructured{} + err := dec.Decode(obj) + if err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("failed to decode YAML content: %w", err) + } + objects = append(objects, obj) + } + + return objects, nil +} + +func IsYamlFile(filename string) bool { + ext := filepath.Ext(filename) + return ext == ".yaml" || ext == ".yml" +} + +func DirectoryExists(path string) bool { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return false + } + return info.IsDir() +} diff --git a/pkg/utils/yaml_test.go b/pkg/utils/yaml_test.go new file mode 100644 index 000000000..905604c5f --- /dev/null +++ b/pkg/utils/yaml_test.go @@ -0,0 +1,80 @@ +package utils + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReadYamlFile(t *testing.T) { + // Create a temporary directory + dir, err := os.MkdirTemp("", "test-yaml") + assert.NoError(t, err) + defer os.RemoveAll(dir) + + // Create a sample YAML file + yamlContent := ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-configmap +data: + key: value +` + filePath := filepath.Join(dir, "test.yaml") + err = os.WriteFile(filePath, []byte(yamlContent), 0644) + assert.NoError(t, err) + + // Test ReadYamlFile + objects, err := ReadYamlFile(filePath) + assert.NoError(t, err) + assert.Len(t, objects, 1) + + obj := objects[0] + assert.Equal(t, "ConfigMap", obj.GetKind()) + assert.Equal(t, "test-configmap", obj.GetName()) +} + +func TestReadYamlFilesFromDir(t *testing.T) { + // Create a temporary directory + dir, err := os.MkdirTemp("", "test-yaml-dir") + assert.NoError(t, err) + defer os.RemoveAll(dir) + + // Create sample YAML files + yamlContent1 := ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-configmap1 +data: + key: value1 +` + yamlContent2 := ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-configmap2 +data: + key: value2 +` + err = os.WriteFile(filepath.Join(dir, "test1.yaml"), []byte(yamlContent1), 0644) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(dir, "test2.yaml"), []byte(yamlContent2), 0644) + assert.NoError(t, err) + + // Test ReadYamlFilesFromDir + objects, err := ReadYamlFilesFromDir(dir) + assert.NoError(t, err) + assert.Len(t, objects, 2) + + obj1 := objects[0] + assert.Equal(t, "ConfigMap", obj1.GetKind()) + assert.Equal(t, "test-configmap1", obj1.GetName()) + + obj2 := objects[1] + assert.Equal(t, "ConfigMap", obj2.GetKind()) + assert.Equal(t, "test-configmap2", obj2.GetName()) +}