fix(ci): fix operator deployments on K8s — inherit rendering, rollout wait, and orchestrator config [release-1.9]

.ci/pipelines/cluster/aks/aks-operator-deployment.sh

@@ -30,6 +30,7 @@ initiate_aks_operator_deployment() {
   setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}"
 
   deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start_K8s.yaml"
+  wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME}"
   patch_and_restart_aks_spot "${namespace}" "$RELEASE_NAME"
 
   apply_aks_operator_ingress "$namespace" "backstage-$RELEASE_NAME"
@@ -57,6 +58,7 @@ initiate_rbac_aks_operator_deployment() {
   setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}"
 
   deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac_K8s.yaml"
+  wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME_RBAC}"
   patch_and_restart_aks_spot_rbac "${namespace}" "$RELEASE_NAME_RBAC"
 
   apply_aks_operator_ingress "$namespace" "backstage-$RELEASE_NAME_RBAC"

.ci/pipelines/cluster/eks/eks-operator-deployment.sh

@@ -28,6 +28,7 @@ initiate_eks_operator_deployment() {
   setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}"
 
   deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start_K8s.yaml"
+  wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME}"
 
   apply_eks_operator_ingress "$namespace" "backstage-$RELEASE_NAME"
 }
@@ -54,6 +55,7 @@ initiate_rbac_eks_operator_deployment() {
   setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}"
 
   deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac_K8s.yaml"
+  wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME_RBAC}"
 
   apply_eks_operator_ingress "$namespace" "backstage-$RELEASE_NAME_RBAC"
 }

.ci/pipelines/cluster/gke/gke-operator-deployment.sh

@@ -33,6 +33,7 @@ initiate_gke_operator_deployment() {
   setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}"
 
   deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start_K8s.yaml"
+  wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME}"
 
   apply_gke_operator_ingress "$namespace" "backstage-$RELEASE_NAME"
 }
@@ -60,6 +61,7 @@ initiate_rbac_gke_operator_deployment() {
   setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}"
 
   deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac_K8s.yaml"
+  wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME_RBAC}"
 
   apply_gke_operator_ingress "$namespace" "backstage-$RELEASE_NAME_RBAC"
 }

.ci/pipelines/jobs/ocp-operator.sh

@@ -22,6 +22,7 @@ initiate_operator_deployments() {
   oc apply -f /tmp/configmap-dynamic-plugins.yaml -n "${NAME_SPACE}"
   deploy_redis_cache "${NAME_SPACE}"
   deploy_rhdh_operator "${NAME_SPACE}" "${DIR}/resources/rhdh-operator/rhdh-start.yaml"
+  wait_for_operator_rollout "${NAME_SPACE}" "backstage-${RELEASE_NAME}"
   # TODO: https://issues.redhat.com/browse/RHDHBUGS-2184 fix orchestrator workflows deployment on operator
   # enable_orchestrator_plugins_op "${NAME_SPACE}"
   # deploy_orchestrator_workflows_operator "${NAME_SPACE}"
@@ -35,6 +36,7 @@ initiate_operator_deployments() {
   create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins-rbac.yaml"
   oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}"
   deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml"
+  wait_for_operator_rollout "${NAME_SPACE_RBAC}" "backstage-${RELEASE_NAME_RBAC}"
   # TODO: https://issues.redhat.com/browse/RHDHBUGS-2184 fix orchestrator workflows deployment on operator
   # enable_orchestrator_plugins_op "${NAME_SPACE_RBAC}"
   # deploy_orchestrator_workflows_operator "${NAME_SPACE_RBAC}"
@@ -59,6 +61,7 @@ initiate_operator_deployments_osd_gcp() {
   oc apply -f /tmp/configmap-dynamic-plugins.yaml -n "${NAME_SPACE}"
   deploy_redis_cache "${NAME_SPACE}"
   deploy_rhdh_operator "${NAME_SPACE}" "${DIR}/resources/rhdh-operator/rhdh-start.yaml"
+  wait_for_operator_rollout "${NAME_SPACE}" "backstage-${RELEASE_NAME}"
 
   # Skip orchestrator plugins and workflows for OSD-GCP
   log::warn "Skipping orchestrator plugins and workflows deployment on OSD-GCP environment"
@@ -77,6 +80,7 @@ initiate_operator_deployments_osd_gcp() {
 
   oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}"
   deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml"
+  wait_for_operator_rollout "${NAME_SPACE_RBAC}" "backstage-${RELEASE_NAME_RBAC}"
 
   # Skip orchestrator plugins and workflows for OSD-GCP RBAC
   log::warn "Skipping orchestrator plugins and workflows deployment on OSD-GCP RBAC environment"
@@ -88,6 +92,7 @@ run_operator_runtime_config_change_tests() {
   oc apply -f "$DIR/resources/postgres-db/dynamic-plugins-root-PVC.yaml" -n "${NAME_SPACE_RUNTIME}"
   create_app_config_map "$DIR/resources/postgres-db/rds-app-config.yaml" "${NAME_SPACE_RUNTIME}"
   deploy_rhdh_operator "${NAME_SPACE_RUNTIME}" "${DIR}/resources/rhdh-operator/rhdh-start-runtime.yaml"
+  wait_for_operator_rollout "${NAME_SPACE_RUNTIME}" "backstage-${RELEASE_NAME}"
   local runtime_url="https://backstage-${RELEASE_NAME}-${NAME_SPACE_RUNTIME}.${K8S_CLUSTER_ROUTER_BASE}"
   run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}"
 }

.ci/pipelines/utils.sh

@@ -765,7 +765,41 @@ metadata:
   name: dynamic-plugins
 data:
   dynamic-plugins.yaml: |" > ${final_file}
-  yq '.global.dynamic' ${base_file} | sed -e 's/^/    /' -e 's/{{ "{{" }}inherit{{ "}}" }}/{{inherit}}/g' >> ${final_file}
+  # Render Helm template escapes: {{ "{{" }}inherit{{ "}}" }} -> {{inherit}}
+  # Value files use Helm escaping for {{inherit}} which Helm renders automatically,
+  # but operator deployments use these files directly without Helm rendering.
+  yq '.global.dynamic' ${base_file} | sed -e 's/^/    /' -e 's/{{ "{{" }}/{{/g' -e 's/{{ "}}" }}/}}/g' >> ${final_file}
+}
+
+# Wait for the RHDH operator to finish reconciling and the deployment rollout to stabilize.
+# The operator may update the Deployment spec multiple times after the Backstage CR is applied,
+# causing multiple ReplicaSets. This function waits for the deployment to exist and for
+# the rollout to complete so only one ReplicaSet is active.
+wait_for_operator_rollout() {
+  local namespace=$1
+  local deployment_name=$2
+  local max_wait=${3:-180}
+
+  log::info "Waiting for deployment '$deployment_name' to be created in namespace '$namespace'..."
+  local elapsed=0
+  while [[ $elapsed -lt $max_wait ]]; do
+    if kubectl get deployment "$deployment_name" -n "$namespace" &> /dev/null; then
+      log::info "Deployment '$deployment_name' found, waiting for rollout to stabilize..."
+      break
+    fi
+    sleep 5
+    elapsed=$((elapsed + 5))
+  done
+
+  if ! kubectl get deployment "$deployment_name" -n "$namespace" &> /dev/null; then
+    log::warn "Deployment '$deployment_name' not found after ${max_wait}s, continuing..."
+    return 0
+  fi
+
+  # Wait for rollout to complete (operator may trigger multiple updates)
+  kubectl rollout status deployment/"$deployment_name" -n "$namespace" --timeout="${max_wait}s" 2> /dev/null || true
+  log::info "Deployment '$deployment_name' rollout stabilized"
+  return 0
 }
 
 create_conditional_policies_operator() {

.ci/pipelines/value_files/diff-values_showcase-rbac_AKS.yaml

@@ -180,4 +180,5 @@ upstream:
     enabled: true
     className: webapprouting.kubernetes.azure.com
     host: ""
-orchestrator: null
+orchestrator:
+  enabled: false

.ci/pipelines/value_files/diff-values_showcase-rbac_EKS.yaml

@@ -172,4 +172,5 @@ upstream:
       alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
       alb.ingress.kubernetes.io/ssl-redirect: "443"
       external-dns.alpha.kubernetes.io/hostname: $EKS_INSTANCE_DOMAIN_NAME
-orchestrator: null
+orchestrator:
+  enabled: false

.ci/pipelines/value_files/diff-values_showcase-rbac_GKE.yaml

@@ -175,4 +175,5 @@ upstream:
       ingress.gcp.kubernetes.io/pre-shared-cert: ""
       networking.gke.io/v1beta1.FrontendConfig: rhdh-gke-ingress-security-config
     className: gce
-orchestrator: null
+orchestrator:
+  enabled: false

.ci/pipelines/value_files/diff-values_showcase-rbac_OSD-GCP.yaml

@@ -4,7 +4,8 @@
 # The only exception is global.dynamic.plugins, that gets merged with the base file.
 
 # Disable orchestrator for OSD-GCP due to infrastructure limitations
-orchestrator: null
+orchestrator:
+  enabled: false
 
 global:
   dynamic:

.ci/pipelines/value_files/diff-values_showcase_AKS.yaml

@@ -58,4 +58,5 @@ upstream:
     enabled: true
     className: webapprouting.kubernetes.azure.com
     host: ""
-orchestrator: null
+orchestrator:
+  enabled: false

.ci/pipelines/value_files/diff-values_showcase_EKS.yaml

@@ -52,4 +52,5 @@ upstream:
       alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
       alb.ingress.kubernetes.io/ssl-redirect: "443"
       external-dns.alpha.kubernetes.io/hostname: $EKS_INSTANCE_DOMAIN_NAME
-orchestrator: null
+orchestrator:
+  enabled: false

.ci/pipelines/value_files/diff-values_showcase_GKE.yaml

@@ -72,4 +72,5 @@ upstream:
       ingress.gcp.kubernetes.io/pre-shared-cert: ""
       networking.gke.io/v1beta1.FrontendConfig: rhdh-gke-ingress-security-config
     className: gce
-orchestrator: null
+orchestrator:
+  enabled: false

.ci/pipelines/value_files/diff-values_showcase_OSD-GCP.yaml

@@ -4,7 +4,8 @@
 # The only exception is global.dynamic.plugins, that gets merged with the base file.
 
 # Disable orchestrator for OSD-GCP due to infrastructure limitations
-orchestrator: null
+orchestrator:
+  enabled: false
 
 global:
   dynamic:

.ci/pipelines/value_files/diff-values_showcase_upgrade.yaml

@@ -1 +1,2 @@
-orchestrator: null
+orchestrator:
+  enabled: false