diff --git a/.ci/pipelines/cluster/aks/aks-operator-deployment.sh b/.ci/pipelines/cluster/aks/aks-operator-deployment.sh index 1c255c03f3..17660a0cd0 100644 --- a/.ci/pipelines/cluster/aks/aks-operator-deployment.sh +++ b/.ci/pipelines/cluster/aks/aks-operator-deployment.sh @@ -30,6 +30,7 @@ initiate_aks_operator_deployment() { setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}" deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start_K8s.yaml" + wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME}" patch_and_restart_aks_spot "${namespace}" "$RELEASE_NAME" apply_aks_operator_ingress "$namespace" "backstage-$RELEASE_NAME" @@ -57,6 +58,7 @@ initiate_rbac_aks_operator_deployment() { setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}" deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac_K8s.yaml" + wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME_RBAC}" patch_and_restart_aks_spot_rbac "${namespace}" "$RELEASE_NAME_RBAC" apply_aks_operator_ingress "$namespace" "backstage-$RELEASE_NAME_RBAC" diff --git a/.ci/pipelines/cluster/eks/eks-operator-deployment.sh b/.ci/pipelines/cluster/eks/eks-operator-deployment.sh index ead56d33b2..0a7eb11e21 100644 --- a/.ci/pipelines/cluster/eks/eks-operator-deployment.sh +++ b/.ci/pipelines/cluster/eks/eks-operator-deployment.sh @@ -28,6 +28,7 @@ initiate_eks_operator_deployment() { setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}" deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start_K8s.yaml" + wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME}" apply_eks_operator_ingress "$namespace" "backstage-$RELEASE_NAME" } @@ -54,6 +55,7 @@ initiate_rbac_eks_operator_deployment() { setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}" deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac_K8s.yaml" + wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME_RBAC}" apply_eks_operator_ingress "$namespace" "backstage-$RELEASE_NAME_RBAC" } diff --git a/.ci/pipelines/cluster/gke/gke-operator-deployment.sh b/.ci/pipelines/cluster/gke/gke-operator-deployment.sh index c279afd2ae..248e4d87f9 100644 --- a/.ci/pipelines/cluster/gke/gke-operator-deployment.sh +++ b/.ci/pipelines/cluster/gke/gke-operator-deployment.sh @@ -33,6 +33,7 @@ initiate_gke_operator_deployment() { setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}" deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start_K8s.yaml" + wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME}" apply_gke_operator_ingress "$namespace" "backstage-$RELEASE_NAME" } @@ -60,6 +61,7 @@ initiate_rbac_gke_operator_deployment() { setup_image_pull_secret "${namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}" deploy_rhdh_operator "${namespace}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac_K8s.yaml" + wait_for_operator_rollout "${namespace}" "backstage-${RELEASE_NAME_RBAC}" apply_gke_operator_ingress "$namespace" "backstage-$RELEASE_NAME_RBAC" } diff --git a/.ci/pipelines/jobs/ocp-operator.sh b/.ci/pipelines/jobs/ocp-operator.sh index bdd6f97ae4..97dd5c536e 100644 --- a/.ci/pipelines/jobs/ocp-operator.sh +++ b/.ci/pipelines/jobs/ocp-operator.sh @@ -22,6 +22,7 @@ initiate_operator_deployments() { oc apply -f /tmp/configmap-dynamic-plugins.yaml -n "${NAME_SPACE}" deploy_redis_cache "${NAME_SPACE}" deploy_rhdh_operator "${NAME_SPACE}" "${DIR}/resources/rhdh-operator/rhdh-start.yaml" + wait_for_operator_rollout "${NAME_SPACE}" "backstage-${RELEASE_NAME}" # TODO: https://issues.redhat.com/browse/RHDHBUGS-2184 fix orchestrator workflows deployment on operator # enable_orchestrator_plugins_op "${NAME_SPACE}" # deploy_orchestrator_workflows_operator "${NAME_SPACE}" @@ -35,6 +36,7 @@ initiate_operator_deployments() { create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins-rbac.yaml" oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}" deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml" + wait_for_operator_rollout "${NAME_SPACE_RBAC}" "backstage-${RELEASE_NAME_RBAC}" # TODO: https://issues.redhat.com/browse/RHDHBUGS-2184 fix orchestrator workflows deployment on operator # enable_orchestrator_plugins_op "${NAME_SPACE_RBAC}" # deploy_orchestrator_workflows_operator "${NAME_SPACE_RBAC}" @@ -59,6 +61,7 @@ initiate_operator_deployments_osd_gcp() { oc apply -f /tmp/configmap-dynamic-plugins.yaml -n "${NAME_SPACE}" deploy_redis_cache "${NAME_SPACE}" deploy_rhdh_operator "${NAME_SPACE}" "${DIR}/resources/rhdh-operator/rhdh-start.yaml" + wait_for_operator_rollout "${NAME_SPACE}" "backstage-${RELEASE_NAME}" # Skip orchestrator plugins and workflows for OSD-GCP log::warn "Skipping orchestrator plugins and workflows deployment on OSD-GCP environment" @@ -77,6 +80,7 @@ initiate_operator_deployments_osd_gcp() { oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}" deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml" + wait_for_operator_rollout "${NAME_SPACE_RBAC}" "backstage-${RELEASE_NAME_RBAC}" # Skip orchestrator plugins and workflows for OSD-GCP RBAC log::warn "Skipping orchestrator plugins and workflows deployment on OSD-GCP RBAC environment" @@ -88,6 +92,7 @@ run_operator_runtime_config_change_tests() { oc apply -f "$DIR/resources/postgres-db/dynamic-plugins-root-PVC.yaml" -n "${NAME_SPACE_RUNTIME}" create_app_config_map "$DIR/resources/postgres-db/rds-app-config.yaml" "${NAME_SPACE_RUNTIME}" deploy_rhdh_operator "${NAME_SPACE_RUNTIME}" "${DIR}/resources/rhdh-operator/rhdh-start-runtime.yaml" + wait_for_operator_rollout "${NAME_SPACE_RUNTIME}" "backstage-${RELEASE_NAME}" local runtime_url="https://backstage-${RELEASE_NAME}-${NAME_SPACE_RUNTIME}.${K8S_CLUSTER_ROUTER_BASE}" run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" } diff --git a/.ci/pipelines/utils.sh b/.ci/pipelines/utils.sh index 399a4a6f8f..1e499c5989 100755 --- a/.ci/pipelines/utils.sh +++ b/.ci/pipelines/utils.sh @@ -765,7 +765,41 @@ metadata: name: dynamic-plugins data: dynamic-plugins.yaml: |" > ${final_file} - yq '.global.dynamic' ${base_file} | sed -e 's/^/ /' -e 's/{{ "{{" }}inherit{{ "}}" }}/{{inherit}}/g' >> ${final_file} + # Render Helm template escapes: {{ "{{" }}inherit{{ "}}" }} -> {{inherit}} + # Value files use Helm escaping for {{inherit}} which Helm renders automatically, + # but operator deployments use these files directly without Helm rendering. + yq '.global.dynamic' ${base_file} | sed -e 's/^/ /' -e 's/{{ "{{" }}/{{/g' -e 's/{{ "}}" }}/}}/g' >> ${final_file} +} + +# Wait for the RHDH operator to finish reconciling and the deployment rollout to stabilize. +# The operator may update the Deployment spec multiple times after the Backstage CR is applied, +# causing multiple ReplicaSets. This function waits for the deployment to exist and for +# the rollout to complete so only one ReplicaSet is active. +wait_for_operator_rollout() { + local namespace=$1 + local deployment_name=$2 + local max_wait=${3:-180} + + log::info "Waiting for deployment '$deployment_name' to be created in namespace '$namespace'..." + local elapsed=0 + while [[ $elapsed -lt $max_wait ]]; do + if kubectl get deployment "$deployment_name" -n "$namespace" &> /dev/null; then + log::info "Deployment '$deployment_name' found, waiting for rollout to stabilize..." + break + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + + if ! kubectl get deployment "$deployment_name" -n "$namespace" &> /dev/null; then + log::warn "Deployment '$deployment_name' not found after ${max_wait}s, continuing..." + return 0 + fi + + # Wait for rollout to complete (operator may trigger multiple updates) + kubectl rollout status deployment/"$deployment_name" -n "$namespace" --timeout="${max_wait}s" 2> /dev/null || true + log::info "Deployment '$deployment_name' rollout stabilized" + return 0 } create_conditional_policies_operator() { diff --git a/.ci/pipelines/value_files/diff-values_showcase-rbac_AKS.yaml b/.ci/pipelines/value_files/diff-values_showcase-rbac_AKS.yaml index 8e88328d4d..a6472cfb5e 100644 --- a/.ci/pipelines/value_files/diff-values_showcase-rbac_AKS.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase-rbac_AKS.yaml @@ -180,4 +180,5 @@ upstream: enabled: true className: webapprouting.kubernetes.azure.com host: "" -orchestrator: null +orchestrator: + enabled: false diff --git a/.ci/pipelines/value_files/diff-values_showcase-rbac_EKS.yaml b/.ci/pipelines/value_files/diff-values_showcase-rbac_EKS.yaml index 445e37c084..700fb7222a 100644 --- a/.ci/pipelines/value_files/diff-values_showcase-rbac_EKS.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase-rbac_EKS.yaml @@ -172,4 +172,5 @@ upstream: alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' alb.ingress.kubernetes.io/ssl-redirect: "443" external-dns.alpha.kubernetes.io/hostname: $EKS_INSTANCE_DOMAIN_NAME -orchestrator: null +orchestrator: + enabled: false diff --git a/.ci/pipelines/value_files/diff-values_showcase-rbac_GKE.yaml b/.ci/pipelines/value_files/diff-values_showcase-rbac_GKE.yaml index 31f7008b81..a6c9568815 100644 --- a/.ci/pipelines/value_files/diff-values_showcase-rbac_GKE.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase-rbac_GKE.yaml @@ -175,4 +175,5 @@ upstream: ingress.gcp.kubernetes.io/pre-shared-cert: "" networking.gke.io/v1beta1.FrontendConfig: rhdh-gke-ingress-security-config className: gce -orchestrator: null +orchestrator: + enabled: false diff --git a/.ci/pipelines/value_files/diff-values_showcase-rbac_OSD-GCP.yaml b/.ci/pipelines/value_files/diff-values_showcase-rbac_OSD-GCP.yaml index 26cc072679..45265caf07 100644 --- a/.ci/pipelines/value_files/diff-values_showcase-rbac_OSD-GCP.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase-rbac_OSD-GCP.yaml @@ -4,7 +4,8 @@ # The only exception is global.dynamic.plugins, that gets merged with the base file. # Disable orchestrator for OSD-GCP due to infrastructure limitations -orchestrator: null +orchestrator: + enabled: false global: dynamic: diff --git a/.ci/pipelines/value_files/diff-values_showcase_AKS.yaml b/.ci/pipelines/value_files/diff-values_showcase_AKS.yaml index f0bd5c1728..4cab4f5322 100644 --- a/.ci/pipelines/value_files/diff-values_showcase_AKS.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase_AKS.yaml @@ -58,4 +58,5 @@ upstream: enabled: true className: webapprouting.kubernetes.azure.com host: "" -orchestrator: null +orchestrator: + enabled: false diff --git a/.ci/pipelines/value_files/diff-values_showcase_EKS.yaml b/.ci/pipelines/value_files/diff-values_showcase_EKS.yaml index a10ff6046c..a02f41fefc 100644 --- a/.ci/pipelines/value_files/diff-values_showcase_EKS.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase_EKS.yaml @@ -52,4 +52,5 @@ upstream: alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' alb.ingress.kubernetes.io/ssl-redirect: "443" external-dns.alpha.kubernetes.io/hostname: $EKS_INSTANCE_DOMAIN_NAME -orchestrator: null +orchestrator: + enabled: false diff --git a/.ci/pipelines/value_files/diff-values_showcase_GKE.yaml b/.ci/pipelines/value_files/diff-values_showcase_GKE.yaml index af5d7ea7bf..0129443106 100644 --- a/.ci/pipelines/value_files/diff-values_showcase_GKE.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase_GKE.yaml @@ -72,4 +72,5 @@ upstream: ingress.gcp.kubernetes.io/pre-shared-cert: "" networking.gke.io/v1beta1.FrontendConfig: rhdh-gke-ingress-security-config className: gce -orchestrator: null +orchestrator: + enabled: false diff --git a/.ci/pipelines/value_files/diff-values_showcase_OSD-GCP.yaml b/.ci/pipelines/value_files/diff-values_showcase_OSD-GCP.yaml index 68072eafb5..2b523290af 100644 --- a/.ci/pipelines/value_files/diff-values_showcase_OSD-GCP.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase_OSD-GCP.yaml @@ -4,7 +4,8 @@ # The only exception is global.dynamic.plugins, that gets merged with the base file. # Disable orchestrator for OSD-GCP due to infrastructure limitations -orchestrator: null +orchestrator: + enabled: false global: dynamic: diff --git a/.ci/pipelines/value_files/diff-values_showcase_upgrade.yaml b/.ci/pipelines/value_files/diff-values_showcase_upgrade.yaml index e89df06861..6d6c9b8016 100644 --- a/.ci/pipelines/value_files/diff-values_showcase_upgrade.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase_upgrade.yaml @@ -1 +1,2 @@ -orchestrator: null +orchestrator: + enabled: false