chore(ci): fix Operator tests, artifact handling, pod selectors

.ci/pipelines/env_variables.sh

@@ -139,7 +139,6 @@ AZURE_DB_CERTIFICATES_PATH="/tmp/secrets/azure-db-certificates.pem"
 
 JUNIT_RESULTS="junit-results.xml"
 
-SLACK_DATA_ROUTER_WEBHOOK_URL=$(cat /tmp/secrets/SLACK_DATA_ROUTER_WEBHOOK_URL)
 REDIS_USERNAME=temp
 REDIS_USERNAME_ENCODED=$(printf "%s" $REDIS_USERNAME | base64 | tr -d '\n')
 REDIS_PASSWORD=test123

.ci/pipelines/install-methods/operator.sh

@@ -51,19 +51,22 @@ prepare_operator() {
   k8s_wait::crd "backstages.rhdh.redhat.com" 300 10 || return 1
 }
 
-deploy_rhdh_operator() {
-  local namespace=$1
-  local backstage_crd_path=$2
-
-  # Ensure PostgresCluster CRD is available before deploying Backstage CR
-  # This is required because the operator relies on CrunchyDB for its internal database
+# Waits for the Crunchy Data PostgreSQL Operator's PostgresCluster CRD to become available.
+# Must be called after the Crunchy DB CRD is created and before RHDH is deployed
+# with internal DB disabled and configured to use Crunchy DB as the external PostgreSQL database.
+wait_for_crunchy_crd() {
   log::info "Verifying PostgresCluster CRD is available before deploying Backstage CR..."
   k8s_wait::crd "postgresclusters.postgres-operator.crunchydata.com" 60 5 || {
     log::error "PostgresCluster CRD not available - operator won't be able to create internal database"
     return 1
   }
+}
+
+deploy_rhdh_operator() {
+  local namespace=$1
+  local backstage_crd_path=$2
 
-  # Verify Backstage CRD is also available
+  # Verify Backstage CRD is available
   k8s_wait::crd "backstages.rhdh.redhat.com" 60 5 || return 1
 
   rendered_yaml=$(envsubst < "$backstage_crd_path")

.ci/pipelines/jobs/ocp-nightly.sh

@@ -27,9 +27,9 @@ handle_ocp_nightly() {
   # Use OSD-GCP specific deployment for osd-gcp jobs (orchestrator disabled)
   if [[ "${JOB_NAME}" == *osd-gcp* ]]; then
     log::info "Detected OSD-GCP job, using OSD-GCP specific deployment (orchestrator disabled)"
-    initiate_deployments_osd_gcp
+    initiate_deployments_osd_gcp "${PW_PROJECT_SHOWCASE}" "${PW_PROJECT_SHOWCASE_RBAC}"
   else
-    initiate_deployments
+    initiate_deployments "${PW_PROJECT_SHOWCASE}" "${PW_PROJECT_SHOWCASE_RBAC}"
   fi
 
   deploy_test_backstage_customization_provider "${NAME_SPACE}"
@@ -61,7 +61,7 @@ run_runtime_config_change_tests() {
 
 run_sanity_plugins_check() {
   local sanity_plugins_url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE_SANITY_PLUGINS_CHECK}.${K8S_CLUSTER_ROUTER_BASE}"
-  initiate_sanity_plugin_checks_deployment "${RELEASE_NAME}" "${NAME_SPACE_SANITY_PLUGINS_CHECK}" "${sanity_plugins_url}"
+  initiate_sanity_plugin_checks_deployment "${RELEASE_NAME}" "${NAME_SPACE_SANITY_PLUGINS_CHECK}" "${sanity_plugins_url}" "${PW_PROJECT_SHOWCASE_SANITY_PLUGINS}"
   testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_SANITY_PLUGINS_CHECK}" "${PW_PROJECT_SHOWCASE_SANITY_PLUGINS}" "${sanity_plugins_url}"
 }
 

.ci/pipelines/jobs/ocp-operator.sh

@@ -36,6 +36,7 @@ initiate_operator_deployments() {
   apply_yaml_files "${DIR}" "${NAME_SPACE_RBAC}" "${rbac_rhdh_base_url}"
   config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins-rbac.yaml"
   oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}"
+  wait_for_crunchy_crd || return 1
   deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml"
   # TODO: https://issues.redhat.com/browse/RHDHBUGS-2184 fix orchestrator workflows deployment on operator
   # enable_orchestrator_plugins_op "${NAME_SPACE_RBAC}"
@@ -47,8 +48,6 @@ initiate_operator_deployments() {
 initiate_operator_deployments_osd_gcp() {
   log::info "Initiating Operator-backed deployments on OSD-GCP (orchestrator disabled)"
 
-  prepare_operator
-
   namespace::configure "${NAME_SPACE}"
   deploy_test_backstage_customization_provider "${NAME_SPACE}"
   local rhdh_base_url="https://backstage-${RELEASE_NAME}-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}"
@@ -78,6 +77,7 @@ initiate_operator_deployments_osd_gcp() {
   common::save_artifact "${PW_PROJECT_SHOWCASE_OPERATOR_RBAC}" "/tmp/configmap-dynamic-plugins-rbac.yaml"
 
   oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}"
+  wait_for_crunchy_crd || return 1
   deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml"
 
   # Skip orchestrator plugins and workflows for OSD-GCP RBAC

.ci/pipelines/jobs/ocp-pull.sh

@@ -22,7 +22,7 @@ handle_ocp_pull() {
   K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//')
   export K8S_CLUSTER_ROUTER_BASE
   cluster_setup_ocp_helm
-  initiate_deployments
+  initiate_deployments "${PW_PROJECT_SHOWCASE}" "${PW_PROJECT_SHOWCASE_RBAC}"
   deploy_test_backstage_customization_provider "${NAME_SPACE}"
   local url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}"
   testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SHOWCASE}" "${url}"

.ci/pipelines/lib/common.sh

@@ -188,19 +188,28 @@ common::retry() {
   return 1
 }
 
-# Save a file to the artifacts directory
-# Args: artifacts_subdir, file_path
+# Save a file or directory to the artifacts directory
+# Args:
+#   $1 - artifacts_subdir: Subdirectory under ARTIFACT_DIR (typically playwright_project)
+#   $2 - file_path: File or directory to save
+#   $3 - subdir: (optional) Additional subdirectory under artifacts_subdir
 common::save_artifact() {
   local artifacts_subdir=$1
   local file=$2
+  local subdir=${3:-}
 
   if [[ -z "$ARTIFACT_DIR" ]]; then
     log::warn "ARTIFACT_DIR not set, skipping artifact save"
     return 0
   fi
 
-  mkdir -p "${ARTIFACT_DIR}/${artifacts_subdir}"
-  rsync -a "$file" "${ARTIFACT_DIR}/${artifacts_subdir}/"
+  local target_dir="${ARTIFACT_DIR}/${artifacts_subdir}"
+  if [[ -n "$subdir" ]]; then
+    target_dir="${target_dir}/${subdir}"
+  fi
+
+  mkdir -p "${target_dir}"
+  rsync -a "$file" "${target_dir}/"
 }
 
 # Export functions for subshell usage (e.g., timeout bash -c "...")

.ci/pipelines/lib/config.sh

@@ -94,7 +94,7 @@ metadata:
 data:
   dynamic-plugins.yaml: |
 EOF
-  yq '.global.dynamic' "${base_file}" | sed -e 's/^/    /' >> "${output_file}"
+  yq '.global.dynamic' "${base_file}" | sed -e 's/^/    /' -e 's/{{ "{{" }}inherit{{ "}}" }}/{{inherit}}/g' >> "${output_file}"
   return $?
 }
 

.ci/pipelines/lib/testing.sh

@@ -87,18 +87,16 @@ testing::run_tests() {
   pkill Xvfb || true
 
   # Use artifacts_subdir for artifact directory to keep artifacts organized
-  mkdir -p "${ARTIFACT_DIR}/${artifacts_subdir}/test-results"
-  mkdir -p "${ARTIFACT_DIR}/${artifacts_subdir}/attachments/screenshots"
-  rsync -a "${e2e_tests_dir}/test-results/" "${ARTIFACT_DIR}/${artifacts_subdir}/test-results/" || true
-  rsync -a "${e2e_tests_dir}/${JUNIT_RESULTS}" "${ARTIFACT_DIR}/${artifacts_subdir}/${JUNIT_RESULTS}" || true
+  common::save_artifact "${artifacts_subdir}" "${e2e_tests_dir}/test-results/" "test-results" || true
+  common::save_artifact "${artifacts_subdir}" "${e2e_tests_dir}/${JUNIT_RESULTS}" || true
   if [[ "${CI}" == "true" ]]; then
     rsync "${ARTIFACT_DIR}/${artifacts_subdir}/${JUNIT_RESULTS}" "${SHARED_DIR}/junit-results-${artifacts_subdir}.xml" || true
   fi
 
-  rsync -a "${e2e_tests_dir}/screenshots/" "${ARTIFACT_DIR}/${artifacts_subdir}/attachments/screenshots/" || true
+  common::save_artifact "${artifacts_subdir}" "${e2e_tests_dir}/screenshots/" "attachments/screenshots" || true
   ansi2html < "/tmp/${LOGFILE}" > "/tmp/${LOGFILE}.html"
-  rsync -a "/tmp/${LOGFILE}.html" "${ARTIFACT_DIR}/${artifacts_subdir}/" || true
-  rsync -a "${e2e_tests_dir}/playwright-report/" "${ARTIFACT_DIR}/${artifacts_subdir}/" || true
+  common::save_artifact "${artifacts_subdir}" "/tmp/${LOGFILE}.html" || true
+  common::save_artifact "${artifacts_subdir}" "${e2e_tests_dir}/playwright-report/" || true
 
   echo "Playwright project '${playwright_project}' in namespace '${namespace}' (artifacts: ${artifacts_subdir}) RESULT: ${test_result}"
   local test_passed="true"
@@ -134,26 +132,23 @@ testing::run_tests() {
 #   $1 - release_name: The Helm release name
 #   $2 - namespace: The namespace where Backstage is deployed
 #   $3 - url: The URL to check
-#   $4 - max_attempts: (optional) Maximum number of attempts (default: 30)
-#   $5 - wait_seconds: (optional) Seconds to wait between attempts (default: 30)
+#   $4 - artifacts_subdir: (optional) Subdirectory for artifacts (defaults to namespace)
+#   $5 - max_attempts: (optional) Maximum number of attempts (default: 30)
+#   $6 - wait_seconds: (optional) Seconds to wait between attempts (default: 30)
 # Returns:
 #   0 - Backstage is running
 #   1 - Backstage is not running or crashed
 testing::check_backstage_running() {
   local release_name=$1
   local namespace=$2
   local url=$3
-  local max_attempts=${4:-30}
-  local wait_seconds=${5:-30}
+  local artifacts_subdir=$4
+  local max_attempts=${5:-30}
+  local wait_seconds=${6:-30}
 
-  if [[ -z "$release_name" || -z "$namespace" ]]; then
+  if [[ -z "$release_name" || -z "$namespace" || -z "$url" || -z "$artifacts_subdir" ]]; then
     log::error "${_TESTING_ERR_MISSING_PARAMS}"
-    log::info "Usage: testing::check_backstage_running <release_name> <namespace> <url> [max_attempts] [wait_seconds]"
-    return 1
-  fi
-
-  if [[ -z "${url}" ]]; then
-    log::error "Error: URL is not set. Please provide a valid URL."
+    log::info "Usage: testing::check_backstage_running <release_name> <namespace> <url> <artifacts_subdir> [max_attempts] [wait_seconds]"
     return 1
   fi
 
@@ -190,8 +185,7 @@ testing::check_backstage_running() {
           || oc logs deployment/${release_name} -n "${namespace}" --tail=100 --all-containers=true 2> /dev/null || true
         log::error "Recent events:"
         oc get events -n "${namespace}" --sort-by='.lastTimestamp' | tail -20
-        mkdir -p "${ARTIFACT_DIR}/${namespace}"
-        rsync -a "/tmp/${LOGFILE}" "${ARTIFACT_DIR}/${namespace}/" || true
+        common::save_artifact "${artifacts_subdir}" "/tmp/${LOGFILE}" || true
         return 1
       fi
 
@@ -201,8 +195,7 @@ testing::check_backstage_running() {
 
   log::error "Failed to reach Backstage at ${url} after ${max_attempts} attempts."
   oc get events -n "${namespace}" --sort-by='.lastTimestamp' | tail -10
-  mkdir -p "${ARTIFACT_DIR}/${namespace}"
-  rsync -a "/tmp/${LOGFILE}" "${ARTIFACT_DIR}/${namespace}/" || true
+  common::save_artifact "${artifacts_subdir}" "/tmp/${LOGFILE}" || true
   return 1
 }
 
@@ -235,7 +228,7 @@ testing::check_and_test() {
     return 1
   fi
 
-  if testing::check_backstage_running "${release_name}" "${namespace}" "${url}" "${max_attempts}" "${wait_seconds}"; then
+  if testing::check_backstage_running "${release_name}" "${namespace}" "${url}" "${artifacts_subdir}" "${max_attempts}" "${wait_seconds}"; then
     echo "Display pods for verification..."
     oc get pods -n "${namespace}"
     if [[ "${SKIP_TESTS:-false}" == "true" ]]; then

.ci/pipelines/utils.sh

@@ -154,8 +154,7 @@ save_all_pod_logs() {
     wait "$pid" 2> /dev/null || true
   done
 
-  mkdir -p "${ARTIFACT_DIR}/${artifacts_subdir}/pod_logs"
-  rsync -a pod_logs/ "${ARTIFACT_DIR}/${artifacts_subdir}/pod_logs/" || true
+  common::save_artifact "${artifacts_subdir}" "pod_logs/" "pod_logs" || true
   set -e
 }
 
@@ -528,6 +527,8 @@ cluster_setup_k8s_helm() {
 # ==============================================================================
 
 base_deployment() {
+  local artifacts_subdir=$1
+
   namespace::configure ${NAME_SPACE}
 
   deploy_redis_cache "${NAME_SPACE}"
@@ -542,8 +543,7 @@ base_deployment() {
     helm::merge_values "merge" "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" "${DIR}/value_files/diff-values_showcase_PR.yaml" "${merged_pr_value_file}"
     disable_orchestrator_plugins_in_values "${merged_pr_value_file}"
 
-    mkdir -p "${ARTIFACT_DIR}/${NAME_SPACE}"
-    rsync -a "${merged_pr_value_file}" "${ARTIFACT_DIR}/${NAME_SPACE}/" || true
+    common::save_artifact "${artifacts_subdir}" "${merged_pr_value_file}" || true
     # shellcheck disable=SC2046
     helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE}" \
       "${HELM_CHART_URL}" --version "${CHART_VERSION}" \
@@ -562,6 +562,8 @@ base_deployment() {
 }
 
 rbac_deployment() {
+  local artifacts_subdir=$1
+
   namespace::configure "${NAME_SPACE_POSTGRES_DB}"
   namespace::configure "${NAME_SPACE_RBAC}"
   configure_external_postgres_db "${NAME_SPACE_RBAC}"
@@ -583,8 +585,7 @@ rbac_deployment() {
     helm::merge_values "merge" "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "${DIR}/value_files/diff-values_showcase-rbac_PR.yaml" "${merged_pr_rbac_value_file}"
     disable_orchestrator_plugins_in_values "${merged_pr_rbac_value_file}"
 
-    mkdir -p "${ARTIFACT_DIR}/${NAME_SPACE_RBAC}"
-    rsync -a "${merged_pr_rbac_value_file}" "${ARTIFACT_DIR}/${NAME_SPACE_RBAC}/" || true
+    common::save_artifact "${artifacts_subdir}" "${merged_pr_rbac_value_file}" || true
     # shellcheck disable=SC2046
     helm upgrade -i "${RELEASE_NAME_RBAC}" -n "${NAME_SPACE_RBAC}" \
       "${HELM_CHART_URL}" --version "${CHART_VERSION}" \
@@ -618,13 +619,18 @@ rbac_deployment() {
 }
 
 initiate_deployments() {
+  local base_artifacts_subdir=$1
+  local rbac_artifacts_subdir=$2
+
   cd "${DIR}"
-  base_deployment
-  rbac_deployment
+  base_deployment "${base_artifacts_subdir}"
+  rbac_deployment "${rbac_artifacts_subdir}"
 }
 
 # OSD-GCP specific deployment functions that merge diff files and skip orchestrator workflows
 base_deployment_osd_gcp() {
+  local artifacts_subdir=$1
+
   namespace::configure ${NAME_SPACE}
 
   deploy_redis_cache "${NAME_SPACE}"
@@ -635,8 +641,7 @@ base_deployment_osd_gcp() {
 
   # Merge base values with OSD-GCP diff file
   helm::merge_values "merge" "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" "${DIR}/value_files/${HELM_CHART_OSD_GCP_DIFF_VALUE_FILE_NAME}" "/tmp/merged-values_showcase_OSD-GCP.yaml"
-  mkdir -p "${ARTIFACT_DIR}/${NAME_SPACE}"
-  rsync -a "/tmp/merged-values_showcase_OSD-GCP.yaml" "${ARTIFACT_DIR}/${NAME_SPACE}/" # Save the final value-file into the artifacts directory.
+  common::save_artifact "${artifacts_subdir}" "/tmp/merged-values_showcase_OSD-GCP.yaml"
 
   log::info "Deploying image from repository: ${QUAY_REPO}, TAG_NAME: ${TAG_NAME}, in NAME_SPACE: ${NAME_SPACE}"
 
@@ -652,6 +657,8 @@ base_deployment_osd_gcp() {
 }
 
 rbac_deployment_osd_gcp() {
+  local artifacts_subdir=$1
+
   namespace::configure "${NAME_SPACE_POSTGRES_DB}"
   namespace::configure "${NAME_SPACE_RBAC}"
   configure_external_postgres_db "${NAME_SPACE_RBAC}"
@@ -662,8 +669,7 @@ rbac_deployment_osd_gcp() {
 
   # Merge RBAC values with OSD-GCP diff file
   helm::merge_values "merge" "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "${DIR}/value_files/${HELM_CHART_RBAC_OSD_GCP_DIFF_VALUE_FILE_NAME}" "/tmp/merged-values_showcase-rbac_OSD-GCP.yaml"
-  mkdir -p "${ARTIFACT_DIR}/${NAME_SPACE_RBAC}"
-  rsync -a "/tmp/merged-values_showcase-rbac_OSD-GCP.yaml" "${ARTIFACT_DIR}/${NAME_SPACE_RBAC}/" # Save the final value-file into the artifacts directory.
+  common::save_artifact "${artifacts_subdir}" "/tmp/merged-values_showcase-rbac_OSD-GCP.yaml"
 
   log::info "Deploying image from repository: ${QUAY_REPO}, TAG_NAME: ${TAG_NAME}, in NAME_SPACE: ${RELEASE_NAME_RBAC}"
 
@@ -679,9 +685,12 @@ rbac_deployment_osd_gcp() {
 }
 
 initiate_deployments_osd_gcp() {
+  local base_artifacts_subdir=$1
+  local rbac_artifacts_subdir=$2
+
   cd "${DIR}"
-  base_deployment_osd_gcp
-  rbac_deployment_osd_gcp
+  base_deployment_osd_gcp "${base_artifacts_subdir}"
+  rbac_deployment_osd_gcp "${rbac_artifacts_subdir}"
 }
 
 # install base RHDH deployment before upgrade
@@ -761,14 +770,14 @@ initiate_sanity_plugin_checks_deployment() {
   local release_name=$1
   local name_space_sanity_plugins_check=$2
   local sanity_plugins_url=$3
+  local artifacts_subdir=$4
 
   namespace::configure "${name_space_sanity_plugins_check}"
   helm::uninstall "${name_space_sanity_plugins_check}" "${release_name}"
   deploy_redis_cache "${name_space_sanity_plugins_check}"
   apply_yaml_files "${DIR}" "${name_space_sanity_plugins_check}" "${sanity_plugins_url}"
   helm::merge_values "overwrite" "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" "${DIR}/value_files/${HELM_CHART_SANITY_PLUGINS_DIFF_VALUE_FILE_NAME}" "/tmp/${HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME}"
-  mkdir -p "${ARTIFACT_DIR}/${name_space_sanity_plugins_check}"
-  rsync -a "/tmp/${HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME}" "${ARTIFACT_DIR}/${name_space_sanity_plugins_check}/" || true # Save the final value-file into the artifacts directory.
+  common::save_artifact "${artifacts_subdir}" "/tmp/${HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME}" || true
   # shellcheck disable=SC2046
   helm upgrade -i "${release_name}" -n "${name_space_sanity_plugins_check}" \
     "${HELM_CHART_URL}" --version "${CHART_VERSION}" \

e2e-tests/.gitignore

@@ -69,6 +69,9 @@ site
 # e2e test results
 junit-results.xml
 
+# Local e2e test run files
+.local-test
+
 # auth-providers local runs files
 **/app-config.test.yaml
 **/dynamic-plugins.test.yaml

e2e-tests/playwright.config.ts

@@ -11,7 +11,10 @@ const isPrOcpHelmJob =
 
 const isOsdGcpJob = process.env.JOB_NAME.includes("osd-gcp");
 
-const shouldSkipOrchestratorTests = isPrOcpHelmJob || isOsdGcpJob;
+const isNonOpenShiftJob = process.env.IS_OPENSHIFT === "false";
+
+const shouldSkipOrchestratorTests =
+  isPrOcpHelmJob || isOsdGcpJob || isNonOpenShiftJob;
 
 // Set LOCALE based on which project is being run
 const args = process.argv;