From 7954e6df43d925516cc06d452867fb83fa36ec31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Wed, 27 Aug 2025 15:25:47 +0200 Subject: [PATCH 1/4] chore(ci): use nounset for CI scripts --- .gitignore | 4 ++++ .ibm/pipelines/openshift-ci-tests.sh | 26 ++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 1ec85b89be..de588d4f82 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,7 @@ dynamic-plugins-root/* # Local Cursor rules context .cursor/rules/*.local.mdc + +# CI/Pipeline directories +.ibm/pipelines/shared_dir/* +.ibm/pipelines/artifact_dir/* diff --git a/.ibm/pipelines/openshift-ci-tests.sh b/.ibm/pipelines/openshift-ci-tests.sh index 48eecec6bc..c5a45509c4 100755 --- a/.ibm/pipelines/openshift-ci-tests.sh +++ b/.ibm/pipelines/openshift-ci-tests.sh @@ -1,11 +1,33 @@ #!/bin/bash -set -e +set -o errexit +set -o errtrace +set -o nounset export PS4='[$(date "+%Y-%m-%d %H:%M:%S")] ' # logs timestamp for every cmd. # Define log file names and directories. LOGFILE="test-log" export DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Set default values for OpenShift CI variables, unless populated by CI +# This prevents nounset errors when running locally +# https://docs.ci.openshift.org/docs/architecture/step-registry/#available-environment-variables +# https://docs.prow.k8s.io/docs/jobs/#job-environment-variables +export JOB_NAME="${JOB_NAME:-unknown-job}" +export OPENSHIFT_CI="${OPENSHIFT_CI:-false}" +export REPO_OWNER="${REPO_OWNER:-redhat-developer}" +export REPO_NAME="${REPO_NAME:-rhdh}" +export PULL_NUMBER="${PULL_NUMBER:-}" +export BUILD_ID="${BUILD_ID:-unknown-build}" +export RELEASE_BRANCH_NAME="${RELEASE_BRANCH_NAME:-main}" +export K8S_CLUSTER_TOKEN="${K8S_CLUSTER_TOKEN:-}" +export K8S_CLUSTER_URL="${K8S_CLUSTER_URL:-}" +export SHARED_DIR="${SHARED_DIR:-$DIR/shared_dir}" +export ARTIFACT_DIR="${ARTIFACT_DIR:-$DIR/artifact_dir}" +mkdir -p "${SHARED_DIR}" +mkdir -p "${ARTIFACT_DIR}" + +# Define variables for reporting export CURRENT_DEPLOYMENT=0 # Counter for current deployment. export STATUS_DEPLOYMENT_NAMESPACE # Array that holds the namespaces of deployments. export STATUS_FAILED_TO_DEPLOY # Array that indicates if deployment failed. false = success, true = failure @@ -18,7 +40,7 @@ save_overall_result 0 # Initialize overall result to 0 (success). export OVERALL_RESULT # Define a cleanup function to be executed upon script exit. -# shellcheck disable=SC2317 +# shellcheck disable=SC2329 cleanup() { if [[ $? -ne 0 ]]; then From 7434b87da9c307f5608cbf2ed2edcbacaeb728ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Fri, 29 Aug 2025 14:21:14 +0200 Subject: [PATCH 2/4] Fix sanity checks --- .ibm/pipelines/jobs/ocp-nightly.sh | 2 +- .ibm/pipelines/utils.sh | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.ibm/pipelines/jobs/ocp-nightly.sh b/.ibm/pipelines/jobs/ocp-nightly.sh index b4d6050516..bfde75b789 100644 --- a/.ibm/pipelines/jobs/ocp-nightly.sh +++ b/.ibm/pipelines/jobs/ocp-nightly.sh @@ -35,7 +35,7 @@ run_runtime_config_change_tests() { } run_sanity_plugins_check() { - initiate_sanity_plugin_checks_deployment "${RELEASE_NAME}" "${NAME_SPACE_SANITY_PLUGINS_CHECK}" local sanity_plugins_url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE_SANITY_PLUGINS_CHECK}.${K8S_CLUSTER_ROUTER_BASE}" + initiate_sanity_plugin_checks_deployment "${RELEASE_NAME}" "${NAME_SPACE_SANITY_PLUGINS_CHECK}" "${sanity_plugins_url}" check_and_test "${RELEASE_NAME}" "${NAME_SPACE_SANITY_PLUGINS_CHECK}" "${sanity_plugins_url}" } diff --git a/.ibm/pipelines/utils.sh b/.ibm/pipelines/utils.sh index 08bb7b6327..bfebf2fa2f 100755 --- a/.ibm/pipelines/utils.sh +++ b/.ibm/pipelines/utils.sh @@ -846,14 +846,18 @@ initiate_runtime_deployment() { } initiate_sanity_plugin_checks_deployment() { - configure_namespace "${NAME_SPACE_SANITY_PLUGINS_CHECK}" - uninstall_helmchart "${NAME_SPACE_SANITY_PLUGINS_CHECK}" "${RELEASE_NAME}" - deploy_redis_cache "${NAME_SPACE_SANITY_PLUGINS_CHECK}" - apply_yaml_files "${DIR}" "${NAME_SPACE_SANITY_PLUGINS_CHECK}" "${sanity_plugins_url}" + local release_name=$1 + local name_space_sanity_plugins_check=$2 + local sanity_plugins_url=$3 + + configure_namespace "${name_space_sanity_plugins_check}" + uninstall_helmchart "${name_space_sanity_plugins_check}" "${release_name}" + deploy_redis_cache "${name_space_sanity_plugins_check}" + apply_yaml_files "${DIR}" "${name_space_sanity_plugins_check}" "${sanity_plugins_url}" yq_merge_value_files "overwrite" "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" "${DIR}/value_files/${HELM_CHART_SANITY_PLUGINS_DIFF_VALUE_FILE_NAME}" "/tmp/${HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME}" - mkdir -p "${ARTIFACT_DIR}/${NAME_SPACE_SANITY_PLUGINS_CHECK}" - cp -a "/tmp/${HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME}" "${ARTIFACT_DIR}/${NAME_SPACE_SANITY_PLUGINS_CHECK}/" || true # Save the final value-file into the artifacts directory. - helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE_SANITY_PLUGINS_CHECK}" \ + mkdir -p "${ARTIFACT_DIR}/${name_space_sanity_plugins_check}" + cp -a "/tmp/${HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME}" "${ARTIFACT_DIR}/${name_space_sanity_plugins_check}/" || true # Save the final value-file into the artifacts directory. + helm upgrade -i "${release_name}" -n "${name_space_sanity_plugins_check}" \ "${HELM_CHART_URL}" --version "${CHART_VERSION}" \ -f "/tmp/${HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME}" \ --set global.clusterRouterBase="${K8S_CLUSTER_ROUTER_BASE}" \ From 4eb46c985e10d0e4a79ee3d196ab4bd0cf15a019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Mon, 1 Sep 2025 10:23:00 +0200 Subject: [PATCH 3/4] Refactor openshift-ci-tests --- .gitignore | 3 +- .ibm/pipelines/env_variables.sh | 23 +++++++ .ibm/pipelines/openshift-ci-tests.sh | 93 +++++++++++++++------------- .ibm/pipelines/reporting.sh | 6 ++ 4 files changed, 80 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index de588d4f82..4836dbf3f7 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,7 @@ dynamic-plugins-root/* # Local Cursor rules context .cursor/rules/*.local.mdc -# CI/Pipeline directories +# CI/Pipeline local overrides .ibm/pipelines/shared_dir/* .ibm/pipelines/artifact_dir/* +.ibm/pipelines/env_override.local.sh diff --git a/.ibm/pipelines/env_variables.sh b/.ibm/pipelines/env_variables.sh index d5be86b00c..a85bf9af91 100755 --- a/.ibm/pipelines/env_variables.sh +++ b/.ibm/pipelines/env_variables.sh @@ -2,6 +2,29 @@ # shellcheck disable=SC2034 set -a # Automatically export all variables +# Define log file names and directories. +LOGFILE="test-log" + +# Populated by OpenShift CI or the initial CI scripts +# Addition to JOB_NAME, TAG_NAME, SHARED_DIR, ARTIFACT_DIR +# This prevents nounset errors when running locally +# https://docs.ci.openshift.org/docs/architecture/step-registry/#available-environment-variables +# https://docs.prow.k8s.io/docs/jobs/#job-environment-variables +JOB_NAME="${JOB_NAME:-unknown-job}" +TAG_NAME="${TAG_NAME:-}" +OPENSHIFT_CI="${OPENSHIFT_CI:-false}" +REPO_OWNER="${REPO_OWNER:-redhat-developer}" +REPO_NAME="${REPO_NAME:-rhdh}" +PULL_NUMBER="${PULL_NUMBER:-}" +BUILD_ID="${BUILD_ID:-unknown-build}" +RELEASE_BRANCH_NAME="${RELEASE_BRANCH_NAME:-main}" +K8S_CLUSTER_TOKEN="${K8S_CLUSTER_TOKEN:-}" +K8S_CLUSTER_URL="${K8S_CLUSTER_URL:-}" +SHARED_DIR="${SHARED_DIR:-$DIR/shared_dir}" +ARTIFACT_DIR="${ARTIFACT_DIR:-$DIR/artifact_dir}" +mkdir -p "${SHARED_DIR}" +mkdir -p "${ARTIFACT_DIR}" + #ENVS and Vault Secrets HELM_CHART_VALUE_FILE_NAME="values_showcase.yaml" HELM_CHART_RBAC_VALUE_FILE_NAME="values_showcase-rbac.yaml" diff --git a/.ibm/pipelines/openshift-ci-tests.sh b/.ibm/pipelines/openshift-ci-tests.sh index c5a45509c4..21c708c489 100755 --- a/.ibm/pipelines/openshift-ci-tests.sh +++ b/.ibm/pipelines/openshift-ci-tests.sh @@ -5,33 +5,20 @@ set -o errtrace set -o nounset export PS4='[$(date "+%Y-%m-%d %H:%M:%S")] ' # logs timestamp for every cmd. -# Define log file names and directories. -LOGFILE="test-log" -export DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export DIR -# Set default values for OpenShift CI variables, unless populated by CI -# This prevents nounset errors when running locally -# https://docs.ci.openshift.org/docs/architecture/step-registry/#available-environment-variables -# https://docs.prow.k8s.io/docs/jobs/#job-environment-variables -export JOB_NAME="${JOB_NAME:-unknown-job}" export OPENSHIFT_CI="${OPENSHIFT_CI:-false}" -export REPO_OWNER="${REPO_OWNER:-redhat-developer}" -export REPO_NAME="${REPO_NAME:-rhdh}" -export PULL_NUMBER="${PULL_NUMBER:-}" -export BUILD_ID="${BUILD_ID:-unknown-build}" -export RELEASE_BRANCH_NAME="${RELEASE_BRANCH_NAME:-main}" -export K8S_CLUSTER_TOKEN="${K8S_CLUSTER_TOKEN:-}" -export K8S_CLUSTER_URL="${K8S_CLUSTER_URL:-}" -export SHARED_DIR="${SHARED_DIR:-$DIR/shared_dir}" -export ARTIFACT_DIR="${ARTIFACT_DIR:-$DIR/artifact_dir}" -mkdir -p "${SHARED_DIR}" -mkdir -p "${ARTIFACT_DIR}" +if [[ -z "${OPENSHIFT_CI}" || "${OPENSHIFT_CI}" == "false" ]]; then + # NOTE: Use this file to override the environment variables for the local testing. + echo "Sourcing env_override.local.sh" + # shellcheck source=.ibm/pipelines/env_override.local.sh + source "${DIR}/env_override.local.sh" +fi -# Define variables for reporting -export CURRENT_DEPLOYMENT=0 # Counter for current deployment. -export STATUS_DEPLOYMENT_NAMESPACE # Array that holds the namespaces of deployments. -export STATUS_FAILED_TO_DEPLOY # Array that indicates if deployment failed. false = success, true = failure -export STATUS_TEST_FAILED # Array that indicates if test run failed. false = success, true = failure +echo "Sourcing env_variables.sh" +# shellcheck source=.ibm/pipelines/env_variables.sh +source "${DIR}/env_variables.sh" echo "Sourcing reporting.sh" # shellcheck source=.ibm/pipelines/reporting.sh @@ -61,25 +48,10 @@ cleanup() { trap cleanup EXIT INT ERR -SCRIPTS=( - "utils.sh" - "env_variables.sh" - "clear-database.sh" -) - -# Source explicitly specified scripts -for SCRIPT in "${SCRIPTS[@]}"; do - source "${DIR}/${SCRIPT}" - echo "Loaded ${SCRIPT}" -done - -# Source all scripts in jobs directory -for SCRIPT in "${DIR}"/jobs/*.sh; do - if [ -f "$SCRIPT" ]; then - source "$SCRIPT" - echo "Loaded ${SCRIPT}" - fi -done +# shellcheck source=.ibm/pipelines/utils.sh +source "${DIR}/utils.sh" +# shellcheck source=.ibm/pipelines/clear-database.sh +source "${DIR}/clear-database.sh" main() { echo "Log file: ${LOGFILE}" @@ -91,46 +63,79 @@ main() { case "$JOB_NAME" in *aks-helm*) + echo "Sourcing aks-helm.sh" + # shellcheck source=.ibm/pipelines/jobs/aks-helm.sh + source "${DIR}/jobs/aks-helm.sh" echo "Calling handle_aks_helm" handle_aks_helm ;; *aks-operator*) - echo "Calling handle_aks_helm" + echo "Sourcing aks-operator.sh" + # shellcheck source=.ibm/pipelines/jobs/aks-operator.sh + source "${DIR}/jobs/aks-operator.sh" + echo "Calling handle_aks_operator" handle_aks_operator ;; *eks-helm*) + echo "Sourcing eks-helm.sh" + # shellcheck source=.ibm/pipelines/jobs/eks-helm.sh + source "${DIR}/jobs/eks-helm.sh" echo "Calling handle_eks_helm" handle_eks_helm ;; *eks-operator*) + echo "Sourcing eks-operator.sh" + # shellcheck source=.ibm/pipelines/jobs/eks-operator.sh + source "${DIR}/jobs/eks-operator.sh" echo "Calling handle_eks_operator" handle_eks_operator ;; *e2e-tests-auth-providers-nightly) + echo "Sourcing auth-providers.sh" + # shellcheck source=.ibm/pipelines/jobs/auth-providers.sh + source "${DIR}/jobs/auth-providers.sh" echo "Calling handle_auth_providers" handle_auth_providers ;; *gke-helm*) + echo "Sourcing gke-helm.sh" + # shellcheck source=.ibm/pipelines/jobs/gke-helm.sh + source "${DIR}/jobs/gke-helm.sh" echo "Calling handle_gke_helm" handle_gke_helm ;; *gke-operator*) + echo "Sourcing gke-operator.sh" + # shellcheck source=.ibm/pipelines/jobs/gke-operator.sh + source "${DIR}/jobs/gke-operator.sh" echo "Calling handle_gke_operator" handle_gke_operator ;; *operator*) + echo "Sourcing ocp-operator.sh" + # shellcheck source=.ibm/pipelines/jobs/ocp-operator.sh + source "${DIR}/jobs/ocp-operator.sh" echo "Calling handle_ocp_operator" handle_ocp_operator ;; *upgrade*) + echo "Sourcing upgrade.sh" + # shellcheck source=.ibm/pipelines/jobs/upgrade.sh + source "${DIR}/jobs/upgrade.sh" echo "Calling helm upgrade" handle_ocp_helm_upgrade ;; *nightly*) + echo "Sourcing ocp-nightly.sh" + # shellcheck source=.ibm/pipelines/jobs/ocp-nightly.sh + source "${DIR}/jobs/ocp-nightly.sh" echo "Calling handle_ocp_nightly" handle_ocp_nightly ;; *pull*) + echo "Sourcing ocp-pull.sh" + # shellcheck source=.ibm/pipelines/jobs/ocp-pull.sh + source "${DIR}/jobs/ocp-pull.sh" echo "Calling handle_ocp_pull" handle_ocp_pull ;; diff --git a/.ibm/pipelines/reporting.sh b/.ibm/pipelines/reporting.sh index ce54f6d3e2..2bb994abdf 100644 --- a/.ibm/pipelines/reporting.sh +++ b/.ibm/pipelines/reporting.sh @@ -1,5 +1,11 @@ #!/bin/bash +# Variables for reporting +export CURRENT_DEPLOYMENT=0 # Counter for current deployment. +export STATUS_DEPLOYMENT_NAMESPACE # Array that holds the namespaces of deployments. +export STATUS_FAILED_TO_DEPLOY # Array that indicates if deployment failed. false = success, true = failure +export STATUS_TEST_FAILED # Array that indicates if test run failed. false = success, true = failure + mkdir -p "$ARTIFACT_DIR/reporting" save_status_deployment_namespace() { From 4122e7537ef0a50899e327796cb2649484e3f48b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Mon, 1 Sep 2025 10:28:38 +0200 Subject: [PATCH 4/4] Refactor cleanup --- .ibm/pipelines/cleanup.sh | 19 +++++++++++++++++++ .ibm/pipelines/openshift-ci-tests.sh | 25 ++++--------------------- .ibm/pipelines/reporting.sh | 1 + 3 files changed, 24 insertions(+), 21 deletions(-) create mode 100644 .ibm/pipelines/cleanup.sh diff --git a/.ibm/pipelines/cleanup.sh b/.ibm/pipelines/cleanup.sh new file mode 100644 index 0000000000..ad1e60fced --- /dev/null +++ b/.ibm/pipelines/cleanup.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +cleanup() { + if [[ $? -ne 0 ]]; then + + echo "Exited with an error, setting OVERALL_RESULT to 1" + save_overall_result 1 + fi + echo "Cleaning up before exiting" + if [[ "${OPENSHIFT_CI}" == "true" ]]; then + case "$JOB_NAME" in + *gke*) + echo "Calling cleanup_gke" + cleanup_gke + ;; + esac + fi + rm -rf ~/tmpbin +} \ No newline at end of file diff --git a/.ibm/pipelines/openshift-ci-tests.sh b/.ibm/pipelines/openshift-ci-tests.sh index 21c708c489..e58b8a1e5e 100755 --- a/.ibm/pipelines/openshift-ci-tests.sh +++ b/.ibm/pipelines/openshift-ci-tests.sh @@ -24,32 +24,16 @@ echo "Sourcing reporting.sh" # shellcheck source=.ibm/pipelines/reporting.sh source "${DIR}/reporting.sh" save_overall_result 0 # Initialize overall result to 0 (success). -export OVERALL_RESULT # Define a cleanup function to be executed upon script exit. -# shellcheck disable=SC2329 -cleanup() { - if [[ $? -ne 0 ]]; then - - echo "Exited with an error, setting OVERALL_RESULT to 1" - save_overall_result 1 - fi - echo "Cleaning up before exiting" - if [[ "${OPENSHIFT_CI}" == "true" ]]; then - case "$JOB_NAME" in - *gke*) - echo "Calling cleanup_gke" - cleanup_gke - ;; - esac - fi - rm -rf ~/tmpbin -} - +source "${DIR}/cleanup.sh" trap cleanup EXIT INT ERR +echo "Sourcing utils.sh" # shellcheck source=.ibm/pipelines/utils.sh source "${DIR}/utils.sh" + +echo "Sourcing clear-database.sh" # shellcheck source=.ibm/pipelines/clear-database.sh source "${DIR}/clear-database.sh" @@ -148,7 +132,6 @@ main() { echo "Main script completed with result: ${OVERALL_RESULT}" exit "${OVERALL_RESULT}" - } main diff --git a/.ibm/pipelines/reporting.sh b/.ibm/pipelines/reporting.sh index 2bb994abdf..3f835859e7 100644 --- a/.ibm/pipelines/reporting.sh +++ b/.ibm/pipelines/reporting.sh @@ -5,6 +5,7 @@ export CURRENT_DEPLOYMENT=0 # Counter for current deployment. export STATUS_DEPLOYMENT_NAMESPACE # Array that holds the namespaces of deployments. export STATUS_FAILED_TO_DEPLOY # Array that indicates if deployment failed. false = success, true = failure export STATUS_TEST_FAILED # Array that indicates if test run failed. false = success, true = failure +export OVERALL_RESULT # Overall result of the test run. 0 = success, 1 = failure mkdir -p "$ARTIFACT_DIR/reporting"