chore(ci): use nounset for CI scripts

.gitignore

@@ -66,3 +66,8 @@ dynamic-plugins-root/*
 
 # Local Cursor rules context
 .cursor/rules/*.local.mdc
+
+# CI/Pipeline local overrides
+.ibm/pipelines/shared_dir/*
+.ibm/pipelines/artifact_dir/*
+.ibm/pipelines/env_override.local.sh

.ibm/pipelines/cleanup.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+cleanup() {
+  if [[ $? -ne 0 ]]; then
+
+    echo "Exited with an error, setting OVERALL_RESULT to 1"
+    save_overall_result 1
+  fi
+  echo "Cleaning up before exiting"
+  if [[ "${OPENSHIFT_CI}" == "true" ]]; then
+    case "$JOB_NAME" in
+      *gke*)
+        echo "Calling cleanup_gke"
+        cleanup_gke
+        ;;
+    esac
+  fi
+  rm -rf ~/tmpbin
+}

.ibm/pipelines/env_variables.sh

@@ -2,6 +2,29 @@
 # shellcheck disable=SC2034
 set -a  # Automatically export all variables
 
+# Define log file names and directories.
+LOGFILE="test-log"
+
+# Populated by OpenShift CI or the initial CI scripts
+# Addition to JOB_NAME, TAG_NAME, SHARED_DIR, ARTIFACT_DIR
+# This prevents nounset errors when running locally
+# https://docs.ci.openshift.org/docs/architecture/step-registry/#available-environment-variables
+# https://docs.prow.k8s.io/docs/jobs/#job-environment-variables
+JOB_NAME="${JOB_NAME:-unknown-job}"
+TAG_NAME="${TAG_NAME:-}"
+OPENSHIFT_CI="${OPENSHIFT_CI:-false}"
+REPO_OWNER="${REPO_OWNER:-redhat-developer}"
+REPO_NAME="${REPO_NAME:-rhdh}"
+PULL_NUMBER="${PULL_NUMBER:-}"
+BUILD_ID="${BUILD_ID:-unknown-build}"
+RELEASE_BRANCH_NAME="${RELEASE_BRANCH_NAME:-main}"
+K8S_CLUSTER_TOKEN="${K8S_CLUSTER_TOKEN:-}"
+K8S_CLUSTER_URL="${K8S_CLUSTER_URL:-}"
+SHARED_DIR="${SHARED_DIR:-$DIR/shared_dir}"
+ARTIFACT_DIR="${ARTIFACT_DIR:-$DIR/artifact_dir}"
+mkdir -p "${SHARED_DIR}"
+mkdir -p "${ARTIFACT_DIR}"
+
 #ENVS and Vault Secrets
 HELM_CHART_VALUE_FILE_NAME="values_showcase.yaml"
 HELM_CHART_RBAC_VALUE_FILE_NAME="values_showcase-rbac.yaml"

.ibm/pipelines/jobs/ocp-nightly.sh

@@ -35,7 +35,7 @@ run_runtime_config_change_tests() {
 }
 
 run_sanity_plugins_check() {
-  initiate_sanity_plugin_checks_deployment "${RELEASE_NAME}" "${NAME_SPACE_SANITY_PLUGINS_CHECK}"
   local sanity_plugins_url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE_SANITY_PLUGINS_CHECK}.${K8S_CLUSTER_ROUTER_BASE}"
+  initiate_sanity_plugin_checks_deployment "${RELEASE_NAME}" "${NAME_SPACE_SANITY_PLUGINS_CHECK}" "${sanity_plugins_url}"
   check_and_test "${RELEASE_NAME}" "${NAME_SPACE_SANITY_PLUGINS_CHECK}" "${sanity_plugins_url}"
 }

.ibm/pipelines/openshift-ci-tests.sh

@@ -1,63 +1,41 @@
 #!/bin/bash
 
-set -e
+set -o errexit
+set -o errtrace
+set -o nounset
 export PS4='[$(date "+%Y-%m-%d %H:%M:%S")] ' # logs timestamp for every cmd.
 
-# Define log file names and directories.
-LOGFILE="test-log"
-export DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-export CURRENT_DEPLOYMENT=0 # Counter for current deployment.
-export STATUS_DEPLOYMENT_NAMESPACE # Array that holds the namespaces of deployments.
-export STATUS_FAILED_TO_DEPLOY # Array that indicates if deployment failed. false = success, true = failure
-export STATUS_TEST_FAILED # Array that indicates if test run failed. false = success, true = failure
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+export DIR
+
+export OPENSHIFT_CI="${OPENSHIFT_CI:-false}"
+if [[ -z "${OPENSHIFT_CI}" || "${OPENSHIFT_CI}" == "false" ]]; then
+  # NOTE: Use this file to override the environment variables for the local testing.
+  echo "Sourcing env_override.local.sh"
+  # shellcheck source=.ibm/pipelines/env_override.local.sh
+  source "${DIR}/env_override.local.sh"
+fi
+
+echo "Sourcing env_variables.sh"
+# shellcheck source=.ibm/pipelines/env_variables.sh
+source "${DIR}/env_variables.sh"
 
 echo "Sourcing reporting.sh"
 # shellcheck source=.ibm/pipelines/reporting.sh
 source "${DIR}/reporting.sh"
 save_overall_result 0 # Initialize overall result to 0 (success).
-export OVERALL_RESULT
 
 # Define a cleanup function to be executed upon script exit.
-# shellcheck disable=SC2317
-cleanup() {
-  if [[ $? -ne 0 ]]; then
-
-    echo "Exited with an error, setting OVERALL_RESULT to 1"
-    save_overall_result 1
-  fi
-  echo "Cleaning up before exiting"
-  if [[ "${OPENSHIFT_CI}" == "true" ]]; then
-    case "$JOB_NAME" in
-      *gke*)
-        echo "Calling cleanup_gke"
-        cleanup_gke
-        ;;
-    esac
-  fi
-  rm -rf ~/tmpbin
-}
-
+source "${DIR}/cleanup.sh"
 trap cleanup EXIT INT ERR
 
-SCRIPTS=(
-  "utils.sh"
-  "env_variables.sh"
-  "clear-database.sh"
-)
-
-# Source explicitly specified scripts
-for SCRIPT in "${SCRIPTS[@]}"; do
-  source "${DIR}/${SCRIPT}"
-  echo "Loaded ${SCRIPT}"
-done
+echo "Sourcing utils.sh"
+# shellcheck source=.ibm/pipelines/utils.sh
+source "${DIR}/utils.sh"
 
-# Source all scripts in jobs directory
-for SCRIPT in "${DIR}"/jobs/*.sh; do
-  if [ -f "$SCRIPT" ]; then
-    source "$SCRIPT"
-    echo "Loaded ${SCRIPT}"
-  fi
-done
+echo "Sourcing clear-database.sh"
+# shellcheck source=.ibm/pipelines/clear-database.sh
+source "${DIR}/clear-database.sh"
 
 main() {
   echo "Log file: ${LOGFILE}"
@@ -69,46 +47,79 @@ main() {
 
   case "$JOB_NAME" in
     *aks-helm*)
+      echo "Sourcing aks-helm.sh"
+      # shellcheck source=.ibm/pipelines/jobs/aks-helm.sh
+      source "${DIR}/jobs/aks-helm.sh"
       echo "Calling handle_aks_helm"
       handle_aks_helm
       ;;
     *aks-operator*)
-      echo "Calling handle_aks_helm"
+      echo "Sourcing aks-operator.sh"
+      # shellcheck source=.ibm/pipelines/jobs/aks-operator.sh
+      source "${DIR}/jobs/aks-operator.sh"
+      echo "Calling handle_aks_operator"
       handle_aks_operator
       ;;
     *eks-helm*)
+      echo "Sourcing eks-helm.sh"
+      # shellcheck source=.ibm/pipelines/jobs/eks-helm.sh
+      source "${DIR}/jobs/eks-helm.sh"
       echo "Calling handle_eks_helm"
       handle_eks_helm
       ;;
     *eks-operator*)
+      echo "Sourcing eks-operator.sh"
+      # shellcheck source=.ibm/pipelines/jobs/eks-operator.sh
+      source "${DIR}/jobs/eks-operator.sh"
       echo "Calling handle_eks_operator"
       handle_eks_operator
       ;;
     *e2e-tests-auth-providers-nightly)
+      echo "Sourcing auth-providers.sh"
+      # shellcheck source=.ibm/pipelines/jobs/auth-providers.sh
+      source "${DIR}/jobs/auth-providers.sh"
       echo "Calling handle_auth_providers"
       handle_auth_providers
       ;;
     *gke-helm*)
+      echo "Sourcing gke-helm.sh"
+      # shellcheck source=.ibm/pipelines/jobs/gke-helm.sh
+      source "${DIR}/jobs/gke-helm.sh"
       echo "Calling handle_gke_helm"
       handle_gke_helm
       ;;
     *gke-operator*)
+      echo "Sourcing gke-operator.sh"
+      # shellcheck source=.ibm/pipelines/jobs/gke-operator.sh
+      source "${DIR}/jobs/gke-operator.sh"
       echo "Calling handle_gke_operator"
       handle_gke_operator
       ;;
     *operator*)
+      echo "Sourcing ocp-operator.sh"
+      # shellcheck source=.ibm/pipelines/jobs/ocp-operator.sh
+      source "${DIR}/jobs/ocp-operator.sh"
       echo "Calling handle_ocp_operator"
       handle_ocp_operator
       ;;
     *upgrade*)
+      echo "Sourcing upgrade.sh"
+      # shellcheck source=.ibm/pipelines/jobs/upgrade.sh
+      source "${DIR}/jobs/upgrade.sh"
       echo "Calling helm upgrade"
       handle_ocp_helm_upgrade
       ;;
     *nightly*)
+      echo "Sourcing ocp-nightly.sh"
+      # shellcheck source=.ibm/pipelines/jobs/ocp-nightly.sh
+      source "${DIR}/jobs/ocp-nightly.sh"
       echo "Calling handle_ocp_nightly"
       handle_ocp_nightly
       ;;
     *pull*)
+      echo "Sourcing ocp-pull.sh"
+      # shellcheck source=.ibm/pipelines/jobs/ocp-pull.sh
+      source "${DIR}/jobs/ocp-pull.sh"
       echo "Calling handle_ocp_pull"
       handle_ocp_pull
       ;;
@@ -121,7 +132,6 @@ main() {
 
   echo "Main script completed with result: ${OVERALL_RESULT}"
   exit "${OVERALL_RESULT}"
-
 }
 
 main

.ibm/pipelines/reporting.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
 
+# Variables for reporting
+export CURRENT_DEPLOYMENT=0 # Counter for current deployment.
+export STATUS_DEPLOYMENT_NAMESPACE # Array that holds the namespaces of deployments.
+export STATUS_FAILED_TO_DEPLOY # Array that indicates if deployment failed. false = success, true = failure
+export STATUS_TEST_FAILED # Array that indicates if test run failed. false = success, true = failure
+export OVERALL_RESULT # Overall result of the test run. 0 = success, 1 = failure
+
 mkdir -p "$ARTIFACT_DIR/reporting"
 
 save_status_deployment_namespace() {

.ibm/pipelines/utils.sh

@@ -846,14 +846,18 @@ initiate_runtime_deployment() {
 }
 
 initiate_sanity_plugin_checks_deployment() {
-  configure_namespace "${NAME_SPACE_SANITY_PLUGINS_CHECK}"
-  uninstall_helmchart "${NAME_SPACE_SANITY_PLUGINS_CHECK}" "${RELEASE_NAME}"
-  deploy_redis_cache "${NAME_SPACE_SANITY_PLUGINS_CHECK}"
-  apply_yaml_files "${DIR}" "${NAME_SPACE_SANITY_PLUGINS_CHECK}" "${sanity_plugins_url}"
+  local release_name=$1
+  local name_space_sanity_plugins_check=$2
+  local sanity_plugins_url=$3
+
+  configure_namespace "${name_space_sanity_plugins_check}"
+  uninstall_helmchart "${name_space_sanity_plugins_check}" "${release_name}"
+  deploy_redis_cache "${name_space_sanity_plugins_check}"
+  apply_yaml_files "${DIR}" "${name_space_sanity_plugins_check}" "${sanity_plugins_url}"
   yq_merge_value_files "overwrite" "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" "${DIR}/value_files/${HELM_CHART_SANITY_PLUGINS_DIFF_VALUE_FILE_NAME}" "/tmp/${HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME}"
-  mkdir -p "${ARTIFACT_DIR}/${NAME_SPACE_SANITY_PLUGINS_CHECK}"
-  cp -a "/tmp/${HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME}" "${ARTIFACT_DIR}/${NAME_SPACE_SANITY_PLUGINS_CHECK}/" || true # Save the final value-file into the artifacts directory.
-  helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE_SANITY_PLUGINS_CHECK}" \
+  mkdir -p "${ARTIFACT_DIR}/${name_space_sanity_plugins_check}"
+  cp -a "/tmp/${HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME}" "${ARTIFACT_DIR}/${name_space_sanity_plugins_check}/" || true # Save the final value-file into the artifacts directory.
+  helm upgrade -i "${release_name}" -n "${name_space_sanity_plugins_check}" \
     "${HELM_CHART_URL}" --version "${CHART_VERSION}" \
     -f "/tmp/${HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME}" \
     --set global.clusterRouterBase="${K8S_CLUSTER_ROUTER_BASE}" \