From 2c2d575dc4c7250f1d7c80406f50d0715fb7d2f2 Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Thu, 2 Jul 2026 16:36:13 +0100 Subject: [PATCH 1/8] feat: parallelize bundle and image processing in prepare-restricted-environment.sh Parallelize all network-bound operations in the script for an estimated ~8x speedup in disconnected environment workflows: - Add MAX_PARALLEL variable (default 10) with --max-parallel CLI flag - Add reusable throttle_parallel() and wait_for_pids() helper functions - Parallelize process_bundles() with per-worker sed files for race-free render.yaml updates and inner parallel image mirroring - Parallelize process_bundles_from_dir() with same pattern, converting find|while-read to mapfile-based iteration for PID tracking - Parallelize mirror_extra_images() and mirror_extra_images_from_dir() with pre-created namespaces to avoid TOCTOU races - Parallelize ocp_prepare_internal_registry() namespace/policy/secret setup - Add top-level fork-join so extra image mirroring runs concurrently with bundle processing - Update traps to clean up background jobs on exit/signal Setting MAX_PARALLEL=1 restores fully sequential behavior for debugging. Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .../scripts/prepare-restricted-environment.sh | 708 ++++++++++++------ 1 file changed, 489 insertions(+), 219 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index f072b3f7b..3d42e68bb 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -17,6 +17,13 @@ IS_HOSTED_CONTROL_PLANE="" NAMESPACE_OPERATOR="rhdh-operator" INDEX_IMAGE="registry.redhat.io/redhat/redhat-operator-index:v4.18" FILTERED_VERSIONS=(*) +CATALOG_PULL_SECRET="" + +MAX_PARALLEL="${MAX_PARALLEL:-10}" +if ! [[ "$MAX_PARALLEL" =~ ^[0-9]+$ ]] || [[ "$MAX_PARALLEL" -lt 1 ]]; then + echo "[ERROR] MAX_PARALLEL must be a positive integer, got: '$MAX_PARALLEL'" >&2 + exit 1 +fi # assume mikefarah version of yq is already available on the path; if 1, then install the version shown INSTALL_YQ=0 @@ -53,6 +60,38 @@ function errorf() { logf "ERROR" "\033[0;31m" "$1" } +function throttle_parallel() { + local -n __tp_pids=$1 + while true; do + local running=0 + for pid in ${__tp_pids[@]+"${__tp_pids[@]}"}; do + if kill -0 "$pid" 2>/dev/null; then + running=$((running + 1)) + fi + done + if [[ $running -lt $MAX_PARALLEL ]]; then + break + fi + sleep 0.2 + done +} + +function wait_for_pids() { + local -n __wfp_pids=$1 + local context="${2:-background job}" + local failed=0 + for pid in ${__wfp_pids[@]+"${__wfp_pids[@]}"}; do + if ! wait "$pid"; then + failed=$((failed + 1)) + fi + done + __wfp_pids=() + if [[ $failed -gt 0 ]]; then + errorf "${failed} ${context}(s) failed" + return 1 + fi +} + function check_tool() { if ! command -v "$1" >/dev/null; then errorf "Error: Required tool '$1' is not installed." @@ -112,6 +151,7 @@ Options: --to-registry together. --oc-mirror-path : Path to the oc-mirror binary (default: 'oc-mirror'). --oc-mirror-flags : Additional flags to pass to all oc-mirror commands. + --max-parallel : Maximum number of parallel image operations (default: 10, env: MAX_PARALLEL) --install-yq : Install yq $YQ_VERSION from https://github.com/mikefarah/yq (not the jq python wrapper) Examples: @@ -265,7 +305,15 @@ while [[ "$#" -gt 0 ]]; do OC_MIRROR_FLAGS="$2" shift 1 ;; - '--install-yq') + '--max-parallel') + MAX_PARALLEL="$2" + if ! [[ "$MAX_PARALLEL" =~ ^[0-9]+$ ]] || [[ "$MAX_PARALLEL" -lt 1 ]]; then + errorf "MAX_PARALLEL must be a positive integer, got: '$MAX_PARALLEL'" + exit 1 + fi + shift 1 + ;; + '--install-yq') INSTALL_YQ=1 ;; '-h' | '--help') usage @@ -382,10 +430,13 @@ fi if [[ -n "${TO_DIR}" ]]; then mkdir -p "${TO_DIR}" TMPDIR="${TO_DIR}" + trap "jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null" EXIT + trap "exit 1" INT TERM else TMPDIR=$(mktemp -d) # shellcheck disable=SC2064 - trap "rm -fr $TMPDIR || true" EXIT + trap "rm -fr $TMPDIR || true; jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null" EXIT + trap "exit 1" INT TERM fi pushd "${TMPDIR}" >/dev/null debugf ">>> WORKING DIR: $TMPDIR <<<" @@ -410,37 +461,66 @@ function ocp_prepare_internal_registry() { # https://access.redhat.com/solutions/6022011 oc patch configs.imageregistry.operator.openshift.io/cluster --patch '{"spec":{"disableRedirect":true}}' --type=merge >&2 my_registry=$(oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}') - skopeo login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 - podman login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 + + skopeo login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 & + local skopeo_login_pid=$! + podman login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 & + local podman_login_pid=$! + + local ns_pids=() for ns in rhdh-operator openshift4 rhdh rhel9 oc-mirror; do - # To be able to push images under this scope in the internal image registry - if ! oc get namespace "$ns" &>/dev/null; then - oc create namespace "$ns" >&2 - fi - oc adm policy add-cluster-role-to-user system:image-signer system:serviceaccount:${ns}:default >&2 || true + ( + if ! oc get namespace "$ns" &>/dev/null; then + oc create namespace "$ns" >&2 + fi + oc adm policy add-cluster-role-to-user system:image-signer "system:serviceaccount:${ns}:default" >&2 || true + ) & + ns_pids+=($!) done + wait_for_pids ns_pids "namespace setup" + + if ! wait "$skopeo_login_pid"; then + errorf "skopeo login failed" + return 1 + fi + if ! wait "$podman_login_pid"; then + errorf "podman login failed" + return 1 + fi + + local secret_pids=() for ns in rhdh-operator openshift-marketplace; do - if oc -n ${ns} get secret internal-reg-ext-auth-for-rhdh &>/dev/null; then - oc -n ${ns} delete secret internal-reg-ext-auth-for-rhdh >&2 - fi - oc -n ${ns} create secret docker-registry internal-reg-ext-auth-for-rhdh \ - --docker-server="${my_registry}" \ - --docker-username=kubeadmin \ - --docker-password="$(oc whoami -t)" \ - --docker-email="admin@internal-registry-ext.example.com" >&2 - if oc -n ${ns} get secret internal-reg-auth-for-rhdh &>/dev/null; then - oc -n ${ns} delete secret internal-reg-auth-for-rhdh >&2 - fi - oc -n ${ns} create secret docker-registry internal-reg-auth-for-rhdh \ - --docker-server="${internal_registry_url}" \ - --docker-username=kubeadmin \ - --docker-password="$(oc whoami -t)" \ - --docker-email="admin@internal-registry.example.com" >&2 - oc adm policy add-cluster-role-to-user system:image-signer system:serviceaccount:${ns}:default >&2 || true + ( + if oc -n "${ns}" get secret internal-reg-ext-auth-for-rhdh &>/dev/null; then + oc -n "${ns}" delete secret internal-reg-ext-auth-for-rhdh >&2 + fi + oc -n "${ns}" create secret docker-registry internal-reg-ext-auth-for-rhdh \ + --docker-server="${my_registry}" \ + --docker-username=kubeadmin \ + --docker-password="$(oc whoami -t)" \ + --docker-email="admin@internal-registry-ext.example.com" >&2 + if oc -n "${ns}" get secret internal-reg-auth-for-rhdh &>/dev/null; then + oc -n "${ns}" delete secret internal-reg-auth-for-rhdh >&2 + fi + oc -n "${ns}" create secret docker-registry internal-reg-auth-for-rhdh \ + --docker-server="${internal_registry_url}" \ + --docker-username=kubeadmin \ + --docker-password="$(oc whoami -t)" \ + --docker-email="admin@internal-registry.example.com" >&2 + oc adm policy add-cluster-role-to-user system:image-signer "system:serviceaccount:${ns}:default" >&2 || true + ) & + secret_pids+=($!) done - oc policy add-role-to-user system:image-puller system:serviceaccount:openshift-marketplace:default -n openshift-marketplace >&2 || true - oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:default -n rhdh-operator >&2 || true - oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:rhdh-operator -n rhdh-operator >&2 || true + wait_for_pids secret_pids "secret setup" + + local policy_pids=() + oc policy add-role-to-user system:image-puller system:serviceaccount:openshift-marketplace:default -n openshift-marketplace >&2 & + policy_pids+=($!) + oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:default -n rhdh-operator >&2 & + policy_pids+=($!) + oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:rhdh-operator -n rhdh-operator >&2 & + policy_pids+=($!) + wait_for_pids policy_pids "policy setup" } function buildRegistryUrl() { @@ -536,16 +616,37 @@ function extract_last_two_elements() { } function mirror_extra_images() { - debugf "Extra images: " "${EXTRA_IMAGES[@]}" + if [[ ${#EXTRA_IMAGES[@]} -eq 0 ]]; then + return + fi + debugf "Extra images (${#EXTRA_IMAGES[@]}, max ${MAX_PARALLEL} parallel): ${EXTRA_IMAGES[*]}" + + if [[ -n "$TO_REGISTRY" ]] && [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then + for img in "${EXTRA_IMAGES[@]}"; do + local lastTwo + if [[ "$img" == *"@sha256:"* ]]; then + lastTwo=$(extract_last_two_elements "${img%@*}") + elif [[ "$img" == *":"* ]]; then + lastTwo=$(extract_last_two_elements "${img%:*}") + else + lastTwo=$(extract_last_two_elements "${img}") + fi + local projectNameForOcpReg=${lastTwo%%/*} + oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" + done + fi + + local pids=() for img in "${EXTRA_IMAGES[@]}"; do + local imgDir imgTag lastTwo targetImg if [[ "$img" == *"@sha256:"* ]]; then - imgDigest="${img##*@sha256:}" + local imgDigest="${img##*@sha256:}" imgDir="./extraImages/${img%@*}/sha256_$imgDigest" lastTwo=$(extract_last_two_elements "${img%@*}") targetImg="$(buildRegistryUrl)/${lastTwo}:$imgDigest" elif [[ "$img" == *":"* ]]; then - imgDir="./extraImages/${img%:*}/tag_$imgTag" imgTag="${img##*:}" + imgDir="./extraImages/${img%:*}/tag_$imgTag" lastTwo=$(extract_last_two_elements "${img%:*}") targetImg="$(buildRegistryUrl)/${lastTwo}:$imgTag" else @@ -555,66 +656,104 @@ function mirror_extra_images() { fi if [[ -n "$TO_REGISTRY" ]]; then - if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then - # Create the corresponding project if it doesn't exist - projectNameForOcpReg=${lastTwo%%/*} - oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" - fi - mirror_image_to_registry "$img" "$targetImg" + throttle_parallel pids + mirror_image_to_registry "$img" "$targetImg" & + pids+=($!) else if [ ! -d "$imgDir" ]; then mkdir -p "${imgDir}" - mirror_image_to_archive "$img" "$imgDir" + throttle_parallel pids + mirror_image_to_archive "$img" "$imgDir" & + pids+=($!) fi fi done + wait_for_pids pids "extra image" debugf "... done." } function mirror_extra_images_from_dir() { - BASE_DIR="${FROM_DIR}/extraImages" + local BASE_DIR="${FROM_DIR}/extraImages" debugf "Extra images from ${BASE_DIR}..." - if [ -d "${BASE_DIR}" ]; then - # Iterate over all directories named "sha256_*" - find "$BASE_DIR" -type d -name "sha256_*" | while read -r sha256_dir; do - relative_path=${sha256_dir#"$BASE_DIR/"} - sha256_hash=${sha256_dir##*/sha256_} + if [ ! -d "${BASE_DIR}" ]; then + return + fi + + local sha256_dirs=() + mapfile -t sha256_dirs < <(find "$BASE_DIR" -type d -name "sha256_*" 2>/dev/null) + local tag_dirs=() + mapfile -t tag_dirs < <(find "$BASE_DIR" -type d -name "tag_*" 2>/dev/null) + + if [[ ${#sha256_dirs[@]} -eq 0 ]] && [[ ${#tag_dirs[@]} -eq 0 ]]; then + return + fi + + infof "Pushing ${#sha256_dirs[@]} digest + ${#tag_dirs[@]} tag extra images from dir (max ${MAX_PARALLEL} parallel)..." + + if [[ -n "$TO_REGISTRY" ]] && [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then + local ns_set=() + for sha256_dir in "${sha256_dirs[@]}"; do + local relative_path=${sha256_dir#"$BASE_DIR/"} + local parent_path parent_path=$(dirname "$relative_path") - debugf "parent_path: $parent_path" + local lastTwo lastTwo=$(extract_last_two_elements "${parent_path}") - extraImg="${lastTwo}:${sha256_hash}" - debugf "Extra-image: $extraImg" - if [[ -n "$TO_REGISTRY" ]]; then - if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then - # Create the corresponding project if it doesn't exist - projectNameForOcpReg=${lastTwo%%/*} - oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" - fi - targetImg="$(buildRegistryUrl)/${extraImg%@*}" - push_image_from_archive "$sha256_dir" "$targetImg" - fi + ns_set+=("${lastTwo%%/*}") done - - # Iterate over all directories named "tag_*" - find "$BASE_DIR" -type d -name "tag_*" | while read -r tag_dir; do - relative_path=${tag_dir#"$BASE_DIR/"} - tag_hash=${tag_dir##*/tag_} + for tag_dir in "${tag_dirs[@]}"; do + local relative_path=${tag_dir#"$BASE_DIR/"} + local parent_path parent_path=$(dirname "$relative_path") - debugf "parent_path: $parent_path" + local lastTwo lastTwo=$(extract_last_two_elements "${parent_path}") - extraImg="${lastTwo}:${tag_hash}" - debugf "Extra-image: $extraImg" - if [[ -n "$TO_REGISTRY" ]]; then - if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then - # Create the corresponding project if it doesn't exist - projectNameForOcpReg=${lastTwo%%/*} - oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" - fi - targetImg="$(buildRegistryUrl)/${extraImg%:*}" - push_image_from_archive "$tag_dir" "$targetImg" - fi + ns_set+=("${lastTwo%%/*}") done + local unique_ns + unique_ns=$(printf '%s\n' "${ns_set[@]}" | sort -u) + while IFS= read -r ns; do + if [[ -n "$ns" ]] && ! oc get namespace "$ns" &>/dev/null; then + oc create namespace "$ns" + fi + done <<<"$unique_ns" fi + + local pids=() + + for sha256_dir in "${sha256_dirs[@]}"; do + local relative_path=${sha256_dir#"$BASE_DIR/"} + local sha256_hash=${sha256_dir##*/sha256_} + local parent_path + parent_path=$(dirname "$relative_path") + local lastTwo + lastTwo=$(extract_last_two_elements "${parent_path}") + local extraImg="${lastTwo}:${sha256_hash}" + if [[ -n "$TO_REGISTRY" ]]; then + local targetImg + targetImg="$(buildRegistryUrl)/${extraImg%@*}" + throttle_parallel pids + push_image_from_archive "$sha256_dir" "$targetImg" & + pids+=($!) + fi + done + + for tag_dir in "${tag_dirs[@]}"; do + local relative_path=${tag_dir#"$BASE_DIR/"} + local tag_hash=${tag_dir##*/tag_} + local parent_path + parent_path=$(dirname "$relative_path") + local lastTwo + lastTwo=$(extract_last_two_elements "${parent_path}") + local extraImg="${lastTwo}:${tag_hash}" + if [[ -n "$TO_REGISTRY" ]]; then + local targetImg + targetImg="$(buildRegistryUrl)/${extraImg%:*}" + throttle_parallel pids + push_image_from_archive "$tag_dir" "$targetImg" & + pids+=($!) + fi + done + + wait_for_pids pids "extra image from dir" } function replaceInternalRegIfNeeded() { @@ -630,96 +769,151 @@ function replaceInternalRegIfNeeded() { echo "$img" } -function process_bundles() { +function process_single_bundle() { + set -euo pipefail - for bundleImg in $(grep -E '^image: .*operator-bundle' "${TMPDIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq); do - debugf "bundleImg=$bundleImg" - originalBundleImg="$bundleImg" - bundleImg=$(replaceInternalRegIfNeeded "$bundleImg") - digest="${bundleImg##*@sha256:}" - if skopeo inspect "docker://$bundleImg" &>/dev/null; then - mkdir -p "bundles/$digest" - debugf "\t copying and unpacking image $bundleImg locally..." - if [ ! -d "./bundles/${digest}/src" ]; then - skopeo copy --remove-signatures "docker://$bundleImg" "oci:./bundles/${digest}/src:latest" + local bundleImg="$1" + local originalBundleImg="$2" + local digest="$3" + local sed_commands_dir="$4" + local bundle_id="$5" + + local bundle_dir="bundles/${digest}" + mkdir -p "${bundle_dir}" + + if ! skopeo copy --remove-signatures "docker://$bundleImg" "oci:./${bundle_dir}/src:latest" 2>"${bundle_dir}/copy.err"; then + debugf "bundle #${bundle_id}: skopeo copy failed, skipping (see ${bundle_dir}/copy.err)" >&2 + return 0 + fi + debugf "bundle #${bundle_id}: pulled ${bundleImg}" >&2 + + umoci unpack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" --rootless + + for file in "./${bundle_dir}/unpacked/rootfs/manifests"/*; do + if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then + "$YQ" eval ' + (.spec.install.spec.deployments[] | select(.name == "rhdh-operator").spec.template.spec) += + {"imagePullSecrets": [{"name": "internal-reg-auth-for-rhdh"},{"name": "internal-reg-ext-auth-for-rhdh"},{"name": "reg-pull-secret"}]} + ' -i "$file" + + local all_related_images=() + local images + mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) + if ((${#images[@]})); then + all_related_images+=("${images[@]}") fi - if [ ! -d "./bundles/${digest}/unpacked" ]; then - umoci unpack --image "./bundles/${digest}/src:latest" "./bundles/${digest}/unpacked" --rootless + local related_images + related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) + if [[ -n "$related_images" ]]; then + while IFS= read -r img; do + [[ -n "$img" ]] && all_related_images+=("$img") + done <<<"$related_images" fi - debugf "\t inspecting related images referenced in bundle image $bundleImg..." - for file in "./bundles/${digest}/unpacked/rootfs/manifests"/*; do - if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then - debugf "\t Adding imagePullSecrets to the CSV file so we can pull from private registries" - "$YQ" eval ' - (.spec.install.spec.deployments[] | select(.name == "rhdh-operator").spec.template.spec) += - {"imagePullSecrets": [{"name": "internal-reg-auth-for-rhdh"},{"name": "internal-reg-ext-auth-for-rhdh"},{"name": "reg-pull-secret"}]} - ' -i "$file" - - all_related_images=() - debugf "\t finding related images in $file to mirror..." - mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) - if ((${#images[@]})); then - all_related_images+=("${images[@]}") - fi - # TODO(rm3l): we should use spec.relatedImages instead, but it seems to be incomplete in some bundles - related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) - if [[ -n "$related_images" ]]; then - while IFS= read -r img; do - [[ -n "$img" ]] && all_related_images+=("$img") - done <<<"$related_images" - fi - for relatedImage in "${all_related_images[@]}"; do - imgDir="./images/" - if [[ "$relatedImage" == *"@sha256:"* ]]; then - relatedImageDigest="${relatedImage##*@sha256:}" - imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - lastTwo=$(extract_last_two_elements "${relatedImage%@*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageDigest" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageDigest" - elif [[ "$relatedImage" == *":"* ]]; then - relatedImageTag="${relatedImage##*:}" - imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - lastTwo=$(extract_last_two_elements "${relatedImage%:*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageTag" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageTag" - else - imgDir+="${relatedImage}/tag_latest" - lastTwo=$(extract_last_two_elements "${relatedImage}") - targetImg="$(buildRegistryUrl)/${lastTwo}:latest" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:latest" - fi + local csv_sed_file="${sed_commands_dir}/${digest}_csv.sed" + : > "$csv_sed_file" + local inner_pids=() + + for relatedImage in "${all_related_images[@]}"; do + local imgDir="./images/" + local lastTwo targetImg internalTargetImg + if [[ "$relatedImage" == *"@sha256:"* ]]; then + local relatedImageDigest="${relatedImage##*@sha256:}" + imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" + lastTwo=$(extract_last_two_elements "${relatedImage%@*}") + targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageDigest" + internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageDigest" + elif [[ "$relatedImage" == *":"* ]]; then + local relatedImageTag="${relatedImage##*:}" + imgDir+="${relatedImage%:*}/tag_$relatedImageTag" + lastTwo=$(extract_last_two_elements "${relatedImage%:*}") + targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageTag" + internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageTag" + else + imgDir+="${relatedImage}/tag_latest" + lastTwo=$(extract_last_two_elements "${relatedImage}") + targetImg="$(buildRegistryUrl)/${lastTwo}:latest" + internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:latest" + fi - if [[ -n "$TO_REGISTRY" ]]; then - mirror_image_to_registry "$relatedImage" "$targetImg" - debugf "replacing $relatedImage in file '${file}' => $internalTargetImg" - sed -i 's#'"$relatedImage"'#'"$internalTargetImg"'#g' "$file" - else - if [ ! -d "$imgDir" ]; then - mkdir -p "${imgDir}" - mirror_image_to_archive "$relatedImage" "$imgDir" - fi - fi - done + if [[ -n "$TO_REGISTRY" ]]; then + echo "s#${relatedImage}#${internalTargetImg}#g" >> "$csv_sed_file" + throttle_parallel inner_pids + mirror_image_to_registry "$relatedImage" "$targetImg" & + inner_pids+=($!) + else + if [ ! -d "$imgDir" ]; then + mkdir -p "${imgDir}" + throttle_parallel inner_pids + mirror_image_to_archive "$relatedImage" "$imgDir" & + inner_pids+=($!) + fi fi done - if [[ -n "$TO_REGISTRY" ]]; then - # repack the image with the changes - debugf "\t Repacking image ./bundles/${digest}/src => ./bundles/${digest}/unpacked..." - umoci repack --image "./bundles/${digest}/src:latest" "./bundles/${digest}/unpacked" - - # Push the bundle to the mirror registry - newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - debugf "\t Pushing updated bundle image: ./bundles/${digest}/src => ${newBundleImage}..." - skopeo copy --remove-signatures --dest-tls-verify=false "oci:./bundles/${digest}/src:latest" "docker://${newBundleImage}" + wait_for_pids inner_pids "related image mirror" - sed -i "s#${originalBundleImg}#${newBundleImageInternal}#g" "./rhdh/rhdh/render.yaml" + if [[ -s "$csv_sed_file" ]]; then + sed -i -f "$csv_sed_file" "$file" fi fi done + if [[ -n "$TO_REGISTRY" ]]; then + umoci repack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" + + local newBundleImage + newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + local newBundleImageInternal + newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 + skopeo copy --remove-signatures --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}" + + echo "s#${originalBundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" + fi +} + +function process_bundles() { + + local bundle_images + bundle_images=$(grep -E '^image: .*operator-bundle' "${TMPDIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq) + + local total_bundles + total_bundles=$(echo "$bundle_images" | wc -l | tr -d ' ') + infof "Processing ${total_bundles} bundles (max ${MAX_PARALLEL} parallel)..." + + local sed_commands_dir="${TMPDIR}/sed_commands_bundles" + mkdir -p "$sed_commands_dir" + + local bundle_count=0 + local pids=() + + for bundleImg in $bundle_images; do + bundle_count=$((bundle_count + 1)) + local originalBundleImg="$bundleImg" + bundleImg=$(replaceInternalRegIfNeeded "$bundleImg") + local digest="${bundleImg##*@sha256:}" + debugf "bundle #${bundle_count}/${total_bundles}: $originalBundleImg => $bundleImg" + + throttle_parallel pids + + process_single_bundle "$bundleImg" "$originalBundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & + pids+=($!) + done + + wait_for_pids pids "bundle" + + local sed_files + sed_files=$(find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' 2>/dev/null || true) + if [[ -n "$sed_files" ]]; then + local combined_sed="${TMPDIR}/combined_bundle_sed.txt" + find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' -exec cat {} + > "$combined_sed" + local replacement_count + replacement_count=$(wc -l < "$combined_sed" | tr -d ' ') + infof "Applying ${replacement_count} image ref replacements to render.yaml..." + sed -i -f "$combined_sed" "./rhdh/rhdh/render.yaml" + fi + if [ ! -f "rhdh/rhdh.Dockerfile" ]; then debugf "\t Regenerating Dockerfile so the index can be rebuilt..." opm generate dockerfile rhdh/rhdh @@ -737,6 +931,97 @@ function process_bundles() { fi } +function process_single_bundle_from_dir() { + set -euo pipefail + + local bundleImg="$1" + local digest="$2" + local sed_commands_dir="$3" + local bundle_id="$4" + + if [ ! -d "${FROM_DIR}/bundles/${digest}/src" ]; then + warnf "missing src image for bundle digest: ${FROM_DIR}/bundles/${digest}/src" >&2 + return 0 + fi + if [ ! -d "${FROM_DIR}/bundles/${digest}/unpacked" ]; then + warnf "missing unpacked image for bundle digest: ${FROM_DIR}/bundles/${digest}/unpacked" >&2 + return 0 + fi + + debugf "bundle #${bundle_id}: handling from ${FROM_DIR}/bundles/${digest}..." >&2 + + for file in "${TMPDIR}/bundles/${digest}/unpacked/rootfs/manifests"/*; do + if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then + local all_related_images=() + local images + mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) + if ((${#images[@]})); then + all_related_images+=("${images[@]}") + fi + local related_images + related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) + if [[ -n "$related_images" ]]; then + while IFS= read -r img; do + [[ -n "$img" ]] && all_related_images+=("$img") + done <<<"$related_images" + fi + + local csv_sed_file="${sed_commands_dir}/${digest}_csv.sed" + : > "$csv_sed_file" + local inner_pids=() + + for relatedImage in "${all_related_images[@]}"; do + local imgDir="${FROM_DIR}/images/" + local targetImg targetImgInternal + if [[ "$relatedImage" == *"@sha256:"* ]]; then + local relatedImageDigest="${relatedImage##*@sha256:}" + imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" + targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" + targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" + elif [[ "$relatedImage" == *":"* ]]; then + local relatedImageTag="${relatedImage##*:}" + imgDir+="${relatedImage%:*}/tag_$relatedImageTag" + targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" + targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" + else + imgDir+="${relatedImage}/tag_latest" + targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage}"):latest" + targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage}"):latest" + fi + if [ ! -d "$imgDir" ]; then + warnf "Skipping related image $relatedImage not found mirrored in dir: $FROM_DIR/images" >&2 + continue + fi + if [[ -n "$TO_REGISTRY" ]]; then + echo "s#${relatedImage}#${targetImgInternal}#g" >> "$csv_sed_file" + throttle_parallel inner_pids + push_image_from_archive "$imgDir" "$targetImg" & + inner_pids+=($!) + fi + done + + wait_for_pids inner_pids "related image push" + + if [[ -s "$csv_sed_file" ]]; then + sed -i -f "$csv_sed_file" "$file" + fi + fi + done + + if [[ -n "$TO_REGISTRY" ]]; then + umoci repack --image "${TMPDIR}/bundles/${digest}/src:latest" "${TMPDIR}/bundles/${digest}/unpacked" + + local newBundleImage + newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + local newBundleImageInternal + newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 + skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" + + echo "s#${bundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" + fi +} + function process_bundles_from_dir() { if [ ! -f "${FROM_DIR}/rhdh/rhdh.Dockerfile" ]; then @@ -748,79 +1033,43 @@ function process_bundles_from_dir() { cp -r "${FROM_DIR}/${d}" "${TMPDIR}/${d}" done - for bundleImg in $(grep -E '^image: .*operator-bundle' "${FROM_DIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq); do - debugf "bundleImg=$bundleImg" - digest="${bundleImg##*@sha256:}" - if [ ! -d "${FROM_DIR}/bundles/${digest}/src" ]; then - warnf "missing src image for bundle digest: ${FROM_DIR}/bundles/${digest}/src" - continue - fi - if [ ! -d "${FROM_DIR}/bundles/${digest}/unpacked" ]; then - warnf "missing unpacked image for bundle digest: ${FROM_DIR}/bundles/${digest}/unpacked" - continue - fi + local bundle_images + bundle_images=$(grep -E '^image: .*operator-bundle' "${FROM_DIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq) - debugf "Handling bundle image from ${FROM_DIR}/bundles/${digest}..." + local total_bundles + total_bundles=$(echo "$bundle_images" | wc -l | tr -d ' ') + infof "Processing ${total_bundles} bundles from dir (max ${MAX_PARALLEL} parallel)..." - debugf "\t inspecting related images referenced in bundle image $bundleImg..." - for file in "${TMPDIR}/bundles/${digest}/unpacked/rootfs/manifests"/*; do - if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then - all_related_images=() - debugf "\t finding related images in $file to mirror..." - mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) - if ((${#images[@]})); then - all_related_images+=("${images[@]}") - fi - # TODO(rm3l): we should use spec.relatedImages instead, but it seems to be incomplete in some bundles - related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) - if [[ -n "$related_images" ]]; then - all_related_images+=("$related_images") - fi - for relatedImage in "${all_related_images[@]}"; do - imgDir="${FROM_DIR}/images/" - if [[ "$relatedImage" == *"@sha256:"* ]]; then - relatedImageDigest="${relatedImage##*@sha256:}" - imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" - elif [[ "$relatedImage" == *":"* ]]; then - relatedImageTag="${relatedImage##*:}" - imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" - else - imgDir+="${relatedImage}/tag_latest" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage}"):latest" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage}"):latest" - fi - if [ ! -d "$imgDir" ]; then - warnf "Skipping related image $relatedImage not found mirrored in dir: $FROM_DIR/images" - continue - fi - if [[ -n "$TO_REGISTRY" ]]; then - push_image_from_archive "$imgDir" "$targetImg" - debugf "replacing $relatedImage in file '${file}' => $targetImgInternal" - sed -i 's#'"$relatedImage"'#'"$targetImgInternal"'#g' "$file" - fi - done - fi - done + local sed_commands_dir="${TMPDIR}/sed_commands_bundles_from_dir" + mkdir -p "$sed_commands_dir" - if [[ -n "$TO_REGISTRY" ]]; then - # repack the image with the changes - debugf "\t Repacking image ./bundles/${digest}/src => ./bundles/${digest}/unpacked..." - umoci repack --image "${TMPDIR}/bundles/${digest}/src:latest" "${TMPDIR}/bundles/${digest}/unpacked" + local bundle_count=0 + local pids=() - # Push the bundle to the mirror registry - newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - debugf "\t Pushing updated bundle image: ./bundles/${digest}/src => ${newBundleImage}..." - skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" + for bundleImg in $bundle_images; do + bundle_count=$((bundle_count + 1)) + local digest="${bundleImg##*@sha256:}" + debugf "bundle #${bundle_count}/${total_bundles}: $bundleImg" - sed -i "s#${bundleImg}#${newBundleImageInternal}#g" "${TMPDIR}/rhdh/rhdh/render.yaml" - fi + throttle_parallel pids + + process_single_bundle_from_dir "$bundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & + pids+=($!) done + wait_for_pids pids "bundle from dir" + + local sed_files + sed_files=$(find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' 2>/dev/null || true) + if [[ -n "$sed_files" ]]; then + local combined_sed="${TMPDIR}/combined_bundle_from_dir_sed.txt" + find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' -exec cat {} + > "$combined_sed" + local replacement_count + replacement_count=$(wc -l < "$combined_sed" | tr -d ' ') + infof "Applying ${replacement_count} image ref replacements to render.yaml..." + sed -i -f "$combined_sed" "${TMPDIR}/rhdh/rhdh/render.yaml" + fi + if [[ -n "$TO_REGISTRY" ]]; then pushd "${TMPDIR}/rhdh" my_operator_index="$(buildCatalogImageUrl)" @@ -1051,13 +1300,34 @@ EOF fi else if [[ -z "${FROM_DIR}" ]]; then + mirror_extra_images & + extra_mirror_pid=$! + render_index process_bundles + + if ! wait "$extra_mirror_pid"; then + errorf "mirror_extra_images failed" + exit 1 + fi else + mirror_extra_images_from_dir & + extra_from_dir_pid=$! + + mirror_extra_images & + extra_mirror_pid=$! + process_bundles_from_dir - mirror_extra_images_from_dir + + if ! wait "$extra_from_dir_pid"; then + errorf "mirror_extra_images_from_dir failed" + exit 1 + fi + if ! wait "$extra_mirror_pid"; then + errorf "mirror_extra_images failed" + exit 1 + fi fi - mirror_extra_images # create OLM resources manifestsTargetDir="${TMPDIR}" From c250c6511db3d1156e977ddb3496875f2c528907 Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Thu, 2 Jul 2026 17:01:35 +0100 Subject: [PATCH 2/8] fix: resolve shellcheck warnings in prepare-restricted-environment.sh Remove unused CATALOG_PULL_SECRET variable (SC2034) and replace useless echo wrapping oc command output in buildRegistryUrl (SC2005). Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .rhdh/scripts/prepare-restricted-environment.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index 3d42e68bb..39cbe94e8 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -17,7 +17,6 @@ IS_HOSTED_CONTROL_PLANE="" NAMESPACE_OPERATOR="rhdh-operator" INDEX_IMAGE="registry.redhat.io/redhat/redhat-operator-index:v4.18" FILTERED_VERSIONS=(*) -CATALOG_PULL_SECRET="" MAX_PARALLEL="${MAX_PARALLEL:-10}" if ! [[ "$MAX_PARALLEL" =~ ^[0-9]+$ ]] || [[ "$MAX_PARALLEL" -lt 1 ]]; then @@ -532,7 +531,7 @@ function buildRegistryUrl() { if [[ "${input}" == "internal" ]]; then echo "image-registry.openshift-image-registry.svc:5000" else - echo "$(oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}')" + oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}' fi else echo "${TO_REGISTRY}" From e4549a190055e3058d9e0cc7e31bf95bfb0b7ffa Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Fri, 3 Jul 2026 10:06:01 +0100 Subject: [PATCH 3/8] refactor: replace throttle_parallel with semaphore-based concurrency in prepare-restricted-environment.sh - Introduced semaphore functions (sem_init, sem_acquire, sem_release) to manage parallel execution. - Removed the throttle_parallel function and its usages, enhancing clarity and control over concurrent processes. - Updated traps to ensure proper cleanup of background jobs and temporary directories. - Adjusted image mirroring and processing functions to utilize the new semaphore mechanism for improved synchronization. Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .../scripts/prepare-restricted-environment.sh | 85 +++++++++++-------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index 39cbe94e8..1196d1e41 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -59,22 +59,27 @@ function errorf() { logf "ERROR" "\033[0;31m" "$1" } -function throttle_parallel() { - local -n __tp_pids=$1 - while true; do - local running=0 - for pid in ${__tp_pids[@]+"${__tp_pids[@]}"}; do - if kill -0 "$pid" 2>/dev/null; then - running=$((running + 1)) - fi - done - if [[ $running -lt $MAX_PARALLEL ]]; then - break - fi - sleep 0.2 +SEM_FD="" + +function sem_init() { + local fifo + fifo=$(mktemp -u) + mkfifo "$fifo" + exec {SEM_FD}<>"$fifo" + rm -f "$fifo" + for ((i = 0; i < MAX_PARALLEL; i++)); do + printf '\n' >&"${SEM_FD}" done } +function sem_acquire() { + IFS= read -r -u "${SEM_FD}" +} + +function sem_release() { + printf '\n' >&"${SEM_FD}" +} + function wait_for_pids() { local -n __wfp_pids=$1 local context="${2:-background job}" @@ -429,17 +434,19 @@ fi if [[ -n "${TO_DIR}" ]]; then mkdir -p "${TO_DIR}" TMPDIR="${TO_DIR}" - trap "jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null" EXIT - trap "exit 1" INT TERM + trap '{ jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null; } || true' EXIT + trap 'exit 1' INT TERM else TMPDIR=$(mktemp -d) # shellcheck disable=SC2064 - trap "rm -fr $TMPDIR || true; jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null" EXIT - trap "exit 1" INT TERM + trap "rm -fr \"$TMPDIR\" || true; { jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null; } || true" EXIT + trap 'exit 1' INT TERM fi pushd "${TMPDIR}" >/dev/null debugf ">>> WORKING DIR: $TMPDIR <<<" +sem_init + if (( INSTALL_YQ )); then YQ=$HOME/.local/bin/yq_mf YQ_BINARY=yq_linux_amd64 @@ -655,13 +662,11 @@ function mirror_extra_images() { fi if [[ -n "$TO_REGISTRY" ]]; then - throttle_parallel pids mirror_image_to_registry "$img" "$targetImg" & pids+=($!) else if [ ! -d "$imgDir" ]; then mkdir -p "${imgDir}" - throttle_parallel pids mirror_image_to_archive "$img" "$imgDir" & pids+=($!) fi @@ -729,7 +734,6 @@ function mirror_extra_images_from_dir() { if [[ -n "$TO_REGISTRY" ]]; then local targetImg targetImg="$(buildRegistryUrl)/${extraImg%@*}" - throttle_parallel pids push_image_from_archive "$sha256_dir" "$targetImg" & pids+=($!) fi @@ -745,8 +749,7 @@ function mirror_extra_images_from_dir() { local extraImg="${lastTwo}:${tag_hash}" if [[ -n "$TO_REGISTRY" ]]; then local targetImg - targetImg="$(buildRegistryUrl)/${extraImg%:*}" - throttle_parallel pids + targetImg="$(buildRegistryUrl)/${extraImg}" push_image_from_archive "$tag_dir" "$targetImg" & pids+=($!) fi @@ -780,10 +783,13 @@ function process_single_bundle() { local bundle_dir="bundles/${digest}" mkdir -p "${bundle_dir}" + sem_acquire if ! skopeo copy --remove-signatures "docker://$bundleImg" "oci:./${bundle_dir}/src:latest" 2>"${bundle_dir}/copy.err"; then + sem_release debugf "bundle #${bundle_id}: skopeo copy failed, skipping (see ${bundle_dir}/copy.err)" >&2 return 0 fi + sem_release debugf "bundle #${bundle_id}: pulled ${bundleImg}" >&2 umoci unpack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" --rootless @@ -837,13 +843,11 @@ function process_single_bundle() { if [[ -n "$TO_REGISTRY" ]]; then echo "s#${relatedImage}#${internalTargetImg}#g" >> "$csv_sed_file" - throttle_parallel inner_pids mirror_image_to_registry "$relatedImage" "$targetImg" & inner_pids+=($!) else if [ ! -d "$imgDir" ]; then mkdir -p "${imgDir}" - throttle_parallel inner_pids mirror_image_to_archive "$relatedImage" "$imgDir" & inner_pids+=($!) fi @@ -866,7 +870,9 @@ function process_single_bundle() { local newBundleImageInternal newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 - skopeo copy --remove-signatures --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}" + sem_acquire + skopeo copy --remove-signatures --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } + sem_release echo "s#${originalBundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" fi @@ -894,8 +900,6 @@ function process_bundles() { local digest="${bundleImg##*@sha256:}" debugf "bundle #${bundle_count}/${total_bundles}: $originalBundleImg => $bundleImg" - throttle_parallel pids - process_single_bundle "$bundleImg" "$originalBundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & pids+=($!) done @@ -993,7 +997,6 @@ function process_single_bundle_from_dir() { fi if [[ -n "$TO_REGISTRY" ]]; then echo "s#${relatedImage}#${targetImgInternal}#g" >> "$csv_sed_file" - throttle_parallel inner_pids push_image_from_archive "$imgDir" "$targetImg" & inner_pids+=($!) fi @@ -1015,7 +1018,9 @@ function process_single_bundle_from_dir() { local newBundleImageInternal newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 - skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" + sem_acquire + skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } + sem_release echo "s#${bundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" fi @@ -1050,8 +1055,6 @@ function process_bundles_from_dir() { local digest="${bundleImg##*@sha256:}" debugf "bundle #${bundle_count}/${total_bundles}: $bundleImg" - throttle_parallel pids - process_single_bundle_from_dir "$bundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & pids+=($!) done @@ -1082,30 +1085,42 @@ function process_bundles_from_dir() { } function mirror_image_to_registry() { + sem_acquire + local rc=0 local src_image src_image=$(replaceInternalRegIfNeeded "$1") local dest_image dest_image=$2 - + echo "Mirroring $src_image to $dest_image..." - skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false docker://"$src_image" docker://"$dest_image" + skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false docker://"$src_image" docker://"$dest_image" || rc=$? + sem_release + return $rc } function mirror_image_to_archive() { + sem_acquire + local rc=0 local src_image src_image=$(replaceInternalRegIfNeeded "$1") local archive_path archive_path="$2" debugf "Saving $src_image to $archive_path..." - skopeo copy --preserve-digests --remove-signatures --all --preserve-digests --dest-tls-verify=false docker://"$src_image" dir:"$archive_path" + skopeo copy --preserve-digests --remove-signatures --all --preserve-digests --dest-tls-verify=false docker://"$src_image" dir:"$archive_path" || rc=$? + sem_release + return $rc } function push_image_from_archive() { + sem_acquire + local rc=0 local archive_path=$1 local dest_image=$2 echo "Pushing $archive_path to $dest_image..." - skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image" + skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image" || rc=$? + sem_release + return $rc } check_tool "yq" From 33e3f78cdf699f95862318f2cdcee1454bd1527a Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Fri, 3 Jul 2026 10:40:20 +0100 Subject: [PATCH 4/8] refactor: enhance registry URL handling and streamline image processing in prepare-restricted-environment.sh - Introduced caching for registry URLs to optimize repeated calls. - Replaced direct calls to buildRegistryUrl with cached values in image processing functions. - Added a new helper function _last_two to simplify extraction of last two path elements. - Updated various functions to utilize the new caching mechanism, improving performance and readability. Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .../scripts/prepare-restricted-environment.sh | 159 +++++++++++------- 1 file changed, 99 insertions(+), 60 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index 1196d1e41..ebf9bde07 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -545,11 +545,27 @@ function buildRegistryUrl() { fi } +CACHED_REGISTRY_URL="" +CACHED_REGISTRY_URL_INTERNAL="" + +function cache_registry_urls() { + if [[ -n "$TO_REGISTRY" ]]; then + CACHED_REGISTRY_URL=$(buildRegistryUrl) + CACHED_REGISTRY_URL_INTERNAL=$(buildRegistryUrl "internal") + fi +} + function buildCatalogImageUrl() { if [[ -n "$TO_REGISTRY" ]]; then tag=${INDEX_IMAGE##*:} [[ "$INDEX_IMAGE" == "$tag" ]] && tag="latest" - echo "$(buildRegistryUrl "${1:-external}")/${2:-rhdh/index}:${tag}" + local reg_url + if [[ "${1:-external}" == "internal" ]]; then + reg_url="${CACHED_REGISTRY_URL_INTERNAL}" + else + reg_url="${CACHED_REGISTRY_URL}" + fi + echo "${reg_url}/${2:-rhdh/index}:${tag}" else echo "" fi @@ -599,7 +615,7 @@ function render_index() { >"${local_index_file}" fi - debugf "Got $(cat "${local_index_file}" | wc -l) lines of JSON from the index!" + debugf "Got $(wc -l < "${local_index_file}") lines of JSON from the index!" if [ ! -s "${local_index_file}" ]; then errorf "[ERROR] 'opm render $INDEX_IMAGE' returned an empty output, which likely means that this index Image does not contain the rhdh operator." @@ -621,6 +637,18 @@ function extract_last_two_elements() { fi } +function _last_two() { + local -n __lt_result=$1 + local __lt_input="$2" + if [[ "$__lt_input" == */*/* ]]; then + local __lt_last="${__lt_input##*/}" + local __lt_rest="${__lt_input%/*}" + __lt_result="${__lt_rest##*/}/${__lt_last}" + else + __lt_result="$__lt_input" + fi +} + function mirror_extra_images() { if [[ ${#EXTRA_IMAGES[@]} -eq 0 ]]; then return @@ -629,13 +657,13 @@ function mirror_extra_images() { if [[ -n "$TO_REGISTRY" ]] && [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then for img in "${EXTRA_IMAGES[@]}"; do - local lastTwo + local lastTwo="" if [[ "$img" == *"@sha256:"* ]]; then - lastTwo=$(extract_last_two_elements "${img%@*}") + _last_two lastTwo "${img%@*}" elif [[ "$img" == *":"* ]]; then - lastTwo=$(extract_last_two_elements "${img%:*}") + _last_two lastTwo "${img%:*}" else - lastTwo=$(extract_last_two_elements "${img}") + _last_two lastTwo "${img}" fi local projectNameForOcpReg=${lastTwo%%/*} oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" @@ -644,21 +672,21 @@ function mirror_extra_images() { local pids=() for img in "${EXTRA_IMAGES[@]}"; do - local imgDir imgTag lastTwo targetImg + local imgDir imgTag lastTwo="" targetImg if [[ "$img" == *"@sha256:"* ]]; then local imgDigest="${img##*@sha256:}" imgDir="./extraImages/${img%@*}/sha256_$imgDigest" - lastTwo=$(extract_last_two_elements "${img%@*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$imgDigest" + _last_two lastTwo "${img%@*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$imgDigest" elif [[ "$img" == *":"* ]]; then imgTag="${img##*:}" imgDir="./extraImages/${img%:*}/tag_$imgTag" - lastTwo=$(extract_last_two_elements "${img%:*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$imgTag" + _last_two lastTwo "${img%:*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$imgTag" else imgDir="./extraImages/${img}/tag_latest" - lastTwo=$(extract_last_two_elements "${img}") - targetImg="$(buildRegistryUrl)/${lastTwo}:latest" + _last_two lastTwo "${img}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" fi if [[ -n "$TO_REGISTRY" ]]; then @@ -700,16 +728,16 @@ function mirror_extra_images_from_dir() { local relative_path=${sha256_dir#"$BASE_DIR/"} local parent_path parent_path=$(dirname "$relative_path") - local lastTwo - lastTwo=$(extract_last_two_elements "${parent_path}") + local lastTwo="" + _last_two lastTwo "${parent_path}" ns_set+=("${lastTwo%%/*}") done for tag_dir in "${tag_dirs[@]}"; do local relative_path=${tag_dir#"$BASE_DIR/"} local parent_path parent_path=$(dirname "$relative_path") - local lastTwo - lastTwo=$(extract_last_two_elements "${parent_path}") + local lastTwo="" + _last_two lastTwo "${parent_path}" ns_set+=("${lastTwo%%/*}") done local unique_ns @@ -728,12 +756,12 @@ function mirror_extra_images_from_dir() { local sha256_hash=${sha256_dir##*/sha256_} local parent_path parent_path=$(dirname "$relative_path") - local lastTwo - lastTwo=$(extract_last_two_elements "${parent_path}") + local lastTwo="" + _last_two lastTwo "${parent_path}" local extraImg="${lastTwo}:${sha256_hash}" if [[ -n "$TO_REGISTRY" ]]; then local targetImg - targetImg="$(buildRegistryUrl)/${extraImg%@*}" + targetImg="${CACHED_REGISTRY_URL}/${extraImg%@*}" push_image_from_archive "$sha256_dir" "$targetImg" & pids+=($!) fi @@ -744,12 +772,12 @@ function mirror_extra_images_from_dir() { local tag_hash=${tag_dir##*/tag_} local parent_path parent_path=$(dirname "$relative_path") - local lastTwo - lastTwo=$(extract_last_two_elements "${parent_path}") + local lastTwo="" + _last_two lastTwo "${parent_path}" local extraImg="${lastTwo}:${tag_hash}" if [[ -n "$TO_REGISTRY" ]]; then local targetImg - targetImg="$(buildRegistryUrl)/${extraImg}" + targetImg="${CACHED_REGISTRY_URL}/${extraImg}" push_image_from_archive "$tag_dir" "$targetImg" & pids+=($!) fi @@ -821,24 +849,24 @@ function process_single_bundle() { for relatedImage in "${all_related_images[@]}"; do local imgDir="./images/" - local lastTwo targetImg internalTargetImg + local lastTwo="" targetImg internalTargetImg if [[ "$relatedImage" == *"@sha256:"* ]]; then local relatedImageDigest="${relatedImage##*@sha256:}" imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - lastTwo=$(extract_last_two_elements "${relatedImage%@*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageDigest" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageDigest" + _last_two lastTwo "${relatedImage%@*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageDigest" + internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageDigest" elif [[ "$relatedImage" == *":"* ]]; then local relatedImageTag="${relatedImage##*:}" imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - lastTwo=$(extract_last_two_elements "${relatedImage%:*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageTag" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageTag" + _last_two lastTwo "${relatedImage%:*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageTag" + internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageTag" else imgDir+="${relatedImage}/tag_latest" - lastTwo=$(extract_last_two_elements "${relatedImage}") - targetImg="$(buildRegistryUrl)/${lastTwo}:latest" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:latest" + _last_two lastTwo "${relatedImage}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" + internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:latest" fi if [[ -n "$TO_REGISTRY" ]]; then @@ -865,10 +893,10 @@ function process_single_bundle() { if [[ -n "$TO_REGISTRY" ]]; then umoci repack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" - local newBundleImage - newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - local newBundleImageInternal - newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + local bundleLastTwo="" + _last_two bundleLastTwo "${bundleImg%@*}" + local newBundleImage="${CACHED_REGISTRY_URL}/${bundleLastTwo}:${digest}" + local newBundleImageInternal="${CACHED_REGISTRY_URL_INTERNAL}/${bundleLastTwo}:${digest}" debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 sem_acquire skopeo copy --remove-signatures --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } @@ -975,21 +1003,24 @@ function process_single_bundle_from_dir() { for relatedImage in "${all_related_images[@]}"; do local imgDir="${FROM_DIR}/images/" - local targetImg targetImgInternal + local lastTwo="" targetImg targetImgInternal if [[ "$relatedImage" == *"@sha256:"* ]]; then local relatedImageDigest="${relatedImage##*@sha256:}" imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" + _last_two lastTwo "${relatedImage%@*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageDigest" + targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageDigest" elif [[ "$relatedImage" == *":"* ]]; then local relatedImageTag="${relatedImage##*:}" imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" + _last_two lastTwo "${relatedImage%:*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageTag" + targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageTag" else imgDir+="${relatedImage}/tag_latest" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage}"):latest" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage}"):latest" + _last_two lastTwo "${relatedImage}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" + targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:latest" fi if [ ! -d "$imgDir" ]; then warnf "Skipping related image $relatedImage not found mirrored in dir: $FROM_DIR/images" >&2 @@ -1013,10 +1044,10 @@ function process_single_bundle_from_dir() { if [[ -n "$TO_REGISTRY" ]]; then umoci repack --image "${TMPDIR}/bundles/${digest}/src:latest" "${TMPDIR}/bundles/${digest}/unpacked" - local newBundleImage - newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - local newBundleImageInternal - newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + local bundleLastTwo="" + _last_two bundleLastTwo "${bundleImg%@*}" + local newBundleImage="${CACHED_REGISTRY_URL}/${bundleLastTwo}:${digest}" + local newBundleImageInternal="${CACHED_REGISTRY_URL_INTERNAL}/${bundleLastTwo}:${digest}" debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 sem_acquire skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } @@ -1087,10 +1118,14 @@ function process_bundles_from_dir() { function mirror_image_to_registry() { sem_acquire local rc=0 - local src_image - src_image=$(replaceInternalRegIfNeeded "$1") - local dest_image - dest_image=$2 + local src_image="$1" + if [[ "${IS_CI_INDEX_IMAGE}" == "true" ]]; then + for reg in registry.stage.redhat.io registry.redhat.io; do + src_image="${src_image/$reg\/rhdh/quay.io\/rhdh}" + done + src_image="${src_image/registry-proxy.engineering.redhat.com\/rh-osbs\/rhdh-/quay.io\/rhdh\/}" + fi + local dest_image=$2 echo "Mirroring $src_image to $dest_image..." skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false docker://"$src_image" docker://"$dest_image" || rc=$? @@ -1101,10 +1136,14 @@ function mirror_image_to_registry() { function mirror_image_to_archive() { sem_acquire local rc=0 - local src_image - src_image=$(replaceInternalRegIfNeeded "$1") - local archive_path - archive_path="$2" + local src_image="$1" + if [[ "${IS_CI_INDEX_IMAGE}" == "true" ]]; then + for reg in registry.stage.redhat.io registry.redhat.io; do + src_image="${src_image/$reg\/rhdh/quay.io\/rhdh}" + done + src_image="${src_image/registry-proxy.engineering.redhat.com\/rh-osbs\/rhdh-/quay.io\/rhdh\/}" + fi + local archive_path="$2" debugf "Saving $src_image to $archive_path..." skopeo copy --preserve-digests --remove-signatures --all --preserve-digests --dest-tls-verify=false docker://"$src_image" dir:"$archive_path" || rc=$? @@ -1138,7 +1177,7 @@ detect_ocp_and_set_env_var if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then ocp_prepare_internal_registry fi - +cache_registry_urls manifestsTargetDir="${TMPDIR}" if [[ -n "${FROM_DIR}" ]]; then @@ -1203,7 +1242,7 @@ EOF mirror_image_to_archive "registry.redhat.io/ubi9/ubi:latest" "${TO_DIR}/rhdh-catalog" fi if [[ -n "$TO_REGISTRY" ]]; then - registryUrl=$(buildRegistryUrl) + registryUrl="${CACHED_REGISTRY_URL}" if [[ "${TO_REGISTRY}" == "OCP_INTERNAL" ]]; then registryUrl+="/oc-mirror" fi @@ -1233,7 +1272,7 @@ EOF exit 1 fi if [[ -n "${TO_REGISTRY}" ]]; then - registryUrl=$(buildRegistryUrl) + registryUrl="${CACHED_REGISTRY_URL}" if [[ "${TO_REGISTRY}" == "OCP_INTERNAL" ]]; then registryUrl+="/oc-mirror" fi @@ -1411,7 +1450,7 @@ EOF if [[ -n "${TO_REGISTRY}" ]]; then # IDMS will only work on regular OCP clusters. It doesn't work on ROSA or clusters with hosted control planes like on IBM Cloud. - registry_url_internal=$(buildRegistryUrl) + registry_url_internal="${CACHED_REGISTRY_URL}" cat <"${manifestsTargetDir}/imageDigestMirrorSet.yaml" apiVersion: config.openshift.io/v1 kind: ImageDigestMirrorSet From a53505f8f6531c64e86d7760b2a7c87f8ac6fffb Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Fri, 3 Jul 2026 10:56:25 +0100 Subject: [PATCH 5/8] docs: update --max-parallel option description in prepare-restricted-environment.sh - Enhanced the documentation for the --max-parallel option to clarify its impact on disk usage and recommend lowering the value when disk space is limited. Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .rhdh/scripts/prepare-restricted-environment.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index ebf9bde07..979a7d511 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -155,7 +155,9 @@ Options: --to-registry together. --oc-mirror-path : Path to the oc-mirror binary (default: 'oc-mirror'). --oc-mirror-flags : Additional flags to pass to all oc-mirror commands. - --max-parallel : Maximum number of parallel image operations (default: 10, env: MAX_PARALLEL) + --max-parallel : Maximum number of parallel image operations (default: 10, env: MAX_PARALLEL). + Lower this value if you are running low on disk space, as fewer concurrent + downloads will reduce peak disk usage. --install-yq : Install yq $YQ_VERSION from https://github.com/mikefarah/yq (not the jq python wrapper) Examples: From a001c0cb2de59b94dbc5b5d79f79d9b502a84b2a Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Fri, 3 Jul 2026 14:05:59 +0100 Subject: [PATCH 6/8] feat: implement retry mechanism for image push in prepare-restricted-environment.sh - Added a retry mechanism for the push_image_from_archive function to handle transient failures during image pushing. - Introduced a locking mechanism to prevent duplicate pushes for the same destination image. - Enhanced error handling with exponential backoff for retries. Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .../scripts/prepare-restricted-environment.sh | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index 979a7d511..085bf34e0 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -1154,12 +1154,31 @@ function mirror_image_to_archive() { } function push_image_from_archive() { - sem_acquire - local rc=0 local archive_path=$1 local dest_image=$2 + + local lock_key="${dest_image//\//__}" + lock_key="${lock_key//:/_}" + if ! mkdir "$TMPDIR/.push_locks/$lock_key" 2>/dev/null; then + debugf "Skipping duplicate push: $dest_image (already handled)" + return 0 + fi + + sem_acquire + local rc=0 + local max_retries=3 + local retry_delay=5 echo "Pushing $archive_path to $dest_image..." - skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image" || rc=$? + for ((attempt=1; attempt<=max_retries; attempt++)); do + rc=0 + skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image" || rc=$? + if [[ $rc -eq 0 ]]; then break; fi + if ((attempt < max_retries)); then + warnf "Push failed (attempt $attempt/$max_retries), retrying in ${retry_delay}s... [$dest_image]" + sleep "$retry_delay" + retry_delay=$((retry_delay * 2)) + fi + done sem_release return $rc } @@ -1180,6 +1199,7 @@ if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then ocp_prepare_internal_registry fi cache_registry_urls +mkdir -p "$TMPDIR/.push_locks" manifestsTargetDir="${TMPDIR}" if [[ -n "${FROM_DIR}" ]]; then From 81585b68138d72102d66eaf752224ca8108cad74 Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Fri, 17 Jul 2026 16:58:43 +0100 Subject: [PATCH 7/8] chore: reset prepare-restricted-environment.sh to upstream main Reset the script to match upstream main before re-applying parallelization changes cleanly. The previous merge introduced broken code (mismatched done/fi, duplicated code blocks in process_bundles_from_dir). Signed-off-by: Fortune Ndlovu --- .../scripts/prepare-restricted-environment.sh | 741 +++++------------- 1 file changed, 189 insertions(+), 552 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index a417641e5..a24a9aace 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -20,12 +20,6 @@ FILTERED_VERSIONS=("*") OLM_VERSION="auto" RESOLVED_OLM_VERSION="" -MAX_PARALLEL="${MAX_PARALLEL:-10}" -if ! [[ "$MAX_PARALLEL" =~ ^[0-9]+$ ]] || [[ "$MAX_PARALLEL" -lt 1 ]]; then - echo "[ERROR] MAX_PARALLEL must be a positive integer, got: '$MAX_PARALLEL'" >&2 - exit 1 -fi - # assume mikefarah version of yq is already available on the path; if 1, then install the version shown INSTALL_YQ=0 YQ_VERSION=v4.45.1 @@ -61,43 +55,6 @@ function errorf() { logf "ERROR" "\033[0;31m" "$1" } -SEM_FD="" - -function sem_init() { - local fifo - fifo=$(mktemp -u) - mkfifo "$fifo" - exec {SEM_FD}<>"$fifo" - rm -f "$fifo" - for ((i = 0; i < MAX_PARALLEL; i++)); do - printf '\n' >&"${SEM_FD}" - done -} - -function sem_acquire() { - IFS= read -r -u "${SEM_FD}" -} - -function sem_release() { - printf '\n' >&"${SEM_FD}" -} - -function wait_for_pids() { - local -n __wfp_pids=$1 - local context="${2:-background job}" - local failed=0 - for pid in ${__wfp_pids[@]+"${__wfp_pids[@]}"}; do - if ! wait "$pid"; then - failed=$((failed + 1)) - fi - done - __wfp_pids=() - if [[ $failed -gt 0 ]]; then - errorf "${failed} ${context}(s) failed" - return 1 - fi -} - function check_tool() { if ! command -v "$1" >/dev/null; then errorf "Error: Required tool '$1' is not installed." @@ -604,19 +561,14 @@ fi if [[ -n "${TO_DIR}" ]]; then mkdir -p "${TO_DIR}" TMPDIR="${TO_DIR}" - trap '{ jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null; } || true' EXIT - trap 'exit 1' INT TERM else TMPDIR=$(mktemp -d) # shellcheck disable=SC2064 - trap "rm -fr \"$TMPDIR\" || true; { jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null; } || true" EXIT - trap 'exit 1' INT TERM + trap "rm -fr $TMPDIR || true" EXIT fi pushd "${TMPDIR}" >/dev/null debugf ">>> WORKING DIR: $TMPDIR <<<" -sem_init - if (( INSTALL_YQ )); then YQ=$HOME/.local/bin/yq_mf YQ_BINARY=yq_linux_amd64 @@ -637,66 +589,37 @@ function ocp_prepare_internal_registry() { # https://access.redhat.com/solutions/6022011 oc patch configs.imageregistry.operator.openshift.io/cluster --patch '{"spec":{"disableRedirect":true}}' --type=merge >&2 my_registry=$(oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}') - - skopeo login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 & - local skopeo_login_pid=$! - podman login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 & - local podman_login_pid=$! - - local ns_pids=() + skopeo login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 + podman login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 for ns in rhdh-operator openshift4 rhdh rhel9 oc-mirror; do - ( - if ! oc get namespace "$ns" &>/dev/null; then - oc create namespace "$ns" >&2 - fi - oc adm policy add-cluster-role-to-user system:image-signer "system:serviceaccount:${ns}:default" >&2 || true - ) & - ns_pids+=($!) + # To be able to push images under this scope in the internal image registry + if ! oc get namespace "$ns" &>/dev/null; then + oc create namespace "$ns" >&2 + fi + oc adm policy add-cluster-role-to-user system:image-signer system:serviceaccount:${ns}:default >&2 || true done - wait_for_pids ns_pids "namespace setup" - - if ! wait "$skopeo_login_pid"; then - errorf "skopeo login failed" - return 1 - fi - if ! wait "$podman_login_pid"; then - errorf "podman login failed" - return 1 - fi - - local secret_pids=() for ns in rhdh-operator openshift-marketplace; do - ( - if oc -n "${ns}" get secret internal-reg-ext-auth-for-rhdh &>/dev/null; then - oc -n "${ns}" delete secret internal-reg-ext-auth-for-rhdh >&2 - fi - oc -n "${ns}" create secret docker-registry internal-reg-ext-auth-for-rhdh \ - --docker-server="${my_registry}" \ - --docker-username=kubeadmin \ - --docker-password="$(oc whoami -t)" \ - --docker-email="admin@internal-registry-ext.example.com" >&2 - if oc -n "${ns}" get secret internal-reg-auth-for-rhdh &>/dev/null; then - oc -n "${ns}" delete secret internal-reg-auth-for-rhdh >&2 - fi - oc -n "${ns}" create secret docker-registry internal-reg-auth-for-rhdh \ - --docker-server="${internal_registry_url}" \ - --docker-username=kubeadmin \ - --docker-password="$(oc whoami -t)" \ - --docker-email="admin@internal-registry.example.com" >&2 - oc adm policy add-cluster-role-to-user system:image-signer "system:serviceaccount:${ns}:default" >&2 || true - ) & - secret_pids+=($!) + if oc -n ${ns} get secret internal-reg-ext-auth-for-rhdh &>/dev/null; then + oc -n ${ns} delete secret internal-reg-ext-auth-for-rhdh >&2 + fi + oc -n ${ns} create secret docker-registry internal-reg-ext-auth-for-rhdh \ + --docker-server="${my_registry}" \ + --docker-username=kubeadmin \ + --docker-password="$(oc whoami -t)" \ + --docker-email="admin@internal-registry-ext.example.com" >&2 + if oc -n ${ns} get secret internal-reg-auth-for-rhdh &>/dev/null; then + oc -n ${ns} delete secret internal-reg-auth-for-rhdh >&2 + fi + oc -n ${ns} create secret docker-registry internal-reg-auth-for-rhdh \ + --docker-server="${internal_registry_url}" \ + --docker-username=kubeadmin \ + --docker-password="$(oc whoami -t)" \ + --docker-email="admin@internal-registry.example.com" >&2 + oc adm policy add-cluster-role-to-user system:image-signer system:serviceaccount:${ns}:default >&2 || true done - wait_for_pids secret_pids "secret setup" - - local policy_pids=() - oc policy add-role-to-user system:image-puller system:serviceaccount:openshift-marketplace:default -n openshift-marketplace >&2 & - policy_pids+=($!) - oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:default -n rhdh-operator >&2 & - policy_pids+=($!) - oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:rhdh-operator -n rhdh-operator >&2 & - policy_pids+=($!) - wait_for_pids policy_pids "policy setup" + oc policy add-role-to-user system:image-puller system:serviceaccount:openshift-marketplace:default -n openshift-marketplace >&2 || true + oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:default -n rhdh-operator >&2 || true + oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:rhdh-operator -n rhdh-operator >&2 || true } function buildRegistryUrl() { @@ -715,27 +638,11 @@ function buildRegistryUrl() { fi } -CACHED_REGISTRY_URL="" -CACHED_REGISTRY_URL_INTERNAL="" - -function cache_registry_urls() { - if [[ -n "$TO_REGISTRY" ]]; then - CACHED_REGISTRY_URL=$(buildRegistryUrl) - CACHED_REGISTRY_URL_INTERNAL=$(buildRegistryUrl "internal") - fi -} - function buildCatalogImageUrl() { if [[ -n "$TO_REGISTRY" ]]; then tag=${INDEX_IMAGE##*:} [[ "$INDEX_IMAGE" == "$tag" ]] && tag="latest" - local reg_url - if [[ "${1:-external}" == "internal" ]]; then - reg_url="${CACHED_REGISTRY_URL_INTERNAL}" - else - reg_url="${CACHED_REGISTRY_URL}" - fi - echo "${reg_url}/${2:-rhdh/index}:${tag}" + echo "$(buildRegistryUrl "${1:-external}")/${2:-rhdh/index}:${tag}" else echo "" fi @@ -785,7 +692,7 @@ function render_index() { >"${local_index_file}" fi - debugf "Got $(wc -l < "${local_index_file}") lines of JSON from the index!" + debugf "Got $(cat "${local_index_file}" | wc -l) lines of JSON from the index!" if [ ! -s "${local_index_file}" ]; then errorf "[ERROR] 'opm render $INDEX_IMAGE' returned an empty output, which likely means that this index Image does not contain the rhdh operator." @@ -807,61 +714,32 @@ function extract_last_two_elements() { fi } -function _last_two() { - local -n __lt_result=$1 - local __lt_input="$2" - if [[ "$__lt_input" == */*/* ]]; then - local __lt_last="${__lt_input##*/}" - local __lt_rest="${__lt_input%/*}" - __lt_result="${__lt_rest##*/}/${__lt_last}" - else - __lt_result="$__lt_input" - fi -} - function mirror_extra_images() { - if [[ ${#EXTRA_IMAGES[@]} -eq 0 ]]; then - return - fi - debugf "Extra images (${#EXTRA_IMAGES[@]}, max ${MAX_PARALLEL} parallel): ${EXTRA_IMAGES[*]}" - - if [[ -n "$TO_REGISTRY" ]] && [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then - for img in "${EXTRA_IMAGES[@]}"; do - local lastTwo="" - if [[ "$img" == *"@sha256:"* ]]; then - _last_two lastTwo "${img%@*}" - elif [[ "$img" == *":"* ]]; then - _last_two lastTwo "${img%:*}" - else - _last_two lastTwo "${img}" - fi - local projectNameForOcpReg=${lastTwo%%/*} - oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" - done - fi - - local pids=() + debugf "Extra images: " "${EXTRA_IMAGES[@]}" for img in "${EXTRA_IMAGES[@]}"; do - local imgDir imgTag lastTwo="" targetImg if [[ "$img" == *"@sha256:"* ]]; then - local imgDigest="${img##*@sha256:}" + imgDigest="${img##*@sha256:}" imgDir="./extraImages/${img%@*}/sha256_$imgDigest" - _last_two lastTwo "${img%@*}" - targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$imgDigest" + lastTwo=$(extract_last_two_elements "${img%@*}") + targetImg="$(buildRegistryUrl)/${lastTwo}:$imgDigest" elif [[ "$img" == *":"* ]]; then - imgTag="${img##*:}" imgDir="./extraImages/${img%:*}/tag_$imgTag" - _last_two lastTwo "${img%:*}" - targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$imgTag" + imgTag="${img##*:}" + lastTwo=$(extract_last_two_elements "${img%:*}") + targetImg="$(buildRegistryUrl)/${lastTwo}:$imgTag" else imgDir="./extraImages/${img}/tag_latest" - _last_two lastTwo "${img}" - targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" + lastTwo=$(extract_last_two_elements "${img}") + targetImg="$(buildRegistryUrl)/${lastTwo}:latest" fi if [[ -n "$TO_REGISTRY" ]]; then - mirror_image_to_registry "$img" "$targetImg" & - pids+=($!) + if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then + # Create the corresponding project if it doesn't exist + projectNameForOcpReg=${lastTwo%%/*} + oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" + fi + mirror_image_to_registry "$img" "$targetImg" else if [ -d "$imgDir" ] && ! is_complete_image_archive "$imgDir"; then warnf "Removing incomplete image archive at ${imgDir} (previous download was interrupted)" @@ -869,95 +747,57 @@ function mirror_extra_images() { fi if [ ! -d "$imgDir" ]; then mkdir -p "${imgDir}" - mirror_image_to_archive "$img" "$imgDir" & - pids+=($!) + mirror_image_to_archive "$img" "$imgDir" fi fi done - wait_for_pids pids "extra image" debugf "... done." } function mirror_extra_images_from_dir() { - local BASE_DIR="${FROM_DIR}/extraImages" + BASE_DIR="${FROM_DIR}/extraImages" debugf "Extra images from ${BASE_DIR}..." - if [ ! -d "${BASE_DIR}" ]; then - return - fi - - local sha256_dirs=() - mapfile -t sha256_dirs < <(find "$BASE_DIR" -type d -name "sha256_*" 2>/dev/null) - local tag_dirs=() - mapfile -t tag_dirs < <(find "$BASE_DIR" -type d -name "tag_*" 2>/dev/null) - - if [[ ${#sha256_dirs[@]} -eq 0 ]] && [[ ${#tag_dirs[@]} -eq 0 ]]; then - return - fi - - infof "Pushing ${#sha256_dirs[@]} digest + ${#tag_dirs[@]} tag extra images from dir (max ${MAX_PARALLEL} parallel)..." - - if [[ -n "$TO_REGISTRY" ]] && [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then - local ns_set=() - for sha256_dir in "${sha256_dirs[@]}"; do - local relative_path=${sha256_dir#"$BASE_DIR/"} - local parent_path + if [ -d "${BASE_DIR}" ]; then + # Iterate over all directories named "sha256_*" + find "$BASE_DIR" -type d -name "sha256_*" | while read -r sha256_dir; do + relative_path=${sha256_dir#"$BASE_DIR/"} + sha256_hash=${sha256_dir##*/sha256_} parent_path=$(dirname "$relative_path") - local lastTwo="" - _last_two lastTwo "${parent_path}" - ns_set+=("${lastTwo%%/*}") + debugf "parent_path: $parent_path" + lastTwo=$(extract_last_two_elements "${parent_path}") + extraImg="${lastTwo}:${sha256_hash}" + debugf "Extra-image: $extraImg" + if [[ -n "$TO_REGISTRY" ]]; then + if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then + # Create the corresponding project if it doesn't exist + projectNameForOcpReg=${lastTwo%%/*} + oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" + fi + targetImg="$(buildRegistryUrl)/${extraImg%@*}" + push_image_from_archive "$sha256_dir" "$targetImg" + fi done - for tag_dir in "${tag_dirs[@]}"; do - local relative_path=${tag_dir#"$BASE_DIR/"} - local parent_path + + # Iterate over all directories named "tag_*" + find "$BASE_DIR" -type d -name "tag_*" | while read -r tag_dir; do + relative_path=${tag_dir#"$BASE_DIR/"} + tag_hash=${tag_dir##*/tag_} parent_path=$(dirname "$relative_path") - local lastTwo="" - _last_two lastTwo "${parent_path}" - ns_set+=("${lastTwo%%/*}") - done - local unique_ns - unique_ns=$(printf '%s\n' "${ns_set[@]}" | sort -u) - while IFS= read -r ns; do - if [[ -n "$ns" ]] && ! oc get namespace "$ns" &>/dev/null; then - oc create namespace "$ns" + debugf "parent_path: $parent_path" + lastTwo=$(extract_last_two_elements "${parent_path}") + extraImg="${lastTwo}:${tag_hash}" + debugf "Extra-image: $extraImg" + if [[ -n "$TO_REGISTRY" ]]; then + if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then + # Create the corresponding project if it doesn't exist + projectNameForOcpReg=${lastTwo%%/*} + oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" + fi + targetImg="$(buildRegistryUrl)/${extraImg%:*}" + push_image_from_archive "$tag_dir" "$targetImg" fi - done <<<"$unique_ns" + done fi - - local pids=() - - for sha256_dir in "${sha256_dirs[@]}"; do - local relative_path=${sha256_dir#"$BASE_DIR/"} - local sha256_hash=${sha256_dir##*/sha256_} - local parent_path - parent_path=$(dirname "$relative_path") - local lastTwo="" - _last_two lastTwo "${parent_path}" - local extraImg="${lastTwo}:${sha256_hash}" - if [[ -n "$TO_REGISTRY" ]]; then - local targetImg - targetImg="${CACHED_REGISTRY_URL}/${extraImg%@*}" - push_image_from_archive "$sha256_dir" "$targetImg" & - pids+=($!) - fi - done - - for tag_dir in "${tag_dirs[@]}"; do - local relative_path=${tag_dir#"$BASE_DIR/"} - local tag_hash=${tag_dir##*/tag_} - local parent_path - parent_path=$(dirname "$relative_path") - local lastTwo="" - _last_two lastTwo "${parent_path}" - local extraImg="${lastTwo}:${tag_hash}" - if [[ -n "$TO_REGISTRY" ]]; then - local targetImg - targetImg="${CACHED_REGISTRY_URL}/${extraImg}" - push_image_from_archive "$tag_dir" "$targetImg" & - pids+=($!) - fi - done - - wait_for_pids pids "extra image from dir" } function replaceInternalRegIfNeeded() { @@ -973,75 +813,65 @@ function replaceInternalRegIfNeeded() { echo "$img" } -function process_single_bundle() { - set -euo pipefail - - local bundleImg="$1" - local originalBundleImg="$2" - local digest="$3" - local sed_commands_dir="$4" - local bundle_id="$5" - - local bundle_dir="bundles/${digest}" - mkdir -p "${bundle_dir}" +function process_bundles() { - sem_acquire - if ! skopeo copy --remove-signatures "docker://$bundleImg" "oci:./${bundle_dir}/src:latest" 2>"${bundle_dir}/copy.err"; then - sem_release - debugf "bundle #${bundle_id}: skopeo copy failed, skipping (see ${bundle_dir}/copy.err)" >&2 - return 0 - fi - sem_release - debugf "bundle #${bundle_id}: pulled ${bundleImg}" >&2 - - umoci unpack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" --rootless - - for file in "./${bundle_dir}/unpacked/rootfs/manifests"/*; do - if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then - "$YQ" eval ' - (.spec.install.spec.deployments[] | select(.name == "rhdh-operator").spec.template.spec) += - {"imagePullSecrets": [{"name": "internal-reg-auth-for-rhdh"},{"name": "internal-reg-ext-auth-for-rhdh"},{"name": "reg-pull-secret"}]} - ' -i "$file" - - local all_related_images=() - local images - mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) - if ((${#images[@]})); then - all_related_images+=("${images[@]}") + for bundleImg in $(grep -E '^image: .*operator-bundle' "${TMPDIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq); do + debugf "bundleImg=$bundleImg" + originalBundleImg="$bundleImg" + bundleImg=$(replaceInternalRegIfNeeded "$bundleImg") + digest="${bundleImg##*@sha256:}" + if skopeo inspect "docker://$bundleImg" &>/dev/null; then + mkdir -p "bundles/$digest" + debugf "\t copying and unpacking image $bundleImg locally..." + if [ ! -d "./bundles/${digest}/src" ]; then + skopeo copy --remove-signatures "docker://$bundleImg" "oci:./bundles/${digest}/src:latest" fi - local related_images - related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) - if [[ -n "$related_images" ]]; then - while IFS= read -r img; do - [[ -n "$img" ]] && all_related_images+=("$img") - done <<<"$related_images" + if [ ! -d "./bundles/${digest}/unpacked" ]; then + umoci unpack --image "./bundles/${digest}/src:latest" "./bundles/${digest}/unpacked" --rootless fi - local csv_sed_file="${sed_commands_dir}/${digest}_csv.sed" - : > "$csv_sed_file" - local inner_pids=() - - for relatedImage in "${all_related_images[@]}"; do - local imgDir="./images/" - local lastTwo="" targetImg internalTargetImg - if [[ "$relatedImage" == *"@sha256:"* ]]; then - local relatedImageDigest="${relatedImage##*@sha256:}" - imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - _last_two lastTwo "${relatedImage%@*}" - targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageDigest" - internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageDigest" - elif [[ "$relatedImage" == *":"* ]]; then - local relatedImageTag="${relatedImage##*:}" - imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - _last_two lastTwo "${relatedImage%:*}" - targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageTag" - internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageTag" - else - imgDir+="${relatedImage}/tag_latest" - _last_two lastTwo "${relatedImage}" - targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" - internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:latest" - fi + debugf "\t inspecting related images referenced in bundle image $bundleImg..." + for file in "./bundles/${digest}/unpacked/rootfs/manifests"/*; do + if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then + debugf "\t Adding imagePullSecrets to the CSV file so we can pull from private registries" + "$YQ" eval ' + (.spec.install.spec.deployments[] | select(.name == "rhdh-operator").spec.template.spec) += + {"imagePullSecrets": [{"name": "internal-reg-auth-for-rhdh"},{"name": "internal-reg-ext-auth-for-rhdh"},{"name": "reg-pull-secret"}]} + ' -i "$file" + + all_related_images=() + debugf "\t finding related images in $file to mirror..." + mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) + if ((${#images[@]})); then + all_related_images+=("${images[@]}") + fi + # TODO(rm3l): we should use spec.relatedImages instead, but it seems to be incomplete in some bundles + related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) + if [[ -n "$related_images" ]]; then + while IFS= read -r img; do + [[ -n "$img" ]] && all_related_images+=("$img") + done <<<"$related_images" + fi + for relatedImage in "${all_related_images[@]}"; do + imgDir="./images/" + if [[ "$relatedImage" == *"@sha256:"* ]]; then + relatedImageDigest="${relatedImage##*@sha256:}" + imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" + lastTwo=$(extract_last_two_elements "${relatedImage%@*}") + targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageDigest" + internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageDigest" + elif [[ "$relatedImage" == *":"* ]]; then + relatedImageTag="${relatedImage##*:}" + imgDir+="${relatedImage%:*}/tag_$relatedImageTag" + lastTwo=$(extract_last_two_elements "${relatedImage%:*}") + targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageTag" + internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageTag" + else + imgDir+="${relatedImage}/tag_latest" + lastTwo=$(extract_last_two_elements "${relatedImage}") + targetImg="$(buildRegistryUrl)/${lastTwo}:latest" + internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:latest" + fi if [[ -n "$TO_REGISTRY" ]]; then mirror_image_to_registry "$relatedImage" "$targetImg" @@ -1061,69 +891,22 @@ function process_single_bundle() { fi done - wait_for_pids inner_pids "related image mirror" + if [[ -n "$TO_REGISTRY" ]]; then + # repack the image with the changes + debugf "\t Repacking image ./bundles/${digest}/src => ./bundles/${digest}/unpacked..." + umoci repack --image "./bundles/${digest}/src:latest" "./bundles/${digest}/unpacked" + + # Push the bundle to the mirror registry + newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + debugf "\t Pushing updated bundle image: ./bundles/${digest}/src => ${newBundleImage}..." + skopeo copy --remove-signatures --dest-tls-verify=false "oci:./bundles/${digest}/src:latest" "docker://${newBundleImage}" - if [[ -s "$csv_sed_file" ]]; then - sed -i -f "$csv_sed_file" "$file" + sed -i "s#${originalBundleImg}#${newBundleImageInternal}#g" "./rhdh/rhdh/render.yaml" fi fi done - if [[ -n "$TO_REGISTRY" ]]; then - umoci repack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" - - local bundleLastTwo="" - _last_two bundleLastTwo "${bundleImg%@*}" - local newBundleImage="${CACHED_REGISTRY_URL}/${bundleLastTwo}:${digest}" - local newBundleImageInternal="${CACHED_REGISTRY_URL_INTERNAL}/${bundleLastTwo}:${digest}" - debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 - sem_acquire - skopeo copy --remove-signatures --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } - sem_release - - echo "s#${originalBundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" - fi -} - -function process_bundles() { - - local bundle_images - bundle_images=$(grep -E '^image: .*operator-bundle' "${TMPDIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq) - - local total_bundles - total_bundles=$(echo "$bundle_images" | wc -l | tr -d ' ') - infof "Processing ${total_bundles} bundles (max ${MAX_PARALLEL} parallel)..." - - local sed_commands_dir="${TMPDIR}/sed_commands_bundles" - mkdir -p "$sed_commands_dir" - - local bundle_count=0 - local pids=() - - for bundleImg in $bundle_images; do - bundle_count=$((bundle_count + 1)) - local originalBundleImg="$bundleImg" - bundleImg=$(replaceInternalRegIfNeeded "$bundleImg") - local digest="${bundleImg##*@sha256:}" - debugf "bundle #${bundle_count}/${total_bundles}: $originalBundleImg => $bundleImg" - - process_single_bundle "$bundleImg" "$originalBundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & - pids+=($!) - done - - wait_for_pids pids "bundle" - - local sed_files - sed_files=$(find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' 2>/dev/null || true) - if [[ -n "$sed_files" ]]; then - local combined_sed="${TMPDIR}/combined_bundle_sed.txt" - find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' -exec cat {} + > "$combined_sed" - local replacement_count - replacement_count=$(wc -l < "$combined_sed" | tr -d ' ') - infof "Applying ${replacement_count} image ref replacements to render.yaml..." - sed -i -f "$combined_sed" "./rhdh/rhdh/render.yaml" - fi - if [ ! -f "rhdh/rhdh.Dockerfile" ]; then debugf "\t Regenerating Dockerfile so the index can be rebuilt..." opm generate dockerfile rhdh/rhdh @@ -1141,101 +924,6 @@ function process_bundles() { fi } -function process_single_bundle_from_dir() { - set -euo pipefail - - local bundleImg="$1" - local digest="$2" - local sed_commands_dir="$3" - local bundle_id="$4" - - if [ ! -d "${FROM_DIR}/bundles/${digest}/src" ]; then - warnf "missing src image for bundle digest: ${FROM_DIR}/bundles/${digest}/src" >&2 - return 0 - fi - if [ ! -d "${FROM_DIR}/bundles/${digest}/unpacked" ]; then - warnf "missing unpacked image for bundle digest: ${FROM_DIR}/bundles/${digest}/unpacked" >&2 - return 0 - fi - - debugf "bundle #${bundle_id}: handling from ${FROM_DIR}/bundles/${digest}..." >&2 - - for file in "${TMPDIR}/bundles/${digest}/unpacked/rootfs/manifests"/*; do - if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then - local all_related_images=() - local images - mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) - if ((${#images[@]})); then - all_related_images+=("${images[@]}") - fi - local related_images - related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) - if [[ -n "$related_images" ]]; then - while IFS= read -r img; do - [[ -n "$img" ]] && all_related_images+=("$img") - done <<<"$related_images" - fi - - local csv_sed_file="${sed_commands_dir}/${digest}_csv.sed" - : > "$csv_sed_file" - local inner_pids=() - - for relatedImage in "${all_related_images[@]}"; do - local imgDir="${FROM_DIR}/images/" - local lastTwo="" targetImg targetImgInternal - if [[ "$relatedImage" == *"@sha256:"* ]]; then - local relatedImageDigest="${relatedImage##*@sha256:}" - imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - _last_two lastTwo "${relatedImage%@*}" - targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageDigest" - targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageDigest" - elif [[ "$relatedImage" == *":"* ]]; then - local relatedImageTag="${relatedImage##*:}" - imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - _last_two lastTwo "${relatedImage%:*}" - targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageTag" - targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageTag" - else - imgDir+="${relatedImage}/tag_latest" - _last_two lastTwo "${relatedImage}" - targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" - targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:latest" - fi - if [ ! -d "$imgDir" ]; then - warnf "Skipping related image $relatedImage not found mirrored in dir: $FROM_DIR/images" >&2 - continue - fi - if [[ -n "$TO_REGISTRY" ]]; then - echo "s#${relatedImage}#${targetImgInternal}#g" >> "$csv_sed_file" - push_image_from_archive "$imgDir" "$targetImg" & - inner_pids+=($!) - fi - done - - wait_for_pids inner_pids "related image push" - - if [[ -s "$csv_sed_file" ]]; then - sed -i -f "$csv_sed_file" "$file" - fi - fi - done - - if [[ -n "$TO_REGISTRY" ]]; then - umoci repack --image "${TMPDIR}/bundles/${digest}/src:latest" "${TMPDIR}/bundles/${digest}/unpacked" - - local bundleLastTwo="" - _last_two bundleLastTwo "${bundleImg%@*}" - local newBundleImage="${CACHED_REGISTRY_URL}/${bundleLastTwo}:${digest}" - local newBundleImageInternal="${CACHED_REGISTRY_URL_INTERNAL}/${bundleLastTwo}:${digest}" - debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 - sem_acquire - skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } - sem_release - - echo "s#${bundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" - fi -} - function process_bundles_from_dir() { if [ ! -f "${FROM_DIR}/rhdh/rhdh.Dockerfile" ]; then @@ -1247,12 +935,19 @@ function process_bundles_from_dir() { cp -r "${FROM_DIR}/${d}" "${TMPDIR}/${d}" done - local bundle_images - bundle_images=$(grep -E '^image: .*operator-bundle' "${FROM_DIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq) + for bundleImg in $(grep -E '^image: .*operator-bundle' "${FROM_DIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq); do + debugf "bundleImg=$bundleImg" + digest="${bundleImg##*@sha256:}" + if [ ! -d "${FROM_DIR}/bundles/${digest}/src" ]; then + warnf "missing src image for bundle digest: ${FROM_DIR}/bundles/${digest}/src" + continue + fi + if [ ! -d "${FROM_DIR}/bundles/${digest}/unpacked" ]; then + warnf "missing unpacked image for bundle digest: ${FROM_DIR}/bundles/${digest}/unpacked" + continue + fi - local total_bundles - total_bundles=$(echo "$bundle_images" | wc -l | tr -d ' ') - infof "Processing ${total_bundles} bundles from dir (max ${MAX_PARALLEL} parallel)..." + debugf "Handling bundle image from ${FROM_DIR}/bundles/${digest}..." debugf "\t inspecting related images referenced in bundle image $bundleImg..." for file in "${TMPDIR}/bundles/${digest}/unpacked/rootfs/manifests"/*; do @@ -1300,31 +995,21 @@ function process_bundles_from_dir() { fi done - local bundle_count=0 - local pids=() + if [[ -n "$TO_REGISTRY" ]]; then + # repack the image with the changes + debugf "\t Repacking image ./bundles/${digest}/src => ./bundles/${digest}/unpacked..." + umoci repack --image "${TMPDIR}/bundles/${digest}/src:latest" "${TMPDIR}/bundles/${digest}/unpacked" - for bundleImg in $bundle_images; do - bundle_count=$((bundle_count + 1)) - local digest="${bundleImg##*@sha256:}" - debugf "bundle #${bundle_count}/${total_bundles}: $bundleImg" + # Push the bundle to the mirror registry + newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + debugf "\t Pushing updated bundle image: ./bundles/${digest}/src => ${newBundleImage}..." + skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" - process_single_bundle_from_dir "$bundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & - pids+=($!) + sed -i "s#${bundleImg}#${newBundleImageInternal}#g" "${TMPDIR}/rhdh/rhdh/render.yaml" + fi done - wait_for_pids pids "bundle from dir" - - local sed_files - sed_files=$(find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' 2>/dev/null || true) - if [[ -n "$sed_files" ]]; then - local combined_sed="${TMPDIR}/combined_bundle_from_dir_sed.txt" - find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' -exec cat {} + > "$combined_sed" - local replacement_count - replacement_count=$(wc -l < "$combined_sed" | tr -d ' ') - infof "Applying ${replacement_count} image ref replacements to render.yaml..." - sed -i -f "$combined_sed" "${TMPDIR}/rhdh/rhdh/render.yaml" - fi - if [[ -n "$TO_REGISTRY" ]]; then pushd "${TMPDIR}/rhdh" my_operator_index="$(buildCatalogImageUrl)" @@ -1338,39 +1023,23 @@ function process_bundles_from_dir() { } function mirror_image_to_registry() { - sem_acquire - local rc=0 - local src_image="$1" - if [[ "${IS_CI_INDEX_IMAGE}" == "true" ]]; then - for reg in registry.stage.redhat.io registry.redhat.io; do - src_image="${src_image/$reg\/rhdh/quay.io\/rhdh}" - done - src_image="${src_image/registry-proxy.engineering.redhat.com\/rh-osbs\/rhdh-/quay.io\/rhdh\/}" - fi - local dest_image=$2 - + local src_image + src_image=$(replaceInternalRegIfNeeded "$1") + local dest_image + dest_image=$2 + echo "Mirroring $src_image to $dest_image..." - skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false docker://"$src_image" docker://"$dest_image" || rc=$? - sem_release - return $rc + skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false docker://"$src_image" docker://"$dest_image" } function mirror_image_to_archive() { - sem_acquire - local rc=0 - local src_image="$1" - if [[ "${IS_CI_INDEX_IMAGE}" == "true" ]]; then - for reg in registry.stage.redhat.io registry.redhat.io; do - src_image="${src_image/$reg\/rhdh/quay.io\/rhdh}" - done - src_image="${src_image/registry-proxy.engineering.redhat.com\/rh-osbs\/rhdh-/quay.io\/rhdh\/}" - fi - local archive_path="$2" + local src_image + src_image=$(replaceInternalRegIfNeeded "$1") + local archive_path + archive_path="$2" debugf "Saving $src_image to $archive_path..." - skopeo copy --preserve-digests --remove-signatures --all --preserve-digests --dest-tls-verify=false docker://"$src_image" dir:"$archive_path" || rc=$? - sem_release - return $rc + skopeo copy --preserve-digests --remove-signatures --all --preserve-digests --dest-tls-verify=false docker://"$src_image" dir:"$archive_path" } function is_complete_image_archive() { @@ -1386,18 +1055,7 @@ function push_image_from_archive() { exit 1 fi echo "Pushing $archive_path to $dest_image..." - for ((attempt=1; attempt<=max_retries; attempt++)); do - rc=0 - skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image" || rc=$? - if [[ $rc -eq 0 ]]; then break; fi - if ((attempt < max_retries)); then - warnf "Push failed (attempt $attempt/$max_retries), retrying in ${retry_delay}s... [$dest_image]" - sleep "$retry_delay" - retry_delay=$((retry_delay * 2)) - fi - done - sem_release - return $rc + skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image" } check_tool "yq" @@ -1497,7 +1155,7 @@ EOF mirror_image_to_archive "registry.redhat.io/ubi9/ubi:latest" "${TO_DIR}/rhdh-catalog" fi if [[ -n "$TO_REGISTRY" ]]; then - registryUrl="${CACHED_REGISTRY_URL}" + registryUrl=$(buildRegistryUrl) if [[ "${TO_REGISTRY}" == "OCP_INTERNAL" ]]; then registryUrl+="/oc-mirror" fi @@ -1527,7 +1185,7 @@ EOF exit 1 fi if [[ -n "${TO_REGISTRY}" ]]; then - registryUrl="${CACHED_REGISTRY_URL}" + registryUrl=$(buildRegistryUrl) if [[ "${TO_REGISTRY}" == "OCP_INTERNAL" ]]; then registryUrl+="/oc-mirror" fi @@ -1643,34 +1301,13 @@ EOF fi else if [[ -z "${FROM_DIR}" ]]; then - mirror_extra_images & - extra_mirror_pid=$! - render_index process_bundles - - if ! wait "$extra_mirror_pid"; then - errorf "mirror_extra_images failed" - exit 1 - fi else - mirror_extra_images_from_dir & - extra_from_dir_pid=$! - - mirror_extra_images & - extra_mirror_pid=$! - process_bundles_from_dir - - if ! wait "$extra_from_dir_pid"; then - errorf "mirror_extra_images_from_dir failed" - exit 1 - fi - if ! wait "$extra_mirror_pid"; then - errorf "mirror_extra_images failed" - exit 1 - fi + mirror_extra_images_from_dir fi + mirror_extra_images # create OLM resources manifestsTargetDir="${TMPDIR}" @@ -1757,7 +1394,7 @@ EOF if [[ -n "${TO_REGISTRY}" ]]; then # IDMS will only work on regular OCP clusters. It doesn't work on ROSA or clusters with hosted control planes like on IBM Cloud. - registry_url_internal="${CACHED_REGISTRY_URL}" + registry_url_internal=$(buildRegistryUrl) cat <"${manifestsTargetDir}/imageDigestMirrorSet.yaml" apiVersion: config.openshift.io/v1 kind: ImageDigestMirrorSet From 295b336c9a64bb5bb49470f4f40bb4b91d5a68c7 Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Fri, 17 Jul 2026 17:12:03 +0100 Subject: [PATCH 8/8] feat: parallelize bundle and image processing for faster execution Add FIFO-based semaphore (sem_init/sem_acquire/sem_release) and wait_for_pids helper for controlled parallel execution. Parallelize ocp_prepare_internal_registry, mirror_extra_images, mirror_extra_images_from_dir, process_bundles, and process_bundles_from_dir. Extract process_single_bundle and process_single_bundle_from_dir worker functions that run in background with per-digest sed command files for lock-free parallel bundle processing. Add registry URL caching, _last_two nameref helper, retry logic for push_image_from_archive, and proper trap handling for background job cleanup. Run mirror_extra_images concurrently with bundle processing in main flow. Signed-off-by: Fortune Ndlovu --- .../scripts/prepare-restricted-environment.sh | 776 ++++++++++++------ 1 file changed, 539 insertions(+), 237 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index a24a9aace..cf1c0ac7d 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -20,6 +20,12 @@ FILTERED_VERSIONS=("*") OLM_VERSION="auto" RESOLVED_OLM_VERSION="" +MAX_PARALLEL="${MAX_PARALLEL:-10}" +if ! [[ "$MAX_PARALLEL" =~ ^[0-9]+$ ]] || [[ "$MAX_PARALLEL" -lt 1 ]]; then + errorf "--max-parallel must be a positive integer, got: '$MAX_PARALLEL'" + exit 1 +fi + # assume mikefarah version of yq is already available on the path; if 1, then install the version shown INSTALL_YQ=0 YQ_VERSION=v4.45.1 @@ -55,6 +61,43 @@ function errorf() { logf "ERROR" "\033[0;31m" "$1" } +SEM_FD="" + +function sem_init() { + local fifo + fifo=$(mktemp -u) + mkfifo "$fifo" + exec {SEM_FD}<>"$fifo" + rm -f "$fifo" + for ((i = 0; i < MAX_PARALLEL; i++)); do + printf '\n' >&"${SEM_FD}" + done +} + +function sem_acquire() { + IFS= read -r -u "${SEM_FD}" +} + +function sem_release() { + printf '\n' >&"${SEM_FD}" +} + +function wait_for_pids() { + local -n __wfp_pids=$1 + local context="${2:-background job}" + local failed=0 + for pid in ${__wfp_pids[@]+"${__wfp_pids[@]}"}; do + if ! wait "$pid"; then + failed=$((failed + 1)) + fi + done + __wfp_pids=() + if [[ $failed -gt 0 ]]; then + errorf "${failed} ${context}(s) failed" + return 1 + fi +} + function check_tool() { if ! command -v "$1" >/dev/null; then errorf "Error: Required tool '$1' is not installed." @@ -304,6 +347,10 @@ while [[ "$#" -gt 0 ]]; do shift 1 done +if [[ "${FILTER_VERSIONS_PROVIDED}" == "false" ]]; then + NO_VERSION_FILTER="true" +fi + function is_openshift() { set -euo pipefail @@ -563,12 +610,28 @@ if [[ -n "${TO_DIR}" ]]; then TMPDIR="${TO_DIR}" else TMPDIR=$(mktemp -d) - # shellcheck disable=SC2064 - trap "rm -fr $TMPDIR || true" EXIT fi + +function cleanup() { + local bg_pids + bg_pids=$(jobs -p 2>/dev/null || true) + if [[ -n "$bg_pids" ]]; then + # shellcheck disable=SC2086 + kill $bg_pids 2>/dev/null || true + wait 2>/dev/null || true + fi + if [[ -z "${TO_DIR}" ]]; then + rm -fr "$TMPDIR" || true + fi +} +trap cleanup EXIT +trap 'exit 1' INT TERM + pushd "${TMPDIR}" >/dev/null debugf ">>> WORKING DIR: $TMPDIR <<<" +sem_init + if (( INSTALL_YQ )); then YQ=$HOME/.local/bin/yq_mf YQ_BINARY=yq_linux_amd64 @@ -589,37 +652,66 @@ function ocp_prepare_internal_registry() { # https://access.redhat.com/solutions/6022011 oc patch configs.imageregistry.operator.openshift.io/cluster --patch '{"spec":{"disableRedirect":true}}' --type=merge >&2 my_registry=$(oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}') - skopeo login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 - podman login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 + + skopeo login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 & + local skopeo_login_pid=$! + podman login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 & + local podman_login_pid=$! + + local ns_pids=() for ns in rhdh-operator openshift4 rhdh rhel9 oc-mirror; do - # To be able to push images under this scope in the internal image registry - if ! oc get namespace "$ns" &>/dev/null; then - oc create namespace "$ns" >&2 - fi - oc adm policy add-cluster-role-to-user system:image-signer system:serviceaccount:${ns}:default >&2 || true + ( + if ! oc get namespace "$ns" &>/dev/null; then + oc create namespace "$ns" >&2 + fi + oc adm policy add-cluster-role-to-user system:image-signer "system:serviceaccount:${ns}:default" >&2 || true + ) & + ns_pids+=($!) done + wait_for_pids ns_pids "namespace setup" + + if ! wait "$skopeo_login_pid"; then + errorf "skopeo login failed" + return 1 + fi + if ! wait "$podman_login_pid"; then + errorf "podman login failed" + return 1 + fi + + local secret_pids=() for ns in rhdh-operator openshift-marketplace; do - if oc -n ${ns} get secret internal-reg-ext-auth-for-rhdh &>/dev/null; then - oc -n ${ns} delete secret internal-reg-ext-auth-for-rhdh >&2 - fi - oc -n ${ns} create secret docker-registry internal-reg-ext-auth-for-rhdh \ - --docker-server="${my_registry}" \ - --docker-username=kubeadmin \ - --docker-password="$(oc whoami -t)" \ - --docker-email="admin@internal-registry-ext.example.com" >&2 - if oc -n ${ns} get secret internal-reg-auth-for-rhdh &>/dev/null; then - oc -n ${ns} delete secret internal-reg-auth-for-rhdh >&2 - fi - oc -n ${ns} create secret docker-registry internal-reg-auth-for-rhdh \ - --docker-server="${internal_registry_url}" \ - --docker-username=kubeadmin \ - --docker-password="$(oc whoami -t)" \ - --docker-email="admin@internal-registry.example.com" >&2 - oc adm policy add-cluster-role-to-user system:image-signer system:serviceaccount:${ns}:default >&2 || true + ( + if oc -n "${ns}" get secret internal-reg-ext-auth-for-rhdh &>/dev/null; then + oc -n "${ns}" delete secret internal-reg-ext-auth-for-rhdh >&2 + fi + oc -n "${ns}" create secret docker-registry internal-reg-ext-auth-for-rhdh \ + --docker-server="${my_registry}" \ + --docker-username=kubeadmin \ + --docker-password="$(oc whoami -t)" \ + --docker-email="admin@internal-registry-ext.example.com" >&2 + if oc -n "${ns}" get secret internal-reg-auth-for-rhdh &>/dev/null; then + oc -n "${ns}" delete secret internal-reg-auth-for-rhdh >&2 + fi + oc -n "${ns}" create secret docker-registry internal-reg-auth-for-rhdh \ + --docker-server="${internal_registry_url}" \ + --docker-username=kubeadmin \ + --docker-password="$(oc whoami -t)" \ + --docker-email="admin@internal-registry.example.com" >&2 + oc adm policy add-cluster-role-to-user system:image-signer "system:serviceaccount:${ns}:default" >&2 || true + ) & + secret_pids+=($!) done - oc policy add-role-to-user system:image-puller system:serviceaccount:openshift-marketplace:default -n openshift-marketplace >&2 || true - oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:default -n rhdh-operator >&2 || true - oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:rhdh-operator -n rhdh-operator >&2 || true + wait_for_pids secret_pids "secret setup" + + local policy_pids=() + oc policy add-role-to-user system:image-puller system:serviceaccount:openshift-marketplace:default -n openshift-marketplace >&2 & + policy_pids+=($!) + oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:default -n rhdh-operator >&2 & + policy_pids+=($!) + oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:rhdh-operator -n rhdh-operator >&2 & + policy_pids+=($!) + wait_for_pids policy_pids "policy setup" } function buildRegistryUrl() { @@ -638,11 +730,27 @@ function buildRegistryUrl() { fi } +CACHED_REGISTRY_URL="" +CACHED_REGISTRY_URL_INTERNAL="" + +function cache_registry_urls() { + if [[ -n "$TO_REGISTRY" ]]; then + CACHED_REGISTRY_URL=$(buildRegistryUrl) + CACHED_REGISTRY_URL_INTERNAL=$(buildRegistryUrl "internal") + fi +} + function buildCatalogImageUrl() { if [[ -n "$TO_REGISTRY" ]]; then tag=${INDEX_IMAGE##*:} [[ "$INDEX_IMAGE" == "$tag" ]] && tag="latest" - echo "$(buildRegistryUrl "${1:-external}")/${2:-rhdh/index}:${tag}" + local reg_url + if [[ "${1:-external}" == "internal" ]]; then + reg_url="${CACHED_REGISTRY_URL_INTERNAL}" + else + reg_url="${CACHED_REGISTRY_URL}" + fi + echo "${reg_url}/${2:-rhdh/index}:${tag}" else echo "" fi @@ -714,32 +822,61 @@ function extract_last_two_elements() { fi } +function _last_two() { + local -n __lt_result=$1 + local __lt_input="$2" + if [[ "$__lt_input" == */*/* ]]; then + local __lt_last="${__lt_input##*/}" + local __lt_rest="${__lt_input%/*}" + __lt_result="${__lt_rest##*/}/${__lt_last}" + else + __lt_result="$__lt_input" + fi +} + function mirror_extra_images() { - debugf "Extra images: " "${EXTRA_IMAGES[@]}" + if [[ ${#EXTRA_IMAGES[@]} -eq 0 ]]; then + return + fi + debugf "Extra images (${#EXTRA_IMAGES[@]}, max ${MAX_PARALLEL} parallel): ${EXTRA_IMAGES[*]}" + + if [[ -n "$TO_REGISTRY" ]] && [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then + for img in "${EXTRA_IMAGES[@]}"; do + local lastTwo="" + if [[ "$img" == *"@sha256:"* ]]; then + _last_two lastTwo "${img%@*}" + elif [[ "$img" == *":"* ]]; then + _last_two lastTwo "${img%:*}" + else + _last_two lastTwo "${img}" + fi + local projectNameForOcpReg=${lastTwo%%/*} + oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" + done + fi + + local pids=() for img in "${EXTRA_IMAGES[@]}"; do + local imgDir imgTag lastTwo="" targetImg if [[ "$img" == *"@sha256:"* ]]; then - imgDigest="${img##*@sha256:}" + local imgDigest="${img##*@sha256:}" imgDir="./extraImages/${img%@*}/sha256_$imgDigest" - lastTwo=$(extract_last_two_elements "${img%@*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$imgDigest" + _last_two lastTwo "${img%@*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$imgDigest" elif [[ "$img" == *":"* ]]; then - imgDir="./extraImages/${img%:*}/tag_$imgTag" imgTag="${img##*:}" - lastTwo=$(extract_last_two_elements "${img%:*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$imgTag" + imgDir="./extraImages/${img%:*}/tag_$imgTag" + _last_two lastTwo "${img%:*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$imgTag" else imgDir="./extraImages/${img}/tag_latest" - lastTwo=$(extract_last_two_elements "${img}") - targetImg="$(buildRegistryUrl)/${lastTwo}:latest" + _last_two lastTwo "${img}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" fi if [[ -n "$TO_REGISTRY" ]]; then - if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then - # Create the corresponding project if it doesn't exist - projectNameForOcpReg=${lastTwo%%/*} - oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" - fi - mirror_image_to_registry "$img" "$targetImg" + mirror_image_to_registry "$img" "$targetImg" & + pids+=($!) else if [ -d "$imgDir" ] && ! is_complete_image_archive "$imgDir"; then warnf "Removing incomplete image archive at ${imgDir} (previous download was interrupted)" @@ -747,57 +884,95 @@ function mirror_extra_images() { fi if [ ! -d "$imgDir" ]; then mkdir -p "${imgDir}" - mirror_image_to_archive "$img" "$imgDir" + mirror_image_to_archive "$img" "$imgDir" & + pids+=($!) fi fi done + wait_for_pids pids "extra image" debugf "... done." } function mirror_extra_images_from_dir() { - BASE_DIR="${FROM_DIR}/extraImages" + local BASE_DIR="${FROM_DIR}/extraImages" debugf "Extra images from ${BASE_DIR}..." - if [ -d "${BASE_DIR}" ]; then - # Iterate over all directories named "sha256_*" - find "$BASE_DIR" -type d -name "sha256_*" | while read -r sha256_dir; do - relative_path=${sha256_dir#"$BASE_DIR/"} - sha256_hash=${sha256_dir##*/sha256_} + if [ ! -d "${BASE_DIR}" ]; then + return + fi + + local sha256_dirs=() + mapfile -t sha256_dirs < <(find "$BASE_DIR" -type d -name "sha256_*" 2>/dev/null) + local tag_dirs=() + mapfile -t tag_dirs < <(find "$BASE_DIR" -type d -name "tag_*" 2>/dev/null) + + if [[ ${#sha256_dirs[@]} -eq 0 ]] && [[ ${#tag_dirs[@]} -eq 0 ]]; then + return + fi + + infof "Pushing ${#sha256_dirs[@]} digest + ${#tag_dirs[@]} tag extra images from dir (max ${MAX_PARALLEL} parallel)..." + + if [[ -n "$TO_REGISTRY" ]] && [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then + local ns_set=() + for sha256_dir in "${sha256_dirs[@]}"; do + local relative_path=${sha256_dir#"$BASE_DIR/"} + local parent_path parent_path=$(dirname "$relative_path") - debugf "parent_path: $parent_path" - lastTwo=$(extract_last_two_elements "${parent_path}") - extraImg="${lastTwo}:${sha256_hash}" - debugf "Extra-image: $extraImg" - if [[ -n "$TO_REGISTRY" ]]; then - if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then - # Create the corresponding project if it doesn't exist - projectNameForOcpReg=${lastTwo%%/*} - oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" - fi - targetImg="$(buildRegistryUrl)/${extraImg%@*}" - push_image_from_archive "$sha256_dir" "$targetImg" - fi + local lastTwo="" + _last_two lastTwo "${parent_path}" + ns_set+=("${lastTwo%%/*}") done - - # Iterate over all directories named "tag_*" - find "$BASE_DIR" -type d -name "tag_*" | while read -r tag_dir; do - relative_path=${tag_dir#"$BASE_DIR/"} - tag_hash=${tag_dir##*/tag_} + for tag_dir in "${tag_dirs[@]}"; do + local relative_path=${tag_dir#"$BASE_DIR/"} + local parent_path parent_path=$(dirname "$relative_path") - debugf "parent_path: $parent_path" - lastTwo=$(extract_last_two_elements "${parent_path}") - extraImg="${lastTwo}:${tag_hash}" - debugf "Extra-image: $extraImg" - if [[ -n "$TO_REGISTRY" ]]; then - if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then - # Create the corresponding project if it doesn't exist - projectNameForOcpReg=${lastTwo%%/*} - oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" - fi - targetImg="$(buildRegistryUrl)/${extraImg%:*}" - push_image_from_archive "$tag_dir" "$targetImg" - fi + local lastTwo="" + _last_two lastTwo "${parent_path}" + ns_set+=("${lastTwo%%/*}") done + local unique_ns + unique_ns=$(printf '%s\n' "${ns_set[@]}" | sort -u) + while IFS= read -r ns; do + if [[ -n "$ns" ]] && ! oc get namespace "$ns" &>/dev/null; then + oc create namespace "$ns" + fi + done <<<"$unique_ns" fi + + local pids=() + + for sha256_dir in "${sha256_dirs[@]}"; do + local relative_path=${sha256_dir#"$BASE_DIR/"} + local sha256_hash=${sha256_dir##*/sha256_} + local parent_path + parent_path=$(dirname "$relative_path") + local lastTwo="" + _last_two lastTwo "${parent_path}" + local extraImg="${lastTwo}:${sha256_hash}" + if [[ -n "$TO_REGISTRY" ]]; then + local targetImg + targetImg="${CACHED_REGISTRY_URL}/${extraImg%@*}" + push_image_from_archive "$sha256_dir" "$targetImg" & + pids+=($!) + fi + done + + for tag_dir in "${tag_dirs[@]}"; do + local relative_path=${tag_dir#"$BASE_DIR/"} + local tag_hash=${tag_dir##*/tag_} + local parent_path + parent_path=$(dirname "$relative_path") + local lastTwo="" + _last_two lastTwo "${parent_path}" + local extraImg="${lastTwo}:${tag_hash}" + if [[ -n "$TO_REGISTRY" ]]; then + local targetImg + targetImg="${CACHED_REGISTRY_URL}/${extraImg}" + push_image_from_archive "$tag_dir" "$targetImg" & + pids+=($!) + fi + done + + wait_for_pids pids "extra image from dir" } function replaceInternalRegIfNeeded() { @@ -813,100 +988,145 @@ function replaceInternalRegIfNeeded() { echo "$img" } -function process_bundles() { +function process_single_bundle() { + set -euo pipefail - for bundleImg in $(grep -E '^image: .*operator-bundle' "${TMPDIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq); do - debugf "bundleImg=$bundleImg" - originalBundleImg="$bundleImg" - bundleImg=$(replaceInternalRegIfNeeded "$bundleImg") - digest="${bundleImg##*@sha256:}" - if skopeo inspect "docker://$bundleImg" &>/dev/null; then - mkdir -p "bundles/$digest" - debugf "\t copying and unpacking image $bundleImg locally..." - if [ ! -d "./bundles/${digest}/src" ]; then - skopeo copy --remove-signatures "docker://$bundleImg" "oci:./bundles/${digest}/src:latest" + local bundleImg="$1" + local originalBundleImg="$2" + local digest="$3" + local sed_commands_dir="$4" + local bundle_id="$5" + + local bundle_dir="bundles/${digest}" + mkdir -p "${bundle_dir}" + + sem_acquire + if ! skopeo copy --remove-signatures "docker://$bundleImg" "oci:./${bundle_dir}/src:latest" 2>"${bundle_dir}/copy.err"; then + sem_release + debugf "bundle #${bundle_id}: skopeo copy failed, skipping (see ${bundle_dir}/copy.err)" >&2 + return 0 + fi + sem_release + debugf "bundle #${bundle_id}: pulled ${bundleImg}" >&2 + + umoci unpack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" --rootless + + for file in "./${bundle_dir}/unpacked/rootfs/manifests"/*; do + if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then + "$YQ" eval ' + (.spec.install.spec.deployments[] | select(.name == "rhdh-operator").spec.template.spec) += + {"imagePullSecrets": [{"name": "internal-reg-auth-for-rhdh"},{"name": "internal-reg-ext-auth-for-rhdh"},{"name": "reg-pull-secret"}]} + ' -i "$file" + + local all_related_images=() + local images + mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) + if ((${#images[@]})); then + all_related_images+=("${images[@]}") fi - if [ ! -d "./bundles/${digest}/unpacked" ]; then - umoci unpack --image "./bundles/${digest}/src:latest" "./bundles/${digest}/unpacked" --rootless + local related_images + related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) + if [[ -n "$related_images" ]]; then + while IFS= read -r img; do + [[ -n "$img" ]] && all_related_images+=("$img") + done <<<"$related_images" fi - debugf "\t inspecting related images referenced in bundle image $bundleImg..." - for file in "./bundles/${digest}/unpacked/rootfs/manifests"/*; do - if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then - debugf "\t Adding imagePullSecrets to the CSV file so we can pull from private registries" - "$YQ" eval ' - (.spec.install.spec.deployments[] | select(.name == "rhdh-operator").spec.template.spec) += - {"imagePullSecrets": [{"name": "internal-reg-auth-for-rhdh"},{"name": "internal-reg-ext-auth-for-rhdh"},{"name": "reg-pull-secret"}]} - ' -i "$file" - - all_related_images=() - debugf "\t finding related images in $file to mirror..." - mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) - if ((${#images[@]})); then - all_related_images+=("${images[@]}") + for relatedImage in "${all_related_images[@]}"; do + local imgDir="./images/" + local lastTwo="" targetImg internalTargetImg + if [[ "$relatedImage" == *"@sha256:"* ]]; then + local relatedImageDigest="${relatedImage##*@sha256:}" + imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" + _last_two lastTwo "${relatedImage%@*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageDigest" + internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageDigest" + elif [[ "$relatedImage" == *":"* ]]; then + local relatedImageTag="${relatedImage##*:}" + imgDir+="${relatedImage%:*}/tag_$relatedImageTag" + _last_two lastTwo "${relatedImage%:*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageTag" + internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageTag" + else + imgDir+="${relatedImage}/tag_latest" + _last_two lastTwo "${relatedImage}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" + internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:latest" + fi + + if [[ -n "$TO_REGISTRY" ]]; then + mirror_image_to_registry "$relatedImage" "$targetImg" + debugf "replacing $relatedImage in file '${file}' => $internalTargetImg" + sed -i 's#'"$relatedImage"'#'"$internalTargetImg"'#g' "$file" + else + if [ -d "$imgDir" ] && ! is_complete_image_archive "$imgDir"; then + warnf "Removing incomplete image archive at ${imgDir} (previous download was interrupted)" + rm -rf "$imgDir" fi - # TODO(rm3l): we should use spec.relatedImages instead, but it seems to be incomplete in some bundles - related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) - if [[ -n "$related_images" ]]; then - while IFS= read -r img; do - [[ -n "$img" ]] && all_related_images+=("$img") - done <<<"$related_images" + if [ ! -d "$imgDir" ]; then + mkdir -p "${imgDir}" + mirror_image_to_archive "$relatedImage" "$imgDir" fi - for relatedImage in "${all_related_images[@]}"; do - imgDir="./images/" - if [[ "$relatedImage" == *"@sha256:"* ]]; then - relatedImageDigest="${relatedImage##*@sha256:}" - imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - lastTwo=$(extract_last_two_elements "${relatedImage%@*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageDigest" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageDigest" - elif [[ "$relatedImage" == *":"* ]]; then - relatedImageTag="${relatedImage##*:}" - imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - lastTwo=$(extract_last_two_elements "${relatedImage%:*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageTag" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageTag" - else - imgDir+="${relatedImage}/tag_latest" - lastTwo=$(extract_last_two_elements "${relatedImage}") - targetImg="$(buildRegistryUrl)/${lastTwo}:latest" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:latest" - fi - - if [[ -n "$TO_REGISTRY" ]]; then - mirror_image_to_registry "$relatedImage" "$targetImg" - debugf "replacing $relatedImage in file '${file}' => $internalTargetImg" - sed -i 's#'"$relatedImage"'#'"$internalTargetImg"'#g' "$file" - else - if [ -d "$imgDir" ] && ! is_complete_image_archive "$imgDir"; then - warnf "Removing incomplete image archive at ${imgDir} (previous download was interrupted)" - rm -rf "$imgDir" - fi - if [ ! -d "$imgDir" ]; then - mkdir -p "${imgDir}" - mirror_image_to_archive "$relatedImage" "$imgDir" - fi - fi - done fi done + fi + done - if [[ -n "$TO_REGISTRY" ]]; then - # repack the image with the changes - debugf "\t Repacking image ./bundles/${digest}/src => ./bundles/${digest}/unpacked..." - umoci repack --image "./bundles/${digest}/src:latest" "./bundles/${digest}/unpacked" + if [[ -n "$TO_REGISTRY" ]]; then + umoci repack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" + + local bundleLastTwo="" + _last_two bundleLastTwo "${bundleImg%@*}" + local newBundleImage="${CACHED_REGISTRY_URL}/${bundleLastTwo}:${digest}" + local newBundleImageInternal="${CACHED_REGISTRY_URL_INTERNAL}/${bundleLastTwo}:${digest}" + debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 + sem_acquire + skopeo copy --remove-signatures --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } + sem_release + + echo "s#${originalBundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" + fi +} - # Push the bundle to the mirror registry - newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - debugf "\t Pushing updated bundle image: ./bundles/${digest}/src => ${newBundleImage}..." - skopeo copy --remove-signatures --dest-tls-verify=false "oci:./bundles/${digest}/src:latest" "docker://${newBundleImage}" +function process_bundles() { - sed -i "s#${originalBundleImg}#${newBundleImageInternal}#g" "./rhdh/rhdh/render.yaml" - fi - fi + local bundle_images + bundle_images=$(grep -E '^image: .*operator-bundle' "${TMPDIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq) + + local total_bundles + total_bundles=$(echo "$bundle_images" | wc -l | tr -d ' ') + infof "Processing ${total_bundles} bundles (max ${MAX_PARALLEL} parallel)..." + + local sed_commands_dir="${TMPDIR}/sed_commands_bundles" + mkdir -p "$sed_commands_dir" + + local bundle_count=0 + local pids=() + + for bundleImg in $bundle_images; do + bundle_count=$((bundle_count + 1)) + local originalBundleImg="$bundleImg" + bundleImg=$(replaceInternalRegIfNeeded "$bundleImg") + local digest="${bundleImg##*@sha256:}" + debugf "bundle #${bundle_count}/${total_bundles}: $originalBundleImg => $bundleImg" + + process_single_bundle "$bundleImg" "$originalBundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & + pids+=($!) done + wait_for_pids pids "bundle" + + local sed_files + sed_files=$(find "$sed_commands_dir" -maxdepth 1 -name '*.sed' 2>/dev/null || true) + if [[ -n "$sed_files" ]]; then + local combined_sed="${TMPDIR}/combined_bundle_sed.txt" + find "$sed_commands_dir" -maxdepth 1 -name '*.sed' -exec cat {} + > "$combined_sed" + local replacement_count + replacement_count=$(wc -l < "$combined_sed" | tr -d ' ') + infof "Applying ${replacement_count} image ref replacements to render.yaml..." + sed -i -f "$combined_sed" "./rhdh/rhdh/render.yaml" + fi + if [ ! -f "rhdh/rhdh.Dockerfile" ]; then debugf "\t Regenerating Dockerfile so the index can be rebuilt..." opm generate dockerfile rhdh/rhdh @@ -924,6 +1144,101 @@ function process_bundles() { fi } +function process_single_bundle_from_dir() { + set -euo pipefail + + local bundleImg="$1" + local digest="$2" + local sed_commands_dir="$3" + local bundle_id="$4" + + if [ ! -d "${FROM_DIR}/bundles/${digest}/src" ]; then + warnf "missing src image for bundle digest: ${FROM_DIR}/bundles/${digest}/src" >&2 + return 0 + fi + if [ ! -d "${FROM_DIR}/bundles/${digest}/unpacked" ]; then + warnf "missing unpacked image for bundle digest: ${FROM_DIR}/bundles/${digest}/unpacked" >&2 + return 0 + fi + + debugf "bundle #${bundle_id}: handling from ${FROM_DIR}/bundles/${digest}..." >&2 + + for file in "${TMPDIR}/bundles/${digest}/unpacked/rootfs/manifests"/*; do + if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then + local all_related_images=() + local images + mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) + if ((${#images[@]})); then + all_related_images+=("${images[@]}") + fi + local related_images + related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) + if [[ -n "$related_images" ]]; then + while IFS= read -r img; do + [[ -n "$img" ]] && all_related_images+=("$img") + done <<<"$related_images" + fi + + local csv_sed_file="${sed_commands_dir}/${digest}_csv.sed" + : > "$csv_sed_file" + local inner_pids=() + + for relatedImage in "${all_related_images[@]}"; do + local imgDir="${FROM_DIR}/images/" + local lastTwo="" targetImg targetImgInternal + if [[ "$relatedImage" == *"@sha256:"* ]]; then + local relatedImageDigest="${relatedImage##*@sha256:}" + imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" + _last_two lastTwo "${relatedImage%@*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageDigest" + targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageDigest" + elif [[ "$relatedImage" == *":"* ]]; then + local relatedImageTag="${relatedImage##*:}" + imgDir+="${relatedImage%:*}/tag_$relatedImageTag" + _last_two lastTwo "${relatedImage%:*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageTag" + targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageTag" + else + imgDir+="${relatedImage}/tag_latest" + _last_two lastTwo "${relatedImage}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" + targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:latest" + fi + if [ ! -d "$imgDir" ]; then + warnf "Skipping related image $relatedImage not found mirrored in dir: $FROM_DIR/images" >&2 + continue + fi + if [[ -n "$TO_REGISTRY" ]]; then + echo "s#${relatedImage}#${targetImgInternal}#g" >> "$csv_sed_file" + push_image_from_archive "$imgDir" "$targetImg" & + inner_pids+=($!) + fi + done + + wait_for_pids inner_pids "related image push" + + if [[ -s "$csv_sed_file" ]]; then + sed -i -f "$csv_sed_file" "$file" + fi + fi + done + + if [[ -n "$TO_REGISTRY" ]]; then + umoci repack --image "${TMPDIR}/bundles/${digest}/src:latest" "${TMPDIR}/bundles/${digest}/unpacked" + + local bundleLastTwo="" + _last_two bundleLastTwo "${bundleImg%@*}" + local newBundleImage="${CACHED_REGISTRY_URL}/${bundleLastTwo}:${digest}" + local newBundleImageInternal="${CACHED_REGISTRY_URL_INTERNAL}/${bundleLastTwo}:${digest}" + debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 + sem_acquire + skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } + sem_release + + echo "s#${bundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" + fi +} + function process_bundles_from_dir() { if [ ! -f "${FROM_DIR}/rhdh/rhdh.Dockerfile" ]; then @@ -935,81 +1250,41 @@ function process_bundles_from_dir() { cp -r "${FROM_DIR}/${d}" "${TMPDIR}/${d}" done - for bundleImg in $(grep -E '^image: .*operator-bundle' "${FROM_DIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq); do - debugf "bundleImg=$bundleImg" - digest="${bundleImg##*@sha256:}" - if [ ! -d "${FROM_DIR}/bundles/${digest}/src" ]; then - warnf "missing src image for bundle digest: ${FROM_DIR}/bundles/${digest}/src" - continue - fi - if [ ! -d "${FROM_DIR}/bundles/${digest}/unpacked" ]; then - warnf "missing unpacked image for bundle digest: ${FROM_DIR}/bundles/${digest}/unpacked" - continue - fi + local bundle_images + bundle_images=$(grep -E '^image: .*operator-bundle' "${FROM_DIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq) - debugf "Handling bundle image from ${FROM_DIR}/bundles/${digest}..." + local total_bundles + total_bundles=$(echo "$bundle_images" | wc -l | tr -d ' ') + infof "Processing ${total_bundles} bundles from dir (max ${MAX_PARALLEL} parallel)..." - debugf "\t inspecting related images referenced in bundle image $bundleImg..." - for file in "${TMPDIR}/bundles/${digest}/unpacked/rootfs/manifests"/*; do - if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then - all_related_images=() - debugf "\t finding related images in $file to mirror..." - mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) - if ((${#images[@]})); then - all_related_images+=("${images[@]}") - fi - # TODO(rm3l): we should use spec.relatedImages instead, but it seems to be incomplete in some bundles - related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) - if [[ -n "$related_images" ]]; then - while IFS= read -r img; do - [[ -n "$img" ]] && all_related_images+=("$img") - done <<<"$related_images" - fi - for relatedImage in "${all_related_images[@]}"; do - imgDir="${FROM_DIR}/images/" - if [[ "$relatedImage" == *"@sha256:"* ]]; then - relatedImageDigest="${relatedImage##*@sha256:}" - imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" - elif [[ "$relatedImage" == *":"* ]]; then - relatedImageTag="${relatedImage##*:}" - imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" - else - imgDir+="${relatedImage}/tag_latest" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage}"):latest" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage}"):latest" - fi - if [ ! -d "$imgDir" ]; then - warnf "Skipping related image $relatedImage not found mirrored in dir: $FROM_DIR/images" - continue - fi - if [[ -n "$TO_REGISTRY" ]]; then - push_image_from_archive "$imgDir" "$targetImg" - debugf "replacing $relatedImage in file '${file}' => $targetImgInternal" - sed -i 's#'"$relatedImage"'#'"$targetImgInternal"'#g' "$file" - fi - done - fi - done + local sed_commands_dir="${TMPDIR}/sed_commands_bundles_from_dir" + mkdir -p "$sed_commands_dir" - if [[ -n "$TO_REGISTRY" ]]; then - # repack the image with the changes - debugf "\t Repacking image ./bundles/${digest}/src => ./bundles/${digest}/unpacked..." - umoci repack --image "${TMPDIR}/bundles/${digest}/src:latest" "${TMPDIR}/bundles/${digest}/unpacked" + local bundle_count=0 + local pids=() - # Push the bundle to the mirror registry - newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - debugf "\t Pushing updated bundle image: ./bundles/${digest}/src => ${newBundleImage}..." - skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" + for bundleImg in $bundle_images; do + bundle_count=$((bundle_count + 1)) + local digest="${bundleImg##*@sha256:}" + debugf "bundle #${bundle_count}/${total_bundles}: $bundleImg" - sed -i "s#${bundleImg}#${newBundleImageInternal}#g" "${TMPDIR}/rhdh/rhdh/render.yaml" - fi + process_single_bundle_from_dir "$bundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & + pids+=($!) done + wait_for_pids pids "bundle from dir" + + local sed_files + sed_files=$(find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' 2>/dev/null || true) + if [[ -n "$sed_files" ]]; then + local combined_sed="${TMPDIR}/combined_bundle_from_dir_sed.txt" + find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' -exec cat {} + > "$combined_sed" + local replacement_count + replacement_count=$(wc -l < "$combined_sed" | tr -d ' ') + infof "Applying ${replacement_count} image ref replacements to render.yaml..." + sed -i -f "$combined_sed" "${TMPDIR}/rhdh/rhdh/render.yaml" + fi + if [[ -n "$TO_REGISTRY" ]]; then pushd "${TMPDIR}/rhdh" my_operator_index="$(buildCatalogImageUrl)" @@ -1027,9 +1302,11 @@ function mirror_image_to_registry() { src_image=$(replaceInternalRegIfNeeded "$1") local dest_image dest_image=$2 - + echo "Mirroring $src_image to $dest_image..." - skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false docker://"$src_image" docker://"$dest_image" + sem_acquire + skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false docker://"$src_image" docker://"$dest_image" || { sem_release; return 1; } + sem_release } function mirror_image_to_archive() { @@ -1039,7 +1316,9 @@ function mirror_image_to_archive() { archive_path="$2" debugf "Saving $src_image to $archive_path..." - skopeo copy --preserve-digests --remove-signatures --all --preserve-digests --dest-tls-verify=false docker://"$src_image" dir:"$archive_path" + sem_acquire + skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false docker://"$src_image" dir:"$archive_path" || { sem_release; return 1; } + sem_release } function is_complete_image_archive() { @@ -1055,7 +1334,21 @@ function push_image_from_archive() { exit 1 fi echo "Pushing $archive_path to $dest_image..." - skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image" + local attempt + for attempt in 1 2 3; do + sem_acquire + if skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image"; then + sem_release + return 0 + fi + sem_release + if [[ $attempt -lt 3 ]]; then + warnf "push_image_from_archive: attempt $attempt failed for $dest_image, retrying..." + sleep $((attempt * 2)) + fi + done + errorf "push_image_from_archive: all 3 attempts failed for $dest_image" + return 1 } check_tool "yq" @@ -1077,6 +1370,8 @@ if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then ocp_prepare_internal_registry fi +cache_registry_urls + if [[ -n "${TO_REGISTRY}" ]]; then resolve_olm_version if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" && "${RESOLVED_OLM_VERSION}" == "v1" ]]; then @@ -1300,6 +1595,9 @@ EOF fi fi else + mirror_extra_images & + mirror_extra_pid=$! + if [[ -z "${FROM_DIR}" ]]; then render_index process_bundles @@ -1307,7 +1605,11 @@ else process_bundles_from_dir mirror_extra_images_from_dir fi - mirror_extra_images + + if ! wait "$mirror_extra_pid"; then + errorf "mirror_extra_images failed" + exit 1 + fi # create OLM resources manifestsTargetDir="${TMPDIR}"