From e80b2fc1feefa1b917920885fd424d7996c6c85c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 14 Mar 2021 15:53:22 +0100 Subject: [PATCH] Better diagnostics for image waiting Sometimes (very rarely) some 'wait for image' pulling steps loop forever (while other steps from parallell jobs pulling the same image have no problems). Example here: Failed step here: * https://github.com/apache/airflow/runs/2106723280?check_suite_focus=true#step:5:349 Another similar step in parallel job had no problems with retrieving the same image earlier: * https://github.com/apache/airflow/runs/2106723269?check_suite_focus=true#step:5:119 Both images pulled the same image: docker.pkg.github.com/apache/airflow/master-python3.6-ci-v2:651461418 This change adds diagnostics information that might provide more information in case this happens again so that we can understand what is going on and mitigate the issue. --- scripts/ci/libraries/_build_images.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index be4833df02966..2deb574bc5da4 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -998,11 +998,18 @@ function build_images::wait_for_image_tag() { start_end::group_start "Wait for image tag ${IMAGE_TO_WAIT_FOR}" while true; do set +e - docker pull "${IMAGE_TO_WAIT_FOR}" 2>/dev/null >/dev/null + echo "${COLOR_BLUE}Docker pull ${IMAGE_TO_WAIT_FOR} ${COLOR_RESET}" >"${OUTPUT_LOG}" + docker pull "${IMAGE_TO_WAIT_FOR}" >>"${OUTPUT_LOG}" 2>&1 set -e - if [[ -z "$(docker images -q "${IMAGE_TO_WAIT_FOR}" 2>/dev/null || true)" ]]; then + local image_hash + echo "${COLOR_BLUE} Docker images -q ${IMAGE_TO_WAIT_FOR}${COLOR_RESET}" >>"${OUTPUT_LOG}" + image_hash="$(docker images -q "${IMAGE_TO_WAIT_FOR}" 2>>"${OUTPUT_LOG}" || true)" + if [[ -z "${image_hash}" ]]; then echo - echo "The image ${IMAGE_TO_WAIT_FOR} is not yet available. Waiting" + echo "The image ${IMAGE_TO_WAIT_FOR} is not yet available. No local hash for the image. Waiting." + echo + echo "Last log:" + cat "${OUTPUT_LOG}" echo sleep 10 else