From beedf459d17f524c0cd10f4c6480d7a47f9cc611 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 12 Mar 2021 01:36:24 +0100 Subject: [PATCH] Fixes force-pulling base python images Sometimes base python image patchlevel might case failure of tests. This happens for example with test_views.py tests fixed in #14719 where CVE fix in all python versions caused our tests to fail. There was an error in our scripts - when --force-pull-images were used, the base python version was not updated to the latest version even if there was a newer one and it caused our images to bounce few times between two latest patchlevels when they were manually refreshed. This change fixes it so that the base python image is always used when a) FORCE_PULL_IMAGES is true or b) UPGRADE_TO_NEWER_DEPENDENCIES is != false This will cause python upgrade in two cases: - when images are rebuilt with --force-pull-images locally - when images are upgraded with newer dependencies on master --- .../workflows/build-images-workflow-run.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/scheduled_quarantined.yml | 2 +- .../ci/libraries/_push_pull_remove_images.sh | 27 ++++++++++--------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml index 33c7a79c3ed87..65ba7aa547b8c 100644 --- a/.github/workflows/build-images-workflow-run.yml +++ b/.github/workflows/build-images-workflow-run.yml @@ -24,7 +24,7 @@ on: # yamllint disable-line rule:truthy env: MOUNT_SELECTED_LOCAL_SOURCES: "false" FORCE_ANSWER_TO_QUESTIONS: "yes" - FORCE_PULL_IMAGES: "true" + FORCE_PULL_IMAGES: "false" CHECK_IMAGE_FOR_REBUILD: "true" SKIP_CHECK_REMOTE_IMAGE: "true" DB_RESET: "true" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5e115b20b3c2..c0523d65f9246 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy env: MOUNT_SELECTED_LOCAL_SOURCES: "false" FORCE_ANSWER_TO_QUESTIONS: "yes" - FORCE_PULL_IMAGES: "true" + FORCE_PULL_IMAGES: "false" CHECK_IMAGE_FOR_REBUILD: "true" SKIP_CHECK_REMOTE_IMAGE: "true" DB_RESET: "true" diff --git a/.github/workflows/scheduled_quarantined.yml b/.github/workflows/scheduled_quarantined.yml index 00663cbb3920d..2f7f05f2ea5f7 100644 --- a/.github/workflows/scheduled_quarantined.yml +++ b/.github/workflows/scheduled_quarantined.yml @@ -25,7 +25,7 @@ on: # yamllint disable-line rule:truthy env: MOUNT_SELECTED_LOCAL_SOURCES: "false" FORCE_ANSWER_TO_QUESTIONS: "yes" - FORCE_PULL_IMAGES: "true" + FORCE_PULL_IMAGES: "false" CHECK_IMAGE_FOR_REBUILD: "true" SKIP_CHECK_REMOTE_IMAGE: "true" DB_RESET: "true" diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh index c48491ddc8f6a..5090c5b0f0f2c 100644 --- a/scripts/ci/libraries/_push_pull_remove_images.sh +++ b/scripts/ci/libraries/_push_pull_remove_images.sh @@ -104,33 +104,34 @@ function push_pull_remove_images::pull_image_github_dockerhub() { set -e } +# Force pulls the python base image +function push_pull_remove_images::force_pull_python_base_image() { + docker pull "${PYTHON_BASE_IMAGE}" + echo "FROM ${PYTHON_BASE_IMAGE}" | \ + docker build \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ + -t "${AIRFLOW_PYTHON_BASE_IMAGE}" - +} + # Pulls the base Python image. This image is used as base for CI and PROD images, depending on the parameters used: # -# * if UPGRADE_TO_NEWER_DEPENDENCIES is noy false, then it pulls the latest Python image available first and +# * if FORCE_PULL_IMAGES is true or UPGRADE_TO_NEWER_DEPENDENCIES != false, then it pulls the latest Python image available first and # adds `org.opencontainers.image.source` label to it, so that it is linked to Airflow repository when # we push it to GHCR registry -# * If UPGRADE_TO_NEWER_DEPENDENCIES it pulls the Python base image from either GitHub registry or from DockerHub +# * Otherwise it pulls the Python base image from either GitHub registry or from DockerHub # depending on USE_GITHUB_REGISTRY variable. In case we pull specific build image (via suffix) # it will pull the right image using the specified suffix function push_pull_remove_images::pull_base_python_image() { echo - echo "Force pull base image ${AIRFLOW_PYTHON_BASE_IMAGE}. Upgrade to newer dependencies: ${UPGRADE_TO_NEWER_DEPENDENCIES}" + echo "Force pull python base image ${AIRFLOW_PYTHON_BASE_IMAGE}. Upgrade to newer dependencies: ${UPGRADE_TO_NEWER_DEPENDENCIES}" echo if [[ -n ${DETECTED_TERMINAL=} ]]; then echo -n " Docker pulling ${AIRFLOW_PYTHON_BASE_IMAGE}. Upgrade to newer dependencies ${UPGRADE_TO_NEWER_DEPENDENCIES} " > "${DETECTED_TERMINAL}" fi - if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then - # Pull latest PYTHON_BASE_IMAGE, so that it is linked to the current project it is build in. - # This is necessary in case we use Google Container registry - we always use the - # Airflow version of the python image with the opencontainers label, so that GHCR can link it - # to the Apache Airflow repository. - docker pull "${PYTHON_BASE_IMAGE}" - echo "FROM ${PYTHON_BASE_IMAGE}" | \ - docker build \ - --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - -t "${AIRFLOW_PYTHON_BASE_IMAGE}" - + if [[ "${FORCE_PULL_IMAGES}" == "true" || ${UPGRADE_TO_NEWER_DEPENDENCIES} != "false" ]]; then + push_pull_remove_images::force_pull_python_base_image else if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then PYTHON_TAG_SUFFIX=""