diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 9970a82e6c213..541c72165997b 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -57,6 +57,7 @@ jobs: pythonVersions: "${{ steps.selective-checks.python-versions }}" upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} + allPythonVersionsListAsString: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} run-tests: ${{ steps.selective-checks.outputs.run-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} @@ -155,19 +156,14 @@ jobs: permissions: packages: write timeout-minutes: 80 - name: "Build CI image ${{matrix.python-version}}" + name: "Build CI images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}" runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} - fail-fast: true if: | needs.build-info.outputs.image-build == 'true' && github.event.pull_request.head.repo.full_name != 'apache/airflow' env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} BACKEND: sqlite outputs: ${{toJSON(needs.build-info.outputs) }} steps: @@ -226,20 +222,25 @@ jobs: - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze free-space - - name: Build & Push CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze build-image --push-image --tag-as-latest + - name: > + Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + run: breeze build-image --push-image --tag-as-latest --run-in-parallel env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} - name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} if: failure() || cancelled() - run: breeze build-image --push-image --empty-image + run: breeze build-image --push-image --empty-image --run-in-parallel env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - name: "Candidates for pip resolver backtrack triggers: ${{ matrix.python-version }}" + - name: "Candidates for pip resolver backtrack triggers" if: failure() || cancelled() - run: breeze find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}" + run: > + breeze find-newer-dependencies --max-age 1 + --python "${{ needs.build-info.outputs.defaultPythonVersion }}" - name: "Fix ownership" run: breeze fix-ownership if: always() @@ -248,19 +249,14 @@ jobs: permissions: packages: write timeout-minutes: 80 - name: "Build PROD image ${{matrix.python-version}}" + name: "Build PROD images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}" runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info, build-ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} - fail-fast: true if: | needs.build-info.outputs.image-build == 'true' && github.event.pull_request.head.repo.full_name != 'apache/airflow' env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} BACKEND: sqlite steps: - name: Cleanup repo @@ -320,7 +316,7 @@ jobs: run: breeze free-space - name: > Pull CI image for PROD build: - ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} + ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest env: # Always use default Python version of CI image for preparing packages @@ -338,9 +334,12 @@ jobs: run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files - - name: Build & Push PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: > + Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} run: > breeze build-prod-image + --run-in-parallel --tag-as-latest --push-image --install-packages-from-context @@ -350,9 +349,10 @@ jobs: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - name: Push empty PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + - name: Push empty PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} if: failure() || cancelled() - run: breeze build-prod-image --cleanup-context --push-image --empty-image + run: breeze build-prod-image --cleanup-context --push-image --empty-image --run-in-parallel env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Fix ownership" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c112a6da727a7..5450833f23b63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,7 @@ jobs: upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} + allPythonVersionsListAsString: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }} kubernetesVersionsListAsString: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} @@ -285,13 +286,11 @@ jobs: permissions: packages: write timeout-minutes: 80 - name: "${{needs.build-info.outputs.buildJobDescription}} CI image ${{matrix.python-version}}" + name: > + ${{needs.build-info.outputs.buildJobDescription}} CI images + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} - fail-fast: true env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} steps: @@ -311,7 +310,7 @@ jobs: if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: "Retrieve DEFAULTS from the _initialization.sh" # We cannot "source" the script here because that would be a security problem (we cannot run - # any code that comes from the sources coming from the PR. Therefore we extract the + # any code that comes from the sources coming from the PR. Therefore, we extract the # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands id: defaults run: | @@ -331,17 +330,21 @@ jobs: - name: "Free space" run: breeze free-space if: needs.build-info.outputs.inWorkflowBuild == 'true' - - name: Build & Push CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze build-image --push-image --tag-as-latest + - name: > + Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + run: breeze build-image --push-image --tag-as-latest --run-in-parallel env: - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} if: needs.build-info.outputs.inWorkflowBuild == 'true' - - name: "Candidates for pip resolver backtrack triggers: ${{ matrix.python-version }}" + - name: "Candidates for pip resolver backtrack triggers" if: failure() || cancelled() - run: breeze find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}" + run: > + breeze find-newer-dependencies --max-age 1 + --python "${{ needs.build-info.outputs.defaultPythonVersion }}" - name: "Fix ownership" run: breeze fix-ownership if: always() && needs.build-info.outputs.inWorkflowBuild == 'true' @@ -350,17 +353,14 @@ jobs: permissions: packages: write timeout-minutes: 80 - name: "${{needs.build-info.outputs.buildJobDescription}} PROD image ${{matrix.python-version}}" + name: > + ${{needs.build-info.outputs.buildJobDescription}} PROD images + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info, build-ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} - fail-fast: true env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} BACKEND: sqlite - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} VERSION_SUFFIX_FOR_PYPI: "dev0" steps: @@ -424,10 +424,13 @@ jobs: - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files if: needs.build-info.outputs.inWorkflowBuild == 'true' - - name: Build & Push PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: > + Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} run: > breeze build-prod-image --tag-as-latest + --run-in-parallel --push-image --install-packages-from-context --disable-airflow-repo-cache @@ -436,6 +439,7 @@ jobs: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: "Fix ownership" run: breeze fix-ownership @@ -1385,7 +1389,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} # We wait for the images to be available either from "build-images.yml' run as pull_request_target - # or from build-prod-image above. + # or from build-prod-images above. # We are utilising single job to wait for all images because this job merely waits # For the images to be available and test them. run: breeze pull-prod-image --verify-image --wait-for-image --run-in-parallel diff --git a/dev/REFRESHING_CI_CACHE.md b/dev/REFRESHING_CI_CACHE.md index 85194845dc9b2..5fd458bf34a43 100644 --- a/dev/REFRESHING_CI_CACHE.md +++ b/dev/REFRESHING_CI_CACHE.md @@ -51,7 +51,7 @@ manual refresh might be needed. # Manually generating constraint files ```bash -breeze build-image --build-multiple-images --upgrade-to-newer-dependencies --answer yes +breeze build-image --run-in-parallel --upgrade-to-newer-dependencies --answer yes breeze generate-constraints --airflow-constraints-mode constraints --run-in-parallel --answer yes breeze generate-constraints --airflow-constraints-mode constraints-source-providers --run-in-parallel --answer yes breeze generate-constraints --airflow-constraints-mode constraints-no-providers --run-in-parallel --answer yes diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 9deac3a276817..859bb2e905b51 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -37,7 +37,6 @@ option_airflow_constraints_mode_ci, option_airflow_constraints_reference_build, option_answer, - option_build_multiple_images, option_debian_version, option_dev_apt_command, option_dev_apt_deps, @@ -115,9 +114,10 @@ ], }, { - "name": "Building multiple images", + "name": "Building images in parallel", "options": [ - "--build-multiple-images", + "--run-in-parallel", + "--parallelism", "--python-versions", ], }, @@ -189,13 +189,46 @@ } +def start_building(ci_image_params: BuildCiParams, dry_run: bool, verbose: bool) -> bool: + """Starts building attempt. Returns false if we should not continue""" + if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies: + if not should_we_run_the_build(build_ci_params=ci_image_params): + return False + if ci_image_params.prepare_buildx_cache or ci_image_params.push_image: + login_to_github_docker_registry(image_params=ci_image_params, dry_run=dry_run, verbose=verbose) + return True + + +def run_build_in_parallel( + image_params_list: List[BuildCiParams], + python_version_list: List[str], + parallelism: int, + dry_run: bool, + verbose: bool, +) -> None: + get_console().print( + f"\n[info]Building with parallelism = {parallelism} for the images: {python_version_list}:" + ) + pool = mp.Pool(parallelism) + results = [ + pool.apply_async( + run_build_ci_image, + args=(verbose, dry_run, image_param, True), + ) + for image_param in image_params_list + ] + check_async_run_results(results) + pool.close() + + @main.command(name='build-image') @option_github_repository @option_verbose @option_dry_run @option_answer @option_python -@option_build_multiple_images +@option_run_in_parallel +@option_parallelism @option_python_versions @option_upgrade_to_newer_dependencies @option_platform @@ -228,7 +261,8 @@ def build_image( verbose: bool, dry_run: bool, - build_multiple_images: bool, + run_in_parallel: bool, + parallelism: int, python_versions: str, answer: str, **kwargs, @@ -236,7 +270,7 @@ def build_image( """Build CI image. Include building multiple images for all python versions (sequentially).""" def run_build(ci_image_params: BuildCiParams) -> None: - return_code, info = build_ci_image( + return_code, info = run_build_ci_image( verbose=verbose, dry_run=dry_run, ci_image_params=ci_image_params, parallel=False ) if return_code != 0: @@ -246,15 +280,26 @@ def run_build(ci_image_params: BuildCiParams) -> None: perform_environment_checks(verbose=verbose) parameters_passed = filter_out_none(**kwargs) parameters_passed['force_build'] = True - if build_multiple_images: + fix_group_permissions(verbose=verbose) + if run_in_parallel: python_version_list = get_python_version_list(python_versions) + params_list: List[BuildCiParams] = [] for python in python_version_list: params = BuildCiParams(**parameters_passed) params.python = python params.answer = answer - run_build(ci_image_params=params) + params_list.append(params) + start_building(params_list[0], dry_run=dry_run, verbose=verbose) + run_build_in_parallel( + image_params_list=params_list, + python_version_list=python_version_list, + parallelism=parallelism, + dry_run=dry_run, + verbose=verbose, + ) else: params = BuildCiParams(**parameters_passed) + start_building(params, dry_run=dry_run, verbose=verbose) run_build(ci_image_params=params) @@ -434,7 +479,7 @@ def should_we_run_the_build(build_ci_params: BuildCiParams) -> bool: sys.exit(1) -def build_ci_image( +def run_build_ci_image( verbose: bool, dry_run: bool, ci_image_params: BuildCiParams, parallel: bool ) -> Tuple[int, str]: """ @@ -465,17 +510,11 @@ def build_ci_image( "preparing buildx cache![/]\n" ) return 1, "Error: building multi-platform image without --push-image." - fix_group_permissions(verbose=verbose) if verbose or dry_run: get_console().print( f"\n[info]Building CI image of airflow from {AIRFLOW_SOURCES_ROOT} " f"python version: {ci_image_params.python}[/]\n" ) - if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies: - if not should_we_run_the_build(build_ci_params=ci_image_params): - return 0, f"Image build: {ci_image_params.python}" - if ci_image_params.prepare_buildx_cache or ci_image_params.push_image: - login_to_github_docker_registry(image_params=ci_image_params, dry_run=dry_run, verbose=verbose) if ci_image_params.prepare_buildx_cache: build_command_result = build_cache( image_params=ci_image_params, dry_run=dry_run, verbose=verbose, parallel=parallel @@ -522,19 +561,6 @@ def build_ci_image( return build_command_result.returncode, f"Image build: {ci_image_params.python}" -def build_ci_image_in_parallel( - verbose: bool, dry_run: bool, parallelism: int, python_version_list: List[str], **kwargs -): - """Run CI image builds in parallel.""" - get_console().print( - f"\n[info]Running with parallelism = {parallelism} for the images: {python_version_list}:" - ) - pool = mp.Pool(parallelism) - results = [pool.apply_async(build_ci_image, args=(verbose, dry_run, False), kwds=kwargs)] - check_async_run_results(results) - pool.close() - - def rebuild_or_pull_ci_image_if_needed( command_params: Union[ShellParams, BuildCiParams], dry_run: bool, verbose: bool ) -> None: @@ -573,4 +599,4 @@ def rebuild_or_pull_ci_image_if_needed( 'Forcing build.[/]' ) ci_image_params.force_build = True - build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params, parallel=False) + run_build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params, parallel=False) diff --git a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py index 116319a2efca3..490a43c5f52a2 100644 --- a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py @@ -547,7 +547,14 @@ def regenerate_command_images(verbose: bool, dry_run: bool): except FileNotFoundError: # when we go to Python 3.8+ we can add missing_ok = True instead of try/except pass - command_to_execute = [sys.executable, "-m", "pre_commit", 'run', 'update-breeze-file', '--all-files'] + command_to_execute = [ + sys.executable, + "-m", + "pre_commit", + 'run', + 'update-breeze-cmd-output', + '--all-files', + ] env = os.environ.copy() run_command( command_to_execute, diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 0b5ed2e68b06d..9ae3791e31264 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. import contextlib +import multiprocessing as mp import os import sys -from typing import Optional, Tuple +from typing import List, Optional, Tuple import click @@ -36,7 +37,6 @@ option_airflow_constraints_mode_prod, option_airflow_constraints_reference_build, option_answer, - option_build_multiple_images, option_debian_version, option_dev_apt_command, option_dev_apt_deps, @@ -74,6 +74,7 @@ prepare_docker_build_from_input, ) from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest +from airflow_breeze.utils.parallel import check_async_run_results from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCKER_CONTEXT_DIR from airflow_breeze.utils.python_versions import get_python_version_list from airflow_breeze.utils.registry import login_to_github_docker_registry @@ -103,9 +104,10 @@ ], }, { - "name": "Building multiple images", + "name": "Building images in parallel", "options": [ - "--build-multiple-images", + "--run-in-parallel", + "--parallelism", "--python-versions", ], }, @@ -192,12 +194,48 @@ } +def start_building(prod_image_params: BuildProdParams, dry_run: bool, verbose: bool): + if prod_image_params.cleanup_context: + clean_docker_context_files(verbose=verbose, dry_run=dry_run) + check_docker_context_files(prod_image_params.install_packages_from_context) + if prod_image_params.prepare_buildx_cache or prod_image_params.push_image: + login_to_github_docker_registry(image_params=prod_image_params, dry_run=dry_run, verbose=verbose) + + +def run_build_in_parallel( + image_params_list: List[BuildProdParams], + python_version_list: List[str], + parallelism: int, + dry_run: bool, + verbose: bool, +) -> None: + get_console().print( + f"\n[info]Building with parallelism = {parallelism} for the images: {python_version_list}:" + ) + pool = mp.Pool(parallelism) + results = [ + pool.apply_async( + run_build_production_image, + args=( + verbose, + dry_run, + image_param, + True, + ), + ) + for image_param in image_params_list + ] + check_async_run_results(results) + pool.close() + + @option_verbose @option_dry_run @option_answer @main.command(name='build-prod-image') @option_python -@option_build_multiple_images +@option_run_in_parallel +@option_parallelism @option_python_versions @option_upgrade_to_newer_dependencies @option_platform @@ -269,7 +307,8 @@ def build_prod_image( verbose: bool, dry_run: bool, - build_multiple_images: bool, + run_in_parallel: bool, + parallelism: int, python_versions: str, answer: Optional[str], **kwargs, @@ -279,8 +318,8 @@ def build_prod_image( """ def run_build(prod_image_params: BuildProdParams) -> None: - return_code, info = build_production_image( - verbose=verbose, dry_run=dry_run, prod_image_params=prod_image_params + return_code, info = run_build_production_image( + verbose=verbose, dry_run=dry_run, prod_image_params=prod_image_params, parallel=False ) if return_code != 0: get_console().print(f"[error]Error when building image! {info}") @@ -288,15 +327,27 @@ def run_build(prod_image_params: BuildProdParams) -> None: perform_environment_checks(verbose=verbose) parameters_passed = filter_out_none(**kwargs) - if build_multiple_images: + + fix_group_permissions(verbose=verbose) + if run_in_parallel: python_version_list = get_python_version_list(python_versions) + params_list: List[BuildProdParams] = [] for python in python_version_list: params = BuildProdParams(**parameters_passed) params.python = python params.answer = answer - run_build(prod_image_params=params) + params_list.append(params) + start_building(prod_image_params=params_list[0], dry_run=dry_run, verbose=verbose) + run_build_in_parallel( + image_params_list=params_list, + python_version_list=python_version_list, + parallelism=parallelism, + dry_run=dry_run, + verbose=verbose, + ) else: params = BuildProdParams(**parameters_passed) + start_building(prod_image_params=params, dry_run=dry_run, verbose=verbose) run_build(prod_image_params=params) @@ -473,8 +524,8 @@ def check_docker_context_files(install_packages_from_context: bool): sys.exit(1) -def build_production_image( - verbose: bool, dry_run: bool, prod_image_params: BuildProdParams +def run_build_production_image( + verbose: bool, dry_run: bool, prod_image_params: BuildProdParams, parallel: bool ) -> Tuple[int, str]: """ Builds PROD image: @@ -505,21 +556,10 @@ def build_production_image( " or preparing buildx cache![/]\n" ) return 1, "Error: building multi-platform image without --push-image." - fix_group_permissions(verbose=verbose) - if verbose or dry_run: - get_console().print( - f"\n[info]Building PROD image of airflow from {AIRFLOW_SOURCES_ROOT} " - f"python version: {prod_image_params.python}[/]\n" - ) - if prod_image_params.cleanup_context: - clean_docker_context_files(verbose=verbose, dry_run=dry_run) - check_docker_context_files(prod_image_params.install_packages_from_context) - if prod_image_params.prepare_buildx_cache or prod_image_params.push_image: - login_to_github_docker_registry(image_params=prod_image_params, dry_run=dry_run, verbose=verbose) get_console().print(f"\n[info]Building PROD Image for Python {prod_image_params.python}\n") if prod_image_params.prepare_buildx_cache: build_command_result = build_cache( - image_params=prod_image_params, dry_run=dry_run, verbose=verbose, parallel=False + image_params=prod_image_params, dry_run=dry_run, verbose=verbose, parallel=parallel ) else: if prod_image_params.empty_image: @@ -547,7 +587,7 @@ def build_production_image( cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True, - enabled_output_group=True, + enabled_output_group=not parallel, ) if build_command_result.returncode == 0: if prod_image_params.tag_as_latest: diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 0bc4d5af9fc86..292e74891dbfa 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -377,12 +377,6 @@ envvar='PARALLELISM', show_default=True, ) -option_build_multiple_images = click.option( - '--build-multiple-images', - help="Run the operation sequentially on all or selected subset of Python versions.", - is_flag=True, - envvar='BUILD_MULTIPLE_IMAGES', -) argument_packages = click.argument( "packages", nargs=-1, diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh index ed5591585aeb0..4a11212b2c82c 100755 --- a/dev/refresh_images.sh +++ b/dev/refresh_images.sh @@ -26,7 +26,7 @@ export GITHUB_TOKEN="" breeze self-upgrade --force breeze build-image \ - --build-multiple-images \ + --run-in-parallel \ --prepare-buildx-cache \ --force-build \ --platform linux/amd64,linux/arm64 \ @@ -44,7 +44,7 @@ breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi mv -v ./dist/*.whl ./docker-context-files breeze build-prod-image \ - --build-multiple-images \ + --run-in-parallel \ --airflow-is-in-context \ --install-packages-from-context \ --prepare-buildx-cache \ diff --git a/images/breeze/output-build-image.svg b/images/breeze/output-build-image.svg index 81703735d0155..389c973d0e273 100644 --- a/images/breeze/output-build-image.svg +++ b/images/breeze/output-build-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: build-image + Command: build-image - + - - -Usage: breeze build-image [OPTIONS] - -Build CI image. Include building multiple images for all python versions (sequentially). - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -│--image-tag-tTag of the image which is used to pull or run the image (implies               │ -│--mount-sources=skip when using to run shell or tests)                         │ -│(TEXT)                                                                         │ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--force-buildForce image build no matter if it is determined as needed.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│--build-multiple-imagesRun the operation sequentially on all or selected subset of Python versions.│ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-constraints-modeMode of constraints for CI image building                              │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--push-imagePush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze build-image [OPTIONS] + +Build CI image. Include building multiple images for all python versions (sequentially). + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag of the image which is used to pull or run the image (implies               │ +│--mount-sources=skip when using to run shell or tests)                         │ +│(TEXT)                                                                         │ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--force-buildForce image build no matter if it is determined as needed.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-constraints-modeMode of constraints for CI image building                              │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--push-imagePush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg index dcbd32a96662c..331cd3808ba34 100644 --- a/images/breeze/output-build-prod-image.svg +++ b/images/breeze/output-build-prod-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: build-prod-image + Command: build-prod-image - + - - -Usage: breeze build-prod-image [OPTIONS] - -Build Production image. Include building multiple images for all or selected Python versions sequentially. - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -│--image-tag-tTag of the image which is used to pull or run the image (implies               │ -│--mount-sources=skip when using to run shell or tests)                         │ -│(TEXT)                                                                         │ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│--build-multiple-imagesRun the operation sequentially on all or selected subset of Python versions.│ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ -│--airflow-constraints-modeMode of constraints for PROD image building                            │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.│ -│--airflow-is-in-contextIf set Airflow is installed from docker-context-files only rather than    │ -│from PyPI or sources.                                                     │ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--push-imagePush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze build-prod-image [OPTIONS] + +Build Production image. Include building multiple images for all or selected Python versions sequentially. + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag of the image which is used to pull or run the image (implies               │ +│--mount-sources=skip when using to run shell or tests)                         │ +│(TEXT)                                                                         │ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ +│--airflow-constraints-modeMode of constraints for PROD image building                            │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.│ +│--airflow-is-in-contextIf set Airflow is installed from docker-context-files only rather than    │ +│from PyPI or sources.                                                     │ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--push-imagePush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index ef6122b57ad37..bf36da39c3388 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -2,4 +2,4 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -82cd0f28c2c53f699a5b571275f8f674 +4051e9cd6b50474e5d2c0fb0ec515f33 diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh index c9f215c53c251..26fbf13f66ed9 100755 --- a/scripts/ci/selective_ci_checks.sh +++ b/scripts/ci/selective_ci_checks.sh @@ -55,6 +55,7 @@ function output_all_basic_variables() { "$(initialization::parameters_to_json "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}")" initialization::ga_output all-python-versions \ "$(initialization::parameters_to_json "${ALL_PYTHON_MAJOR_MINOR_VERSIONS[@]}")" + initialization::ga_output all-python-versions-list-as-string "${ALL_PYTHON_MAJOR_MINOR_VERSIONS[*]}" initialization::ga_output python-versions-list-as-string "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[*]}" initialization::ga_output kubernetes-versions-list-as-string "${CURRENT_KUBERNETES_VERSIONS[*]}" else @@ -64,6 +65,7 @@ function output_all_basic_variables() { # all-python-versions are used in BuildImage Workflow initialization::ga_output all-python-versions \ "$(initialization::parameters_to_json "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}")" + initialization::ga_output all-python-versions-list-as-string "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" initialization::ga_output python-versions-list-as-string "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" initialization::ga_output kubernetes-versions-list-as-string "${DEFAULT_KUBERNETES_VERSION}" fi