diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 04d295d36949e..3f530853d7929 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -223,7 +223,7 @@ jobs: cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Build & Push CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze build-image --push-image env: @@ -237,7 +237,10 @@ jobs: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Candidates for pip resolver backtrack triggers: ${{ matrix.python-version }}" if: failure() || cancelled() - run: airflow-find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}" + run: breeze find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}" + - name: "Fix ownership" + run: breeze fix-ownership + if: always() build-prod-images: permissions: @@ -310,7 +313,7 @@ jobs: cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: > Pull CI image for PROD build: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} @@ -347,3 +350,6 @@ jobs: run: breeze build-prod-image --push-image --empty-image env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: "Fix ownership" + run: breeze fix-ownership + if: always() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8a249c404776..4e9f1338602c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -322,7 +322,7 @@ jobs: echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: "Free space" - run: airflow-freespace + run: breeze free-space if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: Build & Push CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze build-image --push-image @@ -334,7 +334,10 @@ jobs: if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: "Candidates for pip resolver backtrack triggers: ${{ matrix.python-version }}" if: failure() || cancelled() - run: airflow-find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}" + run: breeze find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}" + - name: "Fix ownership" + run: breeze fix-ownership + if: always() && needs.build-info.outputs.inWorkflowBuild == 'true' build-prod-images: permissions: @@ -385,7 +388,7 @@ jobs: - run: ./scripts/ci/install_breeze.sh if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: "Free space" - run: airflow-freespace + run: breeze free-space if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: > Pull CI image for PROD build: @@ -423,6 +426,9 @@ jobs: DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} if: needs.build-info.outputs.inWorkflowBuild == 'true' + - name: "Fix ownership" + run: breeze fix-ownership + if: always() && needs.build-info.outputs.inWorkflowBuild == 'true' run-new-breeze-tests: timeout-minutes: 10 @@ -548,13 +554,16 @@ jobs: cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} id: wait-for-images run: breeze pull-image --run-in-parallel --verify-image --wait-for-image env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: "Fix ownership" + run: breeze fix-ownership + if: always() static-checks: timeout-minutes: 30 @@ -578,7 +587,7 @@ jobs: cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: > Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest @@ -614,6 +623,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" VERBOSE: false SKIP: "identity" COLUMNS: 250 + - name: "Fix ownership" + run: breeze fix-ownership + if: always() # Those checks are run if no image needs to be built for checks. This is for simple changes that # Do not touch any of the python code or any of the important files that might require building @@ -671,6 +683,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --commit-ref "${{ github.sha }}" env: VERBOSE: false + - name: "Fix ownership" + run: breeze fix-ownership + if: always() + docs: timeout-minutes: 45 name: "Build docs" @@ -693,7 +709,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest env: @@ -722,6 +738,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && github.event_name == 'push' run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs + - name: "Fix ownership" + run: breeze fix-ownership + if: always() prepare-test-provider-packages-wheel: timeout-minutes: 40 @@ -746,7 +765,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: > Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest @@ -772,6 +791,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" env: USE_AIRFLOW_VERSION: "2.1.0" PACKAGE_FORMAT: "wheel" + - name: "Fix ownership" + run: breeze fix-ownership + if: always() prepare-test-provider-packages-sdist: timeout-minutes: 40 @@ -796,7 +818,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: > Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest @@ -820,6 +842,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" USE_AIRFLOW_VERSION: "sdist" PACKAGE_FORMAT: "sdist" AIRFLOW_EXTRAS: "all" + - name: "Fix ownership" + run: breeze fix-ownership + if: always() tests-helm: timeout-minutes: 80 @@ -850,7 +875,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: > Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest @@ -881,6 +906,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" name: coverage-helm path: "./files/coverage*.xml" retention-days: 7 + - name: "Fix ownership" + run: breeze fix-ownership + if: always() tests-postgres: timeout-minutes: 130 @@ -915,7 +943,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest env: @@ -949,6 +977,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" name: coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} path: "./files/coverage*.xml" retention-days: 7 + - name: "Fix ownership" + run: breeze fix-ownership + if: always() tests-mysql: timeout-minutes: 130 @@ -982,7 +1013,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest env: @@ -1016,6 +1047,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} path: "./files/coverage*.xml" retention-days: 7 + - name: "Fix ownership" + run: breeze fix-ownership + if: always() tests-mssql: timeout-minutes: 130 @@ -1049,7 +1083,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest env: @@ -1081,6 +1115,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" name: coverage-mssql-${{matrix.python-version}}-${{matrix.mssql-version}} path: "./files/coverage*.xml" retention-days: 7 + - name: "Fix ownership" + run: breeze fix-ownership + if: always() tests-sqlite: timeout-minutes: 130 @@ -1112,7 +1149,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest env: @@ -1144,6 +1181,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" name: coverage-sqlite-${{matrix.python-version}} path: ./files/coverage*.xml retention-days: 7 + - name: "Fix ownership" + run: breeze fix-ownership + if: always() tests-quarantined: timeout-minutes: 60 @@ -1183,7 +1223,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" echo "ISSUE_ID=10128" >> $GITHUB_ENV - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest env: @@ -1220,6 +1260,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" name: coverage-quarantined-${{ matrix.backend }} path: "./files/coverage*.xml" retention-days: 7 + - name: "Fix ownership" + run: breeze fix-ownership + if: always() upload-coverage: timeout-minutes: 15 @@ -1253,6 +1296,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" uses: ./.github/actions/codecov-action with: directory: "./coverage-files" + - name: "Fix ownership" + run: breeze fix-ownership + if: always() wait-for-prod-images: timeout-minutes: 120 @@ -1277,7 +1323,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: "Cache virtualenv environment" uses: actions/cache@v2 with: @@ -1292,6 +1338,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: "Fix ownership" + run: breeze fix-ownership + if: always() test-docker-compose-quick-start: timeout-minutes: 60 @@ -1315,13 +1364,16 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Pull PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-prod-image --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Test docker-compose quick start" run: breeze docker-compose-tests + - name: "Fix ownership" + run: breeze fix-ownership + if: always() tests-kubernetes: timeout-minutes: 70 @@ -1362,7 +1414,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-prod-image --run-in-parallel --tag-as-latest env: @@ -1387,6 +1439,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" name: kind-logs-${{matrix.executor}} path: /tmp/kind_logs_* retention-days: 7 + - name: "Fix ownership" + run: breeze fix-ownership + if: always() tests-helm-executor-upgrade: timeout-minutes: 100 @@ -1422,7 +1477,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-prod-image --run-in-parallel --tag-as-latest env: @@ -1458,6 +1513,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" name: kind-logs-KubernetesExecutor path: /tmp/kind_logs_* retention-days: 7 + - name: "Fix ownership" + run: breeze fix-ownership + if: always() constraints: permissions: @@ -1491,7 +1549,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --run-in-parallel --tag-as-latest env: @@ -1504,7 +1562,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" breeze generate-constraints --run-in-parallel --generate-constraints-mode no-providers env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }} - ANSWER: "yes" - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh @@ -1526,6 +1583,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" github_token: ${{ secrets.GITHUB_TOKEN }} branch: ${{ steps.constraints-branch.outputs.branch }} directory: "repo" + - name: "Fix ownership" + run: breeze fix-ownership + if: always() # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build # is executed as result of direct push to "main" or one of the "vX-Y-test" branches @@ -1561,7 +1621,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" - run: airflow-freespace + run: breeze free-space - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - name: "Build & Push CI image ${{ matrix.python-version }}:latest" @@ -1611,3 +1671,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh if: always() + - name: "Fix ownership" + run: breeze fix-ownership + if: always() diff --git a/BREEZE.rst b/BREEZE.rst index 243b51360b606..5f60b121355ed 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -193,6 +193,9 @@ periodically. For details see On WSL2 you might want to increase your Virtual Hard Disk by following: `Expanding the size of your WSL 2 Virtual Hard Disk `_ +There is a command ``breeze resource-check`` that you can run to check available resources. See below +for details. + Cleaning the environment ------------------------ @@ -445,57 +448,71 @@ Airflow Breeze is a bash script serving as a "swiss-army-knife" of Airflow testi hood it uses other scripts that you can also run manually if you have problem with running the Breeze environment. Breeze script allows performing the following tasks: -Airflow developers tasks ------------------------- +Development tasks +----------------- -Regular development tasks: +Those are commands mostly used by contributors: -* Setup autocomplete for Breeze with ``breeze setup-autocomplete`` command +* Execute arbitrary command in the test environment with ``breeze shell`` command * Enter interactive shell in CI container when ``shell`` (or no command) is specified * Start containerised, development-friendly airflow installation with ``breeze start-airflow`` command * Build documentation with ``breeze build-docs`` command * Initialize local virtualenv with ``./scripts/tools/initialize_virtualenv.py`` command -* Cleanup breeze with ``breeze cleanup`` command * Run static checks with autocomplete support ``breeze static-checks`` command * Run test specified with ``./breeze-legacy tests`` command - -CI Image tasks: - -* Build CI docker image with ``breeze build-image`` command -* Pull CI images in parallel ``breeze pull-image`` command -* Verify CI image ``breeze verify-image`` command - -PROD Image tasks: -* Build PROD image with ``breeze build-prod-image`` command -* Pull PROD image in parallel ``breeze pull-prod-image`` command -* Verify CI image ``breeze verify-prod-image`` command - -Additional management tasks: - * Join running interactive shell with ``./breeze-legacy exec`` command * Stop running interactive environment with ``breeze stop`` command -* Execute arbitrary command in the test environment with ``breeze shell`` command * Execute arbitrary docker-compose command with ``./breeze-legacy docker-compose`` command -Docker compose tests: +Tests +----- * Run docker-compose tests with ``breeze docker-compose-tests`` command. -Kubernetes tests related: +Kubernetes tests +---------------- * Manage KinD Kubernetes cluster and deploy Airflow to KinD cluster ``./breeze-legacy kind-cluster`` commands * Run Kubernetes tests specified with ``./breeze-legacy kind-cluster tests`` command * Enter the interactive kubernetes test environment with ``./breeze-legacy kind-cluster shell`` command -Airflow can also be used for managing Production images - this is a development-only feature, -regular users of Airflow should use ``docker build`` commands to manage the images as described -in the user documentation about `building the image `_ +CI Image tasks +-------------- + +The image building is usually run for users automatically when needed, +but sometimes Breeze users might want to manually build, pull or verify the CI images. + +* Build CI docker image with ``breeze build-image`` command +* Pull CI images in parallel ``breeze pull-image`` command +* Verify CI image ``breeze verify-image`` command -Maintainer tasks +PROD Image tasks ---------------- +Users can also build Production images when they are developing them. However when you want to +use the PROD image, the regular docker build commands are recommended. See +`building the image `_ + +* Build PROD image with ``breeze build-prod-image`` command +* Pull PROD image in parallel ``breeze pull-prod-image`` command +* Verify CI image ``breeze verify-prod-image`` command + +Configuration and maintenance +----------------------------- + +* Cleanup breeze with ``breeze cleanup`` command +* Self-upgrade breeze with ``breeze self-upgrade`` command +* Setup autocomplete for Breeze with ``breeze setup-autocomplete`` command +* Checking available resources for docker with ``breeze resource-check`` command +* Freeing space needed to run CI tests with ``breeze free-space`` command +* Fixing ownership of files in your repository with ``breeze fix-ownership`` command +* Print Breeze version with ``breeze fix-ownership`` command + +Release tasks +------------- + Maintainers also can use Breeze for other purposes (those are commands that regular contributors likely -do not need): +do not need or have no access to run). Those are usually connected with releasing Airflow: * Prepare cache for CI: ``breeze build-image --prepare-build-cache`` and ``breeze build-prod image --prepare-build-cache``(needs buildx plugin and write access to registry ghcr.io) @@ -503,6 +520,9 @@ do not need): * Prepare airflow packages: ``breeze prepare-airflow-package`` (when releasing Airflow) * Prepare provider documentation ``breeze prepare-provider-documentation`` and prepare provider packages ``breeze prepare-provider-packages`` (when releasing provider packages) +* Finding the updated dependencies since the last successful build when we have conflict with + ``breeze find-newer-dependencies`` command + Details of Breeze usage ======================= @@ -1150,6 +1170,49 @@ command but it is very similar to current ``breeze`` command): +Resource check +============== + +Breeze requires certain resources to be available - disk, memory, CPU. When you enter Breeze's shell, +the resources are checked and information if there is enough resources is displayed. However you can +manually run resource check any time by ``breeze resource-check`` command. + +Those are all available flags of ``resource-check`` command: + +.. image:: ./images/breeze/output-resource-check.svg + :width: 100% + :alt: Breeze resource-check + + +Freeing the space +================= + +When our CI runs a job, it needs all memory and disk it can have. We have a Breeze command that frees +the memory and disk space used. You can also use it clear space locally but it performs a few operations +that might be a bit invasive - such are removing swap file and complete pruning of docker disk space used. + +Those are all available flags of ``free-space`` command: + +.. image:: ./images/breeze/output-free-space.svg + :width: 100% + :alt: Breeze free-space + + +Tracking backtracking issues for CI builds +========================================== + +When our CI runs a job, we automatically upgrade our dependencies in the ``main`` build. However, this might +lead to conflicts and ``pip`` backtracking for a long time (possibly forever) for dependency resolution. +Unfortunately those issues are difficult to diagnose so we had to invent our own tool to help us with +diagnosing them. This tool is ``find-newer-dependencies`` and it works in the way that it helps to guess +which new dependency might have caused the backtracking. The whole process is described in +`tracking backtracking issues `_. + +Those are all available flags of ``find-newer-dependencies`` command: + +.. image:: ./images/breeze/output-find-newer-dependencies.svg + :width: 100% + :alt: Breeze find-newer-dependencies Internal details of Breeze ========================== @@ -1240,11 +1303,18 @@ On Linux, there is a problem with propagating ownership of created files (a know files and directories created in the container are not owned by the host user (but by the root user in our case). This may prevent you from switching branches, for example, if files owned by the root user are created within your sources. In case you are on a Linux host and have some files in your sources created -by the root user, you can fix the ownership of those files by running this script: +by the root user, you can fix the ownership of those files by running : .. code-block:: - ./scripts/ci/tools/fix_ownership.sh + breeze fix-ownership + +Those are all available flags of ``fix-ownership`` command: + +.. image:: ./images/breeze/output-fix-ownership.svg + :width: 100% + :alt: Breeze fix-ownership + Mounting Local Sources to Breeze -------------------------------- diff --git a/dev/TRACKING_BACKTRACKING_ISSUES.md b/dev/TRACKING_BACKTRACKING_ISSUES.md index a4c6e14efc60f..212c7dd059a20 100644 --- a/dev/TRACKING_BACKTRACKING_ISSUES.md +++ b/dev/TRACKING_BACKTRACKING_ISSUES.md @@ -192,28 +192,15 @@ You need to install the breeze: * `pipx install -e ./dev/breeze` if you use pipx install. -Then you can run airflow-find-newer-dependencies with optional flags. For example if you know that the build -was likely broken on a given date and time (in your timezone) and you want to check python 3.8 -(because this is the only failing build) you can run: +Then you can run ``breeze find-newer-dependencies`` with optional flags. +For example if you know that the build was likely broken on a given date and time +(in your timezone) and you want to check python 3.8 (because this is the only +failing build) you can run: ``` -airflow-find-newer-dependencies --updated-on-or-after '2022-02-22 10:30:00' --timezone 'CET' --python 3.8 +breeze find-newer-dependencies --updated-on-or-after '2022-02-22 10:30:00' --timezone 'CET' --python 3.8 ``` -Full list of options: +The full list of options for `find-newer-dependencies` can be seen here -``` -Usage: airflow-find-newer-dependencies [OPTIONS] - -Options: - --max-age INTEGER Max age of the last release (used if no updated- - after if specified) - - --updated-on-or-after TEXT Date when the release was updated after - --python [3.7|3.8|3.9,3.10] Python version used - --constraints-branch TEXT Constraint branch to use to find newer - dependencies - - --timezone TEXT Timezone to use during the check - --help Show this message and exit. -``` +![breeze find-newer-dependencies](../images/breeze/output-find-newer-dependencies.svg) diff --git a/dev/breeze/README.md b/dev/breeze/README.md index 604fdb1bf8083..fbf2edcc1efc4 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: ea8611f8d231310d9c8dad11e7f61b5d8dd9f995473451a0ae60856e51d2264a6c1dc0a58be8fdc665673945197018b41d6b4b5aad1fdffcd011aa10c25790bc +Package config hash: a710b1c85a4ab4308cfab0ef5d2e118516f9574708d72648dc4063aae04e3cc1b44d35de446832017801b2b63aeec0a3d381d3050d517acb785e0236fd2b8588 --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg index 79e4d9a3f462a..5ef8675c98db0 100644 --- a/dev/breeze/setup.cfg +++ b/dev/breeze/setup.cfg @@ -72,8 +72,6 @@ where=src [options.entry_points] console_scripts= breeze=airflow_breeze.breeze:main - airflow-freespace=airflow_ci.freespace:main - airflow-find-newer-dependencies=airflow_ci.find_newer_dependencies:main [bdist_wheel] python-tag=py3 diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py index 5e28860649763..381c93560db30 100755 --- a/dev/breeze/src/airflow_breeze/breeze.py +++ b/dev/breeze/src/airflow_breeze/breeze.py @@ -33,6 +33,7 @@ from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.confirm import Answer, set_forced_answer, user_confirm from airflow_breeze.utils.constraints import run_generate_constraints, run_generate_constraints_in_parallel +from airflow_breeze.utils.find_newer_dependencies import find_newer_dependencies from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel from airflow_breeze.utils.reinstall import ask_to_reinstall_breeze, reinstall_breeze, warn_non_editable from airflow_breeze.utils.run_tests import run_docker_compose_tests, verify_an_image @@ -383,6 +384,18 @@ "breeze prepare-provider-documentation": [ {"name": "Provider documentation preparation flags", "options": ["--skip-package-verification"]} ], + "breeze find-newer-dependencies": [ + { + "name": "Find newer dependencies flags", + "options": [ + "--python", + "--timezone", + "--constraints-branch", + "--updated-on-or-after", + "--max-age", + ], + } + ], "breeze generate-constraints": [ { "name": "Generate constraints flags", @@ -431,7 +444,16 @@ }, { "name": "Configuration & maintenance", - "commands": ["cleanup", "self-upgrade", "setup-autocomplete", "config", "version"], + "commands": [ + "cleanup", + "self-upgrade", + "setup-autocomplete", + "config", + "resource-check", + "free-space", + "fix-ownership", + "version", + ], }, { "name": "CI Image tools", @@ -456,6 +478,7 @@ "prepare-provider-packages", "prepare-airflow-package", "generate-constraints", + "find-newer-dependencies", ], }, ] @@ -1753,7 +1776,6 @@ def build_docs( """Build documentation in the container.""" params = BuildCiParams(github_repository=github_repository, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION) ci_image_name = params.airflow_image_name - check_docker_resources(verbose, ci_image_name) doc_builder = DocParams( package_filter=package_filter, docs_only=docs_only, @@ -2142,7 +2164,7 @@ def generate_constraints( synchronize_parameters_with_cache(shell_params, {"python": python}) console.print("\n[yellow]Use this command to build the image:[/]\n") console.print( - f" breeze build-image --python '{shell_params.python}' " + f" breeze build-image --python'{shell_params.python}' " f"--upgrade-to-newer-dependencies true\n" ) sys.exit(1) @@ -2180,6 +2202,106 @@ def generate_constraints( sys.exit(return_code) +@main.command(name="free-space", help="Free space for jobs run in CI.") +@option_verbose +@option_dry_run +@option_answer +def free_space(verbose: bool, dry_run: bool, answer: str): + set_forced_answer(answer) + if user_confirm("Are you sure to run free-space and perform cleanup?", timeout=None) == Answer.YES: + run_command(["sudo", "swapoff", "-a"], verbose=verbose, dry_run=dry_run) + run_command(["sudo", "rm", "-f", "/swapfile"], verbose=verbose, dry_run=dry_run) + run_command(["sudo", "apt-get", "clean"], verbose=verbose, dry_run=dry_run, check=False) + run_command( + ["docker", "system", "prune", "--all", "--force", "--volumes"], verbose=verbose, dry_run=dry_run + ) + run_command(["df", "-h"], verbose=verbose, dry_run=dry_run) + run_command(["docker", "logout", "ghcr.io"], verbose=verbose, dry_run=dry_run, check=False) + + +@main.command(name="resource-check", help="Check if available docker resources are enough.") +@option_verbose +@option_dry_run +def resource_check(verbose: bool, dry_run: bool): + shell_params = ShellParams(verbose=verbose, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION) + check_docker_resources(verbose, shell_params.airflow_image_name, dry_run=dry_run) + + +option_timezone = click.option( + "--timezone", + default="UTC", + type=str, + help="Timezone to use during the check", +) + +option_updated_on_or_after = click.option( + "--updated-on-or-after", + type=str, + help="Date when the release was updated after", +) + +option_max_age = click.option( + "--max-age", + type=int, + default=3, + help="Max age of the last release (used if no updated-on-or-after if specified)", +) + +option_branch = click.option( + "--constraints-branch", + default='constraints-main', + help="Constraint branch to use to find newer dependencies", +) + + +@main.command(name="find-newer-dependencies", help="Finds which dependencies are being upgraded.") +@option_timezone +@option_branch +@option_python +@option_updated_on_or_after +@option_max_age +def breeze_find_newer_dependencies( + constraints_branch: str, python: str, timezone: str, updated_on_or_after: str, max_age: int +): + return find_newer_dependencies( + constraints_branch=constraints_branch, + python=python, + timezone=timezone, + updated_on_or_after=updated_on_or_after, + max_age=max_age, + ) + + +@main.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.") +@option_verbose +@option_dry_run +def fix_ownership(verbose: bool, dry_run: bool): + shell_params = ShellParams( + verbose=verbose, + mount_sources=MOUNT_ALL, + python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION, + ) + extra_docker_flags = get_extra_docker_flags(MOUNT_ALL) + env = construct_env_variables_docker_compose_command(shell_params) + cmd = [ + "docker", + "run", + "-t", + *extra_docker_flags, + "-e", + "GITHUB_ACTIONS=", + "-e", + "SKIP_ENVIRONMENT_INITIALIZATION=true", + "--pull", + "never", + shell_params.airflow_image_name_with_tag, + "/opt/airflow/scripts/in_container/run_fix_ownership.sh", + ] + run_command(cmd, verbose=verbose, dry_run=dry_run, text=True, env=env, check=False) + # Always succeed + sys.exit(0) + + @main.command( name="cleanup", help="Cleans the cache of parameters, docker cache and optionally - currently downloaded images.", diff --git a/dev/breeze/src/airflow_breeze/shell/enter_shell.py b/dev/breeze/src/airflow_breeze/shell/enter_shell.py index 855a24f1d6129..309cf316ea413 100644 --- a/dev/breeze/src/airflow_breeze/shell/enter_shell.py +++ b/dev/breeze/src/airflow_breeze/shell/enter_shell.py @@ -115,11 +115,10 @@ def run_shell_with_build_image_checks( :param dry_run: do not execute "write" commands - just print what would happen :param shell_params: parameters of the execution """ - check_docker_resources(verbose, shell_params.airflow_image_name) + check_docker_resources(verbose, shell_params.airflow_image_name, dry_run=dry_run) build_ci_image_check_cache = Path( BUILD_CACHE_DIR, shell_params.airflow_branch, f".built_{shell_params.python}" ) - ci_image_params = get_ci_image_build_params( {"python": shell_params.python, "upgrade_to_newer_dependencies": "false"} ) diff --git a/dev/breeze/src/airflow_breeze/utils/confirm.py b/dev/breeze/src/airflow_breeze/utils/confirm.py index 5fcfe217c1d3e..45a9026edbf3d 100644 --- a/dev/breeze/src/airflow_breeze/utils/confirm.py +++ b/dev/breeze/src/airflow_breeze/utils/confirm.py @@ -59,6 +59,7 @@ def user_confirm( force = forced_answer or os.environ.get('ANSWER') if force: user_status = force + print(f"Forced answer for '{message}': {force}") else: user_status = inputimeout( prompt=f'\n{message} \nPress {allowed_answers}' diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index f6e59d1ea092c..e100986b9b641 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -110,11 +110,12 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]: def check_docker_resources( - verbose: bool, airflow_image_name: str + verbose: bool, airflow_image_name: str, dry_run: bool ) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]: """ Check if we have enough resources to run docker. This is done via running script embedded in our image. :param verbose: print commands when running + :param dry_run: whether to run it in dry run mode :param airflow_image_name: name of the airflow image to use. """ return run_command( @@ -124,11 +125,14 @@ def check_docker_resources( "-t", "--entrypoint", "/bin/bash", + "-e", + "PYTHONDONTWRITEBYTECODE=true", airflow_image_name, "-c", "python /opt/airflow/scripts/in_container/run_resource_check.py", ], verbose=verbose, + dry_run=dry_run, text=True, ) diff --git a/dev/breeze/src/airflow_ci/find_newer_dependencies.py b/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py similarity index 82% rename from dev/breeze/src/airflow_ci/find_newer_dependencies.py rename to dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py index 7742ff291007e..17d577a090bab 100644 --- a/dev/breeze/src/airflow_ci/find_newer_dependencies.py +++ b/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py @@ -32,75 +32,19 @@ from datetime import timedelta from typing import Any, Dict, List, Tuple -import click -import pendulum -import requests -from dateutil.parser import isoparse -from packaging import version from rich.console import Console from rich.progress import Progress console = Console(width=400, color_system="standard") -option_branch = click.option( - "--constraints-branch", - default='constraints-main', - help="Constraint branch to use to find newer dependencies", -) -option_python = click.option( - "--python", - type=click.Choice(["3.7", "3.8", "3.9", "3.10"]), - default="3.7", - help="Python version used", -) +def find_newer_dependencies( + constraints_branch: str, python: str, timezone: str, updated_on_or_after: str, max_age: int +): + import pendulum + import requests + from packaging import version -option_timezone = click.option( - "--timezone", - default="UTC", - type=str, - help="Timezone to use during the check", -) - -option_updated_on_or_after = click.option( - "--updated-on-or-after", - type=str, - help="Date when the release was updated after", -) - -option_max_age = click.option( - "--max-age", - type=int, - default=3, - help="Max age of the last release (used if no updated-on-or-after if specified)", -) - - -def get_releases_and_upload_times(package, min_date, current_version, tz) -> List[Tuple[str, Any]]: - package_info = json.loads(requests.get(f"https://pypi.python.org/pypi/{package}/json").text) - releases: List[Tuple[Any, Any]] = [] - for release_version, release_info in package_info['releases'].items(): - if release_info and not release_info[0]['yanked']: - parsed_version = version.parse(release_version) - if ( - parsed_version.is_prerelease - or parsed_version.is_devrelease - or parsed_version == current_version - ): - continue - upload_date = tz.convert(isoparse(release_info[0]['upload_time_iso_8601'])).replace(microsecond=0) - if upload_date >= min_date: - releases.append((parsed_version, upload_date)) - return releases - - -@option_timezone -@option_branch -@option_python -@option_updated_on_or_after -@option_max_age -@click.command() -def main(constraints_branch: str, python: str, timezone: str, updated_on_or_after: str, max_age: int): constraints = requests.get( f"https://raw.githubusercontent.com/" f"apache/airflow/{constraints_branch}/constraints-{python}.txt" ).text @@ -153,5 +97,23 @@ def main(constraints_branch: str, python: str, timezone: str, updated_on_or_afte ) -if __name__ == '__main__': - main() # type: ignore[misc] +def get_releases_and_upload_times(package, min_date, current_version, tz) -> List[Tuple[str, Any]]: + import requests + from dateutil.parser import isoparse + from packaging import version + + package_info = json.loads(requests.get(f"https://pypi.python.org/pypi/{package}/json").text) + releases: List[Tuple[Any, Any]] = [] + for release_version, release_info in package_info['releases'].items(): + if release_info and not release_info[0]['yanked']: + parsed_version = version.parse(release_version) + if ( + parsed_version.is_prerelease + or parsed_version.is_devrelease + or parsed_version == current_version + ): + continue + upload_date = tz.convert(isoparse(release_info[0]['upload_time_iso_8601'])).replace(microsecond=0) + if upload_date >= min_date: + releases.append((parsed_version, upload_date)) + return releases diff --git a/dev/breeze/src/airflow_ci/__init__.py b/dev/breeze/src/airflow_ci/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/dev/breeze/src/airflow_ci/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/dev/breeze/src/airflow_ci/freespace.py b/dev/breeze/src/airflow_ci/freespace.py deleted file mode 100755 index 5c235a35bed39..0000000000000 --- a/dev/breeze/src/airflow_ci/freespace.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Cleans up the environment before starting CI.""" - - -import rich_click as click -from rich.console import Console - -from airflow_breeze.utils.run_utils import run_command - -console = Console(force_terminal=True, color_system="standard", width=180) - -option_verbose = click.option( - "--verbose", - envvar='VERBOSE', - is_flag=True, - help="Print verbose information about free space steps", -) - -option_dry_run = click.option( - "--dry-run", - is_flag=True, - help="Just prints commands without executing them", -) - - -@click.command() -@option_verbose -@option_dry_run -def main(verbose, dry_run): - run_command(["sudo", "swapoff", "-a"], verbose=verbose, dry_run=dry_run) - run_command(["sudo", "rm", "-f", "/swapfile"], verbose=verbose, dry_run=dry_run) - run_command(["sudo", "apt-get", "clean"], verbose=verbose, dry_run=dry_run, check=False) - run_command( - ["docker", "system", "prune", "--all", "--force", "--volumes"], verbose=verbose, dry_run=dry_run - ) - run_command(["df", "-h"], verbose=verbose, dry_run=dry_run) - run_command(["docker", "logout", "ghcr.io"], verbose=verbose, dry_run=dry_run, check=False) - - -if __name__ == '__main__': - main() diff --git a/docs/conf.py b/docs/conf.py index ef889f641a096..b1742f0c00202 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -44,10 +44,9 @@ except ImportError: from yaml import SafeLoader # type: ignore[misc] -from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES - import airflow from airflow.configuration import AirflowConfigParser, default_config_yaml +from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES sys.path.append(os.path.join(os.path.dirname(__file__), 'exts')) diff --git a/docs/exts/docs_build/fetch_inventories.py b/docs/exts/docs_build/fetch_inventories.py index 0833b6de46755..d55e28ae14547 100644 --- a/docs/exts/docs_build/fetch_inventories.py +++ b/docs/exts/docs_build/fetch_inventories.py @@ -27,11 +27,11 @@ import requests import urllib3.exceptions -from docs.exts.docs_build.docs_builder import get_available_providers_packages -from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES from requests.adapters import DEFAULT_POOLSIZE from airflow.utils.helpers import partition +from docs.exts.docs_build.docs_builder import get_available_providers_packages +from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES CURRENT_DIR = os.path.dirname(__file__) ROOT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir, os.pardir)) diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 4ef31a5f6fb87..78dcf803f3744 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ -
│ self-upgrade Self upgrade Breeze. │
│ setup-autocomplete Enables autocompletion of breeze commands. │
│ config Show/update configuration (Python, Backend, Cheatsheet, ASCIIART). │
+
│ resource-check Check if available docker resources are enough. │
+
│ free-space Free space for jobs run in CI. │
+
│ fix-ownership Fix ownership of source files to be same as host user. │
│ version Print information about version of apache-airflow-breeze. │
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮
@@ -176,6 +179,7 @@
│ prepare-provider-packages Prepare sdist/whl packages of Airflow Providers. │
│ prepare-airflow-package Prepare sdist/whl package of Airflow. │
│ generate-constraints Generates pinned constraint files with all extras from setup.py in parallel. │
+
│ find-newer-dependencies Finds which dependencies are being upgraded. │
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
diff --git a/images/breeze/output-find-newer-dependencies.svg b/images/breeze/output-find-newer-dependencies.svg new file mode 100644 index 0000000000000..cab8045a1b3da --- /dev/null +++ b/images/breeze/output-find-newer-dependencies.svg @@ -0,0 +1,140 @@ + + + + +
+
+
+ + + + + +
Command: find-newer-dependencies
+
+
+
+
Usage: breeze find-newer-dependencies [OPTIONS]
+
+
Finds which dependencies are being upgraded.
+
+
╭─ Find newer dependencies flags ──────────────────────────────────────────────────────────────────────────────────────╮
+
│ --python -p Python major/minor version used in Airflow image for PROD/CI images. │
+
│ (3.7 | 3.8 | 3.9 | 3.10) │
+
│ --timezone Timezone to use during the check (TEXT) │
+
│ --constraints-branch Constraint branch to use to find newer dependencies (TEXT) │
+
│ --updated-on-or-after Date when the release was updated after (TEXT) │
+
│ --max-age Max age of the last release (used if no updated-on-or-after if specified) (INTEGER) │
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+
│ --help -h Show this message and exit. │
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+
+
+
+
+ + + diff --git a/images/breeze/output-fix-ownership.svg b/images/breeze/output-fix-ownership.svg new file mode 100644 index 0000000000000..2b89bb86e53df --- /dev/null +++ b/images/breeze/output-fix-ownership.svg @@ -0,0 +1,133 @@ + + + + +
+
+
+ + + + + +
Command: fix-ownership
+
+
+
+
Usage: breeze fix-ownership [OPTIONS]
+
+
Fix ownership of source files to be same as host user.
+
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+
│ --verbose -v Print verbose information about performed steps. │
+
│ --dry-run -D If dry-run is set, commands are only printed, not executed. │
+
│ --help -h Show this message and exit. │
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+
+
+
+
+ + + diff --git a/images/breeze/output-free-space.svg b/images/breeze/output-free-space.svg new file mode 100644 index 0000000000000..9250d459c7936 --- /dev/null +++ b/images/breeze/output-free-space.svg @@ -0,0 +1,135 @@ + + + + +
+
+
+ + + + + +
Command: free-space
+
+
+
+
Usage: breeze free-space [OPTIONS]
+
+
Free space for jobs run in CI.
+
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+
│ --verbose -v Print verbose information about performed steps. │
+
│ --dry-run -D If dry-run is set, commands are only printed, not executed. │
+
│ --answer -a Force answer to questions. (y | n | q | yes | no | quit) │
+
│ --help -h Show this message and exit. │
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+
+
+
+
+ + + diff --git a/images/breeze/output-resource-check.svg b/images/breeze/output-resource-check.svg new file mode 100644 index 0000000000000..c5a2bcb473f29 --- /dev/null +++ b/images/breeze/output-resource-check.svg @@ -0,0 +1,133 @@ + + + + +
+
+
+ + + + + +
Command: resource-check
+
+
+
+
Usage: breeze resource-check [OPTIONS]
+
+
Check if available docker resources are enough.
+
+
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+
│ --verbose -v Print verbose information about performed steps. │
+
│ --dry-run -D If dry-run is set, commands are only printed, not executed. │
+
│ --help -h Show this message and exit. │
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+
+
+
+
+ + + diff --git a/kubernetes_tests/test_other_executors.py b/kubernetes_tests/test_other_executors.py index e2b703777923a..9c92dbca4aae7 100644 --- a/kubernetes_tests/test_other_executors.py +++ b/kubernetes_tests/test_other_executors.py @@ -18,6 +18,7 @@ import time import pytest + from kubernetes_tests.test_base import EXECUTOR, TestBase diff --git a/scripts/ci/tools/fix_ownership.sh b/scripts/ci/tools/fix_ownership.sh deleted file mode 100755 index 1dc7b92e4ec56..0000000000000 --- a/scripts/ci/tools/fix_ownership.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# Fixes ownership for files created inside container (files owned by root will be owned by host user) -# -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - -if [[ ${OSTYPE} == "darwin"* ]]; then - # No need to fix ownership on MacOS - the filesystem there takes care about ownership mapping - exit -fi - -declare -a EXTRA_DOCKER_FLAGS - -sanity_checks::sanitize_mounted_files - -read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker_params)" - -if docker image inspect "${AIRFLOW_CI_IMAGE_WITH_TAG}" >/dev/null 2>&1; then - docker_v run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \ - --rm \ - --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \ - "${AIRFLOW_CI_IMAGE_WITH_TAG}" \ - -c /opt/airflow/scripts/in_container/run_fix_ownership.sh || true -else - echo "Skip fixing ownership as seems that you do not have the ${AIRFLOW_CI_IMAGE_WITH_TAG} image yet" -fi diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index bb81e7aa343ab..57f4e4d40d150 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -123,8 +123,18 @@ function in_container_fix_ownership() { "/opt/airflow/dags" "${AIRFLOW_SOURCES}" ) + echo + echo "${COLOR_BLUE}Fixing ownership of generated files as Host OS is ${HOST_OS}${COLOR_RESET}" + echo "${COLOR_BLUE}Directories: ${DIRECTORIES_TO_FIX[*]}${COLOR_RESET}" + echo find "${DIRECTORIES_TO_FIX[@]}" -print0 -user root 2>/dev/null | xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference || true >/dev/null 2>&1 + echo "${COLOR_BLUE}Fixed ownership of generated files." + echo + else + echo + echo "${COLOR_YELLOW}Skip fixing ownership of generated files as Host OS is ${HOST_OS}${COLOR_RESET}" + echo fi }