diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index e656f48a5a300..6fe821ad3909f 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -36,10 +36,6 @@ on: # yamllint disable-line rule:truthy description: "Whether to upgrade to newer dependencies (true/false)" required: true type: string - chicken-egg-providers: - description: "Whether to build chicken-egg provider distributions in the same run (true/false)" - required: true - type: string docker-cache: description: "Docker cache specification to build the image (registry, local, disabled)." required: true @@ -74,7 +70,6 @@ jobs: use-uv: "false" build-provider-distributions: ${{ inputs.default-branch == 'main' }} upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} @@ -91,7 +86,6 @@ jobs: use-uv: "false" build-provider-distributions: ${{ inputs.default-branch == 'main' }} upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 215e587ae8ea7..015e6a772b03f 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -60,10 +60,6 @@ on: # yamllint disable-line rule:truthy description: "Whether to upgrade to newer dependencies (true/false)" required: true type: string - chicken-egg-providers: - description: "List of providers that should be prepared from sources" - required: false - type: string basic-checks-only: description: "Whether to run only basic checks (true/false)" required: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e50e11b048643..160820264e38f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,6 @@ jobs: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} canary-run: ${{ steps.source-run-info.outputs.canary-run }} - chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} core-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.core-test-types-list-as-strings-in-json }} @@ -257,7 +256,6 @@ jobs: # generate no providers constraints only in canary builds - they take quite some time to generate # they are not needed for regular builds, they are only needed to update constraints in canaries generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -282,7 +280,6 @@ jobs: basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} ci-image-build: ${{ needs.build-info.outputs.ci-image-build }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} @@ -693,7 +690,6 @@ jobs: use-uv: ${{ needs.build-info.outputs.use-uv }} build-provider-distributions: ${{ needs.build-info.outputs.default-branch == 'main' }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} @@ -708,7 +704,6 @@ jobs: default-branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index e9b48670c6701..46cb9f250bc1b 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -32,10 +32,6 @@ on: # yamllint disable-line rule:truthy description: "Whether to generate constraints without providers (true/false)" required: true type: string - chicken-egg-providers: - description: "Space-separated list of providers that should be installed from context files" - required: true - type: string debug-resources: description: "Whether to run in debug mode (true/false)" required: true @@ -97,17 +93,12 @@ jobs: # The no providers constraints are only needed when we want to update constraints (in canary builds) # They slow down the start of PROD image builds so we want to only run them when needed. if: inputs.generate-no-providers-constraints == 'true' - - name: "Prepare chicken-eggs provider distributions" - # In case of provider distributions which use latest dev0 version of providers, we should prepare them - # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency - # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 + - name: "Prepare updated provider distributions" + # In case of provider distributions which are not yet released, we build them from sources shell: bash - env: - CHICKEN_EGG_PROVIDERS: ${{ inputs.chicken-egg-providers }} run: > breeze release-management prepare-provider-distributions --include-not-ready-providers - --distribution-format wheel ${CHICKEN_EGG_PROVIDERS} - if: inputs.chicken-egg-providers != '' + --distribution-format wheel - name: "Prepare airflow distributions" shell: bash run: > @@ -119,14 +110,10 @@ jobs: - name: "PyPI constraints" shell: bash timeout-minutes: 25 - env: - CHICKEN_EGG_PROVIDERS: ${{ inputs.chicken-egg-providers }} run: | for PYTHON in $PYTHON_VERSIONS; do - breeze release-management generate-constraints \ - --airflow-constraints-mode constraints --answer yes \ - --chicken-egg-providers "${CHICKEN_EGG_PROVIDERS}" \ - --python "${PYTHON}" + breeze release-management generate-constraints --airflow-constraints-mode constraints \ + --answer yes --python "${PYTHON}" done - name: "Dependency upgrade summary" shell: bash diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 27bf6467377e8..1c8f7504c25f9 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -100,10 +100,6 @@ on: # yamllint disable-line rule:truthy description: "Whether to attempt to upgrade image to newer dependencies (true/false)" required: true type: string - chicken-egg-providers: - description: "Space-separated list of providers that should be installed from context files" - required: true - type: string docker-cache: description: "Docker cache specification to build the image (registry, local, disabled)." required: true @@ -154,18 +150,7 @@ jobs: --distributions-list-file ./prod_image_installed_providers.txt --distribution-format wheel --include-not-ready-providers --skip-tag-check if: > - inputs.upload-package-artifact == 'true' && - inputs.build-provider-distributions == 'true' - - name: "Prepare chicken-eggs provider distributions" - shell: bash - env: - CHICKEN_EGG_PROVIDERS: ${{ inputs.chicken-egg-providers }} - run: > - breeze release-management prepare-provider-distributions - --distribution-format wheel ${CHICKEN_EGG_PROVIDERS} - if: > - inputs.upload-package-artifact == 'true' && - inputs.chicken-egg-providers != '' + inputs.upload-package-artifact == 'true' && inputs.build-provider-distributions == 'true' - name: "Prepare airflow package" shell: bash run: > @@ -238,11 +223,23 @@ jobs: with: name: prod-packages path: ./docker-context-files + - name: "Show downloaded packages" + run: ls -la ./docker-context-files - name: "Download constraints" uses: actions/download-artifact@v4 with: name: constraints path: ./docker-context-files + - name: "Show constraints" + run: | + for file in ./docker-context-files/constraints*/constraints*.txt + do + echo "=== ${file} ===" + echo + cat ${file} + echo + echo "=== END ${file} ===" + done - name: "Login to ghcr.io" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 0b208cb552059..287348d77548c 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -48,10 +48,6 @@ on: # yamllint disable-line rule:truthy description: "Whether to attempt to upgrade image to newer dependencies (false/RANDOM_VALUE)" required: true type: string - chicken-egg-providers: - description: "Space-separated list of providers that should be installed from context files" - required: true - type: string constraints-branch: description: "Branch used to construct constraints URL from." required: true @@ -84,7 +80,6 @@ jobs: use-uv: ${{ inputs.use-uv }} build-provider-distributions: ${{ inputs.build-provider-distributions }} upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} @@ -109,7 +104,6 @@ jobs: use-uv: "false" build-provider-distributions: ${{ inputs.build-provider-distributions }} upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index d94efa2c1265c..710cc905ff613 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -45,7 +45,6 @@ jobs: pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} - chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} skipLatest: ${{ github.event.inputs.skipLatest == '' && ' ' || '--skip-latest' }} limitPlatform: ${{ github.repository == 'apache/airflow' && ' ' || '--limit-platform linux/amd64' }} env: @@ -144,21 +143,6 @@ jobs: run: docker run --privileged --rm tonistiigi/binfmt --install all - name: "Create airflow_cache builder" run: docker buildx create --name airflow_cache - - name: "Prepare chicken-eggs provider distributions" - # In case of provider distributions which use latest rc1 version of providers, we should prepare them - # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency - # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.rc0 - shell: bash - env: - CHICKEN_EGG_PROVIDERS: ${{ needs.build-info.outputs.chicken-egg-providers }} - run: > - breeze release-management prepare-provider-distributions - --distribution-format wheel ${CHICKEN_EGG_PROVIDERS} - if: needs.build-info.outputs.chicken-egg-providers != '' - - name: "Copy dist packages to docker-context files" - shell: bash - run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files - if: needs.build-info.outputs.chicken-egg-providers != '' - name: > Release regular images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} env: @@ -168,15 +152,10 @@ jobs: AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest }} LIMIT_PLATFORM: ${{ needs.build-info.outputs.limitPlatform }} - CHICKEN_EGG_PROVIDERS: ${{ needs.build-info.outputs.chicken-egg-providers }} run: > - breeze release-management release-prod-images - --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" - ${SKIP_LATEST} - ${LIMIT_PLATFORM} + breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} ${LIMIT_PLATFORM} --limit-python ${PYTHON_VERSION} - --chicken-egg-providers "${CHICKEN_EGG_PROVIDERS}" - name: > Release slim images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} env: diff --git a/Dockerfile b/Dockerfile index 8d9f1f7c8d56c..2811b0cbc11d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -462,7 +462,7 @@ function common::get_packaging_tool() { UV_CONCURRENT_DOWNLOADS=$(nproc --all) export UV_CONCURRENT_DOWNLOADS if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then - EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease allow" + EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease if-necessary" fi else echo diff --git a/Dockerfile.ci b/Dockerfile.ci index e68477ab09df0..88170d57525af 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -401,7 +401,7 @@ function common::get_packaging_tool() { UV_CONCURRENT_DOWNLOADS=$(nproc --all) export UV_CONCURRENT_DOWNLOADS if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then - EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease allow" + EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease if-necessary" fi else echo diff --git a/contributing-docs/12_provider_distributions.rst b/contributing-docs/12_provider_distributions.rst index 7ff6016e886a1..bf14b56aca014 100644 --- a/contributing-docs/12_provider_distributions.rst +++ b/contributing-docs/12_provider_distributions.rst @@ -137,20 +137,6 @@ When you develop providers, you might want to reuse some of the code between tes This is possible by placing the code in ``test_utils`` in the ``devel-common/src`` directory. The ``tests_common`` module is installed automatically by uv in the uv workspace. -Chicken-egg providers ---------------------- - -Sometimes, when a provider depends on another provider, and you want to add a new feature that spans across -two providers, you might need to add a new feature to the "dependent" provider, you need -to add a new feature to the "dependency" provider as well. This is a chicken-egg problem and by default -some CI jobs (like generating PyPI constraints) will fail because they cannot use the source version of -the provider distribution. This is handled by adding the "dependent" provider to the chicken-egg list of -"providers" in ``dev/breeze/src/airflow_breeze/global_constants.py``. By doing this, the provider is build -locally from sources rather than downloaded from PyPI when generating constraints. - -More information about the chicken-egg providers and how release is handled can be found in -the `Release providers documentation <../dev/README_RELEASE_PROVIDERS.md#chicken-egg-providers>`_ - Developing community managed providers -------------------------------------- diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 3f420f6050c08..9bbe5abae3ea3 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -40,7 +40,6 @@ - [Publish the final Apache Airflow release](#publish-the-final-apache-airflow-release) - [Summarize the voting for the Apache Airflow release](#summarize-the-voting-for-the-apache-airflow-release) - [Publish release to SVN](#publish-release-to-svn) - - [Remove chicken-egg providers](#remove-chicken-egg-providers) - [Manually prepare production Docker Image](#manually-prepare-production-docker-image) - [Verify production images](#verify-production-images) - [Publish documentation](#publish-documentation) @@ -778,43 +777,10 @@ export AIRFLOW_REPO_ROOT=$(pwd) breeze release-management start-release --release-candidate ${RC} --previous-release ``` -## Remove chicken-egg providers - -For the first MINOR (X.Y.0) release you need to do few more steps if there are new "chicken-egg" providers -that have min-airflow version set to X.Y.0 - -* NOTE! WE MIGHT WANT TO AUTOMATE THAT STEP IN THE FUTURE - -1. Checkout the constraints-2-* branch and update the ``constraints-3*.txt`` file with the new provider - version. Find the place where the provider should be added, add it with the latest provider version. - -``` -apache-airflow-providers-PROVIDER==VERSION -``` - -Commit, push and tag this change with ``constraints-X.Y.Z`` tag: - -```bash -git add -git commit -m "Add chicken-egg provider apache-airflow-providers-PROVIDER" -git tag -s constraints-X.Y.Z --force -git push -f apache constraints-X.Y.Z -``` - - -2. remove providers from ``CHICKEN_EGG_PROVIDERS`` list in ``src/airflow_breeze/global_constants.py`` - that have >= ``X.Y.0`` in the corresponding provider.yaml file. - - -3. In case the provider should also be installed in the image (it is part of ``prod_image_installed_providers.txt``) - it should also be added at this moment to ``Dockerfile`` to the list of default extras in the line with ``AIRFLOW_EXTRAS``: - ```Dockerfile ARG AIRFLOW_EXTRAS=".....,,...." ``` -This change needs to be merged to ``main`` and cherry-picked to ``v2-*-test`` branch before building the image. - 4. Make sure to update Airflow version in ``v3-*-test`` branch after cherry-picking to X.Y.1 in ``airflow/__init__.py`` @@ -829,7 +795,7 @@ When you trigger it you need to pass: * Airflow Version * Optional "true" in skip latest field if you do not want to re-tag the latest image -Make sure you use ``v2-*-test`` branch to run the workflow. +Make sure you use ``v3-*-test`` branch to run the workflow. ![Release prod image](images/release_prod_image.png) diff --git a/dev/README_RELEASE_PROVIDERS.md b/dev/README_RELEASE_PROVIDERS.md index aaf120df3555c..b98bfa5de1edc 100644 --- a/dev/README_RELEASE_PROVIDERS.md +++ b/dev/README_RELEASE_PROVIDERS.md @@ -26,7 +26,6 @@ - [Decide when to release](#decide-when-to-release) - [Provider distributions versioning](#provider-distributions-versioning) - [Possible states of Provider distributions](#possible-states-of-provider-distributions) -- [Chicken-egg providers](#chicken-egg-providers) - [Prepare Regular Provider distributions (RC)](#prepare-regular-provider-distributions-rc) - [Move provider into remove state](#move-provider-into-remove-state) - [Increasing version number](#increasing-version-number) @@ -191,26 +190,6 @@ graph TD; removed -- Remove from the code --> gone; ``` -# Chicken-egg providers - -Sometimes (rare) we release providers that have dependencies on future version of Airflow - which means that -they are released long before they are actually usable and it also means that versions in PyPI should be -released with `apache-airflow >= x.y.z.dev0` version, such providers should have the .dev0 suffix included -in the `apache-airflow` dependency specification, only the final release, just before the final Airflow x.y.z -release should get it changed to `>= x.y.z`. This is a rare case and should be handled with care. - -We call such case chicken-egg providers as it's not clear who should be released first - the provider or -the Airflow. - -Similar case is when provider depends on another provider (usually `common.*`) that is not yet released -because you it contains new feature and you want to release the providers together. - -In such case the "common" provider should be added to the list of "chicken-egg" providers in the -`./dev/breeze/src/airflow_breeze/global_constants.py` file and version of the provider should be bumped -in the PR by author of the PR that adds both the functionality. This will make sure that the -provider is build in CI from sources, when CI jobs are run rather than latest version downloaded from PyPI -when constraints are generated. - # Prepare Regular Provider distributions (RC) ## Move provider into remove state diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index efb0b4db13fe5..c450342f7d199 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -164,89 +164,88 @@ separated by spaces. This is to accommodate for the wau how outputs of this kind GitHub Actions to pass the list of parameters to a command to execute -| Output | Meaning of the output | Example value | List | -|------------------------------------------------|--------------------------------------------------------------------------------------------------------|-----------------------------------------|------| -| all-python-versions | List of all python versions there are available in the form of JSON array | \['3.9', '3.10'\] | | -| all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.9 3.10 | * | -| all-versions | If set to true, then all python, k8s, DB versions are used for tests. | false | | -| basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | -| build_system_changed_in_pyproject_toml | When builds system dependencies changed in pyproject.toml changed in the PR. | false | | -| chicken-egg-providers | List of providers that should be considered as "chicken-egg" - expecting development Airflow version | | | -| ci-image-build | Whether CI image build is needed | true | | +| Output | Meaning of the output | Example value | List | +|---------------------------------------------------------|--------------------------------------------------------------------------------------------------------|-----------------------------------------|------| +| all-python-versions | List of all python versions there are available in the form of JSON array | \['3.9', '3.10'\] | | +| all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.9 3.10 | * | +| all-versions | If set to true, then all python, k8s, DB versions are used for tests. | false | | +| basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | +| build_system_changed_in_pyproject_toml | When builds system dependencies changed in pyproject.toml changed in the PR. | false | | +| ci-image-build | Whether CI image build is needed | true | | | core-test-types-list-as-strings-in-json | Which test types should be run for unit tests for core | API Always Providers | * | -| debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | | -| default-branch | Which branch is default for the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | | -| default-constraints-branch | Which branch is default for the build ("constraints-main" for main branch, "constraints-2-4" etc.) | constraints-main | | -| default-helm-version | Which Helm version to use as default | v3.9.4 | | -| default-kind-version | Which Kind version to use as default | v0.16.0 | | -| default-kubernetes-version | Which Kubernetes version to use as default | v1.25.2 | | -| default-mysql-version | Which MySQL version to use as default | 5.7 | | -| default-postgres-version | Which Postgres version to use as default | 10 | | -| default-python-version | Which Python version to use as default | 3.9 | | -| disable-airflow-repo-cache | Disables cache of the repo main cache in CI - aiflow will be installed without main installation cache | true | | -| docker-cache | Which cache should be used for images ("registry", "local" , "disabled") | registry | | -| docs-build | Whether to build documentation ("true"/"false") | true | | -| docs-list-as-string | What filter to apply to docs building - based on which documentation packages should be built | apache-airflow helm-chart google | * | -| excluded-providers-as-string c | List of providers that should be excluded from the build as space-separated string | amazon google | * | -| force-pip | Whether pip should be forced in the image build instead of uv ("true"/"false") | false | | -| full-tests-needed | Whether this build runs complete set of tests or only subset (for faster PR builds) \[1\] | false | | -| generated-dependencies-changed | Whether generated dependencies have changed ("true"/"false") | false | | -| has-migrations | Whether the PR has migrations ("true"/"false") | false | | -| hatch-build-changed | When hatch build.py changed in the PR. | false | | -| helm-test-packages-list-as-string | List of helm packages to test as JSON array | \["airflow_aux", "airflow_core"\] | * | -| helm-version | Which Helm version to use for tests | v3.15.3 | | -| include-success-outputs | Whether to include outputs of successful parallel tests ("true"/"false") | false | | +| debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | | +| default-branch | Which branch is default for the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | | +| default-constraints-branch | Which branch is default for the build ("constraints-main" for main branch, "constraints-2-4" etc.) | constraints-main | | +| default-helm-version | Which Helm version to use as default | v3.9.4 | | +| default-kind-version | Which Kind version to use as default | v0.16.0 | | +| default-kubernetes-version | Which Kubernetes version to use as default | v1.25.2 | | +| default-mysql-version | Which MySQL version to use as default | 5.7 | | +| default-postgres-version | Which Postgres version to use as default | 10 | | +| default-python-version | Which Python version to use as default | 3.9 | | +| disable-airflow-repo-cache | Disables cache of the repo main cache in CI - aiflow will be installed without main installation cache | true | | +| docker-cache | Which cache should be used for images ("registry", "local" , "disabled") | registry | | +| docs-build | Whether to build documentation ("true"/"false") | true | | +| docs-list-as-string | What filter to apply to docs building - based on which documentation packages should be built | apache-airflow helm-chart google | * | +| excluded-providers-as-string c | List of providers that should be excluded from the build as space-separated string | amazon google | * | +| force-pip | Whether pip should be forced in the image build instead of uv ("true"/"false") | false | | +| full-tests-needed | Whether this build runs complete set of tests or only subset (for faster PR builds) \[1\] | false | | +| generated-dependencies-changed | Whether generated dependencies have changed ("true"/"false") | false | | +| has-migrations | Whether the PR has migrations ("true"/"false") | false | | +| hatch-build-changed | When hatch build.py changed in the PR. | false | | +| helm-test-packages-list-as-string | List of helm packages to test as JSON array | \["airflow_aux", "airflow_core"\] | * | +| helm-version | Which Helm version to use for tests | v3.15.3 | | +| include-success-outputs | Whether to include outputs of successful parallel tests ("true"/"false") | false | | | individual-providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers (individually listed) | Providers[\amazon\] Providers\[google\] | * | -| is-airflow-runner | Whether runner used is an airflow or infrastructure runner (true if airflow/false if infrastructure) | false | | -| is-amd-runner | Whether runner used is an AMD one | true | | -| is-arm-runner | Whether runner used is an ARM one | false | | -| is-committer-build | Whether the build is triggered by a committer | false | | -| is-k8s-runner | Whether the build runs on our k8s infrastructure | false | | -| is-legacy-ui-api-labeled | Whether the PR is labeled as legacy UI/API | false | | -| is-self-hosted-runner | Whether the runner is self-hosted | false | | -| is-vm-runner | Whether the runner uses VM to run | true | | -| kind-version | Which Kind version to use for tests | v0.24.0 | | -| kubernetes-combos-list-as-string | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.9-v1.25.2 3.10-v1.28.13 | * | -| kubernetes-versions | All Kubernetes versions to use for tests as JSON array | \['v1.25.2'\] | | -| kubernetes-versions-list-as-string | All Kubernetes versions to use for tests as space-separated string | v1.25.2 | * | -| latest-versions-only | If set, the number of Python, Kubernetes, DB versions will be limited to the latest ones. | false | | -| mypy-checks | List of folders to be considered for mypy checks | \["airflow_aux", "airflow_core"\] | | -| mysql-exclude | Which versions of MySQL to exclude for tests as JSON array | [] | | -| mysql-versions | Which versions of MySQL to use for tests as JSON array | \['8.0'\] | | -| needs-api-codegen | Whether "api-codegen" are needed to run ("true"/"false") | true | | -| needs-api-tests | Whether "api-tests" are needed to run ("true"/"false") | true | | -| needs-helm-tests | Whether Helm tests are needed to run ("true"/"false") | true | | -| needs-javascript-scans | Whether javascript CodeQL scans should be run ("true"/"false") | true | | -| needs-mypy | Whether mypy check is supposed to run in this build | true | | -| needs-python-scans | Whether Python CodeQL scans should be run ("true"/"false") | true | | -| only-new-ui-files | Whether only new UI files are present in the PR ("true"/"false") | false | | -| postgres-exclude | Which versions of Postgres to exclude for tests as JSON array | [] | | -| postgres-versions | Which versions of Postgres to use for tests as JSON array | \['12'\] | | -| prod-image-build | Whether PROD image build is needed | true | | -| providers-compatibility-tests-matrix | Matrix of providers compatibility tests: (python_version, airflow_version, removed_providers) | \[{}\] | | +| is-airflow-runner | Whether runner used is an airflow or infrastructure runner (true if airflow/false if infrastructure) | false | | +| is-amd-runner | Whether runner used is an AMD one | true | | +| is-arm-runner | Whether runner used is an ARM one | false | | +| is-committer-build | Whether the build is triggered by a committer | false | | +| is-k8s-runner | Whether the build runs on our k8s infrastructure | false | | +| is-legacy-ui-api-labeled | Whether the PR is labeled as legacy UI/API | false | | +| is-self-hosted-runner | Whether the runner is self-hosted | false | | +| is-vm-runner | Whether the runner uses VM to run | true | | +| kind-version | Which Kind version to use for tests | v0.24.0 | | +| kubernetes-combos-list-as-string | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.9-v1.25.2 3.10-v1.28.13 | * | +| kubernetes-versions | All Kubernetes versions to use for tests as JSON array | \['v1.25.2'\] | | +| kubernetes-versions-list-as-string | All Kubernetes versions to use for tests as space-separated string | v1.25.2 | * | +| latest-versions-only | If set, the number of Python, Kubernetes, DB versions will be limited to the latest ones. | false | | +| mypy-checks | List of folders to be considered for mypy checks | \["airflow_aux", "airflow_core"\] | | +| mysql-exclude | Which versions of MySQL to exclude for tests as JSON array | [] | | +| mysql-versions | Which versions of MySQL to use for tests as JSON array | \['8.0'\] | | +| needs-api-codegen | Whether "api-codegen" are needed to run ("true"/"false") | true | | +| needs-api-tests | Whether "api-tests" are needed to run ("true"/"false") | true | | +| needs-helm-tests | Whether Helm tests are needed to run ("true"/"false") | true | | +| needs-javascript-scans | Whether javascript CodeQL scans should be run ("true"/"false") | true | | +| needs-mypy | Whether mypy check is supposed to run in this build | true | | +| needs-python-scans | Whether Python CodeQL scans should be run ("true"/"false") | true | | +| only-new-ui-files | Whether only new UI files are present in the PR ("true"/"false") | false | | +| postgres-exclude | Which versions of Postgres to exclude for tests as JSON array | [] | | +| postgres-versions | Which versions of Postgres to use for tests as JSON array | \['12'\] | | +| prod-image-build | Whether PROD image build is needed | true | | +| providers-compatibility-tests-matrix | Matrix of providers compatibility tests: (python_version, airflow_version, removed_providers) | \[{}\] | | | providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers | Providers Providers\[-google\] | * | -| pyproject-toml-changed | When pyproject.toml changed in the PR. | false | | -| python-versions | List of python versions to use for that build | \['3.9'\] | | -| python-versions-list-as-string | Which versions of MySQL to use for tests as space-separated string | 3.9 | * | -| run-amazon-tests | Whether Amazon tests should be run ("true"/"false") | true | | -| run-kubernetes-tests | Whether Kubernetes tests should be run ("true"/"false") | true | | -| run-system-tests | Whether system tests should be run ("true"/"false") | true | | -| run-task-sdk-tests | Whether Task SDK tests should be run ("true"/"false") | true | | -| run-tests | Whether unit tests should be run ("true"/"false") | true | | -| run-ui-tests | Whether UI tests should be run ("true"/"false") | true | | -| run-www-tests | Whether Legacy WWW tests should be run ("true"/"false") | true | | -| runs-on-as-json-default | List of labels assigned for runners for that build for default runs for that build (as string) | \["ubuntu-22.04"\] | | -| runs-on-as-json-docs-build | List of labels assigned for runners for that build for ddcs build (as string) | \["ubuntu-22.04"\] | | -| runs-on-as-json-self-hosted | List of labels assigned for runners for that build for self hosted runners | \["self-hosted", "Linux", "X64"\] | | -| runs-on-as-json-self-hosted-asf | List of labels assigned for runners for that build for ASF self hosted runners | \["self-hosted", "Linux", "X64"\] | | -| runs-on-as-json-public | List of labels assigned for runners for that build for public runners | \["ubuntu-22.04"\] | | -| selected-providers-list-as-string | List of providers affected when they are selectively affected. | airbyte http | * | -| skip-pre-commits | Which pre-commits should be skipped during the static-checks run | flynt,identity | | -| skip-providers-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | | -| sqlite-exclude | Which versions of Sqlite to exclude for tests as JSON array | [] | | -| testable-core-integrations | List of core integrations that are testable in the build as JSON array | \['celery', 'kerberos'\] | | -| testable-providers-integrations | List of core integrations that are testable in the build as JSON array | \['mongo', 'kafka'\] | | -| upgrade-to-newer-dependencies | Whether the image build should attempt to upgrade all dependencies (true/false or commit hash) | false | | +| pyproject-toml-changed | When pyproject.toml changed in the PR. | false | | +| python-versions | List of python versions to use for that build | \['3.9'\] | | +| python-versions-list-as-string | Which versions of MySQL to use for tests as space-separated string | 3.9 | * | +| run-amazon-tests | Whether Amazon tests should be run ("true"/"false") | true | | +| run-kubernetes-tests | Whether Kubernetes tests should be run ("true"/"false") | true | | +| run-system-tests | Whether system tests should be run ("true"/"false") | true | | +| run-task-sdk-tests | Whether Task SDK tests should be run ("true"/"false") | true | | +| run-tests | Whether unit tests should be run ("true"/"false") | true | | +| run-ui-tests | Whether UI tests should be run ("true"/"false") | true | | +| run-www-tests | Whether Legacy WWW tests should be run ("true"/"false") | true | | +| runs-on-as-json-default | List of labels assigned for runners for that build for default runs for that build (as string) | \["ubuntu-22.04"\] | | +| runs-on-as-json-docs-build | List of labels assigned for runners for that build for ddcs build (as string) | \["ubuntu-22.04"\] | | +| runs-on-as-json-self-hosted | List of labels assigned for runners for that build for self hosted runners | \["self-hosted", "Linux", "X64"\] | | +| runs-on-as-json-self-hosted-asf | List of labels assigned for runners for that build for ASF self hosted runners | \["self-hosted", "Linux", "X64"\] | | +| runs-on-as-json-public | List of labels assigned for runners for that build for public runners | \["ubuntu-22.04"\] | | +| selected-providers-list-as-string | List of providers affected when they are selectively affected. | airbyte http | * | +| skip-pre-commits | Which pre-commits should be skipped during the static-checks run | flynt,identity | | +| skip-providers-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | | +| sqlite-exclude | Which versions of Sqlite to exclude for tests as JSON array | [] | | +| testable-core-integrations | List of core integrations that are testable in the build as JSON array | \['celery', 'kerberos'\] | | +| testable-providers-integrations | List of core integrations that are testable in the build as JSON array | \['mongo', 'kafka'\] | | +| upgrade-to-newer-dependencies | Whether the image build should attempt to upgrade all dependencies (true/false or commit hash) | false | | [1] Note for deciding if `full tests needed` mode is enabled and provider.yaml files. diff --git a/dev/breeze/doc/images/output_release-management_generate-constraints.svg b/dev/breeze/doc/images/output_release-management_generate-constraints.svg index 7e59f4ef47663..4cd4b73b12018 100644 --- a/dev/breeze/doc/images/output_release-management_generate-constraints.svg +++ b/dev/breeze/doc/images/output_release-management_generate-constraints.svg @@ -1,4 +1,4 @@ - +