diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 541c72165997b..e28a9d7619c5a 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -109,18 +109,20 @@ jobs: with: persist-credentials: false submodules: recursive + - name: "Setup python" + uses: actions/setup-python@v2 + with: + # We do not have output from selective checks yet, so we need to hardcode python + python-version: 3.7 + cache: 'pip' + cache-dependency-path: ./dev/breeze/setup* + - run: ./scripts/ci/install_breeze.sh - name: Selective checks id: selective-checks env: - PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pullRequestLabels }} - run: | - if [[ ${GITHUB_EVENT_NAME} == "pull_request_target" ]]; then - # Run selective checks - ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" - else - # Run all checks - ./scripts/ci/selective_ci_checks.sh - fi + PR_LABELS: "$${{ steps.get-latest-pr-labels.outputs.pullRequestLabels }}" + COMMIT_REF: "${{ github.sha }}" + run: breeze selective-check - name: Compute dynamic outputs id: dynamic-outputs run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cea9d687f50c..6545423482912 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,18 +192,20 @@ jobs: fetch-depth: 2 persist-credentials: false if: github.event_name == 'pull_request' + - name: "Setup python" + uses: actions/setup-python@v2 + with: + # We do not have output from selective checks yet, so we need to hardcode python + python-version: 3.7 + cache: 'pip' + cache-dependency-path: ./dev/breeze/setup* + - run: ./scripts/ci/install_breeze.sh - name: Selective checks id: selective-checks env: PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" - run: | - if [[ ${GITHUB_EVENT_NAME} == "pull_request" ]]; then - # Run selective checks - ./scripts/ci/selective_ci_checks.sh "${GITHUB_SHA}" - else - # Run all checks - ./scripts/ci/selective_ci_checks.sh - fi + COMMIT_REF: "${{ github.sha }}" + run: breeze selective-check # Avoid having to specify the runs-on logic every time. We use the custom # env var AIRFLOW_SELF_HOSTED_RUNNER set only on our runners, but never # on the public runners diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6d6f4d02562d5..4e6c7c83f4dc4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,19 +43,19 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Setup python" + uses: actions/setup-python@v2 + with: + # We do not have output from selective checks yet, so we need to hardcode python + python-version: 3.7 + cache: 'pip' + cache-dependency-path: ./dev/breeze/setup* + - run: ./scripts/ci/install_breeze.sh - name: Selective checks id: selective-checks env: - EVENT_NAME: ${{ github.event_name }} - TARGET_COMMIT_SHA: ${{ github.sha }} - run: | - if [[ ${EVENT_NAME} == "pull_request" ]]; then - # Run selective checks - ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" - else - # Run all checks - ./scripts/ci/selective_ci_checks.sh - fi + COMMIT_REF: "${{ github.sha }}" + run: breeze selective-check analyze: name: Analyze diff --git a/BREEZE.rst b/BREEZE.rst index 7ce48270d02ff..64cbd2eef94d7 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -565,13 +565,17 @@ Configuration and maintenance * Cleanup breeze with ``breeze cleanup`` command * Self-upgrade breeze with ``breeze self-upgrade`` command * Setup autocomplete for Breeze with ``breeze setup-autocomplete`` command -* Checking available resources for docker with ``breeze resource-check`` command -* Freeing space needed to run CI tests with ``breeze free-space`` command -* Fixing ownership of files in your repository with ``breeze fix-ownership`` command * Print Breeze version with ``breeze version`` command * Outputs hash of commands defined by ``breeze`` with ``command-hash-export`` (useful to avoid needless regeneration of Breeze images) +CI tasks +-------- +* Freeing space needed to run CI tests with ``breeze free-space`` command +* Fixing ownership of files in your repository with ``breeze fix-ownership`` command +* Checking available resources for docker with ``breeze resource-check`` command +* Deciding which tests should be run with ``breeze selective-check`` command + Release tasks ------------- @@ -1295,8 +1299,8 @@ command but it is very similar to current ``breeze`` command): -Resource check -============== +Running resource check +---------------------- Breeze requires certain resources to be available - disk, memory, CPU. When you enter Breeze's shell, the resources are checked and information if there is enough resources is displayed. However you can @@ -1310,7 +1314,7 @@ Those are all available flags of ``resource-check`` command: Freeing the space -================= +----------------- When our CI runs a job, it needs all memory and disk it can have. We have a Breeze command that frees the memory and disk space used. You can also use it clear space locally but it performs a few operations @@ -1323,8 +1327,26 @@ Those are all available flags of ``free-space`` command: :alt: Breeze free-space +Selective check +--------------- + +When our CI runs a job, it needs to decide which tests to run, whether to build images and how much the test +should be run on multiple combinations of Python, Kubernetes, Backend versions. In order to optimize time +needed to run the CI Builds. You can also use the tool to test what tests will be run when you provide +a specific commit that Breeze should run the tests on. + +More details about the algorithm used to pick the right tests can be +found in `Selective Checks `_. + +Those are all available flags of ``selective-check`` command: + +.. image:: ./images/breeze/output-selective-check.svg + :width: 100% + :alt: Breeze selective-check + + Tracking backtracking issues for CI builds -========================================== +------------------------------------------ When our CI runs a job, we automatically upgrade our dependencies in the ``main`` build. However, this might lead to conflicts and ``pip`` backtracking for a long time (possibly forever) for dependency resolution. diff --git a/SELECTIVE_CHECKS.md b/SELECTIVE_CHECKS.md deleted file mode 100644 index 3a92d9c817987..0000000000000 --- a/SELECTIVE_CHECKS.md +++ /dev/null @@ -1,144 +0,0 @@ - - -# Selective CI Checks - -In order to optimise our CI jobs, we've implemented optimisations to only run selected checks for some -kind of changes. The logic implemented reflects the internal architecture of Airflow 2.0 packages -and it helps to keep down both the usage of jobs in GitHub Actions as well as CI feedback time to -contributors in case of simpler changes. - -We have the following test types (separated by packages in which they are): - -* Always - those are tests that should be always executed (always folder) -* Core - for the core Airflow functionality (core folder) -* API - Tests for the Airflow API (api and api_connexion folders) -* CLI - Tests for the Airflow CLI (cli folder) -* WWW - Tests for the Airflow webserver (www folder) -* Providers - Tests for all Providers of Airflow (providers folder) -* Other - all other tests (all other folders that are not part of any of the above) - -We also have several special kinds of tests that are not separated by packages but they are marked with -pytest markers. They can be found in any of those packages and they can be selected by the appropriate -pytest custom command line options. See `TESTING.rst `_ for details but those are: - -* Integration - tests that require external integration images running in docker-compose -* Quarantined - tests that are flaky and need to be fixed -* Postgres - tests that require Postgres database. They are only run when backend is Postgres -* MySQL - tests that require MySQL database. They are only run when backend is MySQL - -Even if the types are separated, In case they share the same backend version/python version, they are -run sequentially in the same job, on the same CI machine. Each of them in a separate `docker run` command -and with additional docker cleaning between the steps to not fall into the trap of exceeding resource -usage in one big test run, but also not to increase the number of jobs per each Pull Request. - -The logic implemented for the changes works as follows: - -1) In case of direct push (so when PR gets merged) or scheduled run, we always run all tests and checks. - This is in order to make sure that the merge did not miss anything important. The remainder of the logic - is executed only in case of Pull Requests. We do not add providers tests in case DEFAULT_BRANCH is - different than main, because providers are only important in main branch and PRs to main branch. - -2) We retrieve which files have changed in the incoming Merge Commit (github.sha is a merge commit - automatically prepared by GitHub in case of Pull Request, so we can retrieve the list of changed - files from that commit directly). - -3) If any of the important, environment files changed (Dockerfile, ci scripts, setup.py, GitHub workflow - files), then we again run all tests and checks. Those are cases where the logic of the checks changed - or the environment for the checks changed so we want to make sure to check everything. We do not add - providers tests in case DEFAULT_BRANCH is different than main, because providers are only - important in main branch and PRs to main branch. - -4) If any of py files changed: we need to have CI image and run full static checks so we enable image building - -5) If any of docs changed: we need to have CI image so we enable image building - -6) If any of chart files changed, we need to run helm tests so we enable helm unit tests - -7) If any of API files changed, we need to run API tests so we enable them - -8) If any of the relevant source files that trigger the tests have changed at all. Those are airflow - sources, chart, tests and kubernetes_tests. If any of those files changed, we enable tests and we - enable image building, because the CI images are needed to run tests. - -9) Then we determine which types of the tests should be run. We count all the changed files in the - relevant airflow sources (airflow, chart, tests, kubernetes_tests) first and then we count how many - files changed in different packages: - - * in any case tests in `Always` folder are run. Those are special tests that should be run any time - modifications to any Python code occurs. Example test of this type is verifying proper structure of - the project including proper naming of all files. - * if any of the Airflow API files changed we enable `API` test type - * if any of the Airflow CLI files changed we enable `CLI` test type and Kubernetes tests (the - K8S tests depend on CLI changes as helm chart uses CLI to run Airflow). - * if this is a main branch and if any of the Provider files changed we enable `Providers` test type - * if any of the WWW files changed we enable `WWW` test type - * if any of the Kubernetes files changed we enable `Kubernetes` test type - * Then we subtract count of all the `specific` above per-type changed files from the count of - all changed files. In case there are any files changed, then we assume that some unknown files - changed (likely from the core of airflow) and in this case we enable all test types above and the - Core test types - simply because we do not want to risk to miss anything. - * In all cases where tests are enabled we also add Integration and - depending on - the backend used = Postgres or MySQL types of tests. - -10) Quarantined tests are always run when tests are run - we need to run them often to observe how - often they fail so that we can decide to move them out of quarantine. Details about the - Quarantined tests are described in `TESTING.rst `_ - -11) There is a special case of static checks. In case the above logic determines that the CI image - needs to be built, we run long and more comprehensive version of static checks - including - Mypy, Flake8. And those tests are run on all files, no matter how many files changed. - In case the image is not built, we run only simpler set of changes - the longer static checks - that require CI image are skipped, and we only run the tests on the files that changed in the incoming - commit - unlike flake8/mypy, those static checks are per-file based and they should not miss any - important change. - -Similarly to selective tests we also run selective security scans. In Pull requests, -the Python scan will only run when there is a python code change and JavaScript scan will only run if -there is a JavaScript or `yarn.lock` file change. For main builds, all scans are always executed. - -The selective check algorithm is shown here: - - -````mermaid -flowchart TD -A(PR arrives)-->B[Selective Check] -B-->C{Direct push merge?} -C-->|Yes| N[Enable images] -N-->D(Run Full Test
+Quarantined
Run full static checks) -C-->|No| E[Retrieve changed files] -E-->F{Environment files changed?} -F-->|Yes| N -F-->|No| G{Docs changed} -G-->|Yes| O[Enable images building] -O-->I{Chart files changed?} -G-->|No| I -I-->|Yes| P[Enable helm tests] -P-->J{API files changed} -I-->|No| J -J-->|Yes| Q[Enable API tests] -Q-->H{Sources changed?} -J-->|No| H -H-->|Yes| R[Enable Pytests] -R-->K[Determine test type] -K-->S{Core files changed} -S-->|Yes| N -S-->|No| M(Run selected test+
Integration, Quarantined
Full static checks) -H-->|No| L[Skip running test
Run subset of static checks] -``` diff --git a/dev/breeze/README.md b/dev/breeze/README.md index 14a9f089a4834..380eb402b83f4 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: a80a853b2c32c284a68ccd6d468804b892a69f14d2ad1886bdaa892755cf6262660e2b9fc582bcae27ae478910055267a76edea2df658196198a0365150e93e5 +Package config hash: 7279229e03b197f2bbd10ebb7b313f67bba3a704735d3688652efc5bdc1b3a60f2d1e0a144c89a2ecd11268b06888c5302a8774a8f392dc383bb940c99521db3 --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/SELECTIVE_CHECKS.md b/dev/breeze/SELECTIVE_CHECKS.md new file mode 100644 index 0000000000000..4504f8d9cd9cd --- /dev/null +++ b/dev/breeze/SELECTIVE_CHECKS.md @@ -0,0 +1,98 @@ + + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Selective CI Checks](#selective-ci-checks) + +# Selective CI Checks + +In order to optimise our CI jobs, we've implemented optimisations to only run selected checks for some +kind of changes. The logic implemented reflects the internal architecture of Airflow 2.0 packages, +and it helps to keep down both the usage of jobs in GitHub Actions and CI feedback time to +contributors in case of simpler changes. + +We have the following Groups of files for CI that determine which tests are run: + +* `Environment files` - if any of those changes, that forces 'run everything' mode, because changes there might + simply change the whole environment of what is going on in CI (Container image, dependencies) +* `Python and Javascript production files` - this area is useful in CodeQL Security scanning - if any of + the python or javascript files for airflow "production" changed, this means that the security scans should run +* `API tests and codegen files` - those are OpenAPI definition files that impact Open API specification and + determine that we should run dedicated API tests. +* `Helm files` - change in those files impacts helm "rendering" tests +* `Setup files` - change in the setup files indicates that we should run `upgrade to newer dependencies` +* `DOCs files` - change in those files indicate that we should run documentation builds +* `UI and WWW files` - those are files for the UI and WWW part of our UI (useful to determine if UI + tests should run) +* `Kubernetes files` - determine if any of Kubernetes related tests should be run +* `All Python files` - if none of the Python file changed, that indicates that we should not run unit tests +* `All source files` - if none of the sources change, that indicates that we should probably not build + an image and run any image-based static checks + +We have the following unit test types that can be selectively disabled/enabled based on the +content of the incoming PR: + +* Always - those are tests that should be always executed (always folder) +* Core - for the core Airflow functionality (core folder) +* API - Tests for the Airflow API (api and api_connexion folders) +* CLI - Tests for the Airflow CLI (cli folder) +* WWW - Tests for the Airflow webserver (www folder) +* Providers - Tests for all Providers of Airflow (providers folder) + +We also have several special kinds of tests that are not separated by packages, but they are marked with +pytest markers. They can be found in any of those packages and they can be selected by the appropriate +pytest custom command line options. See `TESTING.rst `_ for details but those are: + +* Integration - tests that require external integration images running in docker-compose +* Quarantined - tests that are flaky and need to be fixed +* Postgres - tests that require Postgres database. They are only run when backend is Postgres +* MySQL - tests that require MySQL database. They are only run when backend is MySQL + +Even if the types are separated, In case they share the same backend version/python version, they are +run sequentially in the same job, on the same CI machine. Each of them in a separate `docker run` command +and with additional docker cleaning between the steps to not fall into the trap of exceeding resource +usage in one big test run, but also not to increase the number of jobs per each Pull Request. + +The logic implements the following rules: + +* `Full tests` mode is enabled when the event is PUSH, or SCHEDULE or when "full tests needed" label is set. + That enables all matrix combinations of variables, and all possible tests +* Python, Kubernetes, Backend, Kind, Helm versions are limited to "defaults" only unless `Full tests` mode + is enabled. +* If "Commit" to work on cannot be determined, or `Full Test` mode is enabled or some of the important + environment files (setup.py, setup.cfg, Dockerfile, build scripts) changed - all unit tests are + executed - this is `run everything` mode. No further checks are performed. +* `Python scans`, `Javascript scans`, `API tests/codegen`, `UI`, `WWW`, `Kubernetes` tests and `DOC builds` + are enabled if any of the relevant files have been changed. +* `Helm` tests are run only if relevant files have been changed and if current branch is `main`. +* If no Source files are changed - no tests are run and no further rules below are checked. +* `Image building` is enabled if either test are run, docs are build or kubernetes tests are run. All those + need `CI` or `PROD` images to be built. +* The specific unit test type is enabled only if changed files match the expected patterns for each type + (`API`, `CLI`, `WWW`, `Providers`). The `Always` test type is added always if any unit tests are run. + `Providers` tests are removed if current branch is different than `main` +* If there are no files left in sources after matching the test types and Kubernetes files, + then apparently some Core/Other files have been changed. This automatically adds all test + types to execute. This is done because changes in core might impact all the other test types. +* if `Image building` is disabled, only basic pre-commits are enabled - no 'image-depending` pre-commits + are enabled. +* If there are some setup files changed, `upgrade to newer dependencies` is enabled. diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg index c974560561053..7db0782695ebd 100644 --- a/dev/breeze/setup.cfg +++ b/dev/breeze/setup.cfg @@ -53,6 +53,7 @@ package_dir= =src packages = find: install_requires = + cached_property>=1.5.0;python_version<="3.7" click inputimeout importlib-metadata>=4.4; python_version < "3.8" diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py new file mode 100644 index 0000000000000..c8260698d71aa --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py @@ -0,0 +1,237 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import sys +from typing import Optional, Tuple + +import click + +from airflow_breeze.commands.main_command import main +from airflow_breeze.global_constants import ( + DEFAULT_PYTHON_MAJOR_MINOR_VERSION, + MOUNT_ALL, + GithubEvents, + github_events, +) +from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.common_options import ( + option_airflow_constraints_reference, + option_answer, + option_dry_run, + option_github_repository, + option_max_age, + option_python, + option_timezone, + option_updated_on_or_after, + option_verbose, +) +from airflow_breeze.utils.confirm import Answer, user_confirm +from airflow_breeze.utils.console import get_console +from airflow_breeze.utils.custom_param_types import BetterChoice +from airflow_breeze.utils.docker_command_utils import ( + check_docker_resources, + get_env_variables_for_docker_commands, + get_extra_docker_flags, + perform_environment_checks, +) +from airflow_breeze.utils.find_newer_dependencies import find_newer_dependencies +from airflow_breeze.utils.image import find_available_ci_image +from airflow_breeze.utils.run_utils import run_command + +CI_COMMANDS = { + "name": "CI commands", + "commands": [ + "fix-ownership", + "free-space", + "resource-check", + "selective-check", + "find-newer-dependencies", + ], +} + +CI_PARAMETERS = { + "breeze selective-check": [ + { + "name": "Selective check flags", + "options": [ + "--commit-ref", + "--pr-labels", + "--default-branch", + "--github-event-name", + ], + } + ], + "breeze find-newer-dependencies": [ + { + "name": "Find newer dependencies flags", + "options": [ + "--python", + "--timezone", + "--constraints-branch", + "--updated-on-or-after", + "--max-age", + ], + } + ], +} + + +@main.command(name="free-space", help="Free space for jobs run in CI.") +@option_verbose +@option_dry_run +@option_answer +def free_space(verbose: bool, dry_run: bool, answer: str): + if user_confirm("Are you sure to run free-space and perform cleanup?") == Answer.YES: + run_command(["sudo", "swapoff", "-a"], verbose=verbose, dry_run=dry_run) + run_command(["sudo", "rm", "-f", "/swapfile"], verbose=verbose, dry_run=dry_run) + run_command(["sudo", "apt-get", "clean"], verbose=verbose, dry_run=dry_run, check=False) + run_command( + ["docker", "system", "prune", "--all", "--force", "--volumes"], verbose=verbose, dry_run=dry_run + ) + run_command(["df", "-h"], verbose=verbose, dry_run=dry_run) + run_command(["docker", "logout", "ghcr.io"], verbose=verbose, dry_run=dry_run, check=False) + + +@main.command(name="resource-check", help="Check if available docker resources are enough.") +@option_verbose +@option_dry_run +def resource_check(verbose: bool, dry_run: bool): + perform_environment_checks(verbose=verbose) + shell_params = ShellParams(verbose=verbose, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION) + check_docker_resources(shell_params.airflow_image_name, verbose=verbose, dry_run=dry_run) + + +@main.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.") +@option_github_repository +@option_verbose +@option_dry_run +def fix_ownership(github_repository: str, verbose: bool, dry_run: bool): + perform_environment_checks(verbose=verbose) + shell_params = find_available_ci_image(github_repository, dry_run, verbose) + extra_docker_flags = get_extra_docker_flags(MOUNT_ALL) + env = get_env_variables_for_docker_commands(shell_params) + cmd = [ + "docker", + "run", + "-t", + *extra_docker_flags, + "--pull", + "never", + shell_params.airflow_image_name_with_tag, + "/opt/airflow/scripts/in_container/run_fix_ownership.sh", + ] + run_command( + cmd, verbose=verbose, dry_run=dry_run, text=True, env=env, check=False, enabled_output_group=True + ) + # Always succeed + sys.exit(0) + + +def get_changed_files(commit_ref: Optional[str], dry_run: bool, verbose: bool) -> Tuple[str, ...]: + if commit_ref is None: + return () + cmd = [ + "git", + "diff-tree", + "--no-commit-id", + "--name-only", + "-r", + commit_ref + "^", + commit_ref, + ] + result = run_command(cmd, dry_run=dry_run, verbose=verbose, check=False, capture_output=True, text=True) + if result.returncode != 0: + get_console().print( + f"[warning] Error when running diff-tree command [/]\n{result.stdout}\n{result.stderr}" + ) + return () + changed_files = tuple(result.stdout.splitlines()) if result.stdout else () + get_console().print("\n[info]Changed files:[/]\n") + get_console().print(changed_files) + get_console().print() + return changed_files + + +@main.command(name="selective-check", help="Checks what kind of tests should be run for an incoming commit.") +@click.option( + '--commit-ref', + help="Commit-ish reference to the commit that should be checked", + envvar='COMMIT_REF', +) +@click.option( + '--pr-labels', + help="Space-separate list of labels which are valid for the PR", + default="", + envvar="PR_LABELS", +) +@click.option( + '--default-branch', + help="Branch against which the PR should be run", + default="main", + envvar="DEFAULT_BRANCH", + show_default=True, +) +@click.option( + '--github-event-name', + type=BetterChoice(github_events()), + default=github_events()[0], + help="Name of the GitHub event that triggered the check", + envvar="GITHUB_EVENT_NAME", + show_default=True, +) +@option_verbose +@option_dry_run +def selective_check( + commit_ref: Optional[str], + pr_labels: str, + default_branch: str, + github_event_name: str, + verbose: bool, + dry_run: bool, +): + from airflow_breeze.utils.selective_checks import SelectiveChecks + + github_event = GithubEvents(github_event_name) + if github_event == GithubEvents.PULL_REQUEST: + changed_files = get_changed_files(commit_ref=commit_ref, dry_run=dry_run, verbose=verbose) + else: + changed_files = () + sc = SelectiveChecks( + commit_ref=commit_ref, + files=changed_files, + default_branch=default_branch, + pr_labels=tuple(" ".split(pr_labels)) if pr_labels else (), + github_event=github_event, + ) + print(str(sc)) + + +@main.command(name="find-newer-dependencies", help="Finds which dependencies are being upgraded.") +@option_timezone +@option_airflow_constraints_reference +@option_python +@option_updated_on_or_after +@option_max_age +def breeze_find_newer_dependencies( + airflow_constraints_reference: str, python: str, timezone: str, updated_on_or_after: str, max_age: int +): + return find_newer_dependencies( + constraints_branch=airflow_constraints_reference, + python=python, + timezone=timezone, + updated_on_or_after=updated_on_or_after, + max_age=max_age, + ) diff --git a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py index c11fa7882d9fc..20741e9986c87 100644 --- a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py @@ -28,8 +28,6 @@ from airflow_breeze import NAME, VERSION from airflow_breeze.commands.main_command import main -from airflow_breeze.global_constants import DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_ALL -from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.cache import check_if_cache_exists, delete_cache, touch_cache_file from airflow_breeze.utils.common_options import ( option_answer, @@ -44,13 +42,7 @@ ) from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm from airflow_breeze.utils.console import get_console -from airflow_breeze.utils.docker_command_utils import ( - check_docker_resources, - get_env_variables_for_docker_commands, - get_extra_docker_flags, - perform_environment_checks, -) -from airflow_breeze.utils.image import find_available_ci_image +from airflow_breeze.utils.docker_command_utils import perform_environment_checks from airflow_breeze.utils.path_utils import ( AIRFLOW_SOURCES_ROOT, BUILD_CACHE_DIR, @@ -72,9 +64,6 @@ "self-upgrade", "setup-autocomplete", "config", - "resource-check", - "free-space", - "fix-ownership", "regenerate-command-images", "command-hash-export", "version", @@ -394,31 +383,6 @@ def get_status(file: str): get_console().print() -@main.command(name="free-space", help="Free space for jobs run in CI.") -@option_verbose -@option_dry_run -@option_answer -def free_space(verbose: bool, dry_run: bool, answer: str): - if user_confirm("Are you sure to run free-space and perform cleanup?") == Answer.YES: - run_command(["sudo", "swapoff", "-a"], verbose=verbose, dry_run=dry_run) - run_command(["sudo", "rm", "-f", "/swapfile"], verbose=verbose, dry_run=dry_run) - run_command(["sudo", "apt-get", "clean"], verbose=verbose, dry_run=dry_run, check=False) - run_command( - ["docker", "system", "prune", "--all", "--force", "--volumes"], verbose=verbose, dry_run=dry_run - ) - run_command(["df", "-h"], verbose=verbose, dry_run=dry_run) - run_command(["docker", "logout", "ghcr.io"], verbose=verbose, dry_run=dry_run, check=False) - - -@main.command(name="resource-check", help="Check if available docker resources are enough.") -@option_verbose -@option_dry_run -def resource_check(verbose: bool, dry_run: bool): - perform_environment_checks(verbose=verbose) - shell_params = ShellParams(verbose=verbose, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION) - check_docker_resources(shell_params.airflow_image_name, verbose=verbose, dry_run=dry_run) - - def dict_hash(dictionary: Dict[str, Any]) -> str: """MD5 hash of a dictionary. Sorted and dumped via json to account for random sequence)""" dhash = hashlib.md5() @@ -442,32 +406,6 @@ def command_hash_export(verbose: bool, output: IO): output.write(dict_hash(the_context_dict) + "\n") -@main.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.") -@option_github_repository -@option_verbose -@option_dry_run -def fix_ownership(github_repository: str, verbose: bool, dry_run: bool): - perform_environment_checks(verbose=verbose) - shell_params = find_available_ci_image(github_repository, dry_run, verbose) - extra_docker_flags = get_extra_docker_flags(MOUNT_ALL) - env = get_env_variables_for_docker_commands(shell_params) - cmd = [ - "docker", - "run", - "-t", - *extra_docker_flags, - "--pull", - "never", - shell_params.airflow_image_name_with_tag, - "/opt/airflow/scripts/in_container/run_fix_ownership.sh", - ] - run_command( - cmd, verbose=verbose, dry_run=dry_run, text=True, env=env, check=False, enabled_output_group=True - ) - # Always succeed - sys.exit(0) - - def write_to_shell(command_to_execute: str, dry_run: bool, script_path: str, force_setup: bool) -> bool: skip_check = False script_path_file = Path(script_path) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index caf43d47a409b..17bf70b3af9d1 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -45,14 +45,11 @@ option_github_repository, option_image_tag, option_installation_package_format, - option_max_age, option_package_format, option_parallelism, option_python, option_python_versions, option_run_in_parallel, - option_timezone, - option_updated_on_or_after, option_use_airflow_version, option_use_packages_from_dist, option_verbose, @@ -66,7 +63,6 @@ get_extra_docker_flags, perform_environment_checks, ) -from airflow_breeze.utils.find_newer_dependencies import find_newer_dependencies from airflow_breeze.utils.parallel import check_async_run_results from airflow_breeze.utils.python_versions import get_python_version_list from airflow_breeze.utils.run_utils import RunCommandResult, run_command @@ -140,18 +136,6 @@ ], } ], - "breeze find-newer-dependencies": [ - { - "name": "Find newer dependencies flags", - "options": [ - "--python", - "--timezone", - "--constraints-branch", - "--updated-on-or-after", - "--max-age", - ], - } - ], } RELEASE_MANAGEMENT_COMMANDS = { @@ -163,7 +147,6 @@ "prepare-airflow-package", "release-prod-images", "generate-constraints", - "find-newer-dependencies", ], } @@ -767,21 +750,3 @@ def release_prod_images( verbose=verbose, dry_run=dry_run, ) - - -@main.command(name="find-newer-dependencies", help="Finds which dependencies are being upgraded.") -@option_timezone -@option_airflow_constraints_reference -@option_python -@option_updated_on_or_after -@option_max_age -def breeze_find_newer_dependencies( - airflow_constraints_reference: str, python: str, timezone: str, updated_on_or_after: str, max_age: int -): - return find_newer_dependencies( - constraints_branch=airflow_constraints_reference, - python=python, - timezone=timezone, - updated_on_or_after=updated_on_or_after, - max_age=max_age, - ) diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 3946acd2ab3a9..45df368afa89c 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -28,7 +28,7 @@ import click from airflow_breeze.commands.main_command import main -from airflow_breeze.global_constants import ALLOWED_TEST_TYPES +from airflow_breeze.global_constants import ALLOWED_TEST_TYPE_CHOICES from airflow_breeze.params.build_prod_params import BuildProdParams from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.ci_group import ci_group @@ -249,7 +249,7 @@ def run_with_progress( "--test-type", help="Type of test to run.", default="All", - type=BetterChoice(ALLOWED_TEST_TYPES), + type=BetterChoice(ALLOWED_TEST_TYPE_CHOICES), ) @option_db_reset @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) diff --git a/dev/breeze/src/airflow_breeze/configure_rich_click.py b/dev/breeze/src/airflow_breeze/configure_rich_click.py index bc1963684a8a4..8933d57dbeea3 100644 --- a/dev/breeze/src/airflow_breeze/configure_rich_click.py +++ b/dev/breeze/src/airflow_breeze/configure_rich_click.py @@ -14,14 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - - from airflow_breeze.utils import recording # isort:skip # noqa try: # We handle ImportError so that click autocomplete works import rich_click as click + from airflow_breeze.commands.ci_commands import CI_COMMANDS, CI_PARAMETERS from airflow_breeze.commands.ci_image_commands import CI_IMAGE_TOOLS_COMMANDS, CI_IMAGE_TOOLS_PARAMETERS from airflow_breeze.commands.configuration_and_maintenance_commands import ( CONFIGURATION_AND_MAINTENANCE_COMMANDS, @@ -52,6 +51,7 @@ **CONFIGURATION_AND_MAINTENANCE_PARAMETERS, **CI_IMAGE_TOOLS_PARAMETERS, **PRODUCTION_IMAGE_TOOLS_PARAMETERS, + **CI_PARAMETERS, **RELEASE_MANAGEMENT_PARAMETERS, } click.rich_click.COMMAND_GROUPS = { @@ -61,6 +61,7 @@ CONFIGURATION_AND_MAINTENANCE_COMMANDS, CI_IMAGE_TOOLS_COMMANDS, PRODUCTION_IMAGE_TOOLS_COMMANDS, + CI_COMMANDS, RELEASE_MANAGEMENT_COMMANDS, ] } diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 7eeaf1c8605d6..f6727d7a0a8b1 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -17,8 +17,11 @@ """ Global constants that are used by all other Breeze components. """ +from __future__ import annotations + import platform -from typing import List +from enum import Enum +from functools import lru_cache from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT @@ -67,21 +70,36 @@ ALLOWED_POSTGRES_VERSIONS = ['10', '11', '12', '13', '14'] ALLOWED_MYSQL_VERSIONS = ['5.7', '8'] ALLOWED_MSSQL_VERSIONS = ['2017-latest', '2019-latest'] -ALLOWED_TEST_TYPES = [ - 'All', - 'Always', - 'Core', - 'Providers', - 'API', - 'CLI', - 'Integration', - 'Other', - 'WWW', - 'Postgres', - 'MySQL', - 'Helm', - 'Quarantined', + + +@lru_cache(maxsize=None) +def all_selective_test_types() -> tuple[str, ...]: + return tuple(sorted(e.value for e in SelectiveUnitTestTypes)) + + +class SelectiveUnitTestTypes(Enum): + ALWAYS = 'Always' + API = 'API' + CLI = 'CLI' + CORE = 'Core' + OTHER = 'Other' + INTEGRATION = 'Integration' + PROVIDERS = 'Providers' + WWW = 'WWW' + + +ALLOWED_TEST_TYPE_CHOICES = [ + "All", + "Always", + *all_selective_test_types(), + "Helm", + "Postgres", + "MySQL", + "Integration", + "Other", + "Quarantine", ] + ALLOWED_PACKAGE_FORMATS = ['wheel', 'sdist', 'both'] ALLOWED_INSTALLATION_PACKAGE_FORMATS = ['wheel', 'sdist'] ALLOWED_INSTALLATION_METHODS = ['.', 'apache-airflow'] @@ -114,7 +132,7 @@ ] -def get_available_packages(short_version=False) -> List[str]: +def get_available_packages(short_version=False) -> list[str]: docs_path_content = (AIRFLOW_SOURCES_ROOT / 'docs').glob('*/') available_packages = [x.name for x in docs_path_content if x.is_dir()] package_list = list(set(available_packages) - set(EXCLUDE_DOCS_PACKAGE_FOLDER)) @@ -153,8 +171,12 @@ def get_default_platform_machine() -> str: ALL_PYTHON_MAJOR_MINOR_VERSIONS = ['3.7', '3.8', '3.9', '3.10'] CURRENT_PYTHON_MAJOR_MINOR_VERSIONS = ['3.7', '3.8', '3.9', '3.10'] CURRENT_POSTGRES_VERSIONS = ['10', '11', '12', '13', '14'] +DEFAULT_POSTGRES_VERSION = CURRENT_POSTGRES_VERSIONS[0] CURRENT_MYSQL_VERSIONS = ['5.7', '8'] +DEFAULT_MYSQL_VERSION = CURRENT_MYSQL_VERSIONS[0] CURRENT_MSSQL_VERSIONS = ['2017-latest', '2019-latest'] +DEFAULT_MSSQL_VERSION = CURRENT_MSSQL_VERSIONS[0] + DB_RESET = False START_AIRFLOW = "false" LOAD_EXAMPLES = False @@ -225,10 +247,10 @@ def get_airflow_extras(): CURRENT_HELM_VERSIONS = ['v3.6.3'] CURRENT_EXECUTORS = ['KubernetesExecutor'] -DEFAULT_KUBERNETES_MODES = CURRENT_KUBERNETES_MODES[0] -DEFAULT_KUBERNETES_VERSIONS = CURRENT_KUBERNETES_VERSIONS[0] -DEFAULT_KIND_VERSIONS = CURRENT_KIND_VERSIONS[0] -DEFAULT_HELM_VERSIONS = CURRENT_HELM_VERSIONS[0] +DEFAULT_KUBERNETES_MODE = CURRENT_KUBERNETES_MODES[0] +DEFAULT_KUBERNETES_VERSION = CURRENT_KUBERNETES_VERSIONS[0] +DEFAULT_KIND_VERSION = CURRENT_KIND_VERSIONS[0] +DEFAULT_HELM_VERSION = CURRENT_HELM_VERSIONS[0] DEFAULT_EXECUTOR = CURRENT_EXECUTORS[0] # Initialize image build variables - Have to check if this has to go to ci dataclass @@ -276,3 +298,16 @@ def get_airflow_extras(): "virtualenv", # END OF EXTRAS LIST UPDATED BY PRE COMMIT ] + + +class GithubEvents(Enum): + PULL_REQUEST = "pull_request" + PULL_REQUEST_REVIEW = "pull_request_review" + PULL_REQUEST_TARGET = "pull_request_target" + PUSH = "push" + SCHEDULE = "schedule" + + +@lru_cache(maxsize=None) +def github_events() -> list[str]: + return [e.value for e in GithubEvents] diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py new file mode 100644 index 0000000000000..22e74f455301f --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -0,0 +1,480 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import sys +from enum import Enum + +if sys.version_info >= (3, 8): + from functools import cached_property +else: + from cached_property import cached_property + +from functools import lru_cache +from re import match +from typing import Any, Dict, List, TypeVar + +from airflow_breeze.global_constants import ( + ALL_PYTHON_MAJOR_MINOR_VERSIONS, + CURRENT_HELM_VERSIONS, + CURRENT_KIND_VERSIONS, + CURRENT_KUBERNETES_MODES, + CURRENT_KUBERNETES_VERSIONS, + CURRENT_MSSQL_VERSIONS, + CURRENT_MYSQL_VERSIONS, + CURRENT_POSTGRES_VERSIONS, + CURRENT_PYTHON_MAJOR_MINOR_VERSIONS, + DEFAULT_HELM_VERSION, + DEFAULT_KIND_VERSION, + DEFAULT_KUBERNETES_MODE, + DEFAULT_KUBERNETES_VERSION, + DEFAULT_MSSQL_VERSION, + DEFAULT_MYSQL_VERSION, + DEFAULT_POSTGRES_VERSION, + DEFAULT_PYTHON_MAJOR_MINOR_VERSION, + GithubEvents, + SelectiveUnitTestTypes, + all_selective_test_types, +) +from airflow_breeze.utils.console import get_console + +FULL_TESTS_NEEDED_LABEL = "full tests needed" + + +def get_ga_output(name: str, value: Any) -> str: + output_name = name.replace('_', '-') + printed_value = str(value).lower() if isinstance(value, bool) else value + get_console().print(f"[info]{output_name}[/] = [green]{printed_value}[/]") + return f"::set-output name={output_name}::{printed_value}" + + +class FileGroupForCi(Enum): + ENVIRONMENT_FILES = "environment_files" + PYTHON_PRODUCTION_FILES = "python_scans" + JAVASCRIPT_PRODUCTION_FILES = "javascript_scans" + API_TEST_FILES = "api_test_files" + API_CODEGEN_FILES = "api_codegen_files" + HELM_FILES = "helm_files" + SETUP_FILES = "setup_files" + DOC_FILES = "doc_files" + UI_FILES = "ui_files" + WWW_FILES = "www_files" + KUBERNETES_FILES = "kubernetes_files" + ALL_PYTHON_FILES = "all_python_files" + ALL_SOURCE_FILES = "all_sources_for_tests" + + +T = TypeVar('T', FileGroupForCi, SelectiveUnitTestTypes) + + +class HashableDict(Dict[T, List[str]]): + def __hash__(self): + return hash(frozenset(self)) + + +CI_FILE_GROUP_MATCHES = HashableDict( + { + FileGroupForCi.ENVIRONMENT_FILES: [ + r"^.github/workflows", + r"^dev/breeze", + r"^Dockerfile", + r"^scripts", + r"^setup.py", + r"^setup.cfg", + ], + FileGroupForCi.PYTHON_PRODUCTION_FILES: [ + r"^airflow/.*\.py", + r"^setup.py", + ], + FileGroupForCi.JAVASCRIPT_PRODUCTION_FILES: [ + r"^airflow/.*\.[jt]sx?", + r"^airflow/.*\.lock", + ], + FileGroupForCi.API_TEST_FILES: [ + r"^airflow/api", + ], + FileGroupForCi.API_CODEGEN_FILES: [ + "^airflow/api_connexion/openapi/v1.yaml", + "^clients/gen", + ], + FileGroupForCi.HELM_FILES: [ + "^chart", + ], + FileGroupForCi.SETUP_FILES: [ + r"^pyproject.toml", + r"^setup.cfg", + r"^setup.py", + ], + FileGroupForCi.DOC_FILES: [ + r"^docs", + r"^airflow/.*\.py$", + r"^chart", + r"^providers", + r"^CHANGELOG\.txt", + r"^airflow/config_templates/config\.yml", + r"^chart/RELEASE_NOTES\.txt", + r"^chart/values\.schema\.json", + r"^chart/values\.json", + ], + FileGroupForCi.UI_FILES: [ + r"^airflow/ui/.*\.[tj]sx?$", + r"^airflow/ui/[^/]+\.json$", + r"^airflow/ui/.*\.lock$", + ], + FileGroupForCi.WWW_FILES: [ + r"^airflow/www/.*\.js[x]?$", + r"^airflow/www/[^/]+\.json$", + r"^airflow/www/.*\.lock$", + ], + FileGroupForCi.KUBERNETES_FILES: [ + r"^chart", + r"^kubernetes_tests", + r"^airflow/providers/cncf/kubernetes/", + r"^tests/providers/cncf/kubernetes/", + ], + FileGroupForCi.ALL_PYTHON_FILES: [ + r"\.py$", + ], + FileGroupForCi.ALL_SOURCE_FILES: [ + "^.pre-commit-config.yaml$", + "^airflow", + "^chart", + "^tests", + "^kubernetes_tests", + ], + } +) + + +TEST_TYPE_MATCHES = HashableDict( + { + SelectiveUnitTestTypes.API: [ + r"^airflow/api", + r"^airflow/api_connexion", + r"^tests/api", + r"^tests/api_connexion", + ], + SelectiveUnitTestTypes.CLI: [ + r"^airflow/cli", + r"^tests/cli", + ], + SelectiveUnitTestTypes.PROVIDERS: [ + "^airflow/providers/", + "^tests/providers/", + ], + SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www", "^airflow/ui"], + } +) + + +class SelectiveChecks: + __HASHABLE_FIELDS = {'_files', '_default_branch', '_commit_ref', "_pr_labels", "_github_event"} + + def __init__( + self, + files: tuple[str, ...] = (), + default_branch="main", + commit_ref: str | None = None, + pr_labels: tuple[str, ...] = (), + github_event: GithubEvents = GithubEvents.PULL_REQUEST, + ): + self._files = files + self._default_branch = default_branch + self._commit_ref = commit_ref + self._pr_labels = pr_labels + self._github_event = github_event + + def __important_attributes(self) -> tuple[Any, ...]: + return tuple(getattr(self, f) for f in self.__HASHABLE_FIELDS) + + def __hash__(self): + return hash(self.__important_attributes()) + + def __eq__(self, other): + return isinstance(other, SelectiveChecks) and all( + [getattr(other, f) == getattr(self, f) for f in self.__HASHABLE_FIELDS] + ) + + def __str__(self) -> str: + output = [] + for field_name in dir(self): + if not field_name.startswith('_'): + output.append(get_ga_output(field_name, getattr(self, field_name))) + return "\n".join(output) + + default_python_version = DEFAULT_PYTHON_MAJOR_MINOR_VERSION + default_postgres_version = DEFAULT_POSTGRES_VERSION + default_mysql_version = DEFAULT_MYSQL_VERSION + default_mssql_version = DEFAULT_MSSQL_VERSION + + default_kubernetes_version = DEFAULT_KUBERNETES_VERSION + default_kind_version = DEFAULT_KIND_VERSION + default_helm_version = DEFAULT_HELM_VERSION + + @cached_property + def default_branch(self) -> str: + return self._default_branch + + @cached_property + def _full_tests_needed(self) -> bool: + if self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE]: + get_console().print(f"[warning]Full tests needed because event is {self._github_event}[/]") + return True + if FULL_TESTS_NEEDED_LABEL in self._pr_labels: + get_console().print(f"[warning]Full tests needed because labels are {self._pr_labels}[/]") + return True + return False + + @cached_property + def python_versions(self) -> list[str]: + return ( + CURRENT_PYTHON_MAJOR_MINOR_VERSIONS + if self._full_tests_needed + else [DEFAULT_PYTHON_MAJOR_MINOR_VERSION] + ) + + @cached_property + def python_versions_list_as_string(self) -> str: + return " ".join(self.python_versions) + + @cached_property + def all_python_versions(self) -> list[str]: + return ( + ALL_PYTHON_MAJOR_MINOR_VERSIONS + if self._run_everything or self._full_tests_needed + else [DEFAULT_PYTHON_MAJOR_MINOR_VERSION] + ) + + @cached_property + def all_python_versions_list_as_string(self) -> str: + return " ".join(self.all_python_versions) + + @cached_property + def kubernetes_modes(self): + return CURRENT_KUBERNETES_MODES if self._full_tests_needed else [DEFAULT_KUBERNETES_MODE] + + @cached_property + def postgres_versions(self) -> list[str]: + return CURRENT_POSTGRES_VERSIONS if self._full_tests_needed else [DEFAULT_POSTGRES_VERSION] + + @cached_property + def mysql_versions(self) -> list[str]: + return CURRENT_MYSQL_VERSIONS if self._full_tests_needed else [DEFAULT_MYSQL_VERSION] + + @cached_property + def mssql_versions(self) -> list[str]: + return CURRENT_MSSQL_VERSIONS if self._full_tests_needed else [DEFAULT_MSSQL_VERSION] + + @cached_property + def kind_versions(self) -> list[str]: + return CURRENT_KIND_VERSIONS + + @cached_property + def helm_versions(self) -> list[str]: + return CURRENT_HELM_VERSIONS + + @cached_property + def postgres_exclude(self) -> list[dict[str, str]]: + return [{"python-version": "3.7"}] if self._full_tests_needed else [] + + @cached_property + def mssql_exclude(self) -> list[dict[str, str]]: + return [{"python-version": "3.8"}] if self._full_tests_needed else [] + + @cached_property + def mysql_exclude(self) -> list[dict[str, str]]: + return [{"python-version": "3.10"}] if self._full_tests_needed else [] + + @cached_property + def sqlite_exclude(self) -> list[dict[str, str]]: + return [{"python-version": "3.9"}] if self._full_tests_needed else [] + + @cached_property + def kubernetes_versions(self) -> list[str]: + return CURRENT_KUBERNETES_VERSIONS if self._full_tests_needed else [DEFAULT_KUBERNETES_VERSION] + + @cached_property + def kubernetes_versions_list_as_string(self) -> str: + return " ".join(self.kubernetes_versions) + + def _match_files_with_regexps(self, matched_files, regexps): + for file in self._files: + for regexp in regexps: + if match(regexp, file): + matched_files.append(file) + break + + @lru_cache(maxsize=None) + def _matching_files(self, match_group: T, match_dict: dict[T, list[str]]) -> list[str]: + matched_files: list[str] = [] + regexps = match_dict[match_group] + self._match_files_with_regexps(matched_files, regexps) + count = len(matched_files) + if count > 0: + get_console().print(f"[warning]{match_group} matched {count} files.[/]") + get_console().print(matched_files) + else: + get_console().print(f"[warning]{match_group} did not match any file.[/]") + return matched_files + + @cached_property + def _run_everything(self) -> bool: + if not self._commit_ref: + get_console().print("[warning]Running everything as commit is missing[/]") + return True + if self._full_tests_needed: + get_console().print("[warning]Running everything as full tests are needed[/]") + return True + if len(self._matching_files(FileGroupForCi.ENVIRONMENT_FILES, CI_FILE_GROUP_MATCHES)) > 0: + get_console().print("[warning]Running everything because env files changed[/]") + return True + return False + + def _should_be_run(self, source_area: FileGroupForCi) -> bool: + if self._run_everything: + get_console().print(f"[warning]{source_area} enabled because we are running everything[/]") + return True + matched_files = self._matching_files(source_area, CI_FILE_GROUP_MATCHES) + if len(matched_files) > 0: + get_console().print( + f"[warning]{source_area} enabled because it matched {len(matched_files)} changed files[/]" + ) + return True + else: + get_console().print( + f"[warning]{source_area} disabled because it did not match any changed files[/]" + ) + return False + + @cached_property + def needs_python_scans(self) -> bool: + return self._should_be_run(FileGroupForCi.PYTHON_PRODUCTION_FILES) + + @cached_property + def needs_javascript_scans(self) -> bool: + return self._should_be_run(FileGroupForCi.JAVASCRIPT_PRODUCTION_FILES) + + @cached_property + def needs_api_tests(self) -> bool: + return self._should_be_run(FileGroupForCi.API_TEST_FILES) + + @cached_property + def needs_api_codegen(self) -> bool: + return self._should_be_run(FileGroupForCi.API_CODEGEN_FILES) + + @cached_property + def run_ui_tests(self) -> bool: + return self._should_be_run(FileGroupForCi.UI_FILES) + + @cached_property + def run_www_tests(self) -> bool: + return self._should_be_run(FileGroupForCi.WWW_FILES) + + @cached_property + def run_kubernetes_tests(self) -> bool: + return self._should_be_run(FileGroupForCi.KUBERNETES_FILES) + + @cached_property + def docs_build(self) -> bool: + return self._should_be_run(FileGroupForCi.DOC_FILES) + + @cached_property + def needs_helm_tests(self) -> bool: + return self._should_be_run(FileGroupForCi.HELM_FILES) and self._default_branch == "main" + + @cached_property + def run_tests(self) -> bool: + return self._should_be_run(FileGroupForCi.ALL_SOURCE_FILES) + + @cached_property + def image_build(self) -> bool: + return self.run_tests or self.docs_build or self.run_kubernetes_tests + + def _select_test_type_if_matching( + self, test_types: set[str], test_type: SelectiveUnitTestTypes + ) -> list[str]: + matched_files = self._matching_files(test_type, TEST_TYPE_MATCHES) + count = len(matched_files) + if count > 0: + test_types.add(test_type.value) + get_console().print(f"[warning]{test_type} added because it matched {count} files[/]") + return matched_files + + def _get_test_types_to_run(self) -> list[str]: + candidate_test_types: set[str] = {"Always"} + matched_files: set[str] = set() + matched_files.update( + self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.WWW) + ) + matched_files.update( + self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.PROVIDERS) + ) + matched_files.update( + self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.CLI) + ) + matched_files.update( + self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.API) + ) + + kubernetes_files = self._matching_files(FileGroupForCi.KUBERNETES_FILES, CI_FILE_GROUP_MATCHES) + all_source_files = self._matching_files(FileGroupForCi.ALL_SOURCE_FILES, CI_FILE_GROUP_MATCHES) + + remaining_files = set(all_source_files) - set(matched_files) - set(kubernetes_files) + count_remaining_files = len(remaining_files) + if count_remaining_files > 0: + get_console().print( + f"[warning]We should run all tests. There are {count_remaining_files} changed " + "files that seems to fall into Core/Other category[/]" + ) + get_console().print(remaining_files) + candidate_test_types.update(all_selective_test_types()) + else: + get_console().print( + "[warning]There are no core/other files. Only tests relevant to the changed files are run.[/]" + ) + sorted_candidate_test_types = list(sorted(candidate_test_types)) + get_console().print("[warning]Selected test type candidates to run:[/]") + get_console().print(sorted_candidate_test_types) + return sorted_candidate_test_types + + @cached_property + def test_types(self) -> str: + if not self.run_tests: + return "" + if self._run_everything: + current_test_types = list(all_selective_test_types()) + else: + current_test_types = self._get_test_types_to_run() + if self._default_branch != "main": + if "Providers" in current_test_types: + get_console().print( + "[warning]Removing 'Providers' because the target branch " + f"is {self._default_branch} and not main[/]" + ) + current_test_types.remove("Providers") + return " ".join(sorted(current_test_types)) + + @cached_property + def basic_checks_only(self) -> bool: + return not self.image_build + + @cached_property + def upgrade_to_newer_dependencies(self) -> bool: + return len( + self._matching_files(FileGroupForCi.SETUP_FILES, CI_FILE_GROUP_MATCHES) + ) > 0 or self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE] diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py new file mode 100644 index 0000000000000..492135ebd38b0 --- /dev/null +++ b/dev/breeze/tests/test_selective_checks.py @@ -0,0 +1,464 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Dict, Tuple + +import pytest + +from airflow_breeze.global_constants import GithubEvents +from airflow_breeze.utils.selective_checks import SelectiveChecks + + +def assert_outputs_are_printed(expected_outputs: Dict[str, str], output: str): + for name, value in expected_outputs.items(): + assert f"::set-output name={name}::{value}" in output + + +@pytest.mark.parametrize( + "files, expected_outputs,", + [ + ( + pytest.param( + ("INTHEWILD.md",), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "false", + "needs-helm-tests": "false", + "run-tests": "false", + "docs-build": "false", + "upgrade-to-newer-dependencies": "false", + "test-types": "", + }, + id="No tests on simple change", + ) + ), + ( + pytest.param( + ("airflow/api/file.py",), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "docs-build": "true", + "upgrade-to-newer-dependencies": "false", + "test-types": "API Always", + }, + id="Only API tests and DOCS should run", + ) + ), + ( + pytest.param( + ( + "airflow/api/file.py", + "tests/providers/google/file.py", + ), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "docs-build": "true", + "upgrade-to-newer-dependencies": "false", + "test-types": "API Always Providers", + }, + id="API and providers tests and docs should run", + ) + ), + ( + pytest.param( + ("tests/providers/google/file.py",), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "docs-build": "false", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "test-types": "Always Providers", + }, + id="Providers and docs should run", + ) + ), + ( + pytest.param( + ("docs/file.rst",), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "false", + "docs-build": "true", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "test-types": "", + }, + id="Only docs builds should run - no tests needed", + ) + ), + ( + pytest.param( + ( + "chart/aaaa.txt", + "tests/providers/google/file.py", + ), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "true", + "run-tests": "true", + "docs-build": "true", + "run-kubernetes-tests": "true", + "upgrade-to-newer-dependencies": "false", + "test-types": "Always Providers", + }, + id="Helm tests, providers, kubernetes tests and docs should run", + ) + ), + ( + pytest.param( + ( + "INTHEWILD.md", + "chart/aaaa.txt", + "tests/providers/google/file.py", + ), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "true", + "run-tests": "true", + "docs-build": "true", + "run-kubernetes-tests": "true", + "upgrade-to-newer-dependencies": "false", + "test-types": "Always Providers", + }, + id="Helm tests, providers, kubernetes tests and docs should run even if " + "unimportant files were added", + ) + ), + ( + pytest.param( + ("setup.py",), + { + "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "image-build": "true", + "needs-helm-tests": "true", + "run-tests": "true", + "docs-build": "true", + "upgrade-to-newer-dependencies": "true", + "test-types": "API Always CLI Core Integration Other Providers WWW", + }, + id="Everything should run and upgrading to newer requirements as setup.py changed", + ) + ), + ], +) +def test_expected_output_pull_request_main( + files: Tuple[str, ...], + expected_outputs: Dict[str, str], +): + sc = SelectiveChecks( + files=files, + commit_ref="HEAD", + github_event=GithubEvents.PULL_REQUEST, + pr_labels=(), + default_branch="main", + ) + assert_outputs_are_printed(expected_outputs, str(sc)) + + +@pytest.mark.parametrize( + "files, pr_labels, default_branch, expected_outputs,", + [ + ( + pytest.param( + ("INTHEWILD.md",), + ("full tests needed",), + "main", + { + "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "image-build": "true", + "run-tests": "true", + "docs-build": "true", + "upgrade-to-newer-dependencies": "false", + "test-types": "API Always CLI Core Integration Other Providers WWW", + }, + id="Everything should run when full tests are needed", + ) + ), + ( + pytest.param( + ("INTHEWILD.md",), + ( + "another label", + "full tests needed", + ), + "main", + { + "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "image-build": "true", + "run-tests": "true", + "docs-build": "true", + "upgrade-to-newer-dependencies": "false", + "test-types": "API Always CLI Core Integration Other Providers WWW", + }, + id="Everything should run when full tests are needed even with different label set as well", + ) + ), + ( + pytest.param( + (), + ("full tests needed",), + "main", + { + "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "image-build": "true", + "run-tests": "true", + "docs-build": "true", + "upgrade-to-newer-dependencies": "false", + "test-types": "API Always CLI Core Integration Other Providers WWW", + }, + id="Everything should run when full tests are needed even if no files are changed", + ) + ), + ( + pytest.param( + ("INTHEWILD.md",), + ("full tests needed",), + "v2-3-stable", + { + "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "image-build": "true", + "run-tests": "true", + "docs-build": "true", + "upgrade-to-newer-dependencies": "false", + "test-types": "API Always CLI Core Integration Other WWW", + }, + id="Everything should run except Providers when full tests are needed for non-main branch", + ) + ), + ], +) +def test_expected_output_full_tests_needed( + files: Tuple[str, ...], + pr_labels: Tuple[str, ...], + default_branch: str, + expected_outputs: Dict[str, str], +): + sc = SelectiveChecks( + files=files, + commit_ref="HEAD", + github_event=GithubEvents.PULL_REQUEST, + pr_labels=pr_labels, + default_branch=default_branch, + ) + output = str(sc) + assert_outputs_are_printed(expected_outputs, output) + + +@pytest.mark.parametrize( + "files, expected_outputs,", + [ + pytest.param( + ("INTHEWILD.md",), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "false", + "needs-helm-tests": "false", + "run-tests": "false", + "docs-build": "false", + "upgrade-to-newer-dependencies": "false", + "test-types": "", + }, + id="Everything should run when full tests are needed even if no files are changed", + ), + pytest.param( + ( + "chart/aaaa.txt", + "tests/providers/google/file.py", + ), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "needs-helm-tests": "false", + "image-build": "true", + "run-tests": "true", + "docs-build": "true", + "run-kubernetes-tests": "true", + "upgrade-to-newer-dependencies": "false", + "test-types": "Always", + }, + id="No Helm tests, No providers should run if only chart/providers changed in non-main", + ), + pytest.param( + ( + "airflow/cli/test.py", + "chart/aaaa.txt", + "tests/providers/google/file.py", + ), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "docs-build": "true", + "run-kubernetes-tests": "true", + "upgrade-to-newer-dependencies": "false", + "test-types": "Always CLI", + }, + id="Only CLI tests and Kubernetes tests should run if cli/chart files changed in non-main branch", + ), + pytest.param( + ( + "airflow/file.py", + "tests/providers/google/file.py", + ), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "docs-build": "true", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "test-types": "API Always CLI Core Integration Other WWW", + }, + id="All tests except providers should run if core file changed in non-main branch", + ), + ], +) +def test_expected_output_pull_request_v2_3( + files: Tuple[str, ...], + expected_outputs: Dict[str, str], +): + sc = SelectiveChecks( + files=files, + commit_ref="HEAD", + github_event=GithubEvents.PULL_REQUEST, + pr_labels=(), + default_branch="v2-3-stable", + ) + assert_outputs_are_printed(expected_outputs, str(sc)) + + +@pytest.mark.parametrize( + "files, pr_labels, default_branch, expected_outputs,", + [ + pytest.param( + ("INTHEWILD.md",), + (), + "main", + { + "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "image-build": "true", + "needs-helm-tests": "true", + "run-tests": "true", + "docs-build": "true", + "upgrade-to-newer-dependencies": "true", + "test-types": "API Always CLI Core Integration Other Providers WWW", + }, + id="All tests run on push even if unimportant file changed", + ), + pytest.param( + ("INTHEWILD.md",), + (), + "v2-3-stable", + { + "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "docs-build": "true", + "upgrade-to-newer-dependencies": "true", + "test-types": "API Always CLI Core Integration Other WWW", + }, + id="All tests except Providers and Helm run on push" + " even if unimportant file changed in non-main branch", + ), + pytest.param( + ("airflow/api.py",), + (), + "main", + { + "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "image-build": "true", + "needs-helm-tests": "true", + "run-tests": "true", + "docs-build": "true", + "upgrade-to-newer-dependencies": "true", + "test-types": "API Always CLI Core Integration Other Providers WWW", + }, + id="All tests run on push if core file changed", + ), + ], +) +def test_expected_output_push( + files: Tuple[str, ...], + pr_labels: Tuple[str, ...], + default_branch: str, + expected_outputs: Dict[str, str], +): + sc = SelectiveChecks( + files=files, + commit_ref="HEAD", + github_event=GithubEvents.PUSH, + pr_labels=pr_labels, + default_branch=default_branch, + ) + assert_outputs_are_printed(expected_outputs, str(sc)) + + +def test_no_commit_provided(): + sc = SelectiveChecks( + files=(), + commit_ref="", + github_event=GithubEvents.PULL_REQUEST, + pr_labels=(), + default_branch="main", + ) + assert_outputs_are_printed( + { + "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "image-build": "true", + "needs-helm-tests": "true", + "run-tests": "true", + "docs-build": "true", + "upgrade-to-newer-dependencies": "false", + "test-types": "API Always CLI Core Integration Other Providers WWW", + }, + str(sc), + ) diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 0350c181a1dc3..52b984dfefadc 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -2,4 +2,4 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -2e5e83c884f0fe51f91f573fcbff12ba +088f01c27036e2230099add63afb5f6e diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index e9dd89b22e4dc..2e21ac9804afa 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + - Breeze commands + Breeze commands - + - - -Usage: breeze [OPTIONS] COMMAND [ARGS]... - -╭─ Basic flags for the default (shell) command ────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flags for the default (shell) command ─────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ -│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ -│available in dist folder respectively. Implies --mount-sources `remove`.                │ -│(none | wheel | sdist | <airflow_version>)                                              │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when   │ -│entering breeze.                                                                        │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)[default: wheel]│ -│--force-buildForce image build no matter if it is determined as needed.│ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =    │ -│selected).                                                                              │ -│(selected | all | skip | remove)                                                        │ -│[default: selected]                                                                     │ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Developer tools ────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│shell          Enter breeze.py environment. this is the default command use when no other is selected.            │ -│start-airflow  Enter breeze.py environment and starts all Airflow components in the tmux session.                 │ -│exec           Joins the interactive shell of running airflow container                                           │ -│stop           Stop running breeze environment.                                                                   │ -│build-docs     Build documentation in the container.                                                              │ -│static-checks  Run static checks.                                                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│docker-compose-tests Run docker-compose tests.                                                                    │ -│tests                Run the specified unit test targets. Multiple targets may be specified separated by spaces.  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Configuration & maintenance ────────────────────────────────────────────────────────────────────────────────────────╮ -│cleanup                  Cleans the cache of parameters, docker cache and optionally - currently downloaded       │ -│images.                                                                                  │ -│self-upgrade             Self upgrade Breeze.                                                                     │ -│setup-autocomplete       Enables autocompletion of breeze commands.                                               │ -│config                   Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                       │ -│resource-check           Check if available docker resources are enough.                                          │ -│free-space               Free space for jobs run in CI.                                                           │ -│fix-ownership            Fix ownership of source files to be same as host user.                                   │ -│regenerate-command-imagesRegenerate breeze command images.                                                        │ -│command-hash-export      Outputs hash of all click commands to file or stdout if `-` is used (useful to see if    │ -│images should be regenerated).                                                           │ -│version                  Print information about version of apache-airflow-breeze.                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│build-image   Build CI image. Include building multiple images for all python versions (sequentially).            │ -│pull-image    Pull and optionally verify CI images - possibly in parallel for all Python versions.                │ -│verify-image  Verify CI image.                                                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│build-prod-image Build Production image. Include building multiple images for all or selected Python versions     │ -│sequentially.                                                                                    │ -│pull-prod-image  Pull and optionally verify Production images - possibly in parallel for all Python versions.     │ -│verify-prod-imageVerify Production image.                                                                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ -│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ -│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ -│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ -│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ -│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ -│find-newer-dependencies          Finds which dependencies are being upgraded.                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze [OPTIONS] COMMAND [ARGS]... + +╭─ Basic flags for the default (shell) command ────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ +│all)                                                                                     │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flags for the default (shell) command ─────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ +│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ +│available in dist folder respectively. Implies --mount-sources `remove`.                │ +│(none | wheel | sdist | <airflow_version>)                                              │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when   │ +│entering breeze.                                                                        │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)[default: wheel]│ +│--force-buildForce image build no matter if it is determined as needed.│ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =    │ +│selected).                                                                              │ +│(selected | all | skip | remove)                                                        │ +│[default: selected]                                                                     │ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Developer tools ────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│shell          Enter breeze.py environment. this is the default command use when no other is selected.            │ +│start-airflow  Enter breeze.py environment and starts all Airflow components in the tmux session.                 │ +│exec           Joins the interactive shell of running airflow container                                           │ +│stop           Stop running breeze environment.                                                                   │ +│build-docs     Build documentation in the container.                                                              │ +│static-checks  Run static checks.                                                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│docker-compose-tests Run docker-compose tests.                                                                    │ +│tests                Run the specified unit test targets. Multiple targets may be specified separated by spaces.  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Configuration & maintenance ────────────────────────────────────────────────────────────────────────────────────────╮ +│cleanup                  Cleans the cache of parameters, docker cache and optionally - currently downloaded       │ +│images.                                                                                  │ +│self-upgrade             Self upgrade Breeze.                                                                     │ +│setup-autocomplete       Enables autocompletion of breeze commands.                                               │ +│config                   Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                       │ +│regenerate-command-imagesRegenerate breeze command images.                                                        │ +│command-hash-export      Outputs hash of all click commands to file or stdout if `-` is used (useful to see if    │ +│images should be regenerated).                                                           │ +│version                  Print information about version of apache-airflow-breeze.                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│build-image   Build CI image. Include building multiple images for all python versions (sequentially).            │ +│pull-image    Pull and optionally verify CI images - possibly in parallel for all Python versions.                │ +│verify-image  Verify CI image.                                                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│build-prod-image Build Production image. Include building multiple images for all or selected Python versions     │ +│sequentially.                                                                                    │ +│pull-prod-image  Pull and optionally verify Production images - possibly in parallel for all Python versions.     │ +│verify-prod-imageVerify Production image.                                                                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI commands ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│fix-ownership                  Fix ownership of source files to be same as host user.                             │ +│free-space                     Free space for jobs run in CI.                                                     │ +│resource-check                 Check if available docker resources are enough.                                    │ +│selective-check                Checks what kind of tests should be run for an incoming commit.                    │ +│find-newer-dependencies        Finds which dependencies are being upgraded.                                       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ +│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ +│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ +│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ +│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ +│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-selective-check.svg b/images/breeze/output-selective-check.svg new file mode 100644 index 0000000000000..3ea08a5bf7a29 --- /dev/null +++ b/images/breeze/output-selective-check.svg @@ -0,0 +1,132 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: selective-check + + + + + + + + + + +Usage: breeze selective-check [OPTIONS] + +Checks what kind of tests should be run for an incoming commit. + +╭─ Selective check flags ──────────────────────────────────────────────────────────────────────────────────────────────╮ +│--commit-refCommit-ish reference to the commit that should be checked(TEXT)│ +│--pr-labelsSpace-separate list of labels which are valid for the PR(TEXT)│ +│--default-branchBranch against which the PR should be run(TEXT)[default: main]│ +│--github-event-nameName of the GitHub event that triggered the check                           │ +│(pull_request | pull_request_review | pull_request_target | push | schedule)│ +│[default: pull_request]                                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output-tests.svg b/images/breeze/output-tests.svg index 1d819816f2596..0bb6680bef9ae 100644 --- a/images/breeze/output-tests.svg +++ b/images/breeze/output-tests.svg @@ -19,177 +19,177 @@ font-weight: 700; } - .terminal-3301140235-matrix { + .terminal-2980146706-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3301140235-title { + .terminal-2980146706-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3301140235-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-3301140235-r2 { fill: #c5c8c6 } -.terminal-3301140235-r3 { fill: #d0b344;font-weight: bold } -.terminal-3301140235-r4 { fill: #868887 } -.terminal-3301140235-r5 { fill: #68a0b3;font-weight: bold } -.terminal-3301140235-r6 { fill: #8d7b39 } -.terminal-3301140235-r7 { fill: #98a84b;font-weight: bold } + .terminal-2980146706-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-2980146706-r2 { fill: #c5c8c6 } +.terminal-2980146706-r3 { fill: #d0b344;font-weight: bold } +.terminal-2980146706-r4 { fill: #868887 } +.terminal-2980146706-r5 { fill: #68a0b3;font-weight: bold } +.terminal-2980146706-r6 { fill: #8d7b39 } +.terminal-2980146706-r7 { fill: #98a84b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: tests + Command: tests - + - - -Usage: breeze tests [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Run the specified unit test targets. Multiple targets may be specified separated by spaces. - -╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ -│--integrationIntegration(s) to enable when running (can be more than one).                              │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)│ -│--test-typeType of test to run.                                                                        │ -│(All | Always | Core | Providers | API | CLI | Integration | Other | WWW | Postgres | MySQL │ -│| Helm | Quarantined)                                                                       │ -│--db-reset-dReset DB when entering the container.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ -│--limit-progress-outputLimit progress to percentage only and just show the summary when tests complete.│ -│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip  │ -│when using to run shell or tests)                                                      │ -│(TEXT)                                                                                 │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =   │ -│selected).                                                                             │ -│(selected | all | skip | remove)                                                       │ -│[default: selected]                                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze tests [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Run the specified unit test targets. Multiple targets may be specified separated by spaces. + +╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ +│--integrationIntegration(s) to enable when running (can be more than one).                              │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)│ +│--test-typeType of test to run.                                                                        │ +│(All | Always | API | Always | CLI | Core | Integration | Other | Providers | WWW | Helm |  │ +│Postgres | MySQL | Integration | Other | Quarantine)                                        │ +│--db-reset-dReset DB when entering the container.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ +│--limit-progress-outputLimit progress to percentage only and just show the summary when tests complete.│ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip  │ +│when using to run shell or tests)                                                      │ +│(TEXT)                                                                                 │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =   │ +│selected).                                                                             │ +│(selected | all | skip | remove)                                                       │ +│[default: selected]                                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh deleted file mode 100755 index 26fbf13f66ed9..0000000000000 --- a/scripts/ci/selective_ci_checks.sh +++ /dev/null @@ -1,768 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck source=scripts/ci/libraries/_script_init.sh -. ./scripts/ci/libraries/_script_init.sh - -# Parameter: -# -# $1 - COMMIT SHA of the incoming commit. If this parameter is missing, this script does not check anything, -# it simply sets all the version outputs that determine that all tests should be run. -# This happens in case the even triggering the workflow is 'schedule' or 'push'. -# -# The logic of retrieving changes works by comparing the incoming commit with the target branch -# The commit addresses. -# -# -declare -a pattern_array - -if [[ ${PR_LABELS=} == *"full tests needed"* ]]; then - echo - echo "Found the right PR labels in '${PR_LABELS=}': 'full tests needed''" - echo - FULL_TESTS_NEEDED_LABEL="true" -else - echo - echo "Did not find the right PR labels in '${PR_LABELS=}': 'full tests needed'" - echo - FULL_TESTS_NEEDED_LABEL="false" -fi - -function check_upgrade_to_newer_dependencies_needed() { - if [[ ${GITHUB_EVENT_NAME=} == 'push' || ${GITHUB_EVENT_NAME=} == "scheduled" ]]; then - # Trigger upgrading to latest constraints when we are in push or schedule event - upgrade_to_newer_dependencies="true" - fi -} - -function output_all_basic_variables() { - if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then - initialization::ga_output python-versions \ - "$(initialization::parameters_to_json "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}")" - initialization::ga_output all-python-versions \ - "$(initialization::parameters_to_json "${ALL_PYTHON_MAJOR_MINOR_VERSIONS[@]}")" - initialization::ga_output all-python-versions-list-as-string "${ALL_PYTHON_MAJOR_MINOR_VERSIONS[*]}" - initialization::ga_output python-versions-list-as-string "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[*]}" - initialization::ga_output kubernetes-versions-list-as-string "${CURRENT_KUBERNETES_VERSIONS[*]}" - else - initialization::ga_output python-versions \ - "$(initialization::parameters_to_json "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}")" - # this will work as long as DEFAULT_PYTHON_MAJOR_VERSION is the same on HEAD - # all-python-versions are used in BuildImage Workflow - initialization::ga_output all-python-versions \ - "$(initialization::parameters_to_json "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}")" - initialization::ga_output all-python-versions-list-as-string "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" - initialization::ga_output python-versions-list-as-string "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" - initialization::ga_output kubernetes-versions-list-as-string "${DEFAULT_KUBERNETES_VERSION}" - fi - initialization::ga_output default-python-version "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" - - if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then - initialization::ga_output kubernetes-versions \ - "$(initialization::parameters_to_json "${CURRENT_KUBERNETES_VERSIONS[@]}")" - else - initialization::ga_output kubernetes-versions \ - "$(initialization::parameters_to_json "${KUBERNETES_VERSION}")" - fi - initialization::ga_output default-kubernetes-version "${KUBERNETES_VERSION}" - - initialization::ga_output kubernetes-modes \ - "$(initialization::parameters_to_json "${CURRENT_KUBERNETES_MODES[@]}")" - initialization::ga_output default-kubernetes-mode "${KUBERNETES_MODE}" - - if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then - initialization::ga_output postgres-versions \ - "$(initialization::parameters_to_json "${CURRENT_POSTGRES_VERSIONS[@]}")" - else - initialization::ga_output postgres-versions \ - "$(initialization::parameters_to_json "${POSTGRES_VERSION}")" - fi - initialization::ga_output default-postgres-version "${POSTGRES_VERSION}" - - if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then - initialization::ga_output mysql-versions \ - "$(initialization::parameters_to_json "${CURRENT_MYSQL_VERSIONS[@]}")" - else - initialization::ga_output mysql-versions \ - "$(initialization::parameters_to_json "${MYSQL_VERSION}")" - fi - initialization::ga_output default-mysql-version "${MYSQL_VERSION}" - - if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then - initialization::ga_output mssql-versions \ - "$(initialization::parameters_to_json "${CURRENT_MSSQL_VERSIONS[@]}")" - else - initialization::ga_output mssql-versions \ - "$(initialization::parameters_to_json "${MSSQL_VERSION}")" - fi - initialization::ga_output default-mssql-version "${MSSQL_VERSION}" - - - - initialization::ga_output kind-versions \ - "$(initialization::parameters_to_json "${CURRENT_KIND_VERSIONS[@]}")" - initialization::ga_output default-kind-version "${KIND_VERSION}" - - initialization::ga_output helm-versions \ - "$(initialization::parameters_to_json "${CURRENT_HELM_VERSIONS[@]}")" - initialization::ga_output default-helm-version "${HELM_VERSION}" - - if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then - initialization::ga_output postgres-exclude '[{ "python-version": "3.7" }]' - initialization::ga_output mssql-exclude '[{ "python-version": "3.8" }]' - initialization::ga_output mysql-exclude '[{ "python-version": "3.10" }]' - initialization::ga_output sqlite-exclude '[{ "python-version": "3.9" }]' - else - initialization::ga_output postgres-exclude '[]' - initialization::ga_output mysql-exclude '[]' - initialization::ga_output mssql-exclude '[]' - initialization::ga_output sqlite-exclude '[]' - fi - - - initialization::ga_output default-helm-version "${HELM_VERSION}" - initialization::ga_output kubernetes-exclude '[]' - - initialization::ga_output default-branch "${DEFAULT_BRANCH}" - -} - -function get_changed_files() { - start_end::group_start "Get changed files" - echo - echo "Incoming commit SHA: ${INCOMING_COMMIT_SHA}" - echo - echo "Changed files from ${INCOMING_COMMIT_SHA} vs it's first parent" - echo - CHANGED_FILES=$(git diff-tree --no-commit-id --name-only \ - -r "${INCOMING_COMMIT_SHA}^" "${INCOMING_COMMIT_SHA}" || true) - if [[ -z "${CHANGED_FILES}" ]]; then - echo - echo "${COLOR_YELLOW}WARNING: Could not find any changed files ${COLOR_RESET}" - echo Assuming that we should run all tests in this case - echo - set_outputs_run_everything_and_exit - fi - echo - echo "Changed files:" - echo - echo "${CHANGED_FILES}" - echo - readonly CHANGED_FILES - start_end::group_end -} - -function run_tests() { - initialization::ga_output run-tests "${@}" -} - -function run_kubernetes_tests() { - initialization::ga_output run-kubernetes-tests "${@}" -} - -function needs_helm_tests() { - initialization::ga_output needs-helm-tests "${@}" -} - -function needs_api_tests() { - initialization::ga_output needs-api-tests "${@}" -} - -function needs_api_codegen() { - initialization::ga_output needs-api-codegen "${@}" -} - -function needs_javascript_scans() { - initialization::ga_output needs-javascript-scans "${@}" -} - -function needs_python_scans() { - initialization::ga_output needs-python-scans "${@}" -} - -function set_test_types() { - initialization::ga_output test-types "${@}" -} - -function set_docs_build() { - initialization::ga_output docs-build "${@}" -} - -function set_image_build() { - initialization::ga_output image-build "${@}" -} - -function set_basic_checks_only() { - initialization::ga_output basic-checks-only "${@}" -} - -function set_upgrade_to_newer_dependencies() { - initialization::ga_output upgrade-to-newer-dependencies "${@}" -} - -function needs_ui_tests() { - initialization::ga_output run-ui-tests "${@}" -} - -function needs_www_tests() { - initialization::ga_output run-www-tests "${@}" -} - -if [[ ${DEFAULT_BRANCH} == "main" ]]; then - ALL_TESTS="Always API Core Other CLI Providers WWW Integration" -else - # Skips Provider tests in case current default branch is not main - ALL_TESTS="Always API Core Other CLI WWW Integration" -fi -readonly ALL_TESTS - -function set_outputs_run_everything_and_exit() { - needs_api_tests "true" - needs_api_codegen "true" - needs_helm_tests "true" - needs_javascript_scans "true" - needs_python_scans "true" - run_tests "true" - run_kubernetes_tests "true" - set_test_types "${ALL_TESTS}" - set_basic_checks_only "false" - set_docs_build "true" - set_image_build "true" - set_upgrade_to_newer_dependencies "${upgrade_to_newer_dependencies}" - needs_ui_tests "true" - needs_www_tests "true" - exit -} - -function set_outputs_run_all_python_tests() { - run_tests "true" - run_kubernetes_tests "true" - set_test_types "${ALL_TESTS}" - set_basic_checks_only "false" - set_image_build "true" - kubernetes_tests_needed="true" -} - -function set_output_skip_all_tests_and_docs_and_exit() { - needs_api_tests "false" - needs_api_codegen "false" - needs_helm_tests "false" - needs_javascript_scans "false" - needs_python_scans "false" - run_tests "false" - run_kubernetes_tests "false" - set_test_types "" - set_basic_checks_only "true" - set_docs_build "false" - set_image_build "false" - set_upgrade_to_newer_dependencies "false" - needs_ui_tests "false" - needs_www_tests "false" - exit -} - -function set_output_skip_tests_but_build_images_and_exit() { - needs_api_tests "false" - needs_api_codegen "false" - needs_helm_tests "false" - needs_javascript_scans "false" - needs_python_scans "false" - run_tests "false" - run_kubernetes_tests "false" - set_test_types "" - set_basic_checks_only "false" - set_docs_build "true" - set_image_build "true" - set_upgrade_to_newer_dependencies "${upgrade_to_newer_dependencies}" - needs_ui_tests "false" - needs_www_tests "false" - exit -} - -# Converts array of patterns into single | pattern string -# pattern_array - array storing regexp patterns -# Outputs - pattern string -function get_regexp_from_patterns() { - local test_triggering_regexp="" - local separator="" - local pattern - for pattern in "${pattern_array[@]}"; do - test_triggering_regexp="${test_triggering_regexp}${separator}${pattern}" - separator="|" - done - echo "${test_triggering_regexp}" -} - -# Shows changed files in the commit vs. the target. -# Input: -# pattern_array - array storing regexp patterns -function show_changed_files() { - local the_regexp - the_regexp=$(get_regexp_from_patterns) - echo - echo "Changed files matching the ${the_regexp} pattern:" - echo - echo "${CHANGED_FILES}" | grep -E "${the_regexp}" || true - echo -} - -# Counts changed files in the commit vs. the target -# Input: -# pattern_array - array storing regexp patterns -# Output: -# Count of changed files matching the patterns -function count_changed_files() { - echo "${CHANGED_FILES}" | grep -c -E "$(get_regexp_from_patterns)" || true -} - -function check_if_python_security_scans_should_be_run() { - start_end::group_start "Check Python security scans" - local pattern_array=( - "^airflow/.*\.py" - "^setup.py" - ) - show_changed_files - - if [[ $(count_changed_files) == "0" ]]; then - needs_python_scans "false" - else - needs_python_scans "true" - fi - start_end::group_end -} - -function check_if_setup_files_changed() { - start_end::group_start "Check setup.py/cfg changed" - local pattern_array=( - "^setup.cfg" - "^setup.py" - ) - show_changed_files - - if [[ $(count_changed_files) != "0" ]]; then - # In case the setup files changed, we automatically force upgrading to newer dependencies - # no matter what was set before. - upgrade_to_newer_dependencies="true" - fi - start_end::group_end -} - - -function check_if_javascript_security_scans_should_be_run() { - start_end::group_start "Check JavaScript security scans" - local pattern_array=( - "^airflow/.*\.[jt]sx?" - "^airflow/.*\.lock" - ) - show_changed_files - - if [[ $(count_changed_files) == "0" ]]; then - needs_javascript_scans "false" - else - needs_javascript_scans "true" - fi - start_end::group_end -} - -function check_if_api_tests_should_be_run() { - start_end::group_start "Check API tests" - local pattern_array=( - "^airflow/api" - ) - show_changed_files - - if [[ $(count_changed_files) == "0" ]]; then - needs_api_tests "false" - else - needs_api_tests "true" - fi - start_end::group_end -} - -function check_if_api_codegen_should_be_run() { - start_end::group_start "Check API codegen" - local pattern_array=( - "^airflow/api_connexion/openapi/v1.yaml" - "^clients/gen" - ) - show_changed_files - - if [[ $(count_changed_files) == "0" ]]; then - needs_api_codegen "false" - else - needs_api_codegen "true" - fi - start_end::group_end -} - -function check_if_helm_tests_should_be_run() { - start_end::group_start "Check helm tests" - local pattern_array=( - "^chart" - ) - show_changed_files - - if [[ $(count_changed_files) == "0" ]]; then - needs_helm_tests "false" - else - needs_helm_tests "true" - fi - start_end::group_end -} - -function check_if_docs_should_be_generated() { - start_end::group_start "Check docs" - local pattern_array=( - "^docs" - "^airflow/.*\.py$" - "^CHANGELOG\.txt" - "^airflow/config_templates/config\.yml" - "^chart/UPDATING\.rst" - "^chart/CHANGELOG\.txt" - "^chart/values\.schema\.json" - ) - show_changed_files - - if [[ $(count_changed_files) == "0" ]]; then - echo "None of the docs changed" - else - image_build_needed="true" - docs_build_needed="true" - fi - start_end::group_end -} - -function check_if_ui_tests_should_be_run() { - start_end::group_start "Check UI" - local pattern_array=( - "^airflow/ui/.*\.[tj]sx?$" - # tsconfig.json, package.json, etc. - "^airflow/ui/[^/]+\.json$" - "^airflow/ui/.*\.lock$" - ) - show_changed_files - - if [[ $(count_changed_files) == "0" ]]; then - needs_ui_tests "false" - else - needs_ui_tests "true" - fi - start_end::group_end -} - -function check_if_www_tests_should_be_run() { - start_end::group_start "Check WWW" - local pattern_array=( - "^airflow/www/.*\.js[x]?$" - # tsconfig.json, package.json, etc. - "^airflow/www/[^/]+\.json$" - "^airflow/www/.*\.lock$" - ) - show_changed_files - - if [[ $(count_changed_files) == "0" ]]; then - needs_www_tests "false" - else - needs_www_tests "true" - fi - start_end::group_end -} - - -ANY_PY_FILES_CHANGED=( - "\.py$" -) -readonly ANY_PY_FILES_CHANGED - -function check_if_any_py_files_changed() { - start_end::group_start "Check if any Python files changed" - local pattern_array=("${ANY_PY_FILES_CHANGED[@]}") - show_changed_files - - if [[ $(count_changed_files) != "0" ]]; then - image_build_needed="true" - fi - start_end::group_end -} - - -AIRFLOW_SOURCES_TRIGGERING_TESTS=( - "^.pre-commit-config.yaml$" - "^airflow" - "^chart" - "^tests" - "^kubernetes_tests" -) -readonly AIRFLOW_SOURCES_TRIGGERING_TESTS - -function check_if_tests_are_needed_at_all() { - start_end::group_start "Check tests are needed" - local pattern_array=("${AIRFLOW_SOURCES_TRIGGERING_TESTS[@]}") - show_changed_files - - if [[ $(count_changed_files) == "0" ]]; then - if [[ ${image_build_needed} == "true" ]]; then - echo "No tests needed, Skipping tests but building images." - set_output_skip_tests_but_build_images_and_exit - else - echo "None of the important files changed, Skipping tests" - set_output_skip_all_tests_and_docs_and_exit - fi - else - image_build_needed="true" - tests_needed="true" - fi - start_end::group_end -} - -function run_all_tests_if_environment_files_changed() { - start_end::group_start "Check if everything should be run" - local pattern_array=( - "^.github/workflows/" - "^Dockerfile" - "^scripts" - "^setup.py" - "^setup.cfg" - ) - show_changed_files - - if [[ $(count_changed_files) != "0" ]]; then - echo "Important environment files changed. Running everything" - set_outputs_run_everything_and_exit - fi - if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then - echo "Full tests requested by label on PR. Running everything" - set_outputs_run_everything_and_exit - fi - start_end::group_end -} - -function get_count_all_files() { - start_end::group_start "Count all airflow source files" - local pattern_array=("${AIRFLOW_SOURCES_TRIGGERING_TESTS[@]}") - show_changed_files - COUNT_ALL_CHANGED_FILES=$(count_changed_files) - echo "Files count: ${COUNT_ALL_CHANGED_FILES}" - readonly COUNT_ALL_CHANGED_FILES - start_end::group_end -} - -function get_count_api_files() { - start_end::group_start "Count API files" - local pattern_array=( - "^airflow/api" - "^airflow/api_connexion" - "^tests/api" - "^tests/api_connexion" - ) - show_changed_files - COUNT_API_CHANGED_FILES=$(count_changed_files) - echo "Files count: ${COUNT_API_CHANGED_FILES}" - readonly COUNT_API_CHANGED_FILES - start_end::group_end -} - -function get_count_cli_files() { - start_end::group_start "Count CLI files" - local pattern_array=( - "^airflow/cli" - "^tests/cli" - ) - show_changed_files - COUNT_CLI_CHANGED_FILES=$(count_changed_files) - echo "Files count: ${COUNT_CLI_CHANGED_FILES}" - readonly COUNT_CLI_CHANGED_FILES - start_end::group_end -} - -function get_count_providers_files() { - start_end::group_start "Count providers files" - local pattern_array=( - "^airflow/providers/" - "^tests/providers/" - ) - show_changed_files - COUNT_PROVIDERS_CHANGED_FILES=$(count_changed_files) - echo "Files count: ${COUNT_PROVIDERS_CHANGED_FILES}" - readonly COUNT_PROVIDERS_CHANGED_FILES - start_end::group_end -} - -function get_count_www_files() { - start_end::group_start "Count www files" - local pattern_array=( - "^airflow/www" - "^tests/www" - ) - show_changed_files - COUNT_WWW_CHANGED_FILES=$(count_changed_files) - echo "Files count: ${COUNT_WWW_CHANGED_FILES}" - readonly COUNT_WWW_CHANGED_FILES - start_end::group_end -} - -function get_count_ui_files() { - start_end::group_start "Count ui files" - local pattern_array=( - "^airflow/ui/" - ) - show_changed_files - COUNT_UI_CHANGED_FILES=$(count_changed_files) - echo "Files count: ${COUNT_UI_CHANGED_FILES}" - readonly COUNT_UI_CHANGED_FILES - start_end::group_end -} - -function get_count_kubernetes_files() { - start_end::group_start "Count kubernetes files" - local pattern_array=( - "^chart" - "^kubernetes_tests" - "^airflow/providers/cncf/kubernetes/" - "^tests/providers/cncf/kubernetes/" - ) - show_changed_files - COUNT_KUBERNETES_CHANGED_FILES=$(count_changed_files) - echo "Files count: ${COUNT_KUBERNETES_CHANGED_FILES}" - readonly COUNT_KUBERNETES_CHANGED_FILES - start_end::group_end -} - -function calculate_test_types_to_run() { - start_end::group_start "Count core/other files" - COUNT_CORE_OTHER_CHANGED_FILES=$((COUNT_ALL_CHANGED_FILES - COUNT_WWW_CHANGED_FILES - COUNT_UI_CHANGED_FILES - COUNT_PROVIDERS_CHANGED_FILES - COUNT_CLI_CHANGED_FILES - COUNT_API_CHANGED_FILES - COUNT_KUBERNETES_CHANGED_FILES)) - - readonly COUNT_CORE_OTHER_CHANGED_FILES - echo - echo "Files count: ${COUNT_CORE_OTHER_CHANGED_FILES}" - echo - if [[ ${COUNT_CORE_OTHER_CHANGED_FILES} -gt 0 ]]; then - # Running all tests because some core or other files changed - echo - echo "Looks like ${COUNT_CORE_OTHER_CHANGED_FILES} files changed in the core/other area and" - echo "We have to run all python tests. This will take longer than usual" - echo - set_outputs_run_all_python_tests - else - if [[ ${COUNT_KUBERNETES_CHANGED_FILES} != "0" ]]; then - kubernetes_tests_needed="true" - fi - tests_needed="true" - SELECTED_TESTS="" - if [[ ${COUNT_API_CHANGED_FILES} != "0" ]]; then - echo - echo "Adding API to selected files as ${COUNT_API_CHANGED_FILES} API files changed" - echo - SELECTED_TESTS="${SELECTED_TESTS} API" - fi - if [[ ${COUNT_CLI_CHANGED_FILES} != "0" ]]; then - echo - echo "Adding CLI and Kubernetes (they depend on CLI) to selected files as ${COUNT_CLI_CHANGED_FILES} CLI files changed" - echo - SELECTED_TESTS="${SELECTED_TESTS} CLI" - kubernetes_tests_needed="true" - fi - - if [[ ${DEFAULT_BRANCH} == "main" ]]; then - if [[ ${COUNT_PROVIDERS_CHANGED_FILES} != "0" ]]; then - echo - echo "Adding Providers to selected files as ${COUNT_PROVIDERS_CHANGED_FILES} Provider files changed" - echo - SELECTED_TESTS="${SELECTED_TESTS} Providers" - fi - else - echo - echo "Providers tests are not added because they are only run in case of main branch." - echo - fi - if [[ ${COUNT_WWW_CHANGED_FILES} != "0" ]]; then - echo - echo "Adding WWW to selected files as ${COUNT_WWW_CHANGED_FILES} WWW files changed" - echo - SELECTED_TESTS="${SELECTED_TESTS} WWW" - fi - initialization::ga_output test-types "Always Integration ${SELECTED_TESTS}" - fi - start_end::group_end -} - - - -upgrade_to_newer_dependencies="false" - -if (($# < 1)); then - echo - echo "No Commit SHA - running all tests (likely direct merge, or scheduled run)!" - echo - INCOMING_COMMIT_SHA="" - readonly INCOMING_COMMIT_SHA - # override FULL_TESTS_NEEDED_LABEL in main/scheduled run - FULL_TESTS_NEEDED_LABEL="true" - readonly FULL_TESTS_NEEDED_LABEL - output_all_basic_variables - check_upgrade_to_newer_dependencies_needed - set_outputs_run_everything_and_exit -else - INCOMING_COMMIT_SHA="${1}" - readonly INCOMING_COMMIT_SHA - echo - echo "Commit SHA passed: ${INCOMING_COMMIT_SHA}!" - echo - readonly FULL_TESTS_NEEDED_LABEL -fi - -check_upgrade_to_newer_dependencies_needed - -output_all_basic_variables - -image_build_needed="false" -docs_build_needed="false" -tests_needed="false" -kubernetes_tests_needed="false" - -get_changed_files -check_if_setup_files_changed -run_all_tests_if_environment_files_changed -check_if_any_py_files_changed -check_if_docs_should_be_generated -check_if_helm_tests_should_be_run -check_if_api_tests_should_be_run -check_if_api_codegen_should_be_run -check_if_javascript_security_scans_should_be_run -check_if_python_security_scans_should_be_run -check_if_ui_tests_should_be_run -check_if_www_tests_should_be_run -check_if_tests_are_needed_at_all -get_count_all_files -get_count_api_files -get_count_cli_files -get_count_providers_files -get_count_www_files -get_count_ui_files -get_count_kubernetes_files -calculate_test_types_to_run - -set_image_build "${image_build_needed}" -if [[ ${image_build_needed} == "true" ]]; then - set_basic_checks_only "false" -else - set_basic_checks_only "true" -fi -set_docs_build "${docs_build_needed}" -run_tests "${tests_needed}" -run_kubernetes_tests "${kubernetes_tests_needed}" -set_upgrade_to_newer_dependencies "${upgrade_to_newer_dependencies}" diff --git a/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh b/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh index d120b50ec75c4..bc5bc041ed17d 100755 --- a/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh +++ b/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh @@ -89,7 +89,7 @@ function run_airflow_testing_in_docker() { echo docker-compose -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ "${INTEGRATIONS[@]}" \ - --project-name "airflow-${TEST_TYPE}-${BACKEND}" \ + --project-name "airflow-${TEST_TYPE,,}-${BACKEND}" \ down --remove-orphans \ --volumes --timeout 10 docker-compose --log-level INFO \ @@ -97,7 +97,7 @@ function run_airflow_testing_in_docker() { "${BACKEND_DOCKER_COMPOSE[@]}" \ "${INTEGRATIONS[@]}" \ "${DOCKER_COMPOSE_LOCAL[@]}" \ - --project-name "airflow-${TEST_TYPE}-${BACKEND}" \ + --project-name "airflow-${TEST_TYPE,,}-${BACKEND}" \ run airflow "${@}" exit_code=$? docker ps @@ -112,7 +112,7 @@ function run_airflow_testing_in_docker() { docker-compose --log-level INFO -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ "${INTEGRATIONS[@]}" \ - --project-name "airflow-${TEST_TYPE}-${BACKEND}" \ + --project-name "airflow-${TEST_TYPE,,}-${BACKEND}" \ down --remove-orphans \ --volumes --timeout 10 set -u