diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 420d915219366..48a00f07f21c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -607,6 +607,13 @@ jobs: python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* + - name: Cache pre-commit envs + uses: actions/cache@v2 + with: + path: ~/.cache/pre-commit + key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ +${{ hashFiles('.pre-commit-config.yaml') }}" + restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze free-space @@ -619,18 +626,6 @@ jobs: run: "echo \"::set-output name=host-python-version::$(python -c 'import platform; print(platform.python_version())')\"" id: host-python-version - - name: "Cache pre-commit envs" - uses: actions/cache@v2 - with: - path: ~/.cache/pre-commit - key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} - - name: "Cache eslint" - uses: actions/cache@v2 - with: - path: 'airflow/ui/node_modules' - key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }} - name: "Static checks" run: breeze static-checks --all-files --show-diff-on-failure --color always env: @@ -667,7 +662,16 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* + - name: Cache pre-commit envs + uses: actions/cache@v2 + with: + path: ~/.cache/pre-commit + key: "pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}}-\ +${{ hashFiles('.pre-commit-config.yaml') }}" + restore-keys: pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}} - run: ./scripts/ci/install_breeze.sh + - name: "Free space" + run: breeze free-space - name: Fetch incoming commit ${{ github.sha }} with its parent uses: actions/checkout@v2 with: @@ -678,13 +682,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: "echo \"::set-output name=host-python-version::$(python -c 'import platform; print(platform.python_version())')\"" id: host-python-version - - name: "Cache pre-commit envs" - uses: actions/cache@v2 - with: - path: ~/.cache/pre-commit - key: "pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}} - name: "Static checks: basic checks only" run: > breeze static-checks --all-files --show-diff-on-failure --color always diff --git a/BREEZE.rst b/BREEZE.rst index 5f60b121355ed..25f6c7c85fe6f 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -506,7 +506,7 @@ Configuration and maintenance * Checking available resources for docker with ``breeze resource-check`` command * Freeing space needed to run CI tests with ``breeze free-space`` command * Fixing ownership of files in your repository with ``breeze fix-ownership`` command -* Print Breeze version with ``breeze fix-ownership`` command +* Print Breeze version with ``breeze version`` command Release tasks ------------- diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py index 381c93560db30..4e522c7120420 100755 --- a/dev/breeze/src/airflow_breeze/breeze.py +++ b/dev/breeze/src/airflow_breeze/breeze.py @@ -545,7 +545,7 @@ get_used_sources_setup_metadata_hash, in_autocomplete, ) -from airflow_breeze.utils.run_utils import check_pre_commit_installed, filter_out_none, run_command +from airflow_breeze.utils.run_utils import assert_pre_commit_installed, filter_out_none, run_command from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE find_airflow_sources_root_to_operate_on() @@ -1851,40 +1851,41 @@ def static_checks( files: bool, precommit_args: Tuple, ): - if check_pre_commit_installed(verbose=verbose): - command_to_execute = ['pre-commit', 'run'] - if last_commit and commit_ref: - console.print("\n[red]You cannot specify both --last-commit and --commit-ref[/]\n") - sys.exit(1) - for single_check in type: - command_to_execute.append(single_check) - if all_files: - command_to_execute.append("--all-files") - if show_diff_on_failure: - command_to_execute.append("--show-diff-on-failure") - if last_commit: - command_to_execute.extend(["--from-ref", "HEAD^", "--to-ref", "HEAD"]) - if commit_ref: - command_to_execute.extend(["--from-ref", f"{commit_ref}^", "--to-ref", f"{commit_ref}"]) - if files: - command_to_execute.append("--files") - if verbose or dry_run: - command_to_execute.append("--verbose") - if precommit_args: - command_to_execute.extend(precommit_args) - env = os.environ.copy() - env['GITHUB_REPOSITORY'] = github_repository - static_checks_result = run_command( - command_to_execute, - verbose=verbose, - dry_run=dry_run, - check=False, - no_output_dump_on_exception=True, - text=True, - env=env, - ) - if static_checks_result.returncode != 0: - console.print("[red]There were errors during pre-commit check. They should be fixed[/n]") + assert_pre_commit_installed(verbose=verbose) + command_to_execute = [sys.executable, "-m", "pre_commit", 'run'] + if last_commit and commit_ref: + console.print("\n[red]You cannot specify both --last-commit and --commit-ref[/]\n") + sys.exit(1) + for single_check in type: + command_to_execute.append(single_check) + if all_files: + command_to_execute.append("--all-files") + if show_diff_on_failure: + command_to_execute.append("--show-diff-on-failure") + if last_commit: + command_to_execute.extend(["--from-ref", "HEAD^", "--to-ref", "HEAD"]) + if commit_ref: + command_to_execute.extend(["--from-ref", f"{commit_ref}^", "--to-ref", f"{commit_ref}"]) + if files: + command_to_execute.append("--files") + if verbose or dry_run: + command_to_execute.append("--verbose") + if precommit_args: + command_to_execute.extend(precommit_args) + env = os.environ.copy() + env['GITHUB_REPOSITORY'] = github_repository + static_checks_result = run_command( + command_to_execute, + verbose=verbose, + dry_run=dry_run, + check=False, + no_output_dump_on_exception=True, + text=True, + env=env, + ) + if static_checks_result.returncode != 0: + console.print("[red]There were errors during pre-commit check. They should be fixed[/]") + sys.exit(static_checks_result.returncode) @main.command(name="stop", help="Stop running breeze environment.") diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index d0310cf5a2641..6ef639862a3df 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -18,7 +18,6 @@ import contextlib import os import shlex -import shutil import stat import subprocess import sys @@ -96,7 +95,7 @@ def run_command( return ex -def check_pre_commit_installed(verbose: bool) -> bool: +def assert_pre_commit_installed(verbose: bool): """ Check if pre-commit is installed in the right version. :param verbose: print commands when running @@ -108,36 +107,39 @@ def check_pre_commit_installed(verbose: bool) -> bool: pre_commit_config = yaml.safe_load((AIRFLOW_SOURCES_ROOT / ".pre-commit-config.yaml").read_text()) min_pre_commit_version = pre_commit_config["minimum_pre_commit_version"] - pre_commit_name = "pre-commit" - is_installed = False - if shutil.which(pre_commit_name) is not None: - command_result = run_command( - [pre_commit_name, "--version"], verbose=verbose, capture_output=True, text=True - ) + python_executable = sys.executable + console.print(f"[bright_blue]Checking pre-commit installed for {python_executable}[/]") + command_result = run_command( + [python_executable, "-m", "pre_commit", "--version"], + verbose=verbose, + capture_output=True, + text=True, + check=False, + ) + if command_result.returncode == 0: if command_result.stdout: pre_commit_version = command_result.stdout.split(" ")[-1].strip() if StrictVersion(pre_commit_version) >= StrictVersion(min_pre_commit_version): console.print( - f"\n[green]Package {pre_commit_name} is installed. " + f"\n[green]Package pre_commit is installed. " f"Good version {pre_commit_version} (>= {min_pre_commit_version})[/]\n" ) - is_installed = True else: console.print( - f"\n[red]Package name {pre_commit_name} version is wrong. It should be" + f"\n[red]Package name pre_commit version is wrong. It should be" f"aat least {min_pre_commit_version} and is {pre_commit_version}.[/]\n\n" ) + sys.exit(1) else: console.print( "\n[bright_yellow]Could not determine version of pre-commit. " "You might need to update it![/]\n" ) - is_installed = True else: - console.print(f"\n[red]Error: Package name {pre_commit_name} is not installed.[/]") - if not is_installed: - console.print("\nPlease install using https://pre-commit.com/#install to continue\n") - return is_installed + console.print("\n[red]Error checking for pre-commit-installation:[/]\n") + console.print(command_result.stderr) + console.print("\nMake sure to run:\n breeze self-upgrade\n\n") + sys.exit(1) def get_filesystem_type(filepath):