Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1056,7 +1056,7 @@ repos:
entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
language: python
files: ^airflow/providers/.*/provider\.yaml$
additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml', 'jsonschema', 'filelock', 'markdown-it-py']
additional_dependencies: ['rich>=12.4.4']
require_serial: true
- id: update-migration-references
name: Update migration ref doc
Expand Down
16 changes: 16 additions & 0 deletions dev/breeze/tests/test_shell_params.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,22 @@
},
id="ENABLED_SYSTEMS empty by default even if they are None in ShellParams",
),
pytest.param(
{},
{},
{
"PYTHONWARNINGS": None,
},
id="PYTHONWARNINGS should not be set by default",
),
pytest.param(
{"PYTHONWARNINGS": "default"},
{},
{
"PYTHONWARNINGS": "default",
},
id="PYTHONWARNINGS should be set when specified in environment",
),
],
)
def test_shell_params_to_env_var_conversion(
Expand Down
63 changes: 15 additions & 48 deletions scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,57 +17,24 @@
# under the License.
from __future__ import annotations

import os
import shlex
import sys
from pathlib import Path

if __name__ not in ("__main__", "__mp_main__"):
raise SystemExit(
"This file is intended to be executed as an executable program. You cannot use it as a module."
f"To run this script, run the ./{__file__} command"
)

AIRFLOW_SOURCES = Path(__file__).parents[3].resolve()
GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "apache/airflow")
os.environ["SKIP_GROUP_OUTPUT"] = "true"

if __name__ == "__main__":
sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
from airflow_breeze.global_constants import DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_SELECTED
from airflow_breeze.params.shell_params import ShellParams
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
from airflow_breeze.utils.run_utils import get_ci_image_for_pre_commits, run_command
sys.path.insert(0, str(Path(__file__).parent.resolve()))
from common_precommit_utils import console, initialize_breeze_precommit, run_command_via_breeze_shell

shell_params = ShellParams(python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION)
initialize_breeze_precommit(__name__, __file__)

cmd = "python3 /opt/airflow/scripts/in_container/run_provider_yaml_files_check.py"
if len(sys.argv) > 1:
cmd = cmd + " " + " ".join([shlex.quote(f) for f in sys.argv[1:]])
airflow_image = get_ci_image_for_pre_commits()
cmd_result = run_command(
[
"docker",
"run",
"-t",
*get_extra_docker_flags(mount_sources=MOUNT_SELECTED),
"-e",
"SKIP_ENVIRONMENT_INITIALIZATION=true",
"-e",
"PYTHONWARNINGS=default",
"--pull",
"never",
airflow_image,
"-c",
cmd,
],
check=False,
env=shell_params.env_variables_for_docker_commands,
files_to_test = sys.argv[1:]
cmd_result = run_command_via_breeze_shell(
["python3", "/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py", *files_to_test],
backend="sqlite",
warn_image_upgrade_needed=True,
extra_env={"PYTHONWARNINGS": "default"},
)
if cmd_result.returncode != 0:
console.print(
"[warning]\nIf you see strange stacktraces above, "
"run `breeze ci-image build --python 3.8` and try again."
)
if cmd_result.returncode != 0:
get_console().print(
"[warning]If you see strange stacktraces above, "
"run `breeze ci-image build --python 3.8` and try again."
)
sys.exit(cmd_result.returncode)
sys.exit(cmd_result.returncode)