From 0b79d62ba5294b7058a6eff16f104d420e467fe9 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 20 Jul 2026 00:24:04 +0200 Subject: [PATCH] [v3-3-test] Automate keeping documented "tested with" versions in sync with the test matrix (#69124) The Python, database and Kubernetes versions Airflow is tested with live in global_constants.py, but are also listed by hand in the installation prerequisites doc and the README. Those lists drifted: PostgreSQL 18 (and MySQL 8.4) were added to the test matrix and the README, but the prerequisites doc was forgotten, so it still advertised PostgreSQL up to 17. Add a prek hook that regenerates the tested-versions block in prerequisites.rst and the "Main version (dev)" column of the README from global_constants.py, so this drift becomes an auto-fixable check failure instead of a silent doc bug. Also correct the stale PostgreSQL and MySQL versions in prerequisites.rst. (cherry picked from commit 4dc4ebff93f7a1d60d1c5aa482007e6befca4233) Co-authored-by: Jarek Potiuk Generated-by: Claude Code (Opus 4.8) following the guidelines --- .pre-commit-config.yaml | 6 + .../docs/installation/prerequisites.rst | 4 + scripts/ci/prek/common_prek_utils.py | 36 ++- scripts/ci/prek/update_tested_versions.py | 209 ++++++++++++++++++ .../tests/ci/prek/test_common_prek_utils.py | 40 ++++ .../ci/prek/test_update_tested_versions.py | 123 +++++++++++ 6 files changed, 417 insertions(+), 1 deletion(-) create mode 100755 scripts/ci/prek/update_tested_versions.py create mode 100644 scripts/tests/ci/prek/test_update_tested_versions.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3998ee80bb5b4..49b66ca2079da 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -670,6 +670,12 @@ repos: ^scripts/ci/prek/extract_permissions\.py$| ^airflow-core/docs/security/api_permissions_ref\.rst$ pass_filenames: false + - id: update-tested-versions + name: Update tested Python/DB/Kubernetes versions in docs + entry: ./scripts/ci/prek/update_tested_versions.py + language: python + files: ^airflow-core/docs/installation/prerequisites\.rst$|^README\.md$|^dev/breeze/src/airflow_breeze/global_constants\.py$|^scripts/ci/prek/update_tested_versions\.py$|^scripts/ci/prek/common_prek_utils\.py$ + pass_filenames: false - id: check-revision-heads-map name: Check that the REVISION_HEADS_MAP is up-to-date language: python diff --git a/airflow-core/docs/installation/prerequisites.rst b/airflow-core/docs/installation/prerequisites.rst index 359450d3eeb09..7fdc7f8beba5f 100644 --- a/airflow-core/docs/installation/prerequisites.rst +++ b/airflow-core/docs/installation/prerequisites.rst @@ -20,6 +20,8 @@ Prerequisites Airflow® is tested with: + .. Beginning of the auto-generated tested versions + * Python: 3.10, 3.11, 3.12, 3.13, 3.14 * Databases: @@ -30,6 +32,8 @@ Airflow® is tested with: * Kubernetes: 1.30, 1.31, 1.32, 1.33, 1.34, 1.35 + .. End of the auto-generated tested versions + While we recommend a minimum of 4GB of memory for Airflow, the actual requirements heavily depend on your chosen deployment. .. warning:: diff --git a/scripts/ci/prek/common_prek_utils.py b/scripts/ci/prek/common_prek_utils.py index 3dd9631d08cf0..9c593340fdf3c 100644 --- a/scripts/ci/prek/common_prek_utils.py +++ b/scripts/ci/prek/common_prek_utils.py @@ -130,13 +130,21 @@ def read_airflow_version() -> str: def _read_global_constants_assignment(name: str) -> Any: - """Read a top-level assignment from global_constants.py.""" + """Read a top-level assignment from global_constants.py. + + Handles both plain assignments (``NAME = ...``) and annotated assignments + (``NAME: type = ...``). The value must be a literal so it can be safely + evaluated with ``ast.literal_eval``. + """ tree = ast.parse(GLOBAL_CONSTANTS_PATH.read_text()) for node in tree.body: if isinstance(node, ast.Assign): for target in node.targets: if isinstance(target, ast.Name) and target.id == name: return ast.literal_eval(node.value) + elif isinstance(node, ast.AnnAssign): + if isinstance(node.target, ast.Name) and node.target.id == name and node.value is not None: + return ast.literal_eval(node.value) raise RuntimeError(f"{name} not found in global_constants.py") @@ -149,6 +157,32 @@ def read_allowed_kubernetes_versions() -> list[str]: return [v.lstrip("v") for v in versions] +def read_allowed_python_major_minor_versions() -> list[str]: + """Parse ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS from global_constants.py (single source of truth).""" + return list(_read_global_constants_assignment("ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS")) + + +def read_current_postgres_versions() -> list[str]: + """Parse CURRENT_POSTGRES_VERSIONS from global_constants.py (single source of truth).""" + return list(_read_global_constants_assignment("CURRENT_POSTGRES_VERSIONS")) + + +def read_current_mysql_versions() -> list[str]: + """The MySQL release versions Airflow currently tests with. + + Mirrors how ``CURRENT_MYSQL_VERSIONS`` is built in global_constants.py: the + "old" releases plus the LTS releases, plus an innovation release when one is + configured. Returns the numeric versions only (e.g. ``["8.0", "8.4"]``); the + docs add the textual "Innovation" annotation on top of these. + """ + versions: list[str] = list(_read_global_constants_assignment("MYSQL_OLD_RELEASES")) + versions += list(_read_global_constants_assignment("MYSQL_LTS_RELEASES")) + innovation = _read_global_constants_assignment("MYSQL_INNOVATION_RELEASE") + if innovation: + versions.append(innovation) + return versions + + def read_default_python_major_minor_version_for_images() -> str: """Parse DEFAULT_PYTHON_MAJOR_MINOR_VERSION_FOR_IMAGES from global_constants.py.""" value = _read_global_constants_assignment("DEFAULT_PYTHON_MAJOR_MINOR_VERSION_FOR_IMAGES") diff --git a/scripts/ci/prek/update_tested_versions.py b/scripts/ci/prek/update_tested_versions.py new file mode 100755 index 0000000000000..4b5936de27677 --- /dev/null +++ b/scripts/ci/prek/update_tested_versions.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.10,<3.11" +# dependencies = [] +# /// +"""Keep the documented "tested with" versions in sync with global_constants.py. + +The list of Python, database and Kubernetes versions Airflow is tested with lives +in a single source of truth (``dev/breeze/src/airflow_breeze/global_constants.py``) +but is also rendered for humans in two hand-maintained places: + +* ``airflow-core/docs/installation/prerequisites.rst`` — the "Prerequisites" bullet list. +* ``README.md`` — the "Main version (dev)" column of the Requirements table. + +These used to drift out of sync silently (e.g. PostgreSQL 18 was added to the test +matrix and the README but the prerequisites doc was forgotten). This hook regenerates +both from the constants so the drift becomes a (auto-fixable) prek failure instead of +a documentation bug nobody notices. + +Only versions that have a clean single source in global_constants.py are generated: +Python, PostgreSQL, MySQL (numeric releases) and Kubernetes. SQLite has no constant +and the MySQL "Innovation" annotation is editorial, so both are defined here. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from common_prek_utils import ( + AIRFLOW_ROOT_PATH, + read_allowed_kubernetes_versions, + read_allowed_python_major_minor_versions, + read_current_mysql_versions, + read_current_postgres_versions, +) + +PREREQUISITES_RST = AIRFLOW_ROOT_PATH / "airflow-core" / "docs" / "installation" / "prerequisites.rst" +README_MD = AIRFLOW_ROOT_PATH / "README.md" + +PREREQUISITES_START = " .. Beginning of the auto-generated tested versions\n" +PREREQUISITES_END = " .. End of the auto-generated tested versions\n" + +README_DEV_HEADER = "Main version (dev)" + +# Editorial values without a single source of truth in global_constants.py. +SQLITE_VERSION = "3.15.0+" +MYSQL_INNOVATION_ANNOTATION = "Innovation" +MYSQL_INNOVATION_RST_LINK = ( + "`Innovation `_" +) + + +def kubernetes_major_minor_versions() -> list[str]: + """Kubernetes versions reduced to ``major.minor`` (e.g. ``1.30``) for display.""" + return [".".join(version.split(".")[:2]) for version in read_allowed_kubernetes_versions()] + + +def comma(values: list[str]) -> str: + return ", ".join(values) + + +def render_prerequisites_block() -> str: + """The bullet list rendered between the prerequisites.rst markers.""" + python_versions = comma(read_allowed_python_major_minor_versions()) + postgres_versions = comma(read_current_postgres_versions()) + mysql_versions = comma([*read_current_mysql_versions(), MYSQL_INNOVATION_RST_LINK]) + kubernetes_versions = comma(kubernetes_major_minor_versions()) + return ( + "\n" + f"* Python: {python_versions}\n" + "\n" + "* Databases:\n" + "\n" + f" * PostgreSQL: {postgres_versions}\n" + f" * MySQL: {mysql_versions}\n" + f" * SQLite: {SQLITE_VERSION}\n" + "\n" + f"* Kubernetes: {kubernetes_versions}\n" + "\n" + ) + + +def dev_version_values() -> dict[str, str]: + """Generated value for each README row label whose dev column we keep in sync.""" + return { + "Python": comma(read_allowed_python_major_minor_versions()), + "Kubernetes": comma(kubernetes_major_minor_versions()), + "PostgreSQL": comma(read_current_postgres_versions()), + "MySQL": comma([*read_current_mysql_versions(), MYSQL_INNOVATION_ANNOTATION]), + } + + +def replace_text_between(text: str, start: str, end: str, replacement: str) -> str: + if start not in text or end not in text: + raise RuntimeError(f"Could not find markers {start!r} / {end!r} in the file") + leading = text.split(start)[0] + trailing = text.split(end)[1] + return leading + start + replacement + end + trailing + + +def update_prerequisites() -> bool: + """Regenerate the tested-versions block in prerequisites.rst. Return True if changed.""" + original = PREREQUISITES_RST.read_text() + updated = replace_text_between( + original, PREREQUISITES_START, PREREQUISITES_END, render_prerequisites_block() + ) + if updated != original: + PREREQUISITES_RST.write_text(updated) + return True + return False + + +def update_readme() -> bool: + """Sync the "Main version (dev)" column of the README Requirements table. + + Only the dev cell of the matched rows is rewritten; the label and the historical + "Stable version" columns are preserved byte-for-byte. The dev column is widened (and + only the dev column) if a generated value no longer fits, keeping the table aligned + without touching the stable columns. + """ + lines = README_MD.read_text().splitlines(keepends=True) + header_index = next( + (i for i, line in enumerate(lines) if README_DEV_HEADER in line and line.lstrip().startswith("|")), + None, + ) + if header_index is None: + raise RuntimeError( + f"Could not find the Requirements table header ({README_DEV_HEADER!r}) in README.md" + ) + + # The table spans contiguous lines starting with '|', header first then separator. + table_end = header_index + while table_end < len(lines) and lines[table_end].lstrip().startswith("|"): + table_end += 1 + table_rows = list(range(header_index, table_end)) + separator_index = header_index + 1 + + generated = dev_version_values() + + def cells(line: str) -> list[str]: + # Drop the empty fragments before the first and after the last pipe. + return line.rstrip("\n").split("|")[1:-1] + + # Determine the dev column width: max of the current width and every value we render. + current_width = len(cells(lines[header_index])[1]) - 2 + needed = [current_width] + for idx in table_rows: + if idx == separator_index: + continue + row = cells(lines[idx]) + label = row[0].strip() + value = generated.get(label, row[1].strip()) + needed.append(len(value)) + dev_width = max(needed) + + changed = False + for idx in table_rows: + row = cells(lines[idx]) + if idx == separator_index: + new_dev = "-" * (dev_width + 2) + else: + label = row[0].strip() + value = generated.get(label, row[1].strip()) + new_dev = f" {value.ljust(dev_width)} " + if new_dev == row[1]: + continue + row[1] = new_dev + lines[idx] = "|" + "|".join(row) + "|\n" + changed = True + + if changed: + README_MD.write_text("".join(lines)) + return changed + + +def main() -> int: + changed_files: list[Path] = [] + if update_prerequisites(): + changed_files.append(PREREQUISITES_RST) + if update_readme(): + changed_files.append(README_MD) + if changed_files: + for path in changed_files: + print(f"Updated tested versions in {path.relative_to(AIRFLOW_ROOT_PATH)}") + print("Files were re-generated from global_constants.py - please re-stage them.") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/ci/prek/test_common_prek_utils.py b/scripts/tests/ci/prek/test_common_prek_utils.py index f28961f900694..e23baabc542d2 100644 --- a/scripts/tests/ci/prek/test_common_prek_utils.py +++ b/scripts/tests/ci/prek/test_common_prek_utils.py @@ -33,6 +33,9 @@ pre_process_mypy_files, read_airflow_version, read_allowed_kubernetes_versions, + read_allowed_python_major_minor_versions, + read_current_mysql_versions, + read_current_postgres_versions, read_default_python_major_minor_version_for_images, resolve_github_token, retrieve_gh_token, @@ -386,6 +389,43 @@ def test_version_looks_like_major_minor(self): assert parts[1].isdigit() +class TestReadAllowedPythonMajorMinorVersions: + def test_returns_list_of_versions(self): + versions = read_allowed_python_major_minor_versions() + assert isinstance(versions, list) + assert len(versions) > 0 + + def test_versions_look_like_major_minor(self): + for version in read_allowed_python_major_minor_versions(): + parts = version.split(".") + assert len(parts) == 2, f"Version {version!r} should be major.minor" + assert parts[0].isdigit() + assert parts[1].isdigit() + + +class TestReadCurrentPostgresVersions: + def test_returns_list_of_versions(self): + versions = read_current_postgres_versions() + assert isinstance(versions, list) + assert len(versions) > 0 + + def test_versions_are_numeric_major_versions(self): + for version in read_current_postgres_versions(): + assert version.isdigit(), f"Postgres version {version!r} should be a numeric major version" + + +class TestReadCurrentMysqlVersions: + def test_returns_list_of_versions(self): + versions = read_current_mysql_versions() + assert isinstance(versions, list) + assert len(versions) > 0 + + def test_reads_annotated_assignment(self): + # MYSQL_LTS_RELEASES is an annotated assignment (``NAME: list[str] = [...]``); + # the reader must handle it, so its value must show up here. + assert "8.4" in read_current_mysql_versions() + + class TestConsoleDiff: def test_dump_added_lines(self): diff = ConsoleDiff() diff --git a/scripts/tests/ci/prek/test_update_tested_versions.py b/scripts/tests/ci/prek/test_update_tested_versions.py new file mode 100644 index 0000000000000..08afc23ae7293 --- /dev/null +++ b/scripts/tests/ci/prek/test_update_tested_versions.py @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import textwrap + +from ci.prek import update_tested_versions as hook +from ci.prek.common_prek_utils import ( + read_allowed_python_major_minor_versions, + read_current_postgres_versions, +) + + +def _dev_cell(line: str) -> str: + """The (stripped) "Main version (dev)" cell of a markdown table row.""" + return line.split("|")[1:-1][1].strip() + + +def _stable_cells(line: str) -> list[str]: + """The stable-column cells (everything after the dev column).""" + return [c.strip() for c in line.split("|")[1:-1][2:]] + + +class TestRenderPrerequisitesBlock: + def test_block_reflects_constants(self): + block = hook.render_prerequisites_block() + assert f"* Python: {', '.join(read_allowed_python_major_minor_versions())}\n" in block + assert f" * PostgreSQL: {', '.join(read_current_postgres_versions())}\n" in block + # SQLite has no central constant - it is rendered from the hook's own value. + assert f" * SQLite: {hook.SQLITE_VERSION}\n" in block + # MySQL always carries the editorial "Innovation" annotation. + assert "Innovation" in block + + def test_block_is_wrapped_by_markers_content_only(self): + # The rendered block must not itself contain the markers (those wrap it). + block = hook.render_prerequisites_block() + assert hook.PREREQUISITES_START not in block + assert hook.PREREQUISITES_END not in block + + +class TestUpdateReadme: + TABLE = textwrap.dedent( + """\ + ## Requirements + + | | Main version (dev) | Stable version (3.2.0) | Stable version (2.11.2) | + |------------|--------------------|------------------------|-------------------------| + | Python | WRONG | 3.10, 3.11 | 3.10 | + | Platform | AMD64/ARM64 | AMD64/ARM64 | AMD64/ARM64 | + | PostgreSQL | WRONG | 14, 15 | 12, 13 | + | SQLite | 3.15.0+ | 3.15.0+ | 3.15.0+ | + + text after the table + """ + ) + + def test_syncs_dev_column_and_preserves_stable_columns(self, tmp_path, monkeypatch): + readme = tmp_path / "README.md" + readme.write_text(self.TABLE) + monkeypatch.setattr(hook, "README_MD", readme) + + assert hook.update_readme() is True + + lines = readme.read_text().splitlines() + rows = {line.split("|")[1].strip(): line for line in lines if line.startswith("|")} + + expected_python = ", ".join(read_allowed_python_major_minor_versions()) + expected_postgres = ", ".join(read_current_postgres_versions()) + + # Dev column is now synced from the constants. + assert _dev_cell(rows["Python"]) == expected_python + assert _dev_cell(rows["PostgreSQL"]) == expected_postgres + + # Stable columns are untouched. + assert _stable_cells(rows["Python"]) == ["3.10, 3.11", "3.10"] + assert _stable_cells(rows["PostgreSQL"]) == ["14, 15", "12, 13"] + + # A non-target row keeps its dev value verbatim. + assert _dev_cell(rows["Platform"]) == "AMD64/ARM64" + + # Text outside the table is preserved. + assert "text after the table" in readme.read_text() + + def test_noop_when_already_synced(self, tmp_path, monkeypatch): + readme = tmp_path / "README.md" + readme.write_text(self.TABLE) + monkeypatch.setattr(hook, "README_MD", readme) + + # First pass syncs and reports a change; a second pass is a no-op. + assert hook.update_readme() is True + synced = readme.read_text() + assert hook.update_readme() is False + assert readme.read_text() == synced + + +class TestRepoDocsInSync: + """Regression guard: the committed docs must already match the constants.""" + + def test_prerequisites_in_sync(self): + assert hook.update_prerequisites() is False, ( + "prerequisites.rst is out of sync with global_constants.py - " + "run 'prek run update-tested-versions --all-files'" + ) + + def test_readme_in_sync(self): + assert hook.update_readme() is False, ( + "README.md tested-versions table is out of sync with global_constants.py - " + "run 'prek run update-tested-versions --all-files'" + )