Automate syncing documented tested versions with the test matrix

.pre-commit-config.yaml

@@ -707,6 +707,12 @@ repos:
           ^scripts/ci/prek/extract_permissions\.py$|
           ^airflow-core/docs/security/api_permissions_ref\.rst$
         pass_filenames: false
+      - id: update-tested-versions
+        name: Update tested Python/DB/Kubernetes versions in docs
+        entry: ./scripts/ci/prek/update_tested_versions.py
+        language: python
+        files: ^airflow-core/docs/installation/prerequisites\.rst$|^README\.md$|^dev/breeze/src/airflow_breeze/global_constants\.py$|^scripts/ci/prek/update_tested_versions\.py$|^scripts/ci/prek/common_prek_utils\.py$
+        pass_filenames: false
       - id: check-revision-heads-map
         name: Check that the REVISION_HEADS_MAP is up-to-date
         language: python

airflow-core/docs/installation/prerequisites.rst

@@ -20,6 +20,8 @@ Prerequisites
 
 Airflow® is tested with:
 
+ .. Beginning of the auto-generated tested versions
+
 * Python: 3.10, 3.11, 3.12, 3.13, 3.14
 
 * Databases:
@@ -30,6 +32,8 @@ Airflow® is tested with:
 
 * Kubernetes: 1.30, 1.31, 1.32, 1.33, 1.34, 1.35
 
+ .. End of the auto-generated tested versions
+
 While we recommend a minimum of 4GB of memory for Airflow, the actual requirements heavily depend on your chosen deployment.
 
 .. warning::

scripts/ci/prek/common_prek_utils.py

@@ -133,13 +133,21 @@ def read_airflow_version() -> str:
 
 
 def _read_global_constants_assignment(name: str) -> Any:
-    """Read a top-level assignment from global_constants.py."""
+    """Read a top-level assignment from global_constants.py.
+
+    Handles both plain assignments (``NAME = ...``) and annotated assignments
+    (``NAME: type = ...``). The value must be a literal so it can be safely
+    evaluated with ``ast.literal_eval``.
+    """
     tree = ast.parse(GLOBAL_CONSTANTS_PATH.read_text())
     for node in tree.body:
         if isinstance(node, ast.Assign):
             for target in node.targets:
                 if isinstance(target, ast.Name) and target.id == name:
                     return ast.literal_eval(node.value)
+        elif isinstance(node, ast.AnnAssign):
+            if isinstance(node.target, ast.Name) and node.target.id == name and node.value is not None:
+                return ast.literal_eval(node.value)
     raise RuntimeError(f"{name} not found in global_constants.py")
 
 
@@ -152,6 +160,32 @@ def read_allowed_kubernetes_versions() -> list[str]:
     return [v.lstrip("v") for v in versions]
 
 
+def read_allowed_python_major_minor_versions() -> list[str]:
+    """Parse ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS from global_constants.py (single source of truth)."""
+    return list(_read_global_constants_assignment("ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS"))
+
+
+def read_current_postgres_versions() -> list[str]:
+    """Parse CURRENT_POSTGRES_VERSIONS from global_constants.py (single source of truth)."""
+    return list(_read_global_constants_assignment("CURRENT_POSTGRES_VERSIONS"))
+
+
+def read_current_mysql_versions() -> list[str]:
+    """The MySQL release versions Airflow currently tests with.
+
+    Mirrors how ``CURRENT_MYSQL_VERSIONS`` is built in global_constants.py: the
+    "old" releases plus the LTS releases, plus an innovation release when one is
+    configured. Returns the numeric versions only (e.g. ``["8.0", "8.4"]``); the
+    docs add the textual "Innovation" annotation on top of these.
+    """
+    versions: list[str] = list(_read_global_constants_assignment("MYSQL_OLD_RELEASES"))
+    versions += list(_read_global_constants_assignment("MYSQL_LTS_RELEASES"))
+    innovation = _read_global_constants_assignment("MYSQL_INNOVATION_RELEASE")
+    if innovation:
+        versions.append(innovation)
+    return versions
+
+
 def read_default_python_major_minor_version_for_images() -> str:
     """Parse DEFAULT_PYTHON_MAJOR_MINOR_VERSION_FOR_IMAGES from global_constants.py."""
     value = _read_global_constants_assignment("DEFAULT_PYTHON_MAJOR_MINOR_VERSION_FOR_IMAGES")

scripts/ci/prek/update_tested_versions.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# /// script
+# requires-python = ">=3.10,<3.11"
+# dependencies = []
+# ///
+"""Keep the documented "tested with" versions in sync with global_constants.py.
+
+The list of Python, database and Kubernetes versions Airflow is tested with lives
+in a single source of truth (``dev/breeze/src/airflow_breeze/global_constants.py``)
+but is also rendered for humans in two hand-maintained places:
+
+* ``airflow-core/docs/installation/prerequisites.rst`` — the "Prerequisites" bullet list.
+* ``README.md`` — the "Main version (dev)" column of the Requirements table.
+
+These used to drift out of sync silently (e.g. PostgreSQL 18 was added to the test
+matrix and the README but the prerequisites doc was forgotten). This hook regenerates
+both from the constants so the drift becomes a (auto-fixable) prek failure instead of
+a documentation bug nobody notices.
+
+Only versions that have a clean single source in global_constants.py are generated:
+Python, PostgreSQL, MySQL (numeric releases) and Kubernetes. SQLite has no constant
+and the MySQL "Innovation" annotation is editorial, so both are defined here.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from common_prek_utils import (
+    AIRFLOW_ROOT_PATH,
+    read_allowed_kubernetes_versions,
+    read_allowed_python_major_minor_versions,
+    read_current_mysql_versions,
+    read_current_postgres_versions,
+)
+
+PREREQUISITES_RST = AIRFLOW_ROOT_PATH / "airflow-core" / "docs" / "installation" / "prerequisites.rst"
+README_MD = AIRFLOW_ROOT_PATH / "README.md"
+
+PREREQUISITES_START = " .. Beginning of the auto-generated tested versions\n"
+PREREQUISITES_END = " .. End of the auto-generated tested versions\n"
+
+README_DEV_HEADER = "Main version (dev)"
+
+# Editorial values without a single source of truth in global_constants.py.
+SQLITE_VERSION = "3.15.0+"
+MYSQL_INNOVATION_ANNOTATION = "Innovation"
+MYSQL_INNOVATION_RST_LINK = (
+    "`Innovation <https://dev.mysql.com/blog-archive/"
+    "introducing-mysql-innovation-and-long-term-support-lts-versions>`_"
+)
+
+
+def kubernetes_major_minor_versions() -> list[str]:
+    """Kubernetes versions reduced to ``major.minor`` (e.g. ``1.30``) for display."""
+    return [".".join(version.split(".")[:2]) for version in read_allowed_kubernetes_versions()]
+
+
+def comma(values: list[str]) -> str:
+    return ", ".join(values)
+
+
+def render_prerequisites_block() -> str:
+    """The bullet list rendered between the prerequisites.rst markers."""
+    python_versions = comma(read_allowed_python_major_minor_versions())
+    postgres_versions = comma(read_current_postgres_versions())
+    mysql_versions = comma([*read_current_mysql_versions(), MYSQL_INNOVATION_RST_LINK])
+    kubernetes_versions = comma(kubernetes_major_minor_versions())
+    return (
+        "\n"
+        f"* Python: {python_versions}\n"
+        "\n"
+        "* Databases:\n"
+        "\n"
+        f"  * PostgreSQL: {postgres_versions}\n"
+        f"  * MySQL: {mysql_versions}\n"
+        f"  * SQLite: {SQLITE_VERSION}\n"
+        "\n"
+        f"* Kubernetes: {kubernetes_versions}\n"
+        "\n"
+    )
+
+
+def dev_version_values() -> dict[str, str]:
+    """Generated value for each README row label whose dev column we keep in sync."""
+    return {
+        "Python": comma(read_allowed_python_major_minor_versions()),
+        "Kubernetes": comma(kubernetes_major_minor_versions()),
+        "PostgreSQL": comma(read_current_postgres_versions()),
+        "MySQL": comma([*read_current_mysql_versions(), MYSQL_INNOVATION_ANNOTATION]),
+    }
+
+
+def replace_text_between(text: str, start: str, end: str, replacement: str) -> str:
+    if start not in text or end not in text:
+        raise RuntimeError(f"Could not find markers {start!r} / {end!r} in the file")
+    leading = text.split(start)[0]
+    trailing = text.split(end)[1]
+    return leading + start + replacement + end + trailing
+
+
+def update_prerequisites() -> bool:
+    """Regenerate the tested-versions block in prerequisites.rst. Return True if changed."""
+    original = PREREQUISITES_RST.read_text()
+    updated = replace_text_between(
+        original, PREREQUISITES_START, PREREQUISITES_END, render_prerequisites_block()
+    )
+    if updated != original:
+        PREREQUISITES_RST.write_text(updated)
+        return True
+    return False
+
+
+def update_readme() -> bool:
+    """Sync the "Main version (dev)" column of the README Requirements table.
+
+    Only the dev cell of the matched rows is rewritten; the label and the historical
+    "Stable version" columns are preserved byte-for-byte. The dev column is widened (and
+    only the dev column) if a generated value no longer fits, keeping the table aligned
+    without touching the stable columns.
+    """
+    lines = README_MD.read_text().splitlines(keepends=True)
+    header_index = next(
+        (i for i, line in enumerate(lines) if README_DEV_HEADER in line and line.lstrip().startswith("|")),
+        None,
+    )
+    if header_index is None:
+        raise RuntimeError(
+            f"Could not find the Requirements table header ({README_DEV_HEADER!r}) in README.md"
+        )
+
+    # The table spans contiguous lines starting with '|', header first then separator.
+    table_end = header_index
+    while table_end < len(lines) and lines[table_end].lstrip().startswith("|"):
+        table_end += 1
+    table_rows = list(range(header_index, table_end))
+    separator_index = header_index + 1
+
+    generated = dev_version_values()
+
+    def cells(line: str) -> list[str]:
+        # Drop the empty fragments before the first and after the last pipe.
+        return line.rstrip("\n").split("|")[1:-1]
+
+    # Determine the dev column width: max of the current width and every value we render.
+    current_width = len(cells(lines[header_index])[1]) - 2
+    needed = [current_width]
+    for idx in table_rows:
+        if idx == separator_index:
+            continue
+        row = cells(lines[idx])
+        label = row[0].strip()
+        value = generated.get(label, row[1].strip())
+        needed.append(len(value))
+    dev_width = max(needed)
+
+    changed = False
+    for idx in table_rows:
+        row = cells(lines[idx])
+        if idx == separator_index:
+            new_dev = "-" * (dev_width + 2)
+        else:
+            label = row[0].strip()
+            value = generated.get(label, row[1].strip())
+            new_dev = f" {value.ljust(dev_width)} "
+        if new_dev == row[1]:
+            continue
+        row[1] = new_dev
+        lines[idx] = "|" + "|".join(row) + "|\n"
+        changed = True
+
+    if changed:
+        README_MD.write_text("".join(lines))
+    return changed
+
+
+def main() -> int:
+    changed_files: list[Path] = []
+    if update_prerequisites():
+        changed_files.append(PREREQUISITES_RST)
+    if update_readme():
+        changed_files.append(README_MD)
+    if changed_files:
+        for path in changed_files:
+            print(f"Updated tested versions in {path.relative_to(AIRFLOW_ROOT_PATH)}")
+        print("Files were re-generated from global_constants.py - please re-stage them.")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/tests/ci/prek/test_common_prek_utils.py

@@ -32,6 +32,9 @@
     pre_process_mypy_files,
     read_airflow_version,
     read_allowed_kubernetes_versions,
+    read_allowed_python_major_minor_versions,
+    read_current_mysql_versions,
+    read_current_postgres_versions,
     read_default_python_major_minor_version_for_images,
     resolve_github_token,
     retrieve_gh_token,
@@ -370,6 +373,43 @@ def test_version_looks_like_major_minor(self):
         assert parts[1].isdigit()
 
 
+class TestReadAllowedPythonMajorMinorVersions:
+    def test_returns_list_of_versions(self):
+        versions = read_allowed_python_major_minor_versions()
+        assert isinstance(versions, list)
+        assert len(versions) > 0
+
+    def test_versions_look_like_major_minor(self):
+        for version in read_allowed_python_major_minor_versions():
+            parts = version.split(".")
+            assert len(parts) == 2, f"Version {version!r} should be major.minor"
+            assert parts[0].isdigit()
+            assert parts[1].isdigit()
+
+
+class TestReadCurrentPostgresVersions:
+    def test_returns_list_of_versions(self):
+        versions = read_current_postgres_versions()
+        assert isinstance(versions, list)
+        assert len(versions) > 0
+
+    def test_versions_are_numeric_major_versions(self):
+        for version in read_current_postgres_versions():
+            assert version.isdigit(), f"Postgres version {version!r} should be a numeric major version"
+
+
+class TestReadCurrentMysqlVersions:
+    def test_returns_list_of_versions(self):
+        versions = read_current_mysql_versions()
+        assert isinstance(versions, list)
+        assert len(versions) > 0
+
+    def test_reads_annotated_assignment(self):
+        # MYSQL_LTS_RELEASES is an annotated assignment (``NAME: list[str] = [...]``);
+        # the reader must handle it, so its value must show up here.
+        assert "8.4" in read_current_mysql_versions()
+
+
 class TestConsoleDiff:
     def test_dump_added_lines(self):
         diff = ConsoleDiff()