From ec31a5a3322dd3d7d201f00515d302b49e5e22eb Mon Sep 17 00:00:00 2001 From: Dev-iL <6509619+Dev-iL@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:50:30 +0300 Subject: [PATCH] Move asyncpg to an opt-in extra and default the postgres provider to psycopg3 The async metadata-database driver shipped by the postgres provider now defaults to psycopg3, which is safe behind transaction-mode PgBouncer with no extra configuration. asyncpg moves to the opt-in `asyncpg` extra and remains fully supported via an explicit sql_alchemy_conn_async URL. This pairs with the matching default change in Airflow 3.4.0. --- providers/postgres/README.rst | 3 ++- providers/postgres/docs/changelog.rst | 14 +++++++++++++ providers/postgres/docs/index.rst | 3 ++- providers/postgres/pyproject.toml | 13 +++++++----- .../providers/postgres/hooks/postgres.py | 20 ++----------------- uv.lock | 19 +++++++++--------- 6 files changed, 38 insertions(+), 34 deletions(-) diff --git a/providers/postgres/README.rst b/providers/postgres/README.rst index 3f8b66862114f..a53a92faef0fb 100644 --- a/providers/postgres/README.rst +++ b/providers/postgres/README.rst @@ -58,7 +58,8 @@ PIP package Version required ``apache-airflow-providers-common-sql`` ``>=1.32.0`` ``psycopg2-binary`` ``>=2.9.9; python_version < "3.13"`` ``psycopg2-binary`` ``>=2.9.10; python_version >= "3.13"`` -``asyncpg`` ``>=0.30.0`` +``psycopg[binary]`` ``>=3.2.9; python_version < "3.14"`` +``psycopg[binary]`` ``>=3.3.3; python_version >= "3.14"`` ========================================== ====================================== Optional cross provider package dependencies diff --git a/providers/postgres/docs/changelog.rst b/providers/postgres/docs/changelog.rst index 934ded27a23a7..74effae5e26db 100644 --- a/providers/postgres/docs/changelog.rst +++ b/providers/postgres/docs/changelog.rst @@ -27,6 +27,20 @@ Changelog --------- +.. warning:: + ``asyncpg`` is no longer installed by default — it moved to the ``asyncpg`` optional extra, and + ``psycopg`` (psycopg3) is now installed by default as the async metadata-database driver. The + default derived async connection URL changed from ``postgresql+asyncpg://`` to + ``postgresql+psycopg_async://``, which is safe behind transaction-mode PgBouncer with no extra + configuration. This aligns with the new default async Postgres driver in Airflow 3.4.0. To keep + using asyncpg, install ``apache-airflow-providers-postgres[asyncpg]`` and set + ``[database] sql_alchemy_conn_async = postgresql+asyncpg://...`` explicitly. + + On Airflow versions before 3.4.0 the derived async URL always uses asyncpg (there is no psycopg3 fallback in core), + so on those versions this provider release requires either the ``asyncpg`` extra or an explicitly configured + ``[database] sql_alchemy_conn_async`` — otherwise the async metadata-database engine fails at startup with + ``ModuleNotFoundError: No module named 'asyncpg'``. + 6.8.0 ..... diff --git a/providers/postgres/docs/index.rst b/providers/postgres/docs/index.rst index eef9e8e5eeee3..b10aa40df7646 100644 --- a/providers/postgres/docs/index.rst +++ b/providers/postgres/docs/index.rst @@ -106,7 +106,8 @@ PIP package Version required ``apache-airflow-providers-common-sql`` ``>=1.32.0`` ``psycopg2-binary`` ``>=2.9.9; python_version < "3.13"`` ``psycopg2-binary`` ``>=2.9.10; python_version >= "3.13"`` -``asyncpg`` ``>=0.30.0`` +``psycopg[binary]`` ``>=3.2.9; python_version < "3.14"`` +``psycopg[binary]`` ``>=3.3.3; python_version >= "3.14"`` ========================================== ====================================== Optional cross provider package dependencies diff --git a/providers/postgres/pyproject.toml b/providers/postgres/pyproject.toml index 83b2b92f61053..8adda0b57802b 100644 --- a/providers/postgres/pyproject.toml +++ b/providers/postgres/pyproject.toml @@ -62,9 +62,12 @@ dependencies = [ "apache-airflow>=2.11.0", "apache-airflow-providers-common-compat>=1.12.0", "apache-airflow-providers-common-sql>=1.32.0", + # psycopg2 remains the sync driver for now; its removal and the sync migration to + # psycopg3 are tracked at https://github.com/apache/airflow/issues/68453 "psycopg2-binary>=2.9.9; python_version < '3.13'", "psycopg2-binary>=2.9.10; python_version >= '3.13'", - "asyncpg>=0.30.0", + "psycopg[binary]>=3.2.9; python_version < '3.14'", + "psycopg[binary]>=3.3.3; python_version >= '3.14'", ] # The optional dependencies should be modified in place in the generated file @@ -73,6 +76,9 @@ dependencies = [ "amazon" = [ "apache-airflow-providers-amazon>=2.6.0", ] +"asyncpg" = [ + "asyncpg>=0.30.0", +] "microsoft.azure" = [ "apache-airflow-providers-microsoft-azure>=12.8.0" ] @@ -87,10 +93,6 @@ dependencies = [ "polars" = [ "polars>=1.26.0" ] -"psycopg" = [ - "psycopg[binary]>=3.2.9; python_version < '3.14'", - "psycopg[binary]>=3.3.3; python_version >= '3.14'", -] "sqlalchemy" = [ "sqlalchemy>=1.4.54" ] @@ -108,6 +110,7 @@ dev = [ # Additional devel dependencies (do not remove this line and add extra development dependencies) "apache-airflow-providers-common-sql[pandas]", "apache-airflow-providers-common-sql[polars]", + "apache-airflow-providers-postgres[asyncpg]", "apache-airflow-providers-postgres[sqlalchemy]" ] diff --git a/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py b/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py index 77ca0537b934d..be6519ddade24 100644 --- a/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py +++ b/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py @@ -70,30 +70,14 @@ class CompatConnection(Protocol): - """Protocol for type hinting psycopg2 and psycopg3 connection objects.""" + """Protocol for the common interface shared by psycopg2 and psycopg3 connection objects.""" def cursor(self, *args, **kwargs) -> Any: ... def commit(self) -> None: ... def close(self) -> None: ... - - # Context manager support - def __enter__(self) -> CompatConnection: ... + def __enter__(self) -> Any: ... def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: ... - # Common properties - @property - def notices(self) -> list[Any]: ... - - # psycopg3 specific (optional) - @property - def adapters(self) -> Any: ... - - @property - def row_factory(self) -> Any: ... - - # Optional method for psycopg3 - def add_notice_handler(self, handler: Any) -> None: ... - class PostgresHook(DbApiHook): """ diff --git a/uv.lock b/uv.lock index eac0d996a7baf..d788dfc1e96e1 100644 --- a/uv.lock +++ b/uv.lock @@ -7075,7 +7075,7 @@ dependencies = [ { name = "apache-airflow" }, { name = "apache-airflow-providers-common-compat" }, { name = "apache-airflow-providers-common-sql" }, - { name = "asyncpg" }, + { name = "psycopg", extra = ["binary"] }, { name = "psycopg2-binary" }, ] @@ -7083,6 +7083,9 @@ dependencies = [ amazon = [ { name = "apache-airflow-providers-amazon" }, ] +asyncpg = [ + { name = "asyncpg" }, +] microsoft-azure = [ { name = "apache-airflow-providers-microsoft-azure" }, ] @@ -7095,9 +7098,6 @@ pandas = [ polars = [ { name = "polars" }, ] -psycopg = [ - { name = "psycopg", extra = ["binary"] }, -] sqlalchemy = [ { name = "sqlalchemy" }, ] @@ -7111,7 +7111,7 @@ dev = [ { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, { name = "apache-airflow-providers-microsoft-azure" }, { name = "apache-airflow-providers-openlineage" }, - { name = "apache-airflow-providers-postgres", extra = ["sqlalchemy"] }, + { name = "apache-airflow-providers-postgres", extra = ["asyncpg", "sqlalchemy"] }, { name = "apache-airflow-task-sdk" }, ] docs = [ @@ -7126,18 +7126,18 @@ requires-dist = [ { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "apache-airflow-providers-microsoft-azure", marker = "extra == 'microsoft-azure'", editable = "providers/microsoft/azure" }, { name = "apache-airflow-providers-openlineage", marker = "extra == 'openlineage'", editable = "providers/openlineage" }, - { name = "asyncpg", specifier = ">=0.30.0" }, + { name = "asyncpg", marker = "extra == 'asyncpg'", specifier = ">=0.30.0" }, { name = "pandas", marker = "python_full_version == '3.13.*' and extra == 'pandas'", specifier = ">=2.2.3" }, { name = "pandas", marker = "python_full_version < '3.13' and extra == 'pandas'", specifier = ">=2.1.2" }, { name = "pandas", marker = "python_full_version >= '3.14' and extra == 'pandas'", specifier = ">=2.3.3" }, { name = "polars", marker = "extra == 'polars'", specifier = ">=1.26.0" }, - { name = "psycopg", extras = ["binary"], marker = "python_full_version >= '3.14' and extra == 'psycopg'", specifier = ">=3.3.3" }, - { name = "psycopg", extras = ["binary"], marker = "python_full_version < '3.14' and extra == 'psycopg'", specifier = ">=3.2.9" }, + { name = "psycopg", extras = ["binary"], marker = "python_full_version < '3.14'", specifier = ">=3.2.9" }, + { name = "psycopg", extras = ["binary"], marker = "python_full_version >= '3.14'", specifier = ">=3.3.3" }, { name = "psycopg2-binary", marker = "python_full_version < '3.13'", specifier = ">=2.9.9" }, { name = "psycopg2-binary", marker = "python_full_version >= '3.13'", specifier = ">=2.9.10" }, { name = "sqlalchemy", marker = "extra == 'sqlalchemy'", specifier = ">=1.4.54" }, ] -provides-extras = ["amazon", "microsoft-azure", "openlineage", "pandas", "polars", "psycopg", "sqlalchemy"] +provides-extras = ["amazon", "asyncpg", "microsoft-azure", "openlineage", "pandas", "polars", "sqlalchemy"] [package.metadata.requires-dev] dev = [ @@ -7150,6 +7150,7 @@ dev = [ { name = "apache-airflow-providers-common-sql", extras = ["polars"], editable = "providers/common/sql" }, { name = "apache-airflow-providers-microsoft-azure", editable = "providers/microsoft/azure" }, { name = "apache-airflow-providers-openlineage", editable = "providers/openlineage" }, + { name = "apache-airflow-providers-postgres", extras = ["asyncpg"], editable = "providers/postgres" }, { name = "apache-airflow-providers-postgres", extras = ["sqlalchemy"], editable = "providers/postgres" }, { name = "apache-airflow-task-sdk", editable = "task-sdk" }, ]