From 2447310d3d4c103b8862198ada81a50d3db44950 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Thu, 27 Feb 2025 14:13:02 +0000 Subject: [PATCH 1/3] Add methodtools as a dependency of the common-sql provider Since `apache-airflow-providers-common-sql==1.23.0`, released on 26 February 2025, users started facing the error: ``` from cosmos.operators._asynchronous.bigquery import DbtRunAirflowAsyncBigqueryOperator cosmos/operators/_asynchronous/bigquery.py:8: in from airflow.providers.google.cloud.operators.bigquery import BigQueryInsertJobOperator ../../../.local/share/hatch/env/virtual/astronomer-cosmos/Za_bFbg4/tests.py3.12-2.9/lib/python3.12/site-packages/airflow/providers/google/cloud/operators/bigquery.py:32: in from airflow.providers.common.sql.operators.sql import ( # type: ignore[attr-defined] # for _parse_boolean ../../../.local/share/hatch/env/virtual/astronomer-cosmos/Za_bFbg4/tests.py3.12-2.9/lib/python3.12/site-packages/airflow/providers/common/sql/operators/sql.py:[29](https://github.com/astronomer/astronomer-cosmos/actions/runs/13544957799/job/37854234771#step:6:30): in from airflow.providers.common.sql.hooks.sql import DbApiHook, fetch_all_handler, return_single_query_results ../../../.local/share/hatch/env/virtual/astronomer-cosmos/Za_bFbg4/tests.py3.12-2.9/lib/python3.12/site-packages/airflow/providers/common/sql/hooks/sql.py:37: in from methodtools import lru_cache E ModuleNotFoundError: No module named 'methodtools' ``` It seems this started happening since https://github.com/apache/airflow/pull/41327. This PR aims to solve this issue. Closes: https://github.com/apache/airflow/issues/47147 --- providers/common/sql/provider.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/common/sql/provider.yaml b/providers/common/sql/provider.yaml index b4f74202301c8..09deb140280cc 100644 --- a/providers/common/sql/provider.yaml +++ b/providers/common/sql/provider.yaml @@ -65,6 +65,10 @@ versions: - 1.1.0 - 1.0.0 +dependencies: + # The methodtools dependency is necessary since the introduction of dialects: https://github.com/apache/airflow/pull/41327/files + - methodtools>=0.4.7 + integrations: - integration-name: Common SQL external-doc-url: https://en.wikipedia.org/wiki/SQL From 20e3a233a88e285a2b942dd5f93186603a70ea46 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Thu, 27 Feb 2025 14:29:38 +0000 Subject: [PATCH 2/3] Fix linting --- providers/common/sql/provider.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/common/sql/provider.yaml b/providers/common/sql/provider.yaml index 09deb140280cc..e86bbae1bde58 100644 --- a/providers/common/sql/provider.yaml +++ b/providers/common/sql/provider.yaml @@ -66,7 +66,8 @@ versions: - 1.0.0 dependencies: - # The methodtools dependency is necessary since the introduction of dialects: https://github.com/apache/airflow/pull/41327/files + # The methodtools dependency is necessary since the introduction of dialects: + # https://github.com/apache/airflow/pull/41327/files - methodtools>=0.4.7 integrations: From e7a6619ab9f1d3ddcdd6e1ca8d17f92a26af8aa9 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Thu, 27 Feb 2025 18:04:00 +0000 Subject: [PATCH 3/3] Revert change to provider.yaml and applyc hange to pyproject.toml --- generated/provider_dependencies.json | 1 + providers/common/sql/README.rst | 1 + providers/common/sql/provider.yaml | 5 ----- providers/common/sql/pyproject.toml | 3 +++ .../src/airflow/providers/common/sql/get_provider_info.py | 7 ++++++- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index e31d9c6af0cb0..f8986a4e260ba 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -419,6 +419,7 @@ "common.sql": { "deps": [ "apache-airflow>=2.9.0", + "methodtools>=0.4.7", "more-itertools>=9.0.0", "sqlparse>=0.5.1" ], diff --git a/providers/common/sql/README.rst b/providers/common/sql/README.rst index 1109e6a209fb3..eb58b17020894 100644 --- a/providers/common/sql/README.rst +++ b/providers/common/sql/README.rst @@ -56,6 +56,7 @@ PIP package Version required ``apache-airflow`` ``>=2.9.0`` ``sqlparse`` ``>=0.5.1`` ``more-itertools`` ``>=9.0.0`` +``methodtools`` ``>=0.4.7`` ================== ================== Cross provider package dependencies diff --git a/providers/common/sql/provider.yaml b/providers/common/sql/provider.yaml index e86bbae1bde58..b4f74202301c8 100644 --- a/providers/common/sql/provider.yaml +++ b/providers/common/sql/provider.yaml @@ -65,11 +65,6 @@ versions: - 1.1.0 - 1.0.0 -dependencies: - # The methodtools dependency is necessary since the introduction of dialects: - # https://github.com/apache/airflow/pull/41327/files - - methodtools>=0.4.7 - integrations: - integration-name: Common SQL external-doc-url: https://en.wikipedia.org/wiki/SQL diff --git a/providers/common/sql/pyproject.toml b/providers/common/sql/pyproject.toml index 936cf20debbbf..d405cadf018ad 100644 --- a/providers/common/sql/pyproject.toml +++ b/providers/common/sql/pyproject.toml @@ -58,6 +58,9 @@ dependencies = [ "apache-airflow>=2.9.0", "sqlparse>=0.5.1", "more-itertools>=9.0.0", + # The methodtools dependency is necessary since the introduction of dialects: + # https://github.com/apache/airflow/pull/41327/files + "methodtools>=0.4.7" ] # The optional dependencies should be modified in place in the generated file diff --git a/providers/common/sql/src/airflow/providers/common/sql/get_provider_info.py b/providers/common/sql/src/airflow/providers/common/sql/get_provider_info.py index a7882ee78f709..9ca42a29cd9d4 100644 --- a/providers/common/sql/src/airflow/providers/common/sql/get_provider_info.py +++ b/providers/common/sql/src/airflow/providers/common/sql/get_provider_info.py @@ -111,7 +111,12 @@ def get_provider_info(): "sensors": [ {"integration-name": "Common SQL", "python-modules": ["airflow.providers.common.sql.sensors.sql"]} ], - "dependencies": ["apache-airflow>=2.9.0", "sqlparse>=0.5.1", "more-itertools>=9.0.0"], + "dependencies": [ + "apache-airflow>=2.9.0", + "sqlparse>=0.5.1", + "more-itertools>=9.0.0", + "methodtools>=0.4.7", + ], "optional-dependencies": { "pandas": ["pandas>=2.1.2,<2.2"], "openlineage": ["apache-airflow-providers-openlineage"],