From 6912815b8a57d2e6437db10e2c279c813b964a37 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 27 Jul 2026 18:25:05 +0200 Subject: [PATCH 1/3] Keep DataFrame XComs working on pandas 3 pandas 3 exposes its public classes from the `pandas` namespace, so a DataFrame is now qualified as `pandas.DataFrame` rather than `pandas.core.frame.DataFrame`. The serializer was registered only under the old name, so pushing a DataFrame through XCom raised "cannot serialize object of type ". Both names are registered so values written by either version stay readable. pandas 3 also infers a str column where it used to infer object, and keeps its missing values as NA instead of stringifying them, which the amazon and salesforce tests asserted on. --- .../tests/unit/amazon/aws/transfers/test_sql_to_s3.py | 10 +++++++++- .../tests/unit/salesforce/hooks/test_salesforce.py | 9 +++++++-- task-sdk/src/airflow/sdk/serde/serializers/pandas.py | 4 ++++ task-sdk/tests/task_sdk/serde/test_serializers.py | 10 ++++++++-- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py b/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py index 5fb50fff9fe60..3a960d1fd97d9 100644 --- a/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py +++ b/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py @@ -24,12 +24,15 @@ import pandas as pd import polars as pl import pytest +from packaging.version import Version from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.amazon.aws.transfers.sql_to_s3 import SqlToS3Operator from airflow.providers.common.compat.sdk import AirflowException +PANDAS_3_PLUS = Version(pd.__version__).major >= 3 + class TestSqlToS3Operator: @pytest.mark.parametrize( @@ -156,7 +159,12 @@ def test_fix_dtypes(self, params): ) dirty_df = pd.DataFrame({"strings": ["a", "b", None], "ints": [1, 2, None]}) op._fix_dtypes(df=dirty_df, file_format=op.file_format) - assert dirty_df["strings"].values[2] == params["null_string_result"] + if PANDAS_3_PLUS: + # pandas 3 infers a str column rather than object, and keeps its missing values as NA + # instead of the object None (csv) or the "None" it used to be stringified to (parquet) + assert pd.isna(dirty_df["strings"].values[2]) + else: + assert dirty_df["strings"].values[2] == params["null_string_result"] assert dirty_df["ints"].dtype.kind == "i" @mock.patch("airflow.providers.amazon.aws.transfers.sql_to_s3.S3Hook") diff --git a/providers/salesforce/tests/unit/salesforce/hooks/test_salesforce.py b/providers/salesforce/tests/unit/salesforce/hooks/test_salesforce.py index 39e2e41fde6eb..923d59a39569e 100644 --- a/providers/salesforce/tests/unit/salesforce/hooks/test_salesforce.py +++ b/providers/salesforce/tests/unit/salesforce/hooks/test_salesforce.py @@ -24,12 +24,15 @@ import numpy as np import pandas as pd import pytest +from packaging.version import Version from requests import Session as request_session from simple_salesforce import Salesforce, api from airflow.models.connection import Connection from airflow.providers.salesforce.hooks.salesforce import SalesforceHook +PANDAS_3_PLUS = Version(pd.__version__).major >= 3 + class TestSalesforceHook: def setup_method(self): @@ -349,10 +352,12 @@ def test_write_object_to_file_csv(self, mock_data_frame): data_frame = self.salesforce_hook.write_object_to_file(query_results=[], filename=filename, fmt="csv") mock_data_frame.return_value.to_csv.assert_called_once_with(filename, index=False) - # Note that the latest version of pandas dataframes (1.1.2) returns "nan" rather than "None" here + # Note that the latest version of pandas dataframes (1.1.2) returns "nan" rather than "None" here, + # and pandas 3 keeps missing values as NA instead of stringifying them to "nan" at all + missing = np.nan if PANDAS_3_PLUS else "nan" pd.testing.assert_frame_equal( data_frame, - pd.DataFrame({"test": [1, 2, 3], "dict": ["nan", "nan", str({"foo": "bar"})]}), + pd.DataFrame({"test": [1, 2, 3], "dict": [missing, missing, str({"foo": "bar"})]}), check_index_type=False, ) diff --git a/task-sdk/src/airflow/sdk/serde/serializers/pandas.py b/task-sdk/src/airflow/sdk/serde/serializers/pandas.py index 72a2e818a1132..aaa68580abe9f 100644 --- a/task-sdk/src/airflow/sdk/serde/serializers/pandas.py +++ b/task-sdk/src/airflow/sdk/serde/serializers/pandas.py @@ -22,7 +22,11 @@ from airflow.sdk.module_loading import qualname # lazy loading for performance reasons +# pandas 3 moved the public classes to the `pandas` namespace, so a DataFrame is qualified as +# `pandas.DataFrame` there and as `pandas.core.frame.DataFrame` on pandas 2. Both are registered so +# that XComs serialized by either version stay readable. serializers = [ + "pandas.DataFrame", "pandas.core.frame.DataFrame", ] deserializers = serializers diff --git a/task-sdk/tests/task_sdk/serde/test_serializers.py b/task-sdk/tests/task_sdk/serde/test_serializers.py index 7e2ade64ebdcf..04510eeb1af6b 100644 --- a/task-sdk/tests/task_sdk/serde/test_serializers.py +++ b/task-sdk/tests/task_sdk/serde/test_serializers.py @@ -289,12 +289,18 @@ def test_pandas_serializers(self): @pytest.mark.parametrize( ("klass", "version", "data", "msg"), [ - (pd.DataFrame, 999, "", r"serialized 999 of pandas.core.frame.DataFrame > 1"), # version too new + # pandas 3 qualifies the class as pandas.DataFrame, pandas 2 as pandas.core.frame.DataFrame + ( + pd.DataFrame, + 999, + "", + r"serialized 999 of pandas(\.core\.frame)?\.DataFrame > 1", + ), # version too new ( pd.DataFrame, 1, 123, - r"serialized pandas.core.frame.DataFrame has wrong data type .*", + r"serialized pandas(\.core\.frame)?\.DataFrame has wrong data type .*", ), # bad payload type (str, 1, "", r"do not know how to deserialize builtins.str"), # bad class ], From 3afb8f9879df17f21e3b78512f011c5be9291a2b Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 27 Jul 2026 18:51:10 +0200 Subject: [PATCH 2/3] Document pandas 3 impact on DataFrame XComs Deployments need to know that every component has to carry the pandas 3 support before pandas 3 reaches any worker, that a rollback strands the XComs written in the meantime, and that a pulled DataFrame now takes its dtypes from the reader's pandas version. --- .../newsfragments/70501.significant.rst | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 airflow-core/newsfragments/70501.significant.rst diff --git a/airflow-core/newsfragments/70501.significant.rst b/airflow-core/newsfragments/70501.significant.rst new file mode 100644 index 0000000000000..9dd43544b0dfb --- /dev/null +++ b/airflow-core/newsfragments/70501.significant.rst @@ -0,0 +1,33 @@ +pandas 3 changes how DataFrame XComs are stored and read back + +pandas 3 exposes its public classes from the ``pandas`` namespace, so a DataFrame is qualified as +``pandas.DataFrame`` instead of ``pandas.core.frame.DataFrame``. XComs record that name alongside the +serialized value, so the name written into the metadata database depends on the pandas version of the +component that pushed the value. Airflow registers both names, and a DataFrame written by either +pandas version can be read by either — no configuration change is needed, and existing XComs stay +readable. + +What you should do: + +* **Roll this Airflow version out to every component before pandas 3 reaches any of them** — workers + in particular. A component that predates this change cannot read a DataFrame XCom written under + pandas 3, and fails the pull with: + + .. code-block:: text + + ImportError: pandas.DataFrame was not found in allow list for deserialization imports. + To allow it, add it to allowed_deserialization_classes in the configuration + + The message points at configuration, but the allow list is not the cause and changing it does not + help. The rows are not corrupt: they become readable again as soon as the reader is upgraded. + +* **Treat a downgrade as a one-way door for those XComs.** Rolling back to an Airflow version without + this change strands any DataFrame XCom written while on pandas 3, with the same error, until you + roll forward again. + +* **Review Dags that inspect the dtypes of a pulled DataFrame.** The pandas version of the *reader* + determines what a pulled DataFrame looks like, not the version that wrote it. Under pandas 3, a + column of strings comes back with the ``str`` dtype rather than ``object``, and its missing values + come back as ``nan`` rather than ``None``. Values are unchanged, but downstream code that branches + on ``dtype == "object"``, checks cells with ``is None``, or compares against a reference frame with + ``DataFrame.equals()`` can behave differently after the upgrade. From 0779e0a2b1077e5f931ec4d7de3aa30f14b3ac3c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 27 Jul 2026 21:04:22 +0200 Subject: [PATCH 3/3] Name the pandas 3 newsfragment after its own pull request --- .../{70501.significant.rst => 70558.significant.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename airflow-core/newsfragments/{70501.significant.rst => 70558.significant.rst} (100%) diff --git a/airflow-core/newsfragments/70501.significant.rst b/airflow-core/newsfragments/70558.significant.rst similarity index 100% rename from airflow-core/newsfragments/70501.significant.rst rename to airflow-core/newsfragments/70558.significant.rst