From af650e8d0f5d2b376abf3cb1634ae08d249e325e Mon Sep 17 00:00:00 2001 From: Amogh Atreya Date: Fri, 24 Jul 2026 11:11:06 +0530 Subject: [PATCH 1/2] fix(providers): move S3 and GCE operator template validation out of __init__ (#70296) --- generated/known_airflow_exceptions.txt | 2 +- .../providers/amazon/aws/operators/s3.py | 5 --- .../unit/amazon/aws/operators/test_s3.py | 40 +++++++++---------- .../validate_operators_init_exemptions.txt | 1 - 4 files changed, 21 insertions(+), 27 deletions(-) diff --git a/generated/known_airflow_exceptions.txt b/generated/known_airflow_exceptions.txt index 1fb461854905d..66ee6590bc84e 100644 --- a/generated/known_airflow_exceptions.txt +++ b/generated/known_airflow_exceptions.txt @@ -85,7 +85,7 @@ providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::4 providers/amazon/src/airflow/providers/amazon/aws/operators/rds.py::4 providers/amazon/src/airflow/providers/amazon/aws/operators/redshift_cluster.py::9 providers/amazon/src/airflow/providers/amazon/aws/operators/redshift_data.py::2 -providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py::5 +providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py::4 providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py::22 providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker_unified_studio.py::2 providers/amazon/src/airflow/providers/amazon/aws/operators/step_function.py::2 diff --git a/providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py b/providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py index b74fb60a1b763..381473de9435a 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py @@ -697,11 +697,6 @@ def __init__( self._keys: str | list[str] = "" - if not exactly_one(keys is None, all(var is None for var in [prefix, from_datetime, to_datetime])): - raise AirflowException( - "Either keys or at least one of prefix, from_datetime, to_datetime should be set." - ) - def execute(self, context: Context): if not exactly_one( self.keys is None, all(var is None for var in [self.prefix, self.from_datetime, self.to_datetime]) diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_s3.py b/providers/amazon/tests/unit/amazon/aws/operators/test_s3.py index cd907171927e4..b3d10fa23705c 100644 --- a/providers/amazon/tests/unit/amazon/aws/operators/test_s3.py +++ b/providers/amazon/tests/unit/amazon/aws/operators/test_s3.py @@ -1118,19 +1118,22 @@ def test_s3_delete_empty_string(self): pytest.param(None, None, None, None, id="all-none"), ], ) - def test_validate_keys_and_filters_in_constructor(self, keys, prefix, from_datetime, to_datetime): - with pytest.raises( - AirflowException, - match=r"Either keys or at least one of prefix, from_datetime, to_datetime should be set.", - ): - S3DeleteObjectsOperator( - task_id="test_validate_keys_and_prefix_in_constructor", - bucket="foo-bar-bucket", - keys=keys, - prefix=prefix, - from_datetime=from_datetime, - to_datetime=to_datetime, - ) + def test_no_validation_in_constructor(self, keys, prefix, from_datetime, to_datetime): + # Template fields are rendered after __init__, so keys/prefix/from_datetime/to_datetime + # must not be validated in the constructor — construction must succeed even for combinations + # that are invalid once rendered (validation happens in execute()). + op = S3DeleteObjectsOperator( + task_id="test_no_validation_in_constructor", + bucket="foo-bar-bucket", + keys=keys, + prefix=prefix, + from_datetime=from_datetime, + to_datetime=to_datetime, + ) + assert op.keys == keys + assert op.prefix == prefix + assert op.from_datetime == from_datetime + assert op.to_datetime == to_datetime @pytest.mark.parametrize( ("keys", "prefix", "from_datetime", "to_datetime"), @@ -1162,17 +1165,14 @@ def test_validate_keys_and_prefix_in_execute(self, keys, prefix, from_datetime, conn.create_bucket(Bucket=bucket) conn.upload_fileobj(Bucket=bucket, Key=key_of_test, Fileobj=BytesIO(b"input")) - # Set valid values for constructor, and change them later for emulate rendering template op = S3DeleteObjectsOperator( task_id="test_validate_keys_and_prefix_in_execute", bucket=bucket, - keys="keys-exists", - prefix=None, + keys=keys, + prefix=prefix, + from_datetime=from_datetime, + to_datetime=to_datetime, ) - op.keys = keys - op.prefix = prefix - op.from_datetime = from_datetime - op.to_datetime = to_datetime # The object should be detected before the DELETE action is tested objects_in_dest_bucket = conn.list_objects(Bucket=bucket, Prefix=key_of_test) diff --git a/scripts/ci/prek/validate_operators_init_exemptions.txt b/scripts/ci/prek/validate_operators_init_exemptions.txt index 76ba180595a56..d5b07063177e6 100644 --- a/scripts/ci/prek/validate_operators_init_exemptions.txt +++ b/scripts/ci/prek/validate_operators_init_exemptions.txt @@ -7,7 +7,6 @@ # Burn-down tracked at https://github.com/apache/airflow/issues/70296 providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStartDbClusterOperator providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStopDbClusterOperator -providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py::S3DeleteObjectsOperator providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py::SageMakerCreateNotebookOperator providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py::SageMakerProcessingOperator providers/amazon/src/airflow/providers/amazon/aws/operators/step_function.py::StepFunctionStartExecutionOperator From 7a1b525c33ee97b21ef33b834ac56fa85bf3e6f8 Mon Sep 17 00:00:00 2001 From: Amogh Atreya Date: Mon, 27 Jul 2026 15:21:54 +0530 Subject: [PATCH 2/2] Raises ValueError for conflicting S3DeleteObjectsOperator arguments --- .../providers/amazon/aws/operators/s3.py | 9 +++++ .../unit/amazon/aws/operators/test_s3.py | 40 +++++++++---------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py b/providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py index 381473de9435a..164777c78c136 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py @@ -697,6 +697,15 @@ def __init__( self._keys: str | list[str] = "" + # Checked here and again in execute(): this guard keeps a plain authoring mistake a + # parse-time error, while execute() catches a templated `keys` that renders to None + # (which would otherwise list — and delete from — the whole bucket). + by_scan = prefix is not None or from_datetime is not None or to_datetime is not None + if not exactly_one(keys is not None, by_scan): + raise ValueError( + "Either keys or at least one of prefix, from_datetime, to_datetime should be set." + ) + def execute(self, context: Context): if not exactly_one( self.keys is None, all(var is None for var in [self.prefix, self.from_datetime, self.to_datetime]) diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_s3.py b/providers/amazon/tests/unit/amazon/aws/operators/test_s3.py index b3d10fa23705c..9d062caa4ace8 100644 --- a/providers/amazon/tests/unit/amazon/aws/operators/test_s3.py +++ b/providers/amazon/tests/unit/amazon/aws/operators/test_s3.py @@ -1118,22 +1118,19 @@ def test_s3_delete_empty_string(self): pytest.param(None, None, None, None, id="all-none"), ], ) - def test_no_validation_in_constructor(self, keys, prefix, from_datetime, to_datetime): - # Template fields are rendered after __init__, so keys/prefix/from_datetime/to_datetime - # must not be validated in the constructor — construction must succeed even for combinations - # that are invalid once rendered (validation happens in execute()). - op = S3DeleteObjectsOperator( - task_id="test_no_validation_in_constructor", - bucket="foo-bar-bucket", - keys=keys, - prefix=prefix, - from_datetime=from_datetime, - to_datetime=to_datetime, - ) - assert op.keys == keys - assert op.prefix == prefix - assert op.from_datetime == from_datetime - assert op.to_datetime == to_datetime + def test_validate_keys_and_filters_in_constructor(self, keys, prefix, from_datetime, to_datetime): + with pytest.raises( + ValueError, + match=r"Either keys or at least one of prefix, from_datetime, to_datetime should be set.", + ): + S3DeleteObjectsOperator( + task_id="test_validate_keys_and_prefix_in_constructor", + bucket="foo-bar-bucket", + keys=keys, + prefix=prefix, + from_datetime=from_datetime, + to_datetime=to_datetime, + ) @pytest.mark.parametrize( ("keys", "prefix", "from_datetime", "to_datetime"), @@ -1165,14 +1162,17 @@ def test_validate_keys_and_prefix_in_execute(self, keys, prefix, from_datetime, conn.create_bucket(Bucket=bucket) conn.upload_fileobj(Bucket=bucket, Key=key_of_test, Fileobj=BytesIO(b"input")) + # Set valid values for constructor, and change them later for emulate rendering template op = S3DeleteObjectsOperator( task_id="test_validate_keys_and_prefix_in_execute", bucket=bucket, - keys=keys, - prefix=prefix, - from_datetime=from_datetime, - to_datetime=to_datetime, + keys="keys-exists", + prefix=None, ) + op.keys = keys + op.prefix = prefix + op.from_datetime = from_datetime + op.to_datetime = to_datetime # The object should be detected before the DELETE action is tested objects_in_dest_bucket = conn.list_objects(Bucket=bucket, Prefix=key_of_test)