diff --git a/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py b/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py index 3d46aecb32e0b..30077380eb641 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py @@ -433,7 +433,7 @@ def __init__( ): super().__init__(**kwargs) self.name = name - self.ruleset = ruleset.strip() + self.ruleset = ruleset self.description = description self.update_rule_set = update_rule_set self.data_quality_ruleset_kwargs = data_quality_ruleset_kwargs or {} @@ -449,6 +449,8 @@ def validate_inputs(self) -> None: raise AttributeError("Target table must have DatabaseName and TableName") def execute(self, context: Context): + # ruleset is a template field; strip the rendered value here, not in __init__. + self.ruleset = self.ruleset.strip() self.validate_inputs() config = { diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py b/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py index 230193feeb42b..99d22e3bb5e47 100644 --- a/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py +++ b/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py @@ -17,6 +17,7 @@ from __future__ import annotations from collections.abc import Generator +from datetime import datetime from typing import TYPE_CHECKING from unittest import mock @@ -25,6 +26,7 @@ from boto3 import client from moto import mock_aws +from airflow.models.dag import DAG from airflow.providers.amazon.aws.hooks.glue import GlueDataQualityHook, GlueJobHook from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.amazon.aws.links.glue import GlueJobRunDetailsLink @@ -813,6 +815,28 @@ def test_execute_create_rule(self, glue_data_quality_mock_conn): Ruleset=self.RULE_SET, ) + @mock.patch.object(GlueDataQualityHook, "conn") + def test_execute_strips_rendered_ruleset(self, glue_data_quality_mock_conn): + # ruleset is a template field; execute strips the rendered value (rendering can add whitespace). + with DAG("glue_dq_strip", schedule=None, start_date=datetime(2020, 1, 1)) as dag: + self.operator = GlueDataQualityOperator( + task_id="create_data_quality_ruleset", + name=self.RULE_SET_NAME, + ruleset="{{ params.rules }}", + dag=dag, + ) + self.operator.defer = mock.MagicMock() + self.operator.render_template_fields({"params": {"rules": f" {self.RULE_SET} "}}) + assert self.operator.ruleset == f" {self.RULE_SET} " + + self.operator.execute({}) + + glue_data_quality_mock_conn.create_data_quality_ruleset.assert_called_once_with( + Description="AWS Glue Data Quality Rule Set With Airflow", + Name=self.RULE_SET_NAME, + Ruleset=self.RULE_SET, + ) + @mock.patch.object(GlueDataQualityHook, "conn") def test_execute_create_rule_should_fail_if_rule_already_exists(self, glue_data_quality_mock_conn): self.operator = GlueDataQualityOperator( diff --git a/scripts/ci/prek/validate_operators_init_exemptions.txt b/scripts/ci/prek/validate_operators_init_exemptions.txt index 01fd5bc56dbb7..b6cbde980d002 100644 --- a/scripts/ci/prek/validate_operators_init_exemptions.txt +++ b/scripts/ci/prek/validate_operators_init_exemptions.txt @@ -14,7 +14,6 @@ providers/amazon/src/airflow/providers/amazon/aws/operators/dms.py::DmsModifyTas providers/amazon/src/airflow/providers/amazon/aws/operators/dms.py::DmsStartReplicationOperator providers/amazon/src/airflow/providers/amazon/aws/operators/ecs.py::EcsRunTaskOperator providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py::EmrAddStepsOperator -providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py::GlueDataQualityOperator providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStartDbClusterOperator providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStopDbClusterOperator providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py::S3DeleteObjectsOperator