From 97843133459dfb69afc46b39fdda7c254b64f746 Mon Sep 17 00:00:00 2001 From: Dr Alex Mitre <30060514+mitre88@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:21:36 -0600 Subject: [PATCH] Resolve GCSFileTransformOperator destination fallbacks after rendering destination_bucket/destination_object are template fields, so resolving their 'or source_*' fallbacks in __init__ reads the unrendered value. Move the fallback resolution to execute() where the rendered values are available, keep get_openlineage_facets_on_complete resolving fallbacks locally (it can run outside execute), and remove the operator from the validate_operators_init exemptions list. Related: #70296 --- .../airflow/providers/google/cloud/operators/gcs.py | 10 ++++++---- scripts/ci/prek/validate_operators_init_exemptions.txt | 1 - 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/providers/google/src/airflow/providers/google/cloud/operators/gcs.py b/providers/google/src/airflow/providers/google/cloud/operators/gcs.py index b55709c309eea..5a7dd37d1155e 100644 --- a/providers/google/src/airflow/providers/google/cloud/operators/gcs.py +++ b/providers/google/src/airflow/providers/google/cloud/operators/gcs.py @@ -603,8 +603,8 @@ def __init__( super().__init__(**kwargs) self.source_bucket = source_bucket self.source_object = source_object - self.destination_bucket = destination_bucket or self.source_bucket - self.destination_object = destination_object or self.source_object + self.destination_bucket = destination_bucket + self.destination_object = destination_object self.gcp_conn_id = gcp_conn_id self.transform_script = transform_script @@ -612,6 +612,8 @@ def __init__( self.impersonation_chain = impersonation_chain def execute(self, context: Context) -> None: + self.destination_bucket = self.destination_bucket or self.source_bucket + self.destination_object = self.destination_object or self.source_object hook = GCSHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain) with NamedTemporaryFile() as source_file, NamedTemporaryFile() as destination_file: @@ -658,8 +660,8 @@ def get_openlineage_facets_on_start(self): name=self.source_object, ) output_dataset = Dataset( - namespace=f"gs://{self.destination_bucket}", - name=self.destination_object, + namespace=f"gs://{self.destination_bucket or self.source_bucket}", + name=self.destination_object or self.source_object, ) return OperatorLineage(inputs=[input_dataset], outputs=[output_dataset]) diff --git a/scripts/ci/prek/validate_operators_init_exemptions.txt b/scripts/ci/prek/validate_operators_init_exemptions.txt index 6c4907fbdedcb..4e0f8cbf45b0a 100644 --- a/scripts/ci/prek/validate_operators_init_exemptions.txt +++ b/scripts/ci/prek/validate_operators_init_exemptions.txt @@ -17,7 +17,6 @@ providers/google/src/airflow/providers/google/cloud/operators/cloud_storage_tran providers/google/src/airflow/providers/google/cloud/operators/dataproc.py::DataprocCreateClusterOperator providers/google/src/airflow/providers/google/cloud/operators/dataproc.py::DataprocSubmitJobOperator providers/google/src/airflow/providers/google/cloud/operators/functions.py::CloudFunctionDeployFunctionOperator -providers/google/src/airflow/providers/google/cloud/operators/gcs.py::GCSFileTransformOperator providers/google/src/airflow/providers/google/cloud/sensors/bigquery_dts.py::BigQueryDataTransferServiceTransferRunSensor providers/google/src/airflow/providers/google/cloud/sensors/cloud_composer.py::CloudComposerExternalTaskSensor providers/google/src/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py::AzureFileShareToGCSOperator