From db9809922f480e2a3d4677bae39a7f6f741c6156 Mon Sep 17 00:00:00 2001 From: zachliu Date: Wed, 9 Mar 2022 22:31:57 -0500 Subject: [PATCH 1/3] bugfix --- airflow/providers/amazon/aws/operators/ecs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py index 8a4a039bc2b9b..b02f221b2b65b 100644 --- a/airflow/providers/amazon/aws/operators/ecs.py +++ b/airflow/providers/amazon/aws/operators/ecs.py @@ -450,6 +450,11 @@ def _check_success_task(self) -> None: for task in response['tasks']: if task.get('stopCode', '') == 'TaskFailedToStart': + # Reset task arn here otherwise the retry run will not start + # a new task but keep polling the old dead one + # I'm not resetting it for other exceptions here because + # EcsTaskFailToStart is the only exception that's being retried at the moment + self.arn = None raise EcsTaskFailToStart(f"The task failed to start due to: {task.get('stoppedReason', '')}") # This is a `stoppedReason` that indicates a task has not From acceacd1321536523355d39dda9474afe178d26e Mon Sep 17 00:00:00 2001 From: zachliu Date: Thu, 10 Mar 2022 10:03:44 -0500 Subject: [PATCH 2/3] add another related failure reason --- airflow/providers/amazon/aws/operators/ecs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py index b02f221b2b65b..ca52e0af41868 100644 --- a/airflow/providers/amazon/aws/operators/ecs.py +++ b/airflow/providers/amazon/aws/operators/ecs.py @@ -51,7 +51,10 @@ def should_retry(exception: Exception): def should_retry_eni(exception: Exception): """Check if exception is related to ENI (Elastic Network Interfaces).""" if isinstance(exception, EcsTaskFailToStart): - return any(eni_reason in exception.message for eni_reason in ['network interface provisioning']) + return any( + eni_reason in exception.message + for eni_reason in ['network interface provisioning', 'ResourceInitializationError'] + ) return False From 24bfbca5c7271cfecad729dad4a5a5ef721291ed Mon Sep 17 00:00:00 2001 From: zachliu Date: Thu, 10 Mar 2022 17:36:53 -0500 Subject: [PATCH 3/3] provide visibility during retries --- airflow/providers/amazon/aws/hooks/base_aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/hooks/base_aws.py b/airflow/providers/amazon/aws/hooks/base_aws.py index 0a4c58b5ffee6..16c7385f9c1ef 100644 --- a/airflow/providers/amazon/aws/hooks/base_aws.py +++ b/airflow/providers/amazon/aws/hooks/base_aws.py @@ -584,7 +584,7 @@ def decorator_f(self, *args, **kwargs): min_limit = retry_args.get('min', 1) max_limit = retry_args.get('max', 1) stop_after_delay = retry_args.get('stop_after_delay', 10) - tenacity_logger = tenacity.before_log(self.log, logging.DEBUG) if self.log else None + tenacity_logger = tenacity.before_log(self.log, logging.INFO) if self.log else None default_kwargs = { 'wait': tenacity.wait_exponential(multiplier=multiplier, max=max_limit, min=min_limit), 'retry': tenacity.retry_if_exception(should_retry),