Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions airflow-core/src/airflow/models/taskinstance.py
Original file line number Diff line number Diff line change
Expand Up @@ -1070,12 +1070,18 @@ def next_retry_datetime(self):

delay = self.task.retry_delay
if self.task.retry_exponential_backoff:
# If the min_backoff calculation is below 1, it will be converted to 0 via int. Thus,
# we must round up prior to converting to an int, otherwise a divide by zero error
# will occur in the modded_hash calculation.
# this probably gives unexpected results if a task instance has previously been cleared,
# because try_number can increase without bound
min_backoff = math.ceil(delay.total_seconds() * (2 ** (self.try_number - 1)))
try:
# If the min_backoff calculation is below 1, it will be converted to 0 via int. Thus,
# we must round up prior to converting to an int, otherwise a divide by zero error
# will occur in the modded_hash calculation.
# this probably gives unexpected results if a task instance has previously been cleared,
# because try_number can increase without bound
min_backoff = math.ceil(delay.total_seconds() * (2 ** (self.try_number - 1)))
except OverflowError:
min_backoff = MAX_RETRY_DELAY
Comment thread
perry2of5 marked this conversation as resolved.
self.log.warning(
"OverflowError occurred while calculating min_backoff, using MAX_RETRY_DELAY for min_backoff."
)

# In the case when delay.total_seconds() is 0, min_backoff will not be rounded up to 1.
# To address this, we impose a lower bound of 1 on min_backoff. This effectively makes
Expand Down
25 changes: 25 additions & 0 deletions airflow-core/tests/unit/models/test_taskinstance.py
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,31 @@ def test_next_retry_datetime(self, dag_maker):
date = ti.next_retry_datetime()
assert date == ti.end_date + max_delay

def test_next_retry_datetime_returns_max_for_overflow(self, dag_maker):
delay = datetime.timedelta(seconds=30)
max_delay = datetime.timedelta(minutes=60)

with dag_maker(dag_id="fail_dag"):
task = BashOperator(
task_id="task_with_exp_backoff_and_max_delay",
bash_command="exit 1",
retries=3,
retry_delay=delay,
retry_exponential_backoff=True,
max_retry_delay=max_delay,
)
ti = dag_maker.create_dagrun().task_instances[0]
ti.task = task
ti.end_date = pendulum.instance(timezone.utcnow())

ti.try_number = 5000
date = ti.next_retry_datetime()
assert date == ti.end_date + max_delay

ti.try_number = 50000
date = ti.next_retry_datetime()
assert date == ti.end_date + max_delay

@pytest.mark.parametrize("seconds", [0, 0.5, 1])
def test_next_retry_datetime_short_or_zero_intervals(self, dag_maker, seconds):
delay = datetime.timedelta(seconds=seconds)
Expand Down