diff --git a/processing/src/main/java/org/apache/druid/java/util/common/RetryUtils.java b/processing/src/main/java/org/apache/druid/java/util/common/RetryUtils.java index 98c422b1dcac..a94f8d1df7b5 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/RetryUtils.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/RetryUtils.java @@ -206,9 +206,29 @@ public static void awaitNextRetry( Thread.sleep(sleepMillis); } + /** + * Calculates the duration in milliseconds to sleep before the next attempt of + * a retryable operation. The duration is calculated in an exponential back-off + * manner with a fuzzy multiplier to introduce some variance. + *

+ * Sleep duration in milliseconds for subsequent retries: + *

+ * + * @param nTry Index of the next retry, starting with 1 + * @return Next sleep duration in the range [0, 120,000] millis + */ public static long nextRetrySleepMillis(final int nTry) { + // fuzzyMultiplier in [0, 2] final double fuzzyMultiplier = Math.min(Math.max(1 + 0.2 * ThreadLocalRandom.current().nextGaussian(), 0), 2); + + // sleepMillis in [1 x 2^(nTry-1), 60] * [0, 2] seconds final long sleepMillis = (long) (Math.min(MAX_SLEEP_MILLIS, BASE_SLEEP_MILLIS * Math.pow(2, nTry - 1)) * fuzzyMultiplier); return sleepMillis; diff --git a/processing/src/test/java/org/apache/druid/java/util/common/RetryUtilsTest.java b/processing/src/test/java/org/apache/druid/java/util/common/RetryUtilsTest.java index 7df0aa0687c7..8d0271397d7f 100644 --- a/processing/src/test/java/org/apache/druid/java/util/common/RetryUtilsTest.java +++ b/processing/src/test/java/org/apache/druid/java/util/common/RetryUtilsTest.java @@ -186,4 +186,29 @@ public void testExceptionPredicateForRetryableException() throws Exception Assert.assertEquals(result, "hey"); Assert.assertEquals("count", 2, count.get()); } + + @Test + public void testNextRetrySleepMillis() + { + long totalSleepTimeMillis = 0; + + for (int i = 1; i < 7; ++i) { + final long nextSleepMillis = RetryUtils.nextRetrySleepMillis(i); + Assert.assertTrue(nextSleepMillis >= 0); + Assert.assertTrue(nextSleepMillis <= (2_000 * Math.pow(2, i - 1))); + + totalSleepTimeMillis += nextSleepMillis; + } + + for (int i = 7; i < 11; ++i) { + final long nextSleepMillis = RetryUtils.nextRetrySleepMillis(i); + Assert.assertTrue(nextSleepMillis >= 0); + Assert.assertTrue(nextSleepMillis <= 120_000); + + totalSleepTimeMillis += nextSleepMillis; + } + + Assert.assertTrue(totalSleepTimeMillis > 3 * 60_000); + Assert.assertTrue(totalSleepTimeMillis < 8 * 60_000); + } }