diff --git a/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java b/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java index 9ab4562df5..f363c3169d 100644 --- a/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java +++ b/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java @@ -107,7 +107,7 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException return false; } else if (command.getFailureCount() > retryCountLimit) { logger.warn("Cannot retry after server error, command has exceeded retry limit %1$d: %2$s", retryCountLimit, - command); + command); return false; } else { imposeBackoffExponentialDelay(command.getFailureCount(), "server error: " + command.toString()); @@ -120,8 +120,14 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException } public void imposeBackoffExponentialDelay(long period, int pow, int failureCount, int max, String commandDescription) { + imposeBackoffExponentialDelay(period, period * 10l, pow, failureCount, max, commandDescription); + } + + public void imposeBackoffExponentialDelay(long period, long maxPeriod, int pow, int failureCount, int max, + String commandDescription) { long delayMs = (long) (period * Math.pow(failureCount, pow)); - logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, retryCountLimit, delayMs, commandDescription); + delayMs = delayMs > maxPeriod ? maxPeriod : delayMs; + logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, max, delayMs, commandDescription); try { Thread.sleep(delayMs); } catch (InterruptedException e) { diff --git a/core/src/main/java/org/jclouds/predicates/RetryablePredicate.java b/core/src/main/java/org/jclouds/predicates/RetryablePredicate.java index cf34beda01..a8e284e525 100644 --- a/core/src/main/java/org/jclouds/predicates/RetryablePredicate.java +++ b/core/src/main/java/org/jclouds/predicates/RetryablePredicate.java @@ -37,30 +37,33 @@ import com.google.common.base.Predicate; public class RetryablePredicate implements Predicate { private final long maxWait; private final long period; + private final long maxPeriod; private final Predicate predicate; @Resource protected Logger logger = Logger.NULL; - public RetryablePredicate(Predicate predicate, long maxWait, long period, - TimeUnit unit) { + public RetryablePredicate(Predicate predicate, long maxWait, long period, long maxPeriod, TimeUnit unit) { this.predicate = predicate; this.maxWait = unit.toMillis(maxWait); this.period = unit.toMillis(period); + this.maxPeriod = unit.toMillis(maxPeriod); + } + + public RetryablePredicate(Predicate predicate, long maxWait, long period, TimeUnit unit) { + this(predicate, maxWait, period, period * 10l, unit); } public RetryablePredicate(Predicate predicate, long maxWait) { - this.predicate = predicate; - this.maxWait = maxWait; - this.period = 50l; + this(predicate, maxWait, 50l, 1000l, TimeUnit.MILLISECONDS); } @Override public boolean apply(T input) { try { long i = 1l; - for (Date end = new Date(System.currentTimeMillis() + maxWait); before(end); Thread - .sleep(nextMaxInterval(i++, end))) { + for (Date end = new Date(System.currentTimeMillis() + maxWait); before(end); Thread.sleep(nextMaxInterval(i++, + end))) { if (predicate.apply(input)) { return true; } else if (atOrAfter(end)) { @@ -68,14 +71,14 @@ public class RetryablePredicate implements Predicate { } } } catch (InterruptedException e) { - logger.warn(e, "predicate %s on %s interrupted, returning false", - input, predicate); + logger.warn(e, "predicate %s on %s interrupted, returning false", input, predicate); } return false; } long nextMaxInterval(long attempt, Date end) { long interval = (period * (long) Math.pow(attempt, 1.5)); + interval = interval > maxPeriod ? maxPeriod : interval; long max = end.getTime() - System.currentTimeMillis(); return (interval > max) ? max : interval; } diff --git a/core/src/test/java/org/jclouds/http/handlers/BackoffLimitedRetryHandlerTest.java b/core/src/test/java/org/jclouds/http/handlers/BackoffLimitedRetryHandlerTest.java index 2d70af8be8..984fdf21e8 100644 --- a/core/src/test/java/org/jclouds/http/handlers/BackoffLimitedRetryHandlerTest.java +++ b/core/src/test/java/org/jclouds/http/handlers/BackoffLimitedRetryHandlerTest.java @@ -61,7 +61,7 @@ public class BackoffLimitedRetryHandlerTest { BackoffLimitedRetryHandler handler = new BackoffLimitedRetryHandler(); @Test - void testExponentialBackoffDelay() throws InterruptedException { + void testExponentialBackoffDelayDefaultMaxInterval500() throws InterruptedException { long acceptableDelay = 25; // Delay to forgive if tests run long. long startTime = System.nanoTime(); @@ -85,14 +85,14 @@ public class BackoffLimitedRetryHandlerTest { startTime = System.nanoTime(); handler.imposeBackoffExponentialDelay(4, "TEST FAILURE: 4"); elapsedTime = (System.nanoTime() - startTime) / 1000000; - assert (elapsedTime >= 799) : elapsedTime; - assertTrue(elapsedTime < 800 + acceptableDelay * 2); + assert (elapsedTime >= 499) : elapsedTime; + assertTrue(elapsedTime < 550 + acceptableDelay * 2); startTime = System.nanoTime(); handler.imposeBackoffExponentialDelay(5, "TEST FAILURE: 5"); elapsedTime = (System.nanoTime() - startTime) / 1000000; - assert (elapsedTime >= 1249) : elapsedTime; - assertTrue(elapsedTime < 1250 + acceptableDelay * 2); + assert (elapsedTime >= 499) : elapsedTime; + assertTrue(elapsedTime < 550 + acceptableDelay * 2); } TransformingHttpCommandExecutorServiceImpl executorService; diff --git a/core/src/test/java/org/jclouds/predicates/RetryablePredicateTest.java b/core/src/test/java/org/jclouds/predicates/RetryablePredicateTest.java index 6a7f2c0b66..39ca1f1d36 100644 --- a/core/src/test/java/org/jclouds/predicates/RetryablePredicateTest.java +++ b/core/src/test/java/org/jclouds/predicates/RetryablePredicateTest.java @@ -37,55 +37,63 @@ public class RetryablePredicateTest { @Test void testAlwaysTrue() { - RetryablePredicate predicate = new RetryablePredicate(Predicates - . alwaysTrue(), 3, 1, TimeUnit.SECONDS); - Date startPlusSecond = new Date(System.currentTimeMillis() + 1000); + RetryablePredicate predicate = new RetryablePredicate(Predicates. alwaysTrue(), 3, 1, + TimeUnit.SECONDS); + Date startPlusThird = new Date(System.currentTimeMillis() + 1000); predicate.apply(""); Date now = new Date(); - assert now.compareTo(startPlusSecond) < 0 : String.format("%s should be less than %s", now, - startPlusSecond); + assert now.compareTo(startPlusThird) < 0 : String.format("%s should be less than %s", now, startPlusThird); } @Test void testAlwaysFalseMillis() { - RetryablePredicate predicate = new RetryablePredicate(Predicates - . alwaysFalse(), 3, 1, TimeUnit.SECONDS); - Date startPlus3Seconds = new Date(System.currentTimeMillis() + 3000); - Date startPlus4Seconds = new Date(System.currentTimeMillis() + 4000); + RetryablePredicate predicate = new RetryablePredicate(Predicates. alwaysFalse(), 3, 1, + TimeUnit.SECONDS); + Date startPlus3Thirds = new Date(System.currentTimeMillis() + 3000); + Date startPlus4Thirds = new Date(System.currentTimeMillis() + 4000); predicate.apply(""); Date now = new Date(); - assert now.compareTo(startPlus3Seconds) >= 0 : String.format("%s should be less than %s", - startPlus3Seconds, now); - assert now.compareTo(startPlus4Seconds) <= 0 : String.format("%s should be greater than %s", - startPlus4Seconds, now); + assert now.compareTo(startPlus3Thirds) >= 0 : String.format("%s should be less than %s", startPlus3Thirds, now); + assert now.compareTo(startPlus4Thirds) <= 0 : String + .format("%s should be greater than %s", startPlus4Thirds, now); } - private static class SecondTimeTrue implements Predicate { + private static class ThirdTimeTrue implements Predicate { private int count = 0; @Override public boolean apply(String input) { - return count++ == 1; + return count++ == 2; } } @Test - void testSecondTimeTrue() { - RetryablePredicate predicate = new RetryablePredicate(new SecondTimeTrue(), - 3, 1, TimeUnit.SECONDS); + void testThirdTimeTrue() { + RetryablePredicate predicate = new RetryablePredicate(new ThirdTimeTrue(), 3, 1, TimeUnit.SECONDS); - Date startPlusSecond = new Date(System.currentTimeMillis() + 1000); - Date startPlus2Seconds = new Date(System.currentTimeMillis() + 2000); + Date startPlus = new Date(System.currentTimeMillis() + 1000); + Date startPlus3 = new Date(System.currentTimeMillis() + 3000); predicate.apply(""); Date now = new Date(); - assert now.compareTo(startPlusSecond) >= 0 : String.format("%s should be greater than %s", - now, startPlusSecond); - assert now.compareTo(startPlus2Seconds) <= 0 : String.format("%s should be greater than %s", - startPlus2Seconds, now); + assert now.compareTo(startPlus) >= 0 : String.format("%s should be greater than %s", now, startPlus); + assert now.compareTo(startPlus3) <= 0 : String.format("%s should be greater than %s", startPlus3, now); } + @Test + void testThirdTimeTrueLimitedMaxInterval() { + RetryablePredicate predicate = new RetryablePredicate(new ThirdTimeTrue(), 3, 1, 1, + TimeUnit.SECONDS); + + Date startPlus = new Date(System.currentTimeMillis() + 1000); + Date startPlus2 = new Date(System.currentTimeMillis() + 2000); + + predicate.apply(""); + Date now = new Date(); + assert now.compareTo(startPlus) >= 0 : String.format("%s should be greater than %s", now, startPlus); + assert now.compareTo(startPlus2) <= 0 : String.format("%s should be greater than %s", startPlus2, now); + } } \ No newline at end of file