Issue 489: made a ceiling for retry intervals

core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java

@@ -107,7 +107,7 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException
          return false;
       } else if (command.getFailureCount() > retryCountLimit) {
          logger.warn("Cannot retry after server error, command has exceeded retry limit %1$d: %2$s", retryCountLimit,
-               command);
+                  command);
          return false;
       } else {
          imposeBackoffExponentialDelay(command.getFailureCount(), "server error: " + command.toString());
@@ -120,8 +120,14 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException
    }
 
    public void imposeBackoffExponentialDelay(long period, int pow, int failureCount, int max, String commandDescription) {
+      imposeBackoffExponentialDelay(period, period * 10l, pow, failureCount, max, commandDescription);
+   }
+
+   public void imposeBackoffExponentialDelay(long period, long maxPeriod, int pow, int failureCount, int max,
+            String commandDescription) {
       long delayMs = (long) (period * Math.pow(failureCount, pow));
-      logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, retryCountLimit, delayMs, commandDescription);
+      delayMs = delayMs > maxPeriod ? maxPeriod : delayMs;
+      logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, max, delayMs, commandDescription);
       try {
          Thread.sleep(delayMs);
       } catch (InterruptedException e) {

core/src/main/java/org/jclouds/predicates/RetryablePredicate.java

@@ -37,30 +37,33 @@ import com.google.common.base.Predicate;
 public class RetryablePredicate<T> implements Predicate<T> {
    private final long maxWait;
    private final long period;
+   private final long maxPeriod;
    private final Predicate<T> predicate;
 
    @Resource
    protected Logger logger = Logger.NULL;
 
-   public RetryablePredicate(Predicate<T> predicate, long maxWait, long period,
+   public RetryablePredicate(Predicate<T> predicate, long maxWait, long period, long maxPeriod, TimeUnit unit) {
-         TimeUnit unit) {
       this.predicate = predicate;
       this.maxWait = unit.toMillis(maxWait);
       this.period = unit.toMillis(period);
+      this.maxPeriod = unit.toMillis(maxPeriod);
+   }
+
+   public RetryablePredicate(Predicate<T> predicate, long maxWait, long period, TimeUnit unit) {
+      this(predicate, maxWait, period, period * 10l, unit);
    }
 
    public RetryablePredicate(Predicate<T> predicate, long maxWait) {
-      this.predicate = predicate;
+      this(predicate, maxWait, 50l, 1000l, TimeUnit.MILLISECONDS);
-      this.maxWait = maxWait;
-      this.period = 50l;
    }
 
    @Override
    public boolean apply(T input) {
       try {
          long i = 1l;
-         for (Date end = new Date(System.currentTimeMillis() + maxWait); before(end); Thread
+         for (Date end = new Date(System.currentTimeMillis() + maxWait); before(end); Thread.sleep(nextMaxInterval(i++,
-               .sleep(nextMaxInterval(i++, end))) {
+                  end))) {
             if (predicate.apply(input)) {
                return true;
             } else if (atOrAfter(end)) {
@@ -68,14 +71,14 @@ public class RetryablePredicate<T> implements Predicate<T> {
             }
          }
       } catch (InterruptedException e) {
-         logger.warn(e, "predicate %s on %s interrupted, returning false",
+         logger.warn(e, "predicate %s on %s interrupted, returning false", input, predicate);
-               input, predicate);
       }
       return false;
    }
 
    long nextMaxInterval(long attempt, Date end) {
       long interval = (period * (long) Math.pow(attempt, 1.5));
+      interval = interval > maxPeriod ? maxPeriod : interval;
       long max = end.getTime() - System.currentTimeMillis();
       return (interval > max) ? max : interval;
    }

core/src/test/java/org/jclouds/http/handlers/BackoffLimitedRetryHandlerTest.java

@@ -61,7 +61,7 @@ public class BackoffLimitedRetryHandlerTest {
    BackoffLimitedRetryHandler handler = new BackoffLimitedRetryHandler();
 
    @Test
-   void testExponentialBackoffDelay() throws InterruptedException {
+   void testExponentialBackoffDelayDefaultMaxInterval500() throws InterruptedException {
       long acceptableDelay = 25; // Delay to forgive if tests run long.
 
       long startTime = System.nanoTime();
@@ -85,14 +85,14 @@ public class BackoffLimitedRetryHandlerTest {
       startTime = System.nanoTime();
       handler.imposeBackoffExponentialDelay(4, "TEST FAILURE: 4");
       elapsedTime = (System.nanoTime() - startTime) / 1000000;
-      assert (elapsedTime >= 799) : elapsedTime;
+      assert (elapsedTime >= 499) : elapsedTime;
-      assertTrue(elapsedTime < 800 + acceptableDelay * 2);
+      assertTrue(elapsedTime < 550 + acceptableDelay * 2);
 
       startTime = System.nanoTime();
       handler.imposeBackoffExponentialDelay(5, "TEST FAILURE: 5");
       elapsedTime = (System.nanoTime() - startTime) / 1000000;
-      assert (elapsedTime >= 1249) : elapsedTime;
+      assert (elapsedTime >= 499) : elapsedTime;
-      assertTrue(elapsedTime < 1250 + acceptableDelay * 2);
+      assertTrue(elapsedTime < 550 + acceptableDelay * 2);
    }
 
    TransformingHttpCommandExecutorServiceImpl executorService;

core/src/test/java/org/jclouds/predicates/RetryablePredicateTest.java

@@ -37,55 +37,63 @@ public class RetryablePredicateTest {
 
    @Test
    void testAlwaysTrue() {
-      RetryablePredicate<String> predicate = new RetryablePredicate<String>(Predicates
+      RetryablePredicate<String> predicate = new RetryablePredicate<String>(Predicates.<String> alwaysTrue(), 3, 1,
-               .<String> alwaysTrue(), 3, 1, TimeUnit.SECONDS);
+               TimeUnit.SECONDS);
-      Date startPlusSecond = new Date(System.currentTimeMillis() + 1000);
+      Date startPlusThird = new Date(System.currentTimeMillis() + 1000);
       predicate.apply("");
       Date now = new Date();
-      assert now.compareTo(startPlusSecond) < 0 : String.format("%s should be less than %s", now,
+      assert now.compareTo(startPlusThird) < 0 : String.format("%s should be less than %s", now, startPlusThird);
-               startPlusSecond);
    }
 
    @Test
    void testAlwaysFalseMillis() {
-      RetryablePredicate<String> predicate = new RetryablePredicate<String>(Predicates
+      RetryablePredicate<String> predicate = new RetryablePredicate<String>(Predicates.<String> alwaysFalse(), 3, 1,
-               .<String> alwaysFalse(), 3, 1, TimeUnit.SECONDS);
+               TimeUnit.SECONDS);
-      Date startPlus3Seconds = new Date(System.currentTimeMillis() + 3000);
+      Date startPlus3Thirds = new Date(System.currentTimeMillis() + 3000);
-      Date startPlus4Seconds = new Date(System.currentTimeMillis() + 4000);
+      Date startPlus4Thirds = new Date(System.currentTimeMillis() + 4000);
       predicate.apply("");
       Date now = new Date();
-      assert now.compareTo(startPlus3Seconds) >= 0 : String.format("%s should be less than %s",
+      assert now.compareTo(startPlus3Thirds) >= 0 : String.format("%s should be less than %s", startPlus3Thirds, now);
-               startPlus3Seconds, now);
+      assert now.compareTo(startPlus4Thirds) <= 0 : String
-      assert now.compareTo(startPlus4Seconds) <= 0 : String.format("%s should be greater than %s",
+               .format("%s should be greater than %s", startPlus4Thirds, now);
-               startPlus4Seconds, now);
 
    }
 
-   private static class SecondTimeTrue implements Predicate<String> {
+   private static class ThirdTimeTrue implements Predicate<String> {
 
       private int count = 0;
 
       @Override
       public boolean apply(String input) {
-         return count++ == 1;
+         return count++ == 2;
       }
 
    }
 
    @Test
-   void testSecondTimeTrue() {
+   void testThirdTimeTrue() {
-      RetryablePredicate<String> predicate = new RetryablePredicate<String>(new SecondTimeTrue(),
+      RetryablePredicate<String> predicate = new RetryablePredicate<String>(new ThirdTimeTrue(), 3, 1, TimeUnit.SECONDS);
-               3, 1, TimeUnit.SECONDS);
 
-      Date startPlusSecond = new Date(System.currentTimeMillis() + 1000);
+      Date startPlus = new Date(System.currentTimeMillis() + 1000);
-      Date startPlus2Seconds = new Date(System.currentTimeMillis() + 2000);
+      Date startPlus3 = new Date(System.currentTimeMillis() + 3000);
 
       predicate.apply("");
       Date now = new Date();
-      assert now.compareTo(startPlusSecond) >= 0 : String.format("%s should be greater than %s",
+      assert now.compareTo(startPlus) >= 0 : String.format("%s should be greater than %s", now, startPlus);
-               now, startPlusSecond);
+      assert now.compareTo(startPlus3) <= 0 : String.format("%s should be greater than %s", startPlus3, now);
-      assert now.compareTo(startPlus2Seconds) <= 0 : String.format("%s should be greater than %s",
-               startPlus2Seconds, now);
    }
 
+   @Test
+   void testThirdTimeTrueLimitedMaxInterval() {
+      RetryablePredicate<String> predicate = new RetryablePredicate<String>(new ThirdTimeTrue(), 3, 1, 1,
+               TimeUnit.SECONDS);
+
+      Date startPlus = new Date(System.currentTimeMillis() + 1000);
+      Date startPlus2 = new Date(System.currentTimeMillis() + 2000);
+
+      predicate.apply("");
+      Date now = new Date();
+      assert now.compareTo(startPlus) >= 0 : String.format("%s should be greater than %s", now, startPlus);
+      assert now.compareTo(startPlus2) <= 0 : String.format("%s should be greater than %s", startPlus2, now);
+   }
 }