pulled in stabilization changes from master

core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java

@@ -106,7 +106,7 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException
          return false;
       } else if (command.getFailureCount() > retryCountLimit) {
          logger.warn("Cannot retry after server error, command has exceeded retry limit %1$d: %2$s", retryCountLimit,
-               command);
+                  command);
          return false;
       } else {
          imposeBackoffExponentialDelay(command.getFailureCount(), "server error: " + command.toString());
@@ -119,8 +119,14 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException
    }
 
    public void imposeBackoffExponentialDelay(long period, int pow, int failureCount, int max, String commandDescription) {
+      imposeBackoffExponentialDelay(period, period * 10l, pow, failureCount, max, commandDescription);
+   }
+
+   public void imposeBackoffExponentialDelay(long period, long maxPeriod, int pow, int failureCount, int max,
+            String commandDescription) {
       long delayMs = (long) (period * Math.pow(failureCount, pow));
-      logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, retryCountLimit, delayMs, commandDescription);
+      delayMs = delayMs > maxPeriod ? maxPeriod : delayMs;
+      logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, max, delayMs, commandDescription);
       try {
          Thread.sleep(delayMs);
       } catch (InterruptedException e) {
@@ -129,3 +135,4 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException
    }
 
 }
+

core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java

@@ -18,11 +18,13 @@
  */
 package org.jclouds.http.internal;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Throwables.propagate;
 import static com.google.common.collect.Iterables.getLast;
 import static com.google.common.io.ByteStreams.toByteArray;
 import static com.google.common.io.Closeables.closeQuietly;
+import static org.jclouds.io.Payloads.newInputStreamPayload;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -59,14 +61,13 @@ import org.jclouds.http.handlers.DelegatingErrorHandler;
 import org.jclouds.http.handlers.DelegatingRetryHandler;
 import org.jclouds.io.MutableContentMetadata;
 import org.jclouds.io.Payload;
-import org.jclouds.io.Payloads;
 import org.jclouds.logging.Logger;
 import org.jclouds.rest.internal.RestAnnotationProcessor;
 
 import com.google.common.base.Supplier;
-import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.ImmutableMultimap.Builder;
-import com.google.common.collect.Multimap;
+import com.google.common.io.CountingOutputStream;
 
 /**
  * Basic implementation of a {@link HttpCommandExecutorService}.
@@ -85,11 +86,11 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
 
    @Inject
    public JavaUrlHttpCommandExecutorService(HttpUtils utils,
-         @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor,
+            @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor,
-         DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
+            DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
-         DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
+            DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
-         @Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider) throws SecurityException,
+            @Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider) throws SecurityException,
-         NoSuchFieldException {
+            NoSuchFieldException {
       super(utils, ioWorkerExecutor, retryHandler, ioRetryHandler, errorHandler, wire);
       if (utils.getMaxConnections() > 0)
          System.setProperty("http.maxConnections", String.valueOf(checkNotNull(utils, "utils").getMaxConnections()));
@@ -101,6 +102,7 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
 
    @Override
    protected HttpResponse invoke(HttpURLConnection connection) throws IOException, InterruptedException {
+      HttpResponse.Builder builder = HttpResponse.builder();
       InputStream in = null;
       try {
          in = consumeOnClose(connection.getInputStream());
@@ -112,19 +114,28 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
          assert false : "should have propagated exception";
       }
 
-      if (connection.getResponseCode() == 204) {
+      int responseCode = connection.getResponseCode();
+      if (responseCode == 204) {
          closeQuietly(in);
          in = null;
       }
-      Multimap<String, String> headers = LinkedHashMultimap.create();
+      builder.statusCode(responseCode);
+      builder.message(connection.getResponseMessage());
+
+      Builder<String, String> headerBuilder = ImmutableMultimap.<String, String> builder();
       for (String header : connection.getHeaderFields().keySet()) {
-         headers.putAll(header, connection.getHeaderFields().get(header));
+         // HTTP message comes back as a header without a key
+         if (header != null)
+            headerBuilder.putAll(header, connection.getHeaderFields().get(header));
       }
-      Payload payload = in != null ? Payloads.newInputStreamPayload(in) : null;
+      ImmutableMultimap<String, String> headers = headerBuilder.build();
-      if (payload != null)
+      Payload payload = in != null ? newInputStreamPayload(in) : null;
+      if (payload != null) {
          payload.getContentMetadata().setPropertiesFromHttpHeaders(headers);
-      return new HttpResponse(connection.getResponseCode(), connection.getResponseMessage(), payload,
+         builder.payload(payload);
-            RestAnnotationProcessor.filterOutContentHeaders(headers));
+      }
+      builder.headers(RestAnnotationProcessor.filterOutContentHeaders(headers));
+      return builder.build();
    }
 
    private InputStream bufferAndCloseStream(InputStream inputStream) throws IOException {
@@ -171,6 +182,12 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
          if (utils.trustAllCerts())
             sslCon.setSSLSocketFactory(untrustedSSLContextProvider.get().getSocketFactory());
       }
+      if (utils.getConnectionTimeout() > 0) {
+         connection.setConnectTimeout(utils.getConnectionTimeout());
+      }
+      if (utils.getSocketOpenTimeout() > 0) {
+         connection.setReadTimeout(utils.getSocketOpenTimeout());
+      }
       connection.setDoOutput(true);
       connection.setAllowUserInteraction(false);
       // do not follow redirects since https redirects don't work properly
@@ -184,7 +201,7 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
             methodField.set(connection, request.getMethod());
          } catch (Exception e1) {
             logger.error(e, "could not set request method: ", request.getMethod());
-            Throwables.propagate(e1);
+            propagate(e1);
          }
       }
 
@@ -213,14 +230,20 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
          } else {
             Long length = checkNotNull(md.getContentLength(), "payload.getContentLength");
             connection.setRequestProperty(HttpHeaders.CONTENT_LENGTH, length.toString());
+            // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6755625
+            checkArgument(length < Integer.MAX_VALUE,
+                     "JDK 1.6 does not support >2GB chunks. Use chunked encoding, if possible.");
             connection.setFixedLengthStreamingMode(length.intValue());
+            if (length.intValue() > 0) {
+               connection.setRequestProperty("Expect", "100-continue");
+            }
          }
-         // writeTo will close the output stream
+         CountingOutputStream out = new CountingOutputStream(connection.getOutputStream());
          try {
-            request.getPayload().writeTo(connection.getOutputStream());
+            request.getPayload().writeTo(out);
          } catch (IOException e) {
-            e.printStackTrace();
+            throw new RuntimeException(String.format("error after writing %d/%s bytes to %s", out.getCount(), md
-            throw e;
+                     .getContentLength(), request.getRequestLine()), e);
          }
       } else {
          connection.setRequestProperty(HttpHeaders.CONTENT_LENGTH, "0");

core/src/main/java/org/jclouds/predicates/RetryablePredicate.java

@@ -18,8 +18,12 @@
  */
 package org.jclouds.predicates;
 
+import static org.jclouds.util.Throwables2.getFirstThrowableOfType;
+
 import java.util.Date;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import javax.annotation.Resource;
 
@@ -36,30 +40,33 @@ import com.google.common.base.Predicate;
 public class RetryablePredicate<T> implements Predicate<T> {
    private final long maxWait;
    private final long period;
+   private final long maxPeriod;
    private final Predicate<T> predicate;
 
    @Resource
    protected Logger logger = Logger.NULL;
 
-   public RetryablePredicate(Predicate<T> predicate, long maxWait, long period,
+   public RetryablePredicate(Predicate<T> predicate, long maxWait, long period, long maxPeriod, TimeUnit unit) {
-         TimeUnit unit) {
       this.predicate = predicate;
       this.maxWait = unit.toMillis(maxWait);
       this.period = unit.toMillis(period);
+      this.maxPeriod = unit.toMillis(maxPeriod);
+   }
+
+   public RetryablePredicate(Predicate<T> predicate, long maxWait, long period, TimeUnit unit) {
+      this(predicate, maxWait, period, period * 10l, unit);
    }
 
    public RetryablePredicate(Predicate<T> predicate, long maxWait) {
-      this.predicate = predicate;
+      this(predicate, maxWait, 50l, 1000l, TimeUnit.MILLISECONDS);
-      this.maxWait = maxWait;
-      this.period = 50l;
    }
 
    @Override
    public boolean apply(T input) {
       try {
          long i = 1l;
-         for (Date end = new Date(System.currentTimeMillis() + maxWait); before(end); Thread
+         for (Date end = new Date(System.currentTimeMillis() + maxWait); before(end); Thread.sleep(nextMaxInterval(i++,
-               .sleep(nextMaxInterval(i++, end))) {
+                  end))) {
             if (predicate.apply(input)) {
                return true;
             } else if (atOrAfter(end)) {
@@ -67,14 +74,26 @@ public class RetryablePredicate<T> implements Predicate<T> {
             }
          }
       } catch (InterruptedException e) {
-         logger.warn(e, "predicate %s on %s interrupted, returning false",
+         logger.warn(e, "predicate %s on %s interrupted, returning false", input, predicate);
-               input, predicate);
+      } catch (RuntimeException e) {
+         if (getFirstThrowableOfType(e, ExecutionException.class) != null) {
+            logger.warn(e, "predicate %s on %s errored [%s], returning false", input, predicate, e.getMessage());
+            return false;
+         } else if (getFirstThrowableOfType(e, IllegalStateException.class) != null) {
+            logger.warn(e, "predicate %s on %s illegal state [%s], returning false", input, predicate, e.getMessage());
+            return false;
+         } else if (getFirstThrowableOfType(e, TimeoutException.class) != null) {
+            logger.warn(e, "predicate %s on %s timed out [%s], returning false", input, predicate, e.getMessage());
+            return false;
+         } else
+            throw e;
       }
       return false;
    }
 
    long nextMaxInterval(long attempt, Date end) {
       long interval = (period * (long) Math.pow(attempt, 1.5));
+      interval = interval > maxPeriod ? maxPeriod : interval;
       long max = end.getTime() - System.currentTimeMillis();
       return (interval > max) ? max : interval;
    }
@@ -87,3 +106,4 @@ public class RetryablePredicate<T> implements Predicate<T> {
       return new Date().compareTo(end) >= 0;
    }
 }
+

core/src/test/java/org/jclouds/http/BaseJettyTest.java

@@ -63,6 +63,7 @@ import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Optional;
 import org.testng.annotations.Parameters;
 
+import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.Multimap;
@@ -86,7 +87,7 @@ public abstract class BaseJettyTest {
    static final Pattern actionPattern = Pattern.compile("/objects/(.*)/action/([a-z]*);?(.*)");
 
    @BeforeTest
-   @Parameters({ "test-jetty-port" })
+   @Parameters( { "test-jetty-port" })
    public void setUpJetty(@Optional("8123") final int testPort) throws Exception {
       this.testPort = testPort;
 
@@ -96,62 +97,69 @@ public abstract class BaseJettyTest {
 
       Handler server1Handler = new AbstractHandler() {
          public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
-               throws IOException, ServletException {
+                  throws IOException, ServletException {
-            if (failIfNoContentLength(request, response)) {
+            InputStream body = request.getInputStream();
-               return;
+            try {
-            } else if (target.indexOf("sleep") > 0) {
+               if (failIfNoContentLength(request, response)) {
-               try {
+                  return;
-                  Thread.sleep(100);
+               } else if (target.indexOf("sleep") > 0) {
-               } catch (InterruptedException e) {
+                  try {
-                  propagate(e);
+                     Thread.sleep(100);
-               }
+                  } catch (InterruptedException e) {
-               response.setContentType("text/xml");
+                     propagate(e);
-               response.setStatus(HttpServletResponse.SC_OK);
+                  }
-            } else if (target.indexOf("redirect") > 0) {
+                  response.setContentType("text/xml");
-               response.sendRedirect("https://localhost:" + (testPort + 1) + "/");
+                  response.setStatus(HttpServletResponse.SC_OK);
-            } else if (target.indexOf("101constitutions") > 0) {
+               } else if (target.indexOf("redirect") > 0) {
-               response.setContentType("text/plain");
+                  response.sendRedirect("https://localhost:" + (testPort + 1) + "/");
-               response.setHeader("Content-MD5", md5);
+               } else if (target.indexOf("101constitutions") > 0) {
-               response.setStatus(HttpServletResponse.SC_OK);
+                  response.setContentType("text/plain");
-               copy(oneHundredOneConstitutions.getInput(), response.getOutputStream());
+                  response.setHeader("Content-MD5", md5);
-            } else if (request.getMethod().equals("PUT")) {
+                  response.setStatus(HttpServletResponse.SC_OK);
-               if (request.getContentLength() > 0) {
+                  copy(oneHundredOneConstitutions.getInput(), response.getOutputStream());
+               } else if (request.getMethod().equals("PUT")) {
+                  if (request.getContentLength() > 0) {
+                     response.setStatus(HttpServletResponse.SC_OK);
+                     response.getWriter().println(Strings2.toStringAndClose(body) + "PUT");
+                  } else {
+                     response.sendError(500, "no content");
+                  }
+               } else if (request.getMethod().equals("POST")) {
+                  // don't redirect large objects
+                  if (request.getContentLength() < 10240 && redirectEveryTwentyRequests(request, response))
+                     return;
+                  if (failEveryTenRequests(request, response))
+                     return;
+                  if (request.getContentLength() > 0) {
+                     handlePost(request, response);
+                  } else {
+                     handleAction(request, response);
+                  }
+               } else if (request.getHeader("range") != null) {
+                  response.sendError(404, "no content");
+               } else if (request.getHeader("test") != null) {
+                  response.setContentType("text/plain");
+                  response.setStatus(HttpServletResponse.SC_OK);
+                  response.getWriter().println("test");
+               } else if (request.getMethod().equals("HEAD")) {
+                  /*
+                   * NOTE: by HTML specification, HEAD response MUST NOT include a body
+                   */
+                  response.setContentType("text/xml");
                   response.setStatus(HttpServletResponse.SC_OK);
-                  response.getWriter().println(Strings2.toStringAndClose(request.getInputStream()) + "PUT");
                } else {
-                  response.sendError(500, "no content");
+                  if (failEveryTenRequests(request, response))
+                     return;
+                  response.setContentType("text/xml");
+                  response.setStatus(HttpServletResponse.SC_OK);
+                  response.getWriter().println(XML);
                }
-            } else if (request.getMethod().equals("POST")) {
+               ((Request) request).setHandled(true);
-               // don't redirect large objects
+            } catch (IOException e) {
-               if (request.getContentLength() < 10240 && redirectEveryTwentyRequests(request, response))
+               if (body != null)
-                  return;
+                  closeQuietly(body);
-               if (failEveryTenRequests(request, response))
+               response.sendError(500, Throwables.getStackTraceAsString(e));
-                  return;
-               if (request.getContentLength() > 0) {
-                  handlePost(request, response);
-               } else {
-                  handleAction(request, response);
-               }
-            } else if (request.getHeader("range") != null) {
-               response.sendError(404, "no content");
-            } else if (request.getHeader("test") != null) {
-               response.setContentType("text/plain");
-               response.setStatus(HttpServletResponse.SC_OK);
-               response.getWriter().println("test");
-            } else if (request.getMethod().equals("HEAD")) {
-               /*
-                * NOTE: by HTML specification, HEAD response MUST NOT include a body
-                */
-               response.setContentType("text/xml");
-               response.setStatus(HttpServletResponse.SC_OK);
-            } else {
-               if (failEveryTenRequests(request, response))
-                  return;
-               response.setContentType("text/xml");
-               response.setStatus(HttpServletResponse.SC_OK);
-               response.getWriter().println(XML);
             }
-            ((Request) request).setHandled(true);
          }
 
       };
@@ -172,11 +180,12 @@ public abstract class BaseJettyTest {
    }
 
    private static void handlePost(HttpServletRequest request, HttpServletResponse response) throws IOException {
+      InputStream body = request.getInputStream();
       try {
          if (request.getHeader("Content-MD5") != null) {
             String expectedMd5 = request.getHeader("Content-MD5");
             String realMd5FromRequest;
-            realMd5FromRequest = CryptoStreams.md5Base64(InputSuppliers.of(request.getInputStream()));
+            realMd5FromRequest = CryptoStreams.md5Base64(InputSuppliers.of(body));
             boolean matched = expectedMd5.equals(realMd5FromRequest);
             if (matched) {
                response.setStatus(HttpServletResponse.SC_OK);
@@ -185,52 +194,60 @@ public abstract class BaseJettyTest {
                response.sendError(500, "didn't match");
             }
          } else {
+            String responseString = (request.getContentLength() < 10240) ? Strings2.toStringAndClose(body) + "POST"
+                     : "POST";
+            body = null;
             for (String header : new String[] { "Content-Disposition", HttpHeaders.CONTENT_LANGUAGE,
-                  HttpHeaders.CONTENT_ENCODING })
+                     HttpHeaders.CONTENT_ENCODING })
                if (request.getHeader(header) != null) {
                   response.addHeader("x-" + header, request.getHeader(header));
                }
             response.setStatus(HttpServletResponse.SC_OK);
-            String responseString = "POST";
-            if (request.getContentLength() < 10240) {
-               responseString = Strings2.toStringAndClose(request.getInputStream()) + "POST";
-            } else {
-               closeQuietly(request.getInputStream());
-            }
             response.getWriter().println(responseString);
          }
       } catch (IOException e) {
-         response.sendError(500, e.toString());
+         if (body != null)
+            closeQuietly(body);
+         response.sendError(500, Throwables.getStackTraceAsString(e));
       }
    }
 
    protected void setupAndStartSSLServer(final int testPort) throws Exception {
       Handler server2Handler = new AbstractHandler() {
          public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
-               throws IOException, ServletException {
+                  throws IOException, ServletException {
-            if (request.getMethod().equals("PUT")) {
+            InputStream body = request.getInputStream();
-               if (request.getContentLength() > 0) {
+            try {
+               if (request.getMethod().equals("PUT")) {
+                  String text = Strings2.toStringAndClose(body);
+                  body = null;
+                  if (request.getContentLength() > 0) {
+                     response.setStatus(HttpServletResponse.SC_OK);
+                     response.getWriter().println(text + "PUTREDIRECT");
+                  }
+               } else if (request.getMethod().equals("POST")) {
+                  if (request.getContentLength() > 0) {
+                     handlePost(request, response);
+                  } else {
+                     handleAction(request, response);
+                  }
+               } else if (request.getMethod().equals("HEAD")) {
+                  /*
+                   * NOTE: by HTML specification, HEAD response MUST NOT include a body
+                   */
+                  response.setContentType("text/xml");
                   response.setStatus(HttpServletResponse.SC_OK);
-                  response.getWriter().println(Strings2.toStringAndClose(request.getInputStream()) + "PUTREDIRECT");
-               }
-            } else if (request.getMethod().equals("POST")) {
-               if (request.getContentLength() > 0) {
-                  handlePost(request, response);
                } else {
-                  handleAction(request, response);
+                  response.setContentType("text/xml");
+                  response.setStatus(HttpServletResponse.SC_OK);
+                  response.getWriter().println(XML2);
                }
-            } else if (request.getMethod().equals("HEAD")) {
+               ((Request) request).setHandled(true);
-               /*
+            } catch (IOException e) {
-                * NOTE: by HTML specification, HEAD response MUST NOT include a body
+               if (body != null)
-                */
+                  closeQuietly(body);
-               response.setContentType("text/xml");
+               response.sendError(500, Throwables.getStackTraceAsString(e));
-               response.setStatus(HttpServletResponse.SC_OK);
-            } else {
-               response.setContentType("text/xml");
-               response.setStatus(HttpServletResponse.SC_OK);
-               response.getWriter().println(XML2);
             }
-            ((Request) request).setHandled(true);
          }
       };
 
@@ -261,12 +278,12 @@ public abstract class BaseJettyTest {
    }
 
    public static RestContextBuilder<IntegrationTestClient, IntegrationTestAsyncClient> newBuilder(int testPort,
-         Properties properties, Module... connectionModules) {
+            Properties properties, Module... connectionModules) {
       properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, "true");
       properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, "true");
       RestContextSpec<IntegrationTestClient, IntegrationTestAsyncClient> contextSpec = contextSpec("test",
-            "http://localhost:" + testPort, "1", "", "identity", null, IntegrationTestClient.class,
+               "http://localhost:" + testPort, "1", "", "identity", null, IntegrationTestClient.class,
-            IntegrationTestAsyncClient.class, ImmutableSet.<Module> copyOf(connectionModules));
+               IntegrationTestAsyncClient.class, ImmutableSet.<Module> copyOf(connectionModules));
       return createContextBuilder(contextSpec, properties);
    }
 
@@ -300,7 +317,7 @@ public abstract class BaseJettyTest {
    }
 
    protected boolean redirectEveryTwentyRequests(HttpServletRequest request, HttpServletResponse response)
-         throws IOException {
+            throws IOException {
       if (cycle.incrementAndGet() % 20 == 0) {
          response.sendRedirect("http://localhost:" + (testPort + 1) + "/");
          ((Request) request).setHandled(true);

core/src/test/java/org/jclouds/http/handlers/BackoffLimitedRetryHandlerTest.java

@@ -60,38 +60,40 @@ public class BackoffLimitedRetryHandlerTest {
    BackoffLimitedRetryHandler handler = new BackoffLimitedRetryHandler();
 
    @Test
-   void testExponentialBackoffDelay() throws InterruptedException {
+   void testExponentialBackoffDelayDefaultMaxInterval500() throws InterruptedException {
-      long acceptableDelay = 25; // Delay to forgive if tests run long.
+      long period = 100;
+      long acceptableDelay = period - 1;
 
       long startTime = System.nanoTime();
-      handler.imposeBackoffExponentialDelay(1, "TEST FAILURE: 1");
+      handler.imposeBackoffExponentialDelay(period, 2, 1, 5, "TEST FAILURE: 1");
       long elapsedTime = (System.nanoTime() - startTime) / 1000000;
-      assert (elapsedTime >= 49) : elapsedTime;
+      assert (elapsedTime >= period - 1) : elapsedTime;
-      assertTrue(elapsedTime < 50 + acceptableDelay);
+      assertTrue(elapsedTime < period + acceptableDelay);
 
       startTime = System.nanoTime();
-      handler.imposeBackoffExponentialDelay(2, "TEST FAILURE: 2");
+      handler.imposeBackoffExponentialDelay(period, 2, 2, 5, "TEST FAILURE: 2");
       elapsedTime = (System.nanoTime() - startTime) / 1000000;
-      assert (elapsedTime >= 199) : elapsedTime;
+      assert (elapsedTime >= period * 4 - 1) : elapsedTime;
-      assertTrue(elapsedTime < 200 + acceptableDelay);
+      assertTrue(elapsedTime < period * 9);
 
       startTime = System.nanoTime();
-      handler.imposeBackoffExponentialDelay(3, "TEST FAILURE: 3");
+      handler.imposeBackoffExponentialDelay(period, 2, 3, 5, "TEST FAILURE: 3");
       elapsedTime = (System.nanoTime() - startTime) / 1000000;
-      assert (elapsedTime >= 449) : elapsedTime;
+      assert (elapsedTime >= period * 9 - 1) : elapsedTime;
-      assertTrue(elapsedTime < 450 + acceptableDelay);
+      assertTrue(elapsedTime < period * 10);
 
       startTime = System.nanoTime();
-      handler.imposeBackoffExponentialDelay(4, "TEST FAILURE: 4");
+      handler.imposeBackoffExponentialDelay(period, 2, 4, 5, "TEST FAILURE: 4");
       elapsedTime = (System.nanoTime() - startTime) / 1000000;
-      assert (elapsedTime >= 799) : elapsedTime;
+      assert (elapsedTime >= period * 10 - 1) : elapsedTime;
-      assertTrue(elapsedTime < 800 + acceptableDelay * 2);
+      assertTrue(elapsedTime < period * 11);
 
       startTime = System.nanoTime();
-      handler.imposeBackoffExponentialDelay(5, "TEST FAILURE: 5");
+      handler.imposeBackoffExponentialDelay(period, 2, 5, 5, "TEST FAILURE: 5");
       elapsedTime = (System.nanoTime() - startTime) / 1000000;
-      assert (elapsedTime >= 1249) : elapsedTime;
+      assert (elapsedTime >= period * 10 - 1) : elapsedTime;
-      assertTrue(elapsedTime < 1250 + acceptableDelay * 2);
+      assertTrue(elapsedTime < period * 11);
+
    }
 
    TransformingHttpCommandExecutorServiceImpl executorService;

core/src/test/java/org/jclouds/predicates/RetryablePredicateTest.java

@@ -19,12 +19,15 @@
 package org.jclouds.predicates;
 
 import java.util.Date;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import org.testng.annotations.Test;
 
 import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
+import com.google.common.base.Supplier;
 
 /**
  * 
@@ -33,58 +36,123 @@ import com.google.common.base.Predicates;
  */
 @Test(groups = "unit", sequential = true)
 public class RetryablePredicateTest {
+   public static int SLOW_BUILD_SERVER_GRACE = 100;
+
+   @Test
+   void testFalseOnIllegalStateExeception() {
+      ensureImmediateReturnFor(new IllegalStateException());
+   }
+
+   @SuppressWarnings("serial")
+   @Test
+   void testFalseOnExecutionException() {
+      ensureImmediateReturnFor(new ExecutionException() {
+      });
+   }
+
+   @SuppressWarnings("serial")
+   @Test
+   void testFalseOnTimeoutException() {
+      ensureImmediateReturnFor(new TimeoutException() {
+      });
+   }
+
+   @SuppressWarnings("serial")
+   @Test(expectedExceptions = RuntimeException.class)
+   void testPropagateOnException() {
+      ensureImmediateReturnFor(new Exception() {
+      });
+   }
+
+   private void ensureImmediateReturnFor(final Exception ex) {
+      RetryablePredicate<Supplier<String>> predicate = new RetryablePredicate<Supplier<String>>(
+               new Predicate<Supplier<String>>() {
+
+                  @Override
+                  public boolean apply(Supplier<String> input) {
+                     return "goo".equals(input.get());
+                  }
+
+               }, 3, 1, TimeUnit.SECONDS);
+      Date startPlusThird = new Date(System.currentTimeMillis() + 1000);
+      assert !predicate.apply(new Supplier<String>() {
+
+         @Override
+         public String get() {
+            throw new RuntimeException(ex);
+         }
+
+      });
+      Date now = new Date();
+      assert now.compareTo(startPlusThird) < 0 : String.format("%s should be less than %s", now.getTime(),
+               startPlusThird.getTime());
+   }
 
    @Test
    void testAlwaysTrue() {
-      RetryablePredicate<String> predicate = new RetryablePredicate<String>(Predicates
+      RetryablePredicate<String> predicate = new RetryablePredicate<String>(Predicates.<String> alwaysTrue(), 3, 1,
-               .<String> alwaysTrue(), 3, 1, TimeUnit.SECONDS);
+               TimeUnit.SECONDS);
-      Date startPlusSecond = new Date(System.currentTimeMillis() + 1000);
+      Date startPlusThird = new Date(System.currentTimeMillis() + 1000);
       predicate.apply("");
       Date now = new Date();
-      assert now.compareTo(startPlusSecond) < 0 : String.format("%s should be less than %s", now,
+      assert now.compareTo(startPlusThird) < 0 : String.format("%s should be less than %s", now.getTime(),
-               startPlusSecond);
+               startPlusThird.getTime());
    }
 
    @Test
    void testAlwaysFalseMillis() {
-      RetryablePredicate<String> predicate = new RetryablePredicate<String>(Predicates
+      RetryablePredicate<String> predicate = new RetryablePredicate<String>(Predicates.<String> alwaysFalse(), 3, 1,
-               .<String> alwaysFalse(), 3, 1, TimeUnit.SECONDS);
+               TimeUnit.SECONDS);
       Date startPlus3Seconds = new Date(System.currentTimeMillis() + 3000);
-      Date startPlus4Seconds = new Date(System.currentTimeMillis() + 4000);
+      Date startPlus4Seconds = new Date(System.currentTimeMillis() + 4000 + SLOW_BUILD_SERVER_GRACE);
       predicate.apply("");
       Date now = new Date();
-      assert now.compareTo(startPlus3Seconds) >= 0 : String.format("%s should be less than %s",
+      assert now.compareTo(startPlus3Seconds) >= 0 : String.format("%s should be less than %s", startPlus3Seconds
-               startPlus3Seconds, now);
+               .getTime(), now.getTime());
-      assert now.compareTo(startPlus4Seconds) <= 0 : String.format("%s should be greater than %s",
+      assert now.compareTo(startPlus4Seconds) <= 0 : String.format("%s should be greater than %s", startPlus4Seconds
-               startPlus4Seconds, now);
+               .getTime(), now.getTime());
 
    }
 
-   private static class SecondTimeTrue implements Predicate<String> {
+   private static class ThirdTimeTrue implements Predicate<String> {
 
       private int count = 0;
 
       @Override
       public boolean apply(String input) {
-         return count++ == 1;
+         return count++ == 2;
       }
 
    }
 
    @Test
-   void testSecondTimeTrue() {
+   void testThirdTimeTrue() {
-      RetryablePredicate<String> predicate = new RetryablePredicate<String>(new SecondTimeTrue(),
+      RetryablePredicate<String> predicate = new RetryablePredicate<String>(new ThirdTimeTrue(), 3, 1, TimeUnit.SECONDS);
-               3, 1, TimeUnit.SECONDS);
 
-      Date startPlusSecond = new Date(System.currentTimeMillis() + 1000);
+      Date startPlus = new Date(System.currentTimeMillis() + 1000);
-      Date startPlus2Seconds = new Date(System.currentTimeMillis() + 2000);
+      Date startPlus3 = new Date(System.currentTimeMillis() + 3000 + SLOW_BUILD_SERVER_GRACE);
 
       predicate.apply("");
       Date now = new Date();
-      assert now.compareTo(startPlusSecond) >= 0 : String.format("%s should be greater than %s",
+      assert now.compareTo(startPlus) >= 0 : String.format("%s should be greater than %s", now.getTime(), startPlus
-               now, startPlusSecond);
+               .getTime());
-      assert now.compareTo(startPlus2Seconds) <= 0 : String.format("%s should be greater than %s",
+      assert now.compareTo(startPlus3) <= 0 : String.format("%s should be greater than %s", startPlus3.getTime(), now
-               startPlus2Seconds, now);
+               .getTime());
    }
 
+   @Test
+   void testThirdTimeTrueLimitedMaxInterval() {
+      RetryablePredicate<String> predicate = new RetryablePredicate<String>(new ThirdTimeTrue(), 3, 1, 1,
+               TimeUnit.SECONDS);
+
+      Date startPlus = new Date(System.currentTimeMillis() + 1000);
+      Date startPlus2 = new Date(System.currentTimeMillis() + 2000 + SLOW_BUILD_SERVER_GRACE);
+
+      predicate.apply("");
+      Date now = new Date();
+      assert now.compareTo(startPlus) >= 0 : String.format("%s should be greater than %s", now.getTime(), startPlus
+               .getTime());
+      assert now.compareTo(startPlus2) <= 0 : String.format("%s should be greater than %s", startPlus2.getTime(), now
+               .getTime());
+   }
 }