JCLOUDS-342: Retry on HTTP 408 for swift

To repro issue 342, the following steps were taken:
 - Spawn 500 parallel / 2000 total putBlobs to cloudfiles-uk
 - issue a SIGSTOP
 - wait 60 seconds
 - issue a SIGCONT

Without this patch, there are several hundred 408s.
With this patch, these are retried and complete successfully.

common/openstack/src/main/java/org/jclouds/openstack/keystone/v1_1/handlers/RetryOnRenew.java

@@ -20,11 +20,14 @@ import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
 import static org.jclouds.http.HttpUtils.releasePayload;
 
 import javax.annotation.Resource;
+import javax.inject.Named;
 
+import org.jclouds.Constants;
 import org.jclouds.domain.Credentials;
 import org.jclouds.http.HttpCommand;
 import org.jclouds.http.HttpResponse;
 import org.jclouds.http.HttpRetryHandler;
+import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
 import org.jclouds.logging.Logger;
 import org.jclouds.openstack.keystone.v1_1.domain.Auth;
 import org.jclouds.openstack.reference.AuthHeaders;
@@ -42,14 +45,22 @@ import com.google.inject.Singleton;
  */
 @Singleton
 public class RetryOnRenew implements HttpRetryHandler {
+   @Inject(optional = true)
+   @Named(Constants.PROPERTY_MAX_RETRIES)
+   private int retryCountLimit = 5;
+
    @Resource
    protected Logger logger = Logger.NULL;
 
    private final LoadingCache<Credentials, Auth> authenticationResponseCache;
 
+   private final BackoffLimitedRetryHandler backoffHandler;
+
    @Inject
-   protected RetryOnRenew(LoadingCache<Credentials, Auth> authenticationResponseCache) {
+   protected RetryOnRenew(LoadingCache<Credentials, Auth> authenticationResponseCache,
+           BackoffLimitedRetryHandler backoffHandler) {
       this.authenticationResponseCache = authenticationResponseCache;
+      this.backoffHandler = backoffHandler;
    }
 
    @Override
@@ -74,6 +85,8 @@ public class RetryOnRenew implements HttpRetryHandler {
                   }
                }
                break;
+            case 408:
+               return backoffHandler.shouldRetryRequest(command, response);
          }
          return retry;
 

common/openstack/src/test/java/org/jclouds/openstack/keystone/v1_1/handlers/RetryOnRenewTest.java

@@ -27,6 +27,7 @@ import org.jclouds.domain.Credentials;
 import org.jclouds.http.HttpCommand;
 import org.jclouds.http.HttpRequest;
 import org.jclouds.http.HttpResponse;
+import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
 import org.jclouds.io.Payloads;
 import org.jclouds.openstack.keystone.v1_1.domain.Auth;
 import org.testng.annotations.Test;
@@ -47,6 +48,7 @@ public class RetryOnRenewTest {
       HttpResponse response = createMock(HttpResponse.class);
       @SuppressWarnings("unchecked")
       LoadingCache<Credentials, Auth> cache = createMock(LoadingCache.class);
+      BackoffLimitedRetryHandler backoffHandler = createMock(BackoffLimitedRetryHandler.class);
 
       expect(command.getCurrentRequest()).andReturn(request);
 
@@ -59,13 +61,71 @@ public class RetryOnRenewTest {
       replay(command);
       replay(response);
       replay(cache);
+      replay(backoffHandler);
 
-      RetryOnRenew retry = new RetryOnRenew(cache);
+      RetryOnRenew retry = new RetryOnRenew(cache, backoffHandler);
 
       assertTrue(retry.shouldRetryRequest(command, response));
 
       verify(command);
       verify(response);
       verify(cache);
+      verify(backoffHandler);
+   }
+
+   @Test
+   public void test408ShouldRetry() {
+      HttpCommand command = createMock(HttpCommand.class);
+      HttpRequest request = createMock(HttpRequest.class);
+      HttpResponse response = createMock(HttpResponse.class);
+      @SuppressWarnings("unchecked")
+      LoadingCache<Credentials, Auth> cache = createMock(LoadingCache.class);
+      BackoffLimitedRetryHandler backoffHandler = createMock(BackoffLimitedRetryHandler.class);
+
+      expect(response.getPayload()).andReturn(Payloads.newStringPayload(
+                  "The server has waited too long for the request to be sent by the client.")).times(2);
+      expect(backoffHandler.shouldRetryRequest(command, response)).andReturn(true).once();
+      expect(response.getStatusCode()).andReturn(408).once();
+
+      replay(command);
+      replay(response);
+      replay(cache);
+      replay(backoffHandler);
+
+      RetryOnRenew retry = new RetryOnRenew(cache, backoffHandler);
+
+      assertTrue(retry.shouldRetryRequest(command, response));
+
+      verify(command);
+      verify(response);
+      verify(cache);
+      verify(backoffHandler);
+   }
+
+   @Test
+   public void test404ShouldNotRetry() {
+      HttpCommand command = createMock(HttpCommand.class);
+      HttpRequest request = createMock(HttpRequest.class);
+      HttpResponse response = createMock(HttpResponse.class);
+      @SuppressWarnings("unchecked")
+      LoadingCache<Credentials, Auth> cache = createMock(LoadingCache.class);
+      BackoffLimitedRetryHandler backoffHandler = createMock(BackoffLimitedRetryHandler.class);
+
+      expect(response.getPayload()).andReturn(Payloads.newStringPayload("")).times(2);
+      expect(response.getStatusCode()).andReturn(404).once();
+
+      replay(command);
+      replay(response);
+      replay(cache);
+      replay(backoffHandler);
+
+      RetryOnRenew retry = new RetryOnRenew(cache, backoffHandler);
+
+      assertTrue(!retry.shouldRetryRequest(command, response));
+
+      verify(command);
+      verify(response);
+      verify(cache);
+      verify(backoffHandler);
    }
 }