JCLOUDS-100. Add AWSServerErrorRetryHandler, with an increased back

off time.

apis/s3/src/main/java/org/jclouds/s3/config/S3RestClientModule.java

@@ -27,6 +27,7 @@ import javax.inject.Singleton;
 import org.jclouds.Constants;
 import org.jclouds.aws.config.AWSRestClientModule;
 import org.jclouds.aws.handlers.AWSClientErrorRetryHandler;
+import org.jclouds.aws.handlers.AWSServerErrorRetryHandler;
 import org.jclouds.blobstore.ContainerNotFoundException;
 import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
@@ -185,6 +186,7 @@ public class S3RestClientModule<S extends S3Client, A extends S3AsyncClient> ext
    protected void bindRetryHandlers() {
       bind(HttpRetryHandler.class).annotatedWith(Redirection.class).to(S3RedirectionRetryHandler.class);
       bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(AWSClientErrorRetryHandler.class);
+      bind(HttpRetryHandler.class).annotatedWith(ServerError.class).to(AWSServerErrorRetryHandler.class);
    }
 
    @Provides

apis/sqs/src/main/java/org/jclouds/sqs/config/SQSRestClientModule.java

@@ -22,6 +22,7 @@ import static org.jclouds.reflect.Reflection2.typeToken;
 import java.util.Map;
 
 import org.jclouds.aws.config.FormSigningRestClientModule;
+import org.jclouds.aws.handlers.AWSServerErrorRetryHandler;
 import org.jclouds.http.HttpErrorHandler;
 import org.jclouds.http.HttpRetryHandler;
 import org.jclouds.http.annotation.ClientError;
@@ -68,6 +69,7 @@ public class SQSRestClientModule extends FormSigningRestClientModule<SQSApi, SQS
    @Override
    protected void bindRetryHandlers() {
       bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(SQSErrorRetryHandler.class);
+      bind(HttpRetryHandler.class).annotatedWith(ServerError.class).to(AWSServerErrorRetryHandler.class);
    }
 
 }

apis/sts/src/main/java/org/jclouds/aws/config/AWSHttpApiModule.java

@@ -22,6 +22,7 @@ import java.util.Set;
 import javax.inject.Singleton;
 
 import org.jclouds.aws.handlers.AWSClientErrorRetryHandler;
+import org.jclouds.aws.handlers.AWSServerErrorRetryHandler;
 import org.jclouds.aws.handlers.ParseAWSErrorFromXmlContent;
 import org.jclouds.http.HttpErrorHandler;
 import org.jclouds.http.HttpRetryHandler;
@@ -56,6 +57,13 @@ public abstract class AWSHttpApiModule<A> extends HttpApiModule<A> {
       return ImmutableSet.of("RequestTimeout", "OperationAborted", "SignatureDoesNotMatch");
    }
    
+   @Provides
+   @ServerError
+   @Singleton
+   protected Set<String> provideRetryableServerCodes(){
+      return ImmutableSet.of("RequestLimitExceeded");
+   }
+
    @Override
    protected void bindErrorHandlers() {
       bind(HttpErrorHandler.class).annotatedWith(Redirection.class).to(ParseAWSErrorFromXmlContent.class);
@@ -66,6 +74,7 @@ public abstract class AWSHttpApiModule<A> extends HttpApiModule<A> {
    @Override
    protected void bindRetryHandlers() {
       bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(AWSClientErrorRetryHandler.class);
+      bind(HttpRetryHandler.class).annotatedWith(ServerError.class).to(AWSServerErrorRetryHandler.class);
    }
 
 }

apis/sts/src/main/java/org/jclouds/aws/config/AWSRestClientModule.java

@@ -23,6 +23,7 @@ import java.util.Set;
 import javax.inject.Singleton;
 
 import org.jclouds.aws.handlers.AWSClientErrorRetryHandler;
+import org.jclouds.aws.handlers.AWSServerErrorRetryHandler;
 import org.jclouds.aws.handlers.ParseAWSErrorFromXmlContent;
 import org.jclouds.http.HttpErrorHandler;
 import org.jclouds.http.HttpRetryHandler;
@@ -71,6 +72,13 @@ public abstract class AWSRestClientModule<S, A> extends RestClientModule<S, A> {
       return ImmutableSet.of("RequestTimeout", "OperationAborted", "SignatureDoesNotMatch");
    }
    
+   @Provides
+   @ServerError
+   @Singleton
+   protected Set<String> provideRetryableServerCodes(){
+      return ImmutableSet.of("RequestLimitExceeded");
+   }
+
    @Override
    protected void bindErrorHandlers() {
       bind(HttpErrorHandler.class).annotatedWith(Redirection.class).to(ParseAWSErrorFromXmlContent.class);
@@ -81,6 +89,7 @@ public abstract class AWSRestClientModule<S, A> extends RestClientModule<S, A> {
    @Override
    protected void bindRetryHandlers() {
       bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(AWSClientErrorRetryHandler.class);
+      bind(HttpRetryHandler.class).annotatedWith(ServerError.class).to(AWSServerErrorRetryHandler.class);
    }
 
 }

apis/sts/src/main/java/org/jclouds/aws/handlers/AWSServerErrorRetryHandler.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.aws.handlers;
+
+import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
+
+import java.util.Set;
+
+import org.jclouds.aws.domain.AWSError;
+import org.jclouds.aws.util.AWSUtils;
+import org.jclouds.http.HttpCommand;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.HttpRetryHandler;
+import org.jclouds.http.annotation.ServerError;
+import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+/**
+ * Handles Retryable responses with error codes in the 5xx range
+ * 
+ * @author Andrew Bayer
+ */
+@Singleton
+public class AWSServerErrorRetryHandler extends BackoffLimitedRetryHandler {
+
+   private final AWSUtils utils;
+   private final Set<String> retryableServerCodes;
+
+   @Inject
+   public AWSServerErrorRetryHandler(AWSUtils utils,
+         @ServerError Set<String> retryableServerCodes) {
+      this.utils = utils;
+      this.retryableServerCodes = retryableServerCodes;
+   }
+
+   @Override
+   public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) {
+      if (response.getStatusCode() == 503) {
+         // Content can be null in the case of HEAD requests
+         if (response.getPayload() != null) {
+            closeClientButKeepContentStream(response);
+            AWSError error = utils.parseAWSErrorFromContent(command.getCurrentRequest(), response);
+            if (error != null) {
+               return shouldRetryRequestOnError(command, response, error);
+            }
+         }
+      }
+      return false;
+   }
+
+   protected boolean shouldRetryRequestOnError(HttpCommand command, HttpResponse response, AWSError error) {
+      if (retryableServerCodes.contains(error.getCode()))
+         return super.shouldRetryRequest(command, response);
+      return false;
+   }
+
+   public void imposeBackoffExponentialDelay(long period, int pow, int failureCount, int max, String commandDescription) {
+      imposeBackoffExponentialDelay(period, period * 100l, pow, failureCount, max, commandDescription);
+   }
+
+   
+}

apis/sts/src/test/java/org/jclouds/aws/handlers/AWSServerErrorRetryHandlerTest.java

@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.aws.handlers;
+import static javax.ws.rs.HttpMethod.PUT;
+import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR;
+import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.testng.Assert.assertFalse;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.easymock.IAnswer;
+
+import org.jclouds.aws.domain.AWSError;
+import org.jclouds.aws.util.AWSUtils;
+import org.jclouds.http.HttpCommand;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
+import org.jclouds.io.Payloads;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * Tests behavior of {@code AWSServerErrorRetryHandler}
+ * 
+ * @author Andrew Bayer
+ */
+@Test(groups = "unit", testName = "AWSServerErrorRetryHandlerTest")
+public class AWSServerErrorRetryHandlerTest {
+   @Test
+   public void test500DoesNotRetry() {
+
+      AWSUtils utils = createMock(AWSUtils.class);
+      HttpCommand command = createMock(HttpCommand.class);
+
+      replay(utils, command);
+
+      AWSServerErrorRetryHandler retry = new AWSServerErrorRetryHandler(utils,
+            ImmutableSet.<String> of());
+
+      assertFalse(retry.shouldRetryRequest(command, HttpResponse.builder().statusCode(INTERNAL_SERVER_ERROR.getStatusCode()).build()));
+
+      verify(utils, command);
+
+   }
+
+   @DataProvider(name = "codes")
+   public Object[][] createData() {
+      return new Object[][] { { "RequestLimitExceeded" } };
+   }
+
+   @Test(dataProvider = "codes")
+   public void test503DoesBackoffAndRetryForCode(String code) {
+
+      AWSUtils utils = createMock(AWSUtils.class);
+      HttpCommand command = createMock(HttpCommand.class);
+
+      HttpRequest putBucket = HttpRequest.builder().method(PUT)
+            .endpoint("https://adriancole-blobstore113.s3.amazonaws.com/").build();
+
+      HttpResponse limitExceeded = HttpResponse.builder().statusCode(SERVICE_UNAVAILABLE.getStatusCode())
+            .payload(Payloads.newStringPayload(String.format("<Error><Code>%s</Code></Error>", code))).build();
+
+      expect(command.getCurrentRequest()).andReturn(putBucket);
+      final AtomicInteger counter = new AtomicInteger();
+      expect(command.incrementFailureCount()).andAnswer(new IAnswer<Integer>() {
+         @Override
+         public Integer answer() throws Throwable {
+            return counter.incrementAndGet();
+         }
+      }).anyTimes();
+      expect(command.isReplayable()).andReturn(true).anyTimes();
+      expect(command.getFailureCount()).andAnswer(new IAnswer<Integer>() {
+         @Override
+         public Integer answer() throws Throwable {
+            return counter.get();
+         }
+      }).anyTimes();
+
+      AWSError error = new AWSError();
+      error.setCode(code);
+
+      expect(utils.parseAWSErrorFromContent(putBucket, limitExceeded)).andReturn(error);
+
+      replay(utils, command);
+
+      AWSServerErrorRetryHandler retry = new AWSServerErrorRetryHandler(utils,
+            ImmutableSet.<String> of("RequestLimitExceeded"));
+
+      assert retry.shouldRetryRequest(command, limitExceeded);
+
+      verify(utils, command);
+
+   }
+
+}