Issue 69: added in S3 specific retry handling

git-svn-id: http://jclouds.googlecode.com/svn/trunk@1467 3d8758e0-26b5-11de-8745-db77d3ebf521

aws/s3/core/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java

@@ -33,6 +33,7 @@ import static org.jclouds.command.pool.PoolConstants.PROPERTY_POOL_MAX_SESSION_F
 import static org.jclouds.command.pool.PoolConstants.PROPERTY_POOL_REQUEST_INVOKER_THREADS;
 import static org.jclouds.http.HttpConstants.PROPERTY_HTTP_ADDRESS;
 import static org.jclouds.http.HttpConstants.PROPERTY_HTTP_MAX_RETRIES;
+import static org.jclouds.http.HttpConstants.PROPERTY_HTTP_MAX_REDIRECTS;
 import static org.jclouds.http.HttpConstants.PROPERTY_HTTP_PORT;
 import static org.jclouds.http.HttpConstants.PROPERTY_HTTP_SECURE;
 
@@ -96,6 +97,7 @@ public class S3ContextFactory {
       properties.setProperty(PROPERTY_HTTP_ADDRESS, "s3.amazonaws.com");
       properties.setProperty(PROPERTY_HTTP_SECURE, "true");
       properties.setProperty(PROPERTY_HTTP_MAX_RETRIES, "5");
+      properties.setProperty(PROPERTY_HTTP_MAX_REDIRECTS, "5");
       properties.setProperty(PROPERTY_POOL_MAX_CONNECTION_REUSE, "75");
       properties.setProperty(PROPERTY_POOL_MAX_SESSION_FAILURES, "2");
       properties.setProperty(PROPERTY_POOL_REQUEST_INVOKER_THREADS, "1");
@@ -168,6 +170,12 @@ public class S3ContextFactory {
       return this;
    }
 
+   public S3ContextFactory withHttpMaxRedirects(int httpMaxRedirects) {
+      properties.setProperty(PROPERTY_HTTP_MAX_REDIRECTS, Integer.toString(httpMaxRedirects));
+      return this;
+   }
+
+   
    public S3ContextFactory withHttpPort(int httpPort) {
       properties.setProperty(PROPERTY_HTTP_PORT, Integer.toString(httpPort));
       return this;

aws/s3/core/src/main/java/org/jclouds/aws/s3/config/LiveS3ConnectionModule.java

@@ -32,11 +32,14 @@ import javax.annotation.Resource;
 
 import org.jclouds.aws.s3.S3Connection;
 import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
+import org.jclouds.aws.s3.handlers.AWSClientErrorRetryHandler;
+import org.jclouds.aws.s3.handlers.AWSRedirectionRetryHandler;
 import org.jclouds.aws.s3.handlers.ParseAWSErrorFromXmlContent;
 import org.jclouds.aws.s3.internal.LiveS3Connection;
 import org.jclouds.http.HttpConstants;
 import org.jclouds.http.HttpErrorHandler;
 import org.jclouds.http.HttpRequestFilter;
+import org.jclouds.http.HttpRetryHandler;
 import org.jclouds.http.annotation.ClientError;
 import org.jclouds.http.annotation.Redirection;
 import org.jclouds.http.annotation.ServerError;
@@ -74,17 +77,25 @@ public class LiveS3ConnectionModule extends AbstractModule {
 
       bind(S3Connection.class).to(LiveS3Connection.class).in(Scopes.SINGLETON);
       bindErrorHandlers();
+      bindRetryHandlers();
       requestInjection(this);
       logger.info("S3 Context = %1$s://%2$s:%3$s", (isSecure ? "https" : "http"), address, port);
    }
 
    protected void bindErrorHandlers() {
       bind(HttpErrorHandler.class).annotatedWith(Redirection.class).to(
-               ParseAWSErrorFromXmlContent.class).in(Scopes.SINGLETON);
+               ParseAWSErrorFromXmlContent.class);
-      bind(HttpErrorHandler.class).annotatedWith(ServerError.class).to(
-               ParseAWSErrorFromXmlContent.class).in(Scopes.SINGLETON);
       bind(HttpErrorHandler.class).annotatedWith(ClientError.class).to(
-               ParseAWSErrorFromXmlContent.class).in(Scopes.SINGLETON);
+               ParseAWSErrorFromXmlContent.class);
+      bind(HttpErrorHandler.class).annotatedWith(ServerError.class).to(
+               ParseAWSErrorFromXmlContent.class);
+   }
+
+   protected void bindRetryHandlers() {
+      bind(HttpRetryHandler.class).annotatedWith(Redirection.class).to(
+               AWSRedirectionRetryHandler.class);
+      bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(
+               AWSClientErrorRetryHandler.class);
    }
 
    @Provides

aws/s3/core/src/main/java/org/jclouds/aws/s3/handlers/AWSClientErrorRetryHandler.java

@@ -0,0 +1,78 @@
+/**
+ *
+ * Copyright (C) 2009 Global Cloud Specialists, Inc. <info@globalcloudspecialists.com>
+ *
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ */
+package org.jclouds.aws.s3.handlers;
+
+import javax.annotation.Resource;
+
+import org.jclouds.aws.domain.AWSError;
+import org.jclouds.aws.s3.util.S3Utils;
+import org.jclouds.aws.s3.xml.S3ParserFactory;
+import org.jclouds.http.HttpException;
+import org.jclouds.http.HttpFutureCommand;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.HttpRetryHandler;
+import org.jclouds.logging.Logger;
+
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+
+/**
+ * Handles Retryable responses with error codes in the 3xx range
+ * 
+ * @author Adrian Cole
+ */
+public class AWSClientErrorRetryHandler implements HttpRetryHandler {
+   private final S3ParserFactory parserFactory;
+
+   private final int retryCountLimit;
+
+   @Resource
+   protected Logger logger = Logger.NULL;
+
+   @Inject
+   public AWSClientErrorRetryHandler(S3ParserFactory parserFactory,
+            @Named("jclouds.http.max-retries") int retryCountLimit) {
+      this.retryCountLimit = retryCountLimit;
+      this.parserFactory = parserFactory;
+   }
+
+   public boolean shouldRetryRequest(HttpFutureCommand<?> command, HttpResponse response) {
+      if (command.getFailureCount() > retryCountLimit)
+         return false;
+      if (response.getStatusCode() == 400) {
+         byte[] content = S3Utils.closeConnectionButKeepContentStream(response);
+         command.incrementRedirectCount();
+         try {
+            AWSError error = S3Utils.parseAWSErrorFromContent(parserFactory, command, response,
+                     new String(content));
+            if ("RequestTimeout".equals(error.getCode())) {
+               return true;
+            }
+         } catch (HttpException e) {
+            logger.warn(e, "error parsing response: %s", new String(content));
+         }
+      }
+      return false;
+   }
+}

aws/s3/core/src/main/java/org/jclouds/aws/s3/handlers/AWSRedirectionRetryHandler.java

@@ -0,0 +1,85 @@
+/**
+ *
+ * Copyright (C) 2009 Global Cloud Specialists, Inc. <info@globalcloudspecialists.com>
+ *
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ */
+package org.jclouds.aws.s3.handlers;
+
+import java.net.URI;
+
+import org.jclouds.aws.domain.AWSError;
+import org.jclouds.aws.s3.reference.S3Constants;
+import org.jclouds.aws.s3.util.S3Utils;
+import org.jclouds.aws.s3.xml.S3ParserFactory;
+import org.jclouds.http.HttpException;
+import org.jclouds.http.HttpFutureCommand;
+import org.jclouds.http.HttpMethod;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.handlers.RedirectionRetryHandler;
+import org.jclouds.util.Utils;
+
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+
+/**
+ * Handles Retryable responses with error codes in the 3xx range
+ * 
+ * @author Adrian Cole
+ */
+public class AWSRedirectionRetryHandler extends RedirectionRetryHandler {
+   private final S3ParserFactory parserFactory;
+
+   @Inject
+   public AWSRedirectionRetryHandler(S3ParserFactory parserFactory,
+            @Named("jclouds.http.max-redirects") int retryCountLimit) {
+      super(retryCountLimit);
+      this.parserFactory = parserFactory;
+   }
+
+   public boolean shouldRetryRequest(HttpFutureCommand<?> command, HttpResponse response) {
+      if (response.getStatusCode() == 301) {
+         byte[] content = S3Utils.closeConnectionButKeepContentStream(response);
+         if (command.getRequest().getMethod() == HttpMethod.HEAD) {
+            command.getRequest().setMethod(HttpMethod.GET);
+            return true;
+         } else {
+            command.incrementRedirectCount();
+            try {
+               AWSError error = S3Utils.parseAWSErrorFromContent(parserFactory, command, response,
+                        new String(content));
+               String host = error.getDetails().get(S3Constants.ENDPOINT);
+               if (host != null) {
+                  URI endPoint = command.getRequest().getEndPoint();
+                  endPoint = Utils.replaceHostInEndPoint(endPoint, host);
+                  command.getRequest().setEndPoint(endPoint);
+                  return true;
+               } else {
+                  return false;
+               }
+            } catch (HttpException e) {
+               return false;
+            }
+         }
+      } else {
+         return super.shouldRetryRequest(command, response);
+      }
+   }
+}

aws/s3/core/src/main/java/org/jclouds/aws/s3/handlers/ParseAWSErrorFromXmlContent.java

@@ -23,17 +23,13 @@
  */
 package org.jclouds.aws.s3.handlers;
 
-import java.io.ByteArrayInputStream;
-
 import javax.annotation.Resource;
 
 import org.jclouds.aws.AWSResponseException;
 import org.jclouds.aws.domain.AWSError;
-import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
+import org.jclouds.aws.s3.util.S3Utils;
-import org.jclouds.aws.s3.reference.S3Headers;
 import org.jclouds.aws.s3.xml.S3ParserFactory;
 import org.jclouds.http.HttpErrorHandler;
-import org.jclouds.http.HttpException;
 import org.jclouds.http.HttpFutureCommand;
 import org.jclouds.http.HttpResponse;
 import org.jclouds.http.HttpResponseException;
@@ -68,7 +64,8 @@ public class ParseAWSErrorFromXmlContent implements HttpErrorHandler {
          if (content != null) {
             try {
                if (content.indexOf('<') >= 0) {
-                  AWSError error = parseAWSErrorFromContent(command, response, content);
+                  AWSError error = S3Utils.parseAWSErrorFromContent(parserFactory, command,
+                           response, content);
                   command.setException(new AWSResponseException(command, response, error));
                } else {
                   command.setException(new HttpResponseException(command, response, content));
@@ -86,15 +83,4 @@ public class ParseAWSErrorFromXmlContent implements HttpErrorHandler {
       }
    }
 
-   private AWSError parseAWSErrorFromContent(HttpFutureCommand<?> command, HttpResponse response,
-            String content) throws HttpException {
-      AWSError error = parserFactory.createErrorParser().parse(
-               new ByteArrayInputStream(content.getBytes()));
-      error.setRequestId(response.getFirstHeaderOrNull(S3Headers.REQUEST_ID));
-      error.setRequestToken(response.getFirstHeaderOrNull(S3Headers.REQUEST_TOKEN));
-      if ("SignatureDoesNotMatch".equals(error.getCode()))
-         error.setStringSigned(RequestAuthorizeSignature.createStringToSign(command.getRequest()));
-      return error;
-   }
-
 }

aws/s3/core/src/main/java/org/jclouds/aws/s3/reference/S3Constants.java

@@ -40,10 +40,11 @@ public interface S3Constants extends AWSConstants, S3Headers {
     * time to pause before retrying a transient failure
     */
    public static final String PROPERTY_S3_MAP_RETRY = "jclouds.s3.map.retry";
-   
+
    /**
     * S3 service's XML Namespace, as used in XML request and response documents.
     */
    public static final String S3_REST_API_XML_NAMESPACE = "http://s3.amazonaws.com/doc/2006-03-01/";
+   public static final String ENDPOINT = "Endpoint";
 
 }

aws/s3/core/src/main/java/org/jclouds/aws/s3/util/S3Utils.java

@@ -28,8 +28,15 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.output.ByteArrayOutputStream;
 import org.bouncycastle.crypto.digests.MD5Digest;
+import org.jclouds.aws.domain.AWSError;
 import org.jclouds.aws.s3.domain.S3Object;
+import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
+import org.jclouds.aws.s3.reference.S3Headers;
+import org.jclouds.aws.s3.xml.S3ParserFactory;
 import org.jclouds.aws.util.AWSUtils;
+import org.jclouds.http.HttpException;
+import org.jclouds.http.HttpFutureCommand;
+import org.jclouds.http.HttpResponse;
 
 import java.io.*;
 
@@ -40,6 +47,25 @@ import java.io.*;
  */
 public class S3Utils extends AWSUtils {
 
+   public static AWSError parseAWSErrorFromContent(S3ParserFactory parserFactory,
+            HttpFutureCommand<?> command, HttpResponse response, InputStream content)
+            throws HttpException {
+      AWSError error = parserFactory.createErrorParser().parse(content);
+      error.setRequestId(response.getFirstHeaderOrNull(S3Headers.REQUEST_ID));
+      error.setRequestToken(response.getFirstHeaderOrNull(S3Headers.REQUEST_TOKEN));
+      if ("SignatureDoesNotMatch".equals(error.getCode()))
+         error.setStringSigned(RequestAuthorizeSignature.createStringToSign(command.getRequest()));
+      return error;
+
+   }
+
+   public static AWSError parseAWSErrorFromContent(S3ParserFactory parserFactory,
+            HttpFutureCommand<?> command, HttpResponse response, String content)
+            throws HttpException {
+      return parseAWSErrorFromContent(parserFactory, command, response, new ByteArrayInputStream(
+               content.getBytes()));
+   }
+
    public static String validateBucketName(String bucketName) {
       checkNotNull(bucketName, "bucketName");
       checkArgument(bucketName.matches("^[a-z0-9].*"),

aws/s3/core/src/test/java/org/jclouds/aws/s3/config/S3ContextModuleTest.java

@@ -25,11 +25,14 @@ package org.jclouds.aws.s3.config;
 
 import static org.testng.Assert.assertEquals;
 
+import org.jclouds.aws.s3.handlers.AWSClientErrorRetryHandler;
+import org.jclouds.aws.s3.handlers.AWSRedirectionRetryHandler;
 import org.jclouds.aws.s3.handlers.ParseAWSErrorFromXmlContent;
 import org.jclouds.aws.s3.reference.S3Constants;
 import org.jclouds.aws.s3.xml.config.S3ParserModule;
 import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
 import org.jclouds.http.handlers.DelegatingErrorHandler;
+import org.jclouds.http.handlers.DelegatingRetryHandler;
 import org.testng.annotations.Test;
 
 import com.google.inject.Guice;
@@ -61,6 +64,8 @@ public class S3ContextModuleTest {
                               .to("false");
                      bindConstant().annotatedWith(
                               Names.named(S3Constants.PROPERTY_HTTP_MAX_RETRIES)).to("5");
+                     bindConstant().annotatedWith(
+                              Names.named(S3Constants.PROPERTY_HTTP_MAX_REDIRECTS)).to("5");
                      super.configure();
                   }
                }, new JavaUrlHttpFutureCommandClientModule());
@@ -78,4 +83,16 @@ public class S3ContextModuleTest {
       assertEquals(handler.getClientErrorHandler().getClass(), ParseAWSErrorFromXmlContent.class);
    }
 
+   @Test
+   void testClientRetryHandler() {
+      DelegatingRetryHandler handler = createInjector().getInstance(DelegatingRetryHandler.class);
+      assertEquals(handler.getClientErrorRetryHandler().getClass(), AWSClientErrorRetryHandler.class);
+   }
+   
+   @Test
+   void testRedirectionRetryHandler() {
+      DelegatingRetryHandler handler = createInjector().getInstance(DelegatingRetryHandler.class);
+      assertEquals(handler.getRedirectionRetryHandler().getClass(), AWSRedirectionRetryHandler.class);
+   }
+   
 }

core/src/main/java/org/jclouds/http/HttpConstants.java

@@ -29,8 +29,9 @@ package org.jclouds.http;
  * @author Adrian Cole
  */
 public interface HttpConstants extends HttpHeaders, ContentTypes {
-    public static final String PROPERTY_HTTP_SECURE = "jclouds.http.secure";
+   public static final String PROPERTY_HTTP_SECURE = "jclouds.http.secure";
-    public static final String PROPERTY_HTTP_PORT = "jclouds.http.port";
+   public static final String PROPERTY_HTTP_PORT = "jclouds.http.port";
-    public static final String PROPERTY_HTTP_ADDRESS = "jclouds.http.address";
+   public static final String PROPERTY_HTTP_ADDRESS = "jclouds.http.address";
-    public static final String PROPERTY_HTTP_MAX_RETRIES = "jclouds.http.max-retries";
+   public static final String PROPERTY_HTTP_MAX_RETRIES = "jclouds.http.max-retries";
+   public static final String PROPERTY_HTTP_MAX_REDIRECTS = "jclouds.http.max-redirects";
 }

core/src/main/java/org/jclouds/http/HttpRequest.java

@@ -41,10 +41,12 @@ import org.jclouds.util.Utils;
  */
 public class HttpRequest extends HttpMessage implements Request<URI> {
 
+   // mutable for purposes of redirects
    private URI endPoint;
-   private final HttpMethod method;
+   private HttpMethod method;
+   
    private final String uri;
-   Object payload;
+   private Object payload;
 
    @Resource
    protected Logger logger = Logger.NULL;
@@ -115,4 +117,8 @@ public class HttpRequest extends HttpMessage implements Request<URI> {
       return endPoint;
    }
 
+   public void setMethod(HttpMethod method) {
+      this.method = method;
+   }
+
 }

core/src/main/java/org/jclouds/http/handlers/DelegatingRetryHandler.java

@@ -75,4 +75,15 @@ public class DelegatingRetryHandler implements HttpRetryHandler {
       return retryRequest;
    }
 
+   public HttpRetryHandler getRedirectionRetryHandler() {
+      return redirectionRetryHandler;
+   }
+
+   public HttpRetryHandler getClientErrorRetryHandler() {
+      return clientErrorRetryHandler;
+   }
+
+   public HttpRetryHandler getServerErrorRetryHandler() {
+      return serverErrorRetryHandler;
+   }
 }

core/src/main/java/org/jclouds/http/handlers/RedirectionRetryHandler.java

@@ -48,7 +48,7 @@ import com.google.inject.name.Named;
  * @author Adrian Cole
  */
 public class RedirectionRetryHandler implements HttpRetryHandler {
-   private final int retryCountLimit;
+   protected final int retryCountLimit;
 
    @Resource
    protected Logger logger = Logger.NULL;

core/src/main/java/org/jclouds/util/Utils.java

@@ -23,15 +23,20 @@
  */
 package org.jclouds.util;
 
+import static com.google.common.base.Preconditions.checkState;
+
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
+import java.net.URI;
 import java.net.URLEncoder;
 import java.util.concurrent.ExecutionException;
 
 import javax.annotation.Resource;
 
 import org.apache.commons.io.IOUtils;
+import org.jclouds.http.HttpResponse;
 import org.jclouds.logging.Logger;
 
 /**
@@ -45,6 +50,43 @@ public class Utils {
    @Resource
    protected static Logger logger = Logger.NULL;
 
+   /**
+    * Content stream may need to be read. However, we should always close the http stream.
+    */
+   public static byte[] closeConnectionButKeepContentStream(HttpResponse response) {
+      if (response.getContent() != null) {
+         try {
+            byte[] data = IOUtils.toByteArray(response.getContent());
+            response.setContent(new ByteArrayInputStream(data));
+            return data;
+         } catch (IOException e) {
+            logger.error(e, "Error consuming input");
+         } finally {
+            IOUtils.closeQuietly(response.getContent());
+         }
+      }
+      return null;
+   }
+
+   public static URI parseEndPoint(String hostHeader) {
+      URI redirectURI = URI.create(hostHeader);
+      String scheme = redirectURI.getScheme();
+
+      checkState(redirectURI.getScheme().startsWith("http"), String.format(
+               "header %s didn't parse an http scheme: [%s]", hostHeader, scheme));
+      int port = redirectURI.getPort() > 0 ? redirectURI.getPort() : redirectURI.getScheme()
+               .equals("https") ? 443 : 80;
+      String host = redirectURI.getHost();
+      checkState(!host.matches("[/]"), String.format(
+               "header %s didn't parse an http host correctly: [%s]", hostHeader, host));
+      URI endPoint = URI.create(String.format("%s://%s:%d", scheme, host, port));
+      return endPoint;
+   }
+
+   public static URI replaceHostInEndPoint(URI endPoint, String host) {
+      return URI.create(endPoint.toString().replace(endPoint.getHost(), host));
+   }
+
    /**
     * 
     * @param <E>

extensions/gae/src/main/java/org/jclouds/gae/URLFetchServiceClient.java

@@ -115,7 +115,7 @@ public class URLFetchServiceClient extends BaseHttpFutureCommandClient<HTTPReque
       URL url = new URL(request.getEndPoint().toURL(), request.getUri());
 
       FetchOptions options = disallowTruncate();
-      followRedirectsUnlessRequestContainsPayload(request, options);
+      options.doNotFollowRedirects();
 
       HTTPRequest gaeRequest = new HTTPRequest(url, HTTPMethod.valueOf(request.getMethod()
                .toString()), options);
@@ -153,15 +153,6 @@ public class URLFetchServiceClient extends BaseHttpFutureCommandClient<HTTPReque
       }
    }
 
-   private void followRedirectsUnlessRequestContainsPayload(HttpRequest request,
-            FetchOptions options) {
-      if (request.getPayload() != null || request.getMethod().equals(HTTPMethod.PUT)
-               || request.getMethod().equals(HTTPMethod.POST))
-         options.doNotFollowRedirects();
-      else
-         options.followRedirects();
-   }
-
    /**
     * nothing to clean up.
     */