Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 69: added in S3 specific retry handling 

git-svn-id: http://jclouds.googlecode.com/svn/trunk@1467 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
adrian.f.cole 2009-06-26 20:53:55 +00:00
parent d20b5d9f5a
commit 2a9ca61b36
14 changed files with 302 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
aws/s3/core/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java
View File

@ -33,6 +33,7 @@ import static org.jclouds.command.pool.PoolConstants.PROPERTY_POOL_MAX_SESSION_F
import static org.jclouds.command.pool.PoolConstants.PROPERTY_POOL_REQUEST_INVOKER_THREADS;
import static org.jclouds.http.HttpConstants.PROPERTY_HTTP_ADDRESS;
import static org.jclouds.http.HttpConstants.PROPERTY_HTTP_MAX_RETRIES;
import static org.jclouds.http.HttpConstants.PROPERTY_HTTP_MAX_REDIRECTS;
import static org.jclouds.http.HttpConstants.PROPERTY_HTTP_PORT;
import static org.jclouds.http.HttpConstants.PROPERTY_HTTP_SECURE;
@ -96,6 +97,7 @@ public class S3ContextFactory {
properties.setProperty(PROPERTY_HTTP_ADDRESS, "s3.amazonaws.com");
properties.setProperty(PROPERTY_HTTP_SECURE, "true");
properties.setProperty(PROPERTY_HTTP_MAX_RETRIES, "5");
properties.setProperty(PROPERTY_HTTP_MAX_REDIRECTS, "5");
properties.setProperty(PROPERTY_POOL_MAX_CONNECTION_REUSE, "75");
properties.setProperty(PROPERTY_POOL_MAX_SESSION_FAILURES, "2");
properties.setProperty(PROPERTY_POOL_REQUEST_INVOKER_THREADS, "1");
@ -168,6 +170,12 @@ public class S3ContextFactory {
return this;
}
public S3ContextFactory withHttpMaxRedirects(int httpMaxRedirects) {
properties.setProperty(PROPERTY_HTTP_MAX_REDIRECTS, Integer.toString(httpMaxRedirects));
return this;
}
public S3ContextFactory withHttpPort(int httpPort) {
properties.setProperty(PROPERTY_HTTP_PORT, Integer.toString(httpPort));
return this;

19
aws/s3/core/src/main/java/org/jclouds/aws/s3/config/LiveS3ConnectionModule.java
View File

@ -32,11 +32,14 @@ import javax.annotation.Resource;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
import org.jclouds.aws.s3.handlers.AWSClientErrorRetryHandler;
import org.jclouds.aws.s3.handlers.AWSRedirectionRetryHandler;
import org.jclouds.aws.s3.handlers.ParseAWSErrorFromXmlContent;
import org.jclouds.aws.s3.internal.LiveS3Connection;
import org.jclouds.http.HttpConstants;
import org.jclouds.http.HttpErrorHandler;
import org.jclouds.http.HttpRequestFilter;
import org.jclouds.http.HttpRetryHandler;
import org.jclouds.http.annotation.ClientError;
import org.jclouds.http.annotation.Redirection;
import org.jclouds.http.annotation.ServerError;
@ -74,17 +77,25 @@ public class LiveS3ConnectionModule extends AbstractModule {
bind(S3Connection.class).to(LiveS3Connection.class).in(Scopes.SINGLETON);
bindErrorHandlers();
bindRetryHandlers();
requestInjection(this);
logger.info("S3 Context = %1$s://%2$s:%3$s", (isSecure ? "https" : "http"), address, port);
}
protected void bindErrorHandlers() {
bind(HttpErrorHandler.class).annotatedWith(Redirection.class).to(
ParseAWSErrorFromXmlContent.class).in(Scopes.SINGLETON);
bind(HttpErrorHandler.class).annotatedWith(ServerError.class).to(
ParseAWSErrorFromXmlContent.class).in(Scopes.SINGLETON);
ParseAWSErrorFromXmlContent.class);
bind(HttpErrorHandler.class).annotatedWith(ClientError.class).to(
ParseAWSErrorFromXmlContent.class).in(Scopes.SINGLETON);
ParseAWSErrorFromXmlContent.class);
bind(HttpErrorHandler.class).annotatedWith(ServerError.class).to(
ParseAWSErrorFromXmlContent.class);
}
protected void bindRetryHandlers() {
bind(HttpRetryHandler.class).annotatedWith(Redirection.class).to(
AWSRedirectionRetryHandler.class);
bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(
AWSClientErrorRetryHandler.class);
}
@Provides

78
aws/s3/core/src/main/java/org/jclouds/aws/s3/handlers/AWSClientErrorRetryHandler.java Normal file
View File

@ -0,0 +1,78 @@
/**
*
* Copyright (C) 2009 Global Cloud Specialists, Inc. <info@globalcloudspecialists.com>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.handlers;
import javax.annotation.Resource;
import org.jclouds.aws.domain.AWSError;
import org.jclouds.aws.s3.util.S3Utils;
import org.jclouds.aws.s3.xml.S3ParserFactory;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.HttpRetryHandler;
import org.jclouds.logging.Logger;
import com.google.inject.Inject;
import com.google.inject.name.Named;
/**
* Handles Retryable responses with error codes in the 3xx range
*
* @author Adrian Cole
*/
public class AWSClientErrorRetryHandler implements HttpRetryHandler {
private final S3ParserFactory parserFactory;
private final int retryCountLimit;
@Resource
protected Logger logger = Logger.NULL;
@Inject
public AWSClientErrorRetryHandler(S3ParserFactory parserFactory,
@Named("jclouds.http.max-retries") int retryCountLimit) {
this.retryCountLimit = retryCountLimit;
this.parserFactory = parserFactory;
}
public boolean shouldRetryRequest(HttpFutureCommand<?> command, HttpResponse response) {
if (command.getFailureCount() > retryCountLimit)
return false;
if (response.getStatusCode() == 400) {
byte[] content = S3Utils.closeConnectionButKeepContentStream(response);
command.incrementRedirectCount();
try {
AWSError error = S3Utils.parseAWSErrorFromContent(parserFactory, command, response,
new String(content));
if ("RequestTimeout".equals(error.getCode())) {
return true;
}
} catch (HttpException e) {
logger.warn(e, "error parsing response: %s", new String(content));
}
}
return false;
}
}

85
aws/s3/core/src/main/java/org/jclouds/aws/s3/handlers/AWSRedirectionRetryHandler.java Normal file
View File

@ -0,0 +1,85 @@
/**
*
* Copyright (C) 2009 Global Cloud Specialists, Inc. <info@globalcloudspecialists.com>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.handlers;
import java.net.URI;
import org.jclouds.aws.domain.AWSError;
import org.jclouds.aws.s3.reference.S3Constants;
import org.jclouds.aws.s3.util.S3Utils;
import org.jclouds.aws.s3.xml.S3ParserFactory;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpMethod;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.handlers.RedirectionRetryHandler;
import org.jclouds.util.Utils;
import com.google.inject.Inject;
import com.google.inject.name.Named;
/**
* Handles Retryable responses with error codes in the 3xx range
*
* @author Adrian Cole
*/
public class AWSRedirectionRetryHandler extends RedirectionRetryHandler {
private final S3ParserFactory parserFactory;
@Inject
public AWSRedirectionRetryHandler(S3ParserFactory parserFactory,
@Named("jclouds.http.max-redirects") int retryCountLimit) {
super(retryCountLimit);
this.parserFactory = parserFactory;
}
public boolean shouldRetryRequest(HttpFutureCommand<?> command, HttpResponse response) {
if (response.getStatusCode() == 301) {
byte[] content = S3Utils.closeConnectionButKeepContentStream(response);
if (command.getRequest().getMethod() == HttpMethod.HEAD) {
command.getRequest().setMethod(HttpMethod.GET);
return true;
} else {
command.incrementRedirectCount();
try {
AWSError error = S3Utils.parseAWSErrorFromContent(parserFactory, command, response,
new String(content));
String host = error.getDetails().get(S3Constants.ENDPOINT);
if (host != null) {
URI endPoint = command.getRequest().getEndPoint();
endPoint = Utils.replaceHostInEndPoint(endPoint, host);
command.getRequest().setEndPoint(endPoint);
return true;
} else {
return false;
}
} catch (HttpException e) {
return false;
}
}
} else {
return super.shouldRetryRequest(command, response);
}
}
}

20
aws/s3/core/src/main/java/org/jclouds/aws/s3/handlers/ParseAWSErrorFromXmlContent.java
View File

@ -23,17 +23,13 @@
*/
package org.jclouds.aws.s3.handlers;
import java.io.ByteArrayInputStream;
import javax.annotation.Resource;
import org.jclouds.aws.AWSResponseException;
import org.jclouds.aws.domain.AWSError;
import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
import org.jclouds.aws.s3.reference.S3Headers;
import org.jclouds.aws.s3.util.S3Utils;
import org.jclouds.aws.s3.xml.S3ParserFactory;
import org.jclouds.http.HttpErrorHandler;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.HttpResponseException;
@ -68,7 +64,8 @@ public class ParseAWSErrorFromXmlContent implements HttpErrorHandler {
if (content != null) {
try {
if (content.indexOf('<') >= 0) {
AWSError error = parseAWSErrorFromContent(command, response, content);
AWSError error = S3Utils.parseAWSErrorFromContent(parserFactory, command,
response, content);
command.setException(new AWSResponseException(command, response, error));
} else {
command.setException(new HttpResponseException(command, response, content));
@ -86,15 +83,4 @@ public class ParseAWSErrorFromXmlContent implements HttpErrorHandler {
}
}
private AWSError parseAWSErrorFromContent(HttpFutureCommand<?> command, HttpResponse response,
String content) throws HttpException {
AWSError error = parserFactory.createErrorParser().parse(
new ByteArrayInputStream(content.getBytes()));
error.setRequestId(response.getFirstHeaderOrNull(S3Headers.REQUEST_ID));
error.setRequestToken(response.getFirstHeaderOrNull(S3Headers.REQUEST_TOKEN));
if ("SignatureDoesNotMatch".equals(error.getCode()))
error.setStringSigned(RequestAuthorizeSignature.createStringToSign(command.getRequest()));
return error;
}
}

1
aws/s3/core/src/main/java/org/jclouds/aws/s3/reference/S3Constants.java
View File

@ -45,5 +45,6 @@ public interface S3Constants extends AWSConstants, S3Headers {
* S3 service's XML Namespace, as used in XML request and response documents.
*/
public static final String S3_REST_API_XML_NAMESPACE = "http://s3.amazonaws.com/doc/2006-03-01/";
public static final String ENDPOINT = "Endpoint";
}

26
aws/s3/core/src/main/java/org/jclouds/aws/s3/util/S3Utils.java
View File

@ -28,8 +28,15 @@ import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.bouncycastle.crypto.digests.MD5Digest;
import org.jclouds.aws.domain.AWSError;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
import org.jclouds.aws.s3.reference.S3Headers;
import org.jclouds.aws.s3.xml.S3ParserFactory;
import org.jclouds.aws.util.AWSUtils;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpResponse;
import java.io.*;
@ -40,6 +47,25 @@ import java.io.*;
*/
public class S3Utils extends AWSUtils {
public static AWSError parseAWSErrorFromContent(S3ParserFactory parserFactory,
HttpFutureCommand<?> command, HttpResponse response, InputStream content)
throws HttpException {
AWSError error = parserFactory.createErrorParser().parse(content);
error.setRequestId(response.getFirstHeaderOrNull(S3Headers.REQUEST_ID));
error.setRequestToken(response.getFirstHeaderOrNull(S3Headers.REQUEST_TOKEN));
if ("SignatureDoesNotMatch".equals(error.getCode()))
error.setStringSigned(RequestAuthorizeSignature.createStringToSign(command.getRequest()));
return error;
}
public static AWSError parseAWSErrorFromContent(S3ParserFactory parserFactory,
HttpFutureCommand<?> command, HttpResponse response, String content)
throws HttpException {
return parseAWSErrorFromContent(parserFactory, command, response, new ByteArrayInputStream(
content.getBytes()));
}
public static String validateBucketName(String bucketName) {
checkNotNull(bucketName, "bucketName");
checkArgument(bucketName.matches("^[a-z0-9].*"),

17
aws/s3/core/src/test/java/org/jclouds/aws/s3/config/S3ContextModuleTest.java
View File

@ -25,11 +25,14 @@ package org.jclouds.aws.s3.config;
import static org.testng.Assert.assertEquals;
import org.jclouds.aws.s3.handlers.AWSClientErrorRetryHandler;
import org.jclouds.aws.s3.handlers.AWSRedirectionRetryHandler;
import org.jclouds.aws.s3.handlers.ParseAWSErrorFromXmlContent;
import org.jclouds.aws.s3.reference.S3Constants;
import org.jclouds.aws.s3.xml.config.S3ParserModule;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import org.jclouds.http.handlers.DelegatingErrorHandler;
import org.jclouds.http.handlers.DelegatingRetryHandler;
import org.testng.annotations.Test;
import com.google.inject.Guice;
@ -61,6 +64,8 @@ public class S3ContextModuleTest {
.to("false");
bindConstant().annotatedWith(
Names.named(S3Constants.PROPERTY_HTTP_MAX_RETRIES)).to("5");
bindConstant().annotatedWith(
Names.named(S3Constants.PROPERTY_HTTP_MAX_REDIRECTS)).to("5");
super.configure();
}
}, new JavaUrlHttpFutureCommandClientModule());
@ -78,4 +83,16 @@ public class S3ContextModuleTest {
assertEquals(handler.getClientErrorHandler().getClass(), ParseAWSErrorFromXmlContent.class);
}
@Test
void testClientRetryHandler() {
DelegatingRetryHandler handler = createInjector().getInstance(DelegatingRetryHandler.class);
assertEquals(handler.getClientErrorRetryHandler().getClass(), AWSClientErrorRetryHandler.class);
}
@Test
void testRedirectionRetryHandler() {
DelegatingRetryHandler handler = createInjector().getInstance(DelegatingRetryHandler.class);
assertEquals(handler.getRedirectionRetryHandler().getClass(), AWSRedirectionRetryHandler.class);
}
}

9
core/src/main/java/org/jclouds/http/HttpConstants.java
View File

@ -29,8 +29,9 @@ package org.jclouds.http;
* @author Adrian Cole
*/
public interface HttpConstants extends HttpHeaders, ContentTypes {
public static final String PROPERTY_HTTP_SECURE = "jclouds.http.secure";
public static final String PROPERTY_HTTP_PORT = "jclouds.http.port";
public static final String PROPERTY_HTTP_ADDRESS = "jclouds.http.address";
public static final String PROPERTY_HTTP_MAX_RETRIES = "jclouds.http.max-retries";
public static final String PROPERTY_HTTP_SECURE = "jclouds.http.secure";
public static final String PROPERTY_HTTP_PORT = "jclouds.http.port";
public static final String PROPERTY_HTTP_ADDRESS = "jclouds.http.address";
public static final String PROPERTY_HTTP_MAX_RETRIES = "jclouds.http.max-retries";
public static final String PROPERTY_HTTP_MAX_REDIRECTS = "jclouds.http.max-redirects";
}

10
core/src/main/java/org/jclouds/http/HttpRequest.java
View File

@ -41,10 +41,12 @@ import org.jclouds.util.Utils;
*/
public class HttpRequest extends HttpMessage implements Request<URI> {
// mutable for purposes of redirects
private URI endPoint;
private final HttpMethod method;
private HttpMethod method;
private final String uri;
Object payload;
private Object payload;
@Resource
protected Logger logger = Logger.NULL;
@ -115,4 +117,8 @@ public class HttpRequest extends HttpMessage implements Request<URI> {
return endPoint;
}
public void setMethod(HttpMethod method) {
this.method = method;
}
}

11
core/src/main/java/org/jclouds/http/handlers/DelegatingRetryHandler.java
View File

@ -75,4 +75,15 @@ public class DelegatingRetryHandler implements HttpRetryHandler {
return retryRequest;
}
public HttpRetryHandler getRedirectionRetryHandler() {
return redirectionRetryHandler;
}
public HttpRetryHandler getClientErrorRetryHandler() {
return clientErrorRetryHandler;
}
public HttpRetryHandler getServerErrorRetryHandler() {
return serverErrorRetryHandler;
}
}

2
core/src/main/java/org/jclouds/http/handlers/RedirectionRetryHandler.java
View File

@ -48,7 +48,7 @@ import com.google.inject.name.Named;
* @author Adrian Cole
*/
public class RedirectionRetryHandler implements HttpRetryHandler {
private final int retryCountLimit;
protected final int retryCountLimit;
@Resource
protected Logger logger = Logger.NULL;

42
core/src/main/java/org/jclouds/util/Utils.java
View File

@ -23,15 +23,20 @@
*/
package org.jclouds.util;
import static com.google.common.base.Preconditions.checkState;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URLEncoder;
import java.util.concurrent.ExecutionException;
import javax.annotation.Resource;
import org.apache.commons.io.IOUtils;
import org.jclouds.http.HttpResponse;
import org.jclouds.logging.Logger;
/**
@ -45,6 +50,43 @@ public class Utils {
@Resource
protected static Logger logger = Logger.NULL;
/**
* Content stream may need to be read. However, we should always close the http stream.
*/
public static byte[] closeConnectionButKeepContentStream(HttpResponse response) {
if (response.getContent() != null) {
try {
byte[] data = IOUtils.toByteArray(response.getContent());
response.setContent(new ByteArrayInputStream(data));
return data;
} catch (IOException e) {
logger.error(e, "Error consuming input");
} finally {
IOUtils.closeQuietly(response.getContent());
}
}
return null;
}
public static URI parseEndPoint(String hostHeader) {
URI redirectURI = URI.create(hostHeader);
String scheme = redirectURI.getScheme();
checkState(redirectURI.getScheme().startsWith("http"), String.format(
"header %s didn't parse an http scheme: [%s]", hostHeader, scheme));
int port = redirectURI.getPort() > 0 ? redirectURI.getPort() : redirectURI.getScheme()
.equals("https") ? 443 : 80;
String host = redirectURI.getHost();
checkState(!host.matches("[/]"), String.format(
"header %s didn't parse an http host correctly: [%s]", hostHeader, host));
URI endPoint = URI.create(String.format("%s://%s:%d", scheme, host, port));
return endPoint;
}
public static URI replaceHostInEndPoint(URI endPoint, String host) {
return URI.create(endPoint.toString().replace(endPoint.getHost(), host));
}
/**
*
* @param <E>

11
extensions/gae/src/main/java/org/jclouds/gae/URLFetchServiceClient.java
View File

@ -115,7 +115,7 @@ public class URLFetchServiceClient extends BaseHttpFutureCommandClient<HTTPReque
URL url = new URL(request.getEndPoint().toURL(), request.getUri());
FetchOptions options = disallowTruncate();
followRedirectsUnlessRequestContainsPayload(request, options);
options.doNotFollowRedirects();
HTTPRequest gaeRequest = new HTTPRequest(url, HTTPMethod.valueOf(request.getMethod()
.toString()), options);
@ -153,15 +153,6 @@ public class URLFetchServiceClient extends BaseHttpFutureCommandClient<HTTPReque
}
}
private void followRedirectsUnlessRequestContainsPayload(HttpRequest request,
FetchOptions options) {
if (request.getPayload() != null || request.getMethod().equals(HTTPMethod.PUT)
|| request.getMethod().equals(HTTPMethod.POST))
options.doNotFollowRedirects();
else
options.followRedirects();
}
/**
* nothing to clean up.
*/