From db14a0232d18f4e08c623b7a16f0994419dddd28 Mon Sep 17 00:00:00 2001 From: "adrian.f.cole" Date: Sat, 9 May 2009 20:31:55 +0000 Subject: [PATCH] refactored out s3nio module and centralized http handling code git-svn-id: http://jclouds.googlecode.com/svn/trunk@485 3d8758e0-26b5-11de-8745-db77d3ebf521 --- .../pool/FutureCommandConnectionPool.java | 4 +- .../http/BaseHttpFutureCommandClient.java | 87 ++++++++++ .../CloseContentAndSetExceptionHandler.java | 42 +++++ .../org/jclouds/http/HttpFutureCommand.java | 9 ++ .../jclouds/http/HttpFutureCommandClient.java | 13 +- .../org/jclouds/http/HttpRequestFilter.java | 1 - .../org/jclouds/http/HttpResponseHandler.java | 21 +-- .../http/JavaUrlHttpFutureCommandClient.java | 38 +---- .../http/annotation/ClientErrorHandler.java | 44 ++++++ .../http/annotation/RedirectHandler.java | 44 ++++++ .../http/annotation/ServerErrorHandler.java | 44 ++++++ .../java/org/jclouds/http/commands/Head.java | 4 +- .../commands/callables/ReturnStringIf200.java | 7 +- ...urnTrueIf200.java => ReturnTrueIf2xx.java} | 17 +- .../http/BaseHttpFutureCommandClientTest.java | 24 +-- .../callables/ReturnStringIf200Test.java | 6 +- .../jclouds/http/httpnio/HttpNioUtils.java | 26 +-- .../HttpNioFutureCommandExecutionHandler.java | 67 ++++---- extensions/s3nio/pom.xml | 93 ----------- ...3HttpNioFutureCommandExecutionHandler.java | 91 ----------- .../S3HttpNioConnectionPoolClientModule.java | 51 ------ .../aws/s3/nio/NioS3InputStreamMapTest.java | 61 -------- .../aws/s3/nio/NioS3ObjectMapTest.java | 61 -------- ...pNioFutureCommandExecutionHandlerTest.java | 148 ------------------ ...HttpNioConnectionPoolClientModuleTest.java | 64 -------- .../jclouds/gae/URLFetchServiceClient.java | 61 +++----- .../gae/URLFetchServiceClientTest.java | 10 -- pom.xml | 1 - s3/perftest/pom.xml | 2 +- ...rmance.java => AmazonPerformanceTest.java} | 10 +- .../java/com/amazon/s3/BasePerformance.java | 1 + .../java/com/amazon/s3/DateServiceTest.java | 2 +- ...ce.java => JCloudsNioPerformanceTest.java} | 15 +- ...mance.java => JCloudsPerformanceTest.java} | 4 +- ...rmance.java => Jets3tPerformanceTest.java} | 22 ++- .../test/java/com/amazon/s3/S3ParserTest.java | 2 +- .../test/java/com/amazon/s3/S3UtilsTest.java | 2 +- .../java/org/jclouds/aws/s3/S3Constants.java | 2 +- .../org/jclouds/aws/s3/S3ContextFactory.java | 6 +- .../jclouds/aws/s3/S3ResponseException.java | 33 ++-- .../main/java/org/jclouds/aws/s3/S3Utils.java | 98 ++++++++++-- .../jclouds/aws/s3/commands/BucketExists.java | 46 +++++- .../jclouds/aws/s3/commands/DeleteBucket.java | 44 +++++- .../jclouds/aws/s3/commands/DeleteObject.java | 4 +- .../jclouds/aws/s3/commands/GetObject.java | 4 +- .../jclouds/aws/s3/commands/HeadMetaData.java | 43 ++++- .../jclouds/aws/s3/commands/PutBucket.java | 4 +- .../jclouds/aws/s3/commands/PutObject.java | 10 +- .../callables/DeleteBucketCallable.java | 60 ------- ...lable.java => ParseMd5FromETagHeader.java} | 26 +-- ...ble.java => ParseMetadataFromHeaders.java} | 81 +++++----- ...ParseObjectFromHeadersAndHttpContent.java} | 50 +++--- .../commands/callables/PutObjectCallable.java | 65 -------- .../aws/s3/config/S3ContextModule.java | 33 ++++ ...3JavaUrlHttpFutureCommandClientModule.java | 46 ------ .../org/jclouds/aws/s3/domain/S3Object.java | 54 ++----- .../ParseS3ErrorFromXmlContent.java} | 48 +++--- .../s3/filters/RequestAuthorizeSignature.java | 28 +--- .../jclouds/aws/s3/internal/BaseS3Map.java | 49 +++--- .../aws/s3/internal/LiveS3InputStreamMap.java | 16 +- .../org/jclouds/aws/s3/S3IntegrationTest.java | 7 +- .../org/jclouds/aws/s3/S3ObjectMapTest.java | 4 +- .../s3/commands/PutObjectIntegrationTest.java | 22 +-- .../aws/s3/commands/S3CommandFactoryTest.java | 16 +- .../callables/DeleteBucketCallableTest.java | 137 ++++++++-------- .../functest/GoogleAppEngineTest.java | 13 +- .../config/GuiceServletConfig.java | 1 - 67 files changed, 978 insertions(+), 1271 deletions(-) create mode 100644 core/src/main/java/org/jclouds/http/BaseHttpFutureCommandClient.java create mode 100644 core/src/main/java/org/jclouds/http/CloseContentAndSetExceptionHandler.java rename s3/src/main/java/org/jclouds/aws/s3/commands/callables/DeleteCallable.java => core/src/main/java/org/jclouds/http/HttpResponseHandler.java (72%) create mode 100644 core/src/main/java/org/jclouds/http/annotation/ClientErrorHandler.java create mode 100644 core/src/main/java/org/jclouds/http/annotation/RedirectHandler.java create mode 100644 core/src/main/java/org/jclouds/http/annotation/ServerErrorHandler.java rename core/src/main/java/org/jclouds/http/commands/callables/{ReturnTrueIf200.java => ReturnTrueIf2xx.java} (80%) delete mode 100644 extensions/s3nio/pom.xml delete mode 100644 extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandler.java delete mode 100644 extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModule.java delete mode 100644 extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3InputStreamMapTest.java delete mode 100644 extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3ObjectMapTest.java delete mode 100644 extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandlerTest.java delete mode 100644 extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModuleTest.java rename s3/perftest/src/test/java/com/amazon/s3/{AmazonPerformance.java => AmazonPerformanceTest.java} (94%) rename s3/perftest/src/test/java/com/amazon/s3/{JCloudsNioPerformance.java => JCloudsNioPerformanceTest.java} (85%) rename s3/perftest/src/test/java/com/amazon/s3/{JCloudsPerformance.java => JCloudsPerformanceTest.java} (91%) rename s3/perftest/src/test/java/com/amazon/s3/{Jets3tPerformance.java => Jets3tPerformanceTest.java} (85%) delete mode 100644 s3/src/main/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallable.java rename s3/src/main/java/org/jclouds/aws/s3/commands/callables/{PutBucketCallable.java => ParseMd5FromETagHeader.java} (67%) rename s3/src/main/java/org/jclouds/aws/s3/commands/callables/{HeadMetaDataCallable.java => ParseMetadataFromHeaders.java} (62%) rename s3/src/main/java/org/jclouds/aws/s3/commands/callables/{GetObjectCallable.java => ParseObjectFromHeadersAndHttpContent.java} (59%) delete mode 100644 s3/src/main/java/org/jclouds/aws/s3/commands/callables/PutObjectCallable.java delete mode 100644 s3/src/main/java/org/jclouds/aws/s3/config/S3JavaUrlHttpFutureCommandClientModule.java rename s3/src/main/java/org/jclouds/aws/s3/{internal/S3JavaUrlHttpFutureCommandClient.java => filters/ParseS3ErrorFromXmlContent.java} (65%) diff --git a/core/src/main/java/org/jclouds/command/pool/FutureCommandConnectionPool.java b/core/src/main/java/org/jclouds/command/pool/FutureCommandConnectionPool.java index a3bbaa4e29..61def42642 100644 --- a/core/src/main/java/org/jclouds/command/pool/FutureCommandConnectionPool.java +++ b/core/src/main/java/org/jclouds/command/pool/FutureCommandConnectionPool.java @@ -89,10 +89,10 @@ public abstract class FutureCommandConnectionPool + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.http; + +import java.net.URL; +import java.util.Collections; +import java.util.List; + +import javax.annotation.Resource; + +import org.jclouds.http.annotation.ClientErrorHandler; +import org.jclouds.http.annotation.RedirectHandler; +import org.jclouds.http.annotation.ServerErrorHandler; +import org.jclouds.logging.Logger; + +import com.google.inject.Inject; + +public abstract class BaseHttpFutureCommandClient implements HttpFutureCommandClient { + + protected final URL target; + + @Resource + protected Logger logger = Logger.NULL; + + @Inject(optional = true) + protected List requestFilters = Collections.emptyList(); + @RedirectHandler + @Inject(optional = true) + protected HttpResponseHandler redirectHandler = new CloseContentAndSetExceptionHandler(); + @ClientErrorHandler + @Inject(optional = true) + protected HttpResponseHandler clientErrorHandler = new CloseContentAndSetExceptionHandler(); + @ServerErrorHandler + @Inject(optional = true) + protected HttpResponseHandler serverErrorHandler = new CloseContentAndSetExceptionHandler(); + + @Inject + public BaseHttpFutureCommandClient(URL target) { + this.target = target; + } + + protected boolean isRetryable(HttpFutureCommand command, + HttpResponse response) { + int code = response.getStatusCode(); + if (command.getRequest().isReplayable() && code >= 500) { + logger.info("resubmitting command: %1s", command); + return true; + } + return false; + } + + protected void handleResponse(HttpFutureCommand command, HttpResponse response) { + int code = response.getStatusCode(); + if (code >= 500) { + this.serverErrorHandler.handle(command, response); + } else if (code >= 400 && code < 500) { + this.clientErrorHandler.handle(command, response); + } else if (code >= 300 && code < 400) { + this.redirectHandler.handle(command, response); + } else { + command.getResponseFuture().setResponse(response); + command.getResponseFuture().run(); + } + } + +} \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/http/CloseContentAndSetExceptionHandler.java b/core/src/main/java/org/jclouds/http/CloseContentAndSetExceptionHandler.java new file mode 100644 index 0000000000..59a741167e --- /dev/null +++ b/core/src/main/java/org/jclouds/http/CloseContentAndSetExceptionHandler.java @@ -0,0 +1,42 @@ +/** + * + * Copyright (C) 2009 Adrian Cole + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.http; + +import java.io.IOException; + +import org.apache.commons.io.IOUtils; + +/** + * + * @author Adrian Cole + */ +public class CloseContentAndSetExceptionHandler implements HttpResponseHandler { + + public void handle(HttpFutureCommand command, HttpResponse response) { + String message = String.format("Command: %2s failed; response: %1s", + response, command); + command.setException(new IOException(message)); + IOUtils.closeQuietly(response.getContent()); + } +} diff --git a/core/src/main/java/org/jclouds/http/HttpFutureCommand.java b/core/src/main/java/org/jclouds/http/HttpFutureCommand.java index 049227a54b..a9bf2a51ce 100644 --- a/core/src/main/java/org/jclouds/http/HttpFutureCommand.java +++ b/core/src/main/java/org/jclouds/http/HttpFutureCommand.java @@ -27,6 +27,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import javax.annotation.Resource; +import org.apache.commons.io.IOUtils; import org.jclouds.command.FutureCommand; import org.jclouds.logging.Logger; @@ -64,6 +65,14 @@ public class HttpFutureCommand extends @Resource protected Logger logger = Logger.NULL; + public void checkCode() { + int code = getResponse().getStatusCode(); + if (code >= 300){ + IOUtils.closeQuietly(getResponse().getContent()); + throw new IllegalStateException("incorrect code for this operation: "+getResponse()); + } + } + private HttpResponse response; public HttpResponse getResponse() { diff --git a/core/src/main/java/org/jclouds/http/HttpFutureCommandClient.java b/core/src/main/java/org/jclouds/http/HttpFutureCommandClient.java index a6b69564da..5111db5582 100644 --- a/core/src/main/java/org/jclouds/http/HttpFutureCommandClient.java +++ b/core/src/main/java/org/jclouds/http/HttpFutureCommandClient.java @@ -23,22 +23,13 @@ */ package org.jclouds.http; -import java.util.List; - import org.jclouds.command.FutureCommandClient; -import com.google.inject.Inject; - /** * // TODO: Adrian: Document this! * * @author Adrian Cole */ -public interface HttpFutureCommandClient - extends FutureCommandClient> { - List getRequestFilters(); - - @Inject - void setRequestFilters(List requestFilters); - +public interface HttpFutureCommandClient extends + FutureCommandClient> { } diff --git a/core/src/main/java/org/jclouds/http/HttpRequestFilter.java b/core/src/main/java/org/jclouds/http/HttpRequestFilter.java index 8c12b0f905..69e6cea044 100644 --- a/core/src/main/java/org/jclouds/http/HttpRequestFilter.java +++ b/core/src/main/java/org/jclouds/http/HttpRequestFilter.java @@ -30,5 +30,4 @@ package org.jclouds.http; */ public interface HttpRequestFilter { void filter(HttpRequest request) throws HttpException; - } diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/DeleteCallable.java b/core/src/main/java/org/jclouds/http/HttpResponseHandler.java similarity index 72% rename from s3/src/main/java/org/jclouds/aws/s3/commands/callables/DeleteCallable.java rename to core/src/main/java/org/jclouds/http/HttpResponseHandler.java index c2589790c2..dc18ba4df6 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/DeleteCallable.java +++ b/core/src/main/java/org/jclouds/http/HttpResponseHandler.java @@ -21,23 +21,18 @@ * under the License. * ==================================================================== */ -package org.jclouds.aws.s3.commands.callables; - -import org.jclouds.http.HttpException; -import org.jclouds.http.HttpFutureCommand; +package org.jclouds.http; /** * // TODO: Adrian: Document this! * * @author Adrian Cole */ -public class DeleteCallable extends HttpFutureCommand.ResponseCallable { - - public Boolean call() throws HttpException { - if (getResponse().getStatusCode() == 204) { - return true; - } else { - throw new HttpException("Error deleting bucket " + getResponse()); +public interface HttpResponseHandler { + public static final HttpResponseHandler NOOP = new HttpResponseHandler() { + public void handle(HttpFutureCommand command, HttpResponse response) { } - } -} \ No newline at end of file + }; + + void handle(HttpFutureCommand command, HttpResponse response); +} diff --git a/core/src/main/java/org/jclouds/http/JavaUrlHttpFutureCommandClient.java b/core/src/main/java/org/jclouds/http/JavaUrlHttpFutureCommandClient.java index 6de20fa532..9c2ac6264c 100644 --- a/core/src/main/java/org/jclouds/http/JavaUrlHttpFutureCommandClient.java +++ b/core/src/main/java/org/jclouds/http/JavaUrlHttpFutureCommandClient.java @@ -32,51 +32,33 @@ import java.io.OutputStreamWriter; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; -import java.util.Collections; -import java.util.List; - -import javax.annotation.Resource; import org.apache.commons.io.IOUtils; -import org.jclouds.logging.Logger; import com.google.inject.Inject; /** - * // TODO: Adrian: Document this! + * Basic implementation of a {@link HttpFutureCommandClient}. * * @author Adrian Cole */ -public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient { - private URL target; - private List requestFilters = Collections.emptyList(); - @Resource - protected Logger logger = Logger.NULL; - - public List getRequestFilters() { - return requestFilters; - } - - @Inject(optional = true) - public void setRequestFilters(List requestFilters) { - this.requestFilters = requestFilters; - } +public class JavaUrlHttpFutureCommandClient extends BaseHttpFutureCommandClient { @Inject public JavaUrlHttpFutureCommandClient(URL target) throws MalformedURLException { - this.target = target; + super(target); } public void submit(HttpFutureCommand command) { HttpRequest request = command.getRequest(); HttpURLConnection connection = null; try { - for (HttpRequestFilter filter : getRequestFilters()) { - filter.filter(request); - } HttpResponse response = null; for (;;) { + for (HttpRequestFilter filter : requestFilters) { + filter.filter(request); + } logger.trace("%1s - converting request %2s", target, request); connection = openJavaConnection(request); logger @@ -84,15 +66,11 @@ public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient { connection); response = getResponse(connection); logger.trace("%1s - received response %2s", target, response); - if (command.getRequest().isReplayable() - && response.getStatusCode() >= 500) { - logger.info("resubmitting command: %1s", command); + if (isRetryable(command, response)) continue; - } break; } - command.getResponseFuture().setResponse(response); - command.getResponseFuture().run(); + handleResponse(command, response); } catch (Exception e) { command.setException(e); } finally { diff --git a/core/src/main/java/org/jclouds/http/annotation/ClientErrorHandler.java b/core/src/main/java/org/jclouds/http/annotation/ClientErrorHandler.java new file mode 100644 index 0000000000..71d2666292 --- /dev/null +++ b/core/src/main/java/org/jclouds/http/annotation/ClientErrorHandler.java @@ -0,0 +1,44 @@ +/** + * + * Copyright (C) 2009 Adrian Cole + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.http.annotation; + +import com.google.inject.BindingAnnotation; +import java.lang.annotation.Target; +import java.lang.annotation.Retention; +import static java.lang.annotation.RetentionPolicy.RUNTIME; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; + +/** + * Implies that the object can address {@link HttpResponse}s that contain status + * code 4xx. + * + * @author Adrian Cole + */ +@BindingAnnotation +@Target( { FIELD, PARAMETER, METHOD }) +@Retention(RUNTIME) +public @interface ClientErrorHandler { +} diff --git a/core/src/main/java/org/jclouds/http/annotation/RedirectHandler.java b/core/src/main/java/org/jclouds/http/annotation/RedirectHandler.java new file mode 100644 index 0000000000..e926adc601 --- /dev/null +++ b/core/src/main/java/org/jclouds/http/annotation/RedirectHandler.java @@ -0,0 +1,44 @@ +/** + * + * Copyright (C) 2009 Adrian Cole + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.http.annotation; + +import com.google.inject.BindingAnnotation; +import java.lang.annotation.Target; +import java.lang.annotation.Retention; +import static java.lang.annotation.RetentionPolicy.RUNTIME; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; + +/** + * Implies that the object can address {@link HttpResponse}s that contain status + * code 3xx. + * + * @author Adrian Cole + */ +@BindingAnnotation +@Target( { FIELD, PARAMETER, METHOD }) +@Retention(RUNTIME) +public @interface RedirectHandler { +} diff --git a/core/src/main/java/org/jclouds/http/annotation/ServerErrorHandler.java b/core/src/main/java/org/jclouds/http/annotation/ServerErrorHandler.java new file mode 100644 index 0000000000..285b497caa --- /dev/null +++ b/core/src/main/java/org/jclouds/http/annotation/ServerErrorHandler.java @@ -0,0 +1,44 @@ +/** + * + * Copyright (C) 2009 Adrian Cole + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.http.annotation; + +import com.google.inject.BindingAnnotation; +import java.lang.annotation.Target; +import java.lang.annotation.Retention; +import static java.lang.annotation.RetentionPolicy.RUNTIME; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; + +/** + * Implies that the object can address {@link HttpResponse}s that contain status + * code 5xx. + * + * @author Adrian Cole + */ +@BindingAnnotation +@Target( { FIELD, PARAMETER, METHOD }) +@Retention(RUNTIME) +public @interface ServerErrorHandler { +} diff --git a/core/src/main/java/org/jclouds/http/commands/Head.java b/core/src/main/java/org/jclouds/http/commands/Head.java index 5a1164959e..75c117dc76 100644 --- a/core/src/main/java/org/jclouds/http/commands/Head.java +++ b/core/src/main/java/org/jclouds/http/commands/Head.java @@ -26,7 +26,7 @@ package org.jclouds.http.commands; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import org.jclouds.http.HttpFutureCommand; -import org.jclouds.http.commands.callables.ReturnTrueIf200; +import org.jclouds.http.commands.callables.ReturnTrueIf2xx; /** * // TODO: Adrian: Document this! @@ -36,7 +36,7 @@ import org.jclouds.http.commands.callables.ReturnTrueIf200; public class Head extends HttpFutureCommand { @Inject - public Head(ReturnTrueIf200 callable, @Assisted String uri) { + public Head(ReturnTrueIf2xx callable, @Assisted String uri) { super("HEAD", uri, callable); } } \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/http/commands/callables/ReturnStringIf200.java b/core/src/main/java/org/jclouds/http/commands/callables/ReturnStringIf200.java index 460a78b068..504642c626 100644 --- a/core/src/main/java/org/jclouds/http/commands/callables/ReturnStringIf200.java +++ b/core/src/main/java/org/jclouds/http/commands/callables/ReturnStringIf200.java @@ -46,11 +46,8 @@ public class ReturnStringIf200 extends } public String call() throws HttpException { - int code = getResponse().getStatusCode(); - if (code >= 400 && code < 500) { - throw new HttpException(String.format("Content not found - %1s", - getResponse())); - } else if (code == 200) { + checkCode(); + if (getResponse().getStatusCode() == 200) { InputStream entity = getResponse().getContent(); if (entity == null) throw new HttpException("no content"); diff --git a/core/src/main/java/org/jclouds/http/commands/callables/ReturnTrueIf200.java b/core/src/main/java/org/jclouds/http/commands/callables/ReturnTrueIf2xx.java similarity index 80% rename from core/src/main/java/org/jclouds/http/commands/callables/ReturnTrueIf200.java rename to core/src/main/java/org/jclouds/http/commands/callables/ReturnTrueIf2xx.java index 41a583cd6c..bfc8c068af 100644 --- a/core/src/main/java/org/jclouds/http/commands/callables/ReturnTrueIf200.java +++ b/core/src/main/java/org/jclouds/http/commands/callables/ReturnTrueIf2xx.java @@ -23,31 +23,28 @@ */ package org.jclouds.http.commands.callables; +import org.apache.commons.io.IOUtils; import org.jclouds.http.HttpException; import org.jclouds.http.HttpFutureCommand; import com.google.inject.Inject; /** - * // TODO: Adrian: Document this! + * Simply returns true when the http response code is in the range 200-299. * * @author Adrian Cole */ -public class ReturnTrueIf200 extends +public class ReturnTrueIf2xx extends HttpFutureCommand.ResponseCallable { @Inject - public ReturnTrueIf200() { + public ReturnTrueIf2xx() { super(); } public Boolean call() throws HttpException { - if (getResponse().getStatusCode() == 200) { - return true; - } else if (getResponse().getStatusCode() == 404) { - return false; - } else { - throw new HttpException("Error checking bucket " + getResponse()); - } + checkCode(); + IOUtils.closeQuietly(getResponse().getContent()); + return true; } } \ No newline at end of file diff --git a/core/src/test/java/org/jclouds/http/BaseHttpFutureCommandClientTest.java b/core/src/test/java/org/jclouds/http/BaseHttpFutureCommandClientTest.java index dc85982b03..043bd0806f 100644 --- a/core/src/test/java/org/jclouds/http/BaseHttpFutureCommandClientTest.java +++ b/core/src/test/java/org/jclouds/http/BaseHttpFutureCommandClientTest.java @@ -68,7 +68,7 @@ import com.google.inject.name.Names; * * @author Adrian Cole */ -@Test(threadPoolSize = 100) +@Test(threadPoolSize = 10) public abstract class BaseHttpFutureCommandClientTest { protected static final String XML = "whoppers"; protected Server server = null; @@ -158,9 +158,9 @@ public abstract class BaseHttpFutureCommandClientTest { server.stop(); } - @Test(invocationCount = 500, timeOut = 1500) - public void testRequestFilter() throws MalformedURLException, ExecutionException, - InterruptedException, TimeoutException { + @Test(invocationCount = 50, timeOut = 1500) + public void testRequestFilter() throws MalformedURLException, + ExecutionException, InterruptedException, TimeoutException { GetString get = factory.createGetString("/"); get.getRequest().getHeaders().put("filterme", "filterme"); client.submit(get); @@ -169,7 +169,7 @@ public abstract class BaseHttpFutureCommandClientTest { TimeUnit.SECONDS)); } - @Test(invocationCount = 500, timeOut = 1500) + @Test(invocationCount = 50, timeOut = 1500) public void testGetStringWithHeader() throws MalformedURLException, ExecutionException, InterruptedException, TimeoutException { GetString get = factory.createGetString("/"); @@ -180,9 +180,9 @@ public abstract class BaseHttpFutureCommandClientTest { TimeUnit.SECONDS)); } - @Test(invocationCount = 500, timeOut = 1500) - public void testGetString() throws MalformedURLException, ExecutionException, - InterruptedException, TimeoutException { + @Test(invocationCount = 50, timeOut = 1500) + public void testGetString() throws MalformedURLException, + ExecutionException, InterruptedException, TimeoutException { GetString get = factory.createGetString("/"); assert get != null; client.submit(get); @@ -191,7 +191,7 @@ public abstract class BaseHttpFutureCommandClientTest { TimeUnit.SECONDS)); } - @Test(invocationCount = 500, timeOut = 1500) + @Test(invocationCount = 50, timeOut = 1500) public void testHead() throws MalformedURLException, ExecutionException, InterruptedException, TimeoutException { Head head = factory.createHead("/"); @@ -200,9 +200,9 @@ public abstract class BaseHttpFutureCommandClientTest { assert head.get(10, TimeUnit.SECONDS); } - @Test(invocationCount = 500, timeOut = 1500) - public void testGetAndParseSax() throws MalformedURLException, ExecutionException, - InterruptedException, TimeoutException { + @Test(invocationCount = 50, timeOut = 1500) + public void testGetAndParseSax() throws MalformedURLException, + ExecutionException, InterruptedException, TimeoutException { GetAndParseSax getAndParseSax = factory.createGetAndParseSax("/", new ParseSax.HandlerWithResult() { @Override diff --git a/core/src/test/java/org/jclouds/http/commands/callables/ReturnStringIf200Test.java b/core/src/test/java/org/jclouds/http/commands/callables/ReturnStringIf200Test.java index 37884988b7..59433a0715 100644 --- a/core/src/test/java/org/jclouds/http/commands/callables/ReturnStringIf200Test.java +++ b/core/src/test/java/org/jclouds/http/commands/callables/ReturnStringIf200Test.java @@ -58,7 +58,7 @@ public class ReturnStringIf200Test { public void testExceptionWhenNoContentOn200() throws ExecutionException, InterruptedException, TimeoutException, IOException { HttpResponse response = createMock(HttpResponse.class); - expect(response.getStatusCode()).andReturn(200); + expect(response.getStatusCode()).andReturn(200).atLeastOnce(); expect(response.getContent()).andReturn(null); replay(response); callable.setResponse(response); @@ -74,7 +74,7 @@ public class ReturnStringIf200Test { public void testExceptionWhenIOExceptionOn200() throws ExecutionException, InterruptedException, TimeoutException, IOException { HttpResponse response = createMock(HttpResponse.class); - expect(response.getStatusCode()).andReturn(200); + expect(response.getStatusCode()).andReturn(200).atLeastOnce(); RuntimeException exception = new RuntimeException("bad"); expect(response.getContent()).andThrow(exception); replay(response); @@ -90,7 +90,7 @@ public class ReturnStringIf200Test { @Test public void testResponseOk() throws Exception { HttpResponse response = createMock(HttpResponse.class); - expect(response.getStatusCode()).andReturn(200); + expect(response.getStatusCode()).andReturn(200).atLeastOnce(); expect(response.getContent()).andReturn(IOUtils.toInputStream("hello")); replay(response); callable.setResponse(response); diff --git a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/HttpNioUtils.java b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/HttpNioUtils.java index fa3a645620..ab02cdbf16 100644 --- a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/HttpNioUtils.java +++ b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/HttpNioUtils.java @@ -45,20 +45,26 @@ public class HttpNioUtils { HttpRequest object) { BasicHttpEntityEnclosingRequest apacheRequest = new BasicHttpEntityEnclosingRequest( object.getMethod(), object.getUri(), HttpVersion.HTTP_1_1); + + Object content = object.getPayload(); + + // Since we may remove headers, ensure they are added to the apache + // request after this block + if (content != null) { + long contentLength = Long.parseLong(object + .getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH)); + object.getHeaders().removeAll(HttpHeaders.CONTENT_LENGTH); + String contentType = object + .getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE); + object.getHeaders().removeAll(HttpHeaders.CONTENT_TYPE); + addEntityForContent(apacheRequest, content, contentType, + contentLength); + } + for (String header : object.getHeaders().keySet()) { for (String value : object.getHeaders().get(header)) apacheRequest.addHeader(header, value); } - Object content = object.getPayload(); - if (content != null) { - addEntityForContent( - apacheRequest, - content, - object.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE), - Long - .parseLong(object - .getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH))); - } return apacheRequest; } diff --git a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/pool/HttpNioFutureCommandExecutionHandler.java b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/pool/HttpNioFutureCommandExecutionHandler.java index 6e658686d0..194c4c6e9d 100644 --- a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/pool/HttpNioFutureCommandExecutionHandler.java +++ b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/pool/HttpNioFutureCommandExecutionHandler.java @@ -35,8 +35,13 @@ import org.apache.http.HttpResponse; import org.apache.http.nio.entity.ConsumingNHttpEntity; import org.apache.http.nio.protocol.NHttpRequestExecutionHandler; import org.apache.http.protocol.HttpContext; +import org.jclouds.http.CloseContentAndSetExceptionHandler; import org.jclouds.http.HttpFutureCommand; import org.jclouds.http.HttpRequest; +import org.jclouds.http.HttpResponseHandler; +import org.jclouds.http.annotation.ClientErrorHandler; +import org.jclouds.http.annotation.RedirectHandler; +import org.jclouds.http.annotation.ServerErrorHandler; import org.jclouds.http.httpnio.HttpNioUtils; import org.jclouds.logging.Logger; @@ -55,6 +60,18 @@ public class HttpNioFutureCommandExecutionHandler implements private final ConsumingNHttpEntityFactory entityFactory; private final BlockingQueue> commandQueue; + @RedirectHandler + @Inject(optional = true) + private HttpResponseHandler redirectHandler = new CloseContentAndSetExceptionHandler(); + + @ClientErrorHandler + @Inject(optional = true) + private HttpResponseHandler clientErrorHandler = new CloseContentAndSetExceptionHandler(); + + @ServerErrorHandler + @Inject(optional = true) + private HttpResponseHandler serverErrorHandler = new CloseContentAndSetExceptionHandler(); + public interface ConsumingNHttpEntityFactory { public ConsumingNHttpEntity create(HttpEntity httpEntity); } @@ -88,27 +105,29 @@ public class HttpNioFutureCommandExecutionHandler implements return entityFactory.create(response.getEntity()); } - public void handleResponse(HttpResponse response, HttpContext context) + public void handleResponse(HttpResponse apacheResponse, HttpContext context) throws IOException { HttpNioFutureCommandConnectionHandle handle = (HttpNioFutureCommandConnectionHandle) context .removeAttribute("command-handle"); if (handle != null) { try { HttpFutureCommand command = handle.getCommand(); - int code = response.getStatusLine().getStatusCode(); - // normal codes for rest commands - if ((code >= 200 && code < 300) || code == 404) { - processResponse(response, command); - } else { - if (isRetryable(response)) { - attemptReplay(command); + org.jclouds.http.HttpResponse response = HttpNioUtils + .convertToJavaCloudsResponse(apacheResponse); + + int code = response.getStatusCode(); + if (code >= 500) { + if (isRetryable(command)) { + commandQueue.add(command); } else { - String message = String - .format( - "response is not retryable: %1s; Command: %2s failed", - response.getStatusLine(), command); - command.setException(new IOException(message)); + this.serverErrorHandler.handle(command, response); } + } else if (code >= 400 && code < 500) { + this.clientErrorHandler.handle(command, response); + } else if (code >= 300 && code < 400) { + this.redirectHandler.handle(command, response); + } else { + processResponse(response, command); } } finally { releaseConnectionToPool(handle); @@ -119,18 +138,12 @@ public class HttpNioFutureCommandExecutionHandler implements } } - protected void attemptReplay(HttpFutureCommand command) { - if (command.getRequest().isReplayable()) - commandQueue.add(command); - else - command.setException(new IOException(String.format( - "%1s: command failed and request is not replayable: %2s", - command, command.getRequest()))); - } - - protected boolean isRetryable(HttpResponse response) throws IOException { - int code = response.getStatusLine().getStatusCode(); - return code == 500 || code == 503; + protected boolean isRetryable(HttpFutureCommand command) { + if (command.getRequest().isReplayable()) { + logger.info("resubmitting command: %1s", command); + return true; + } + return false; } protected void releaseConnectionToPool( @@ -142,10 +155,8 @@ public class HttpNioFutureCommandExecutionHandler implements } } - protected void processResponse(HttpResponse apacheResponse, + protected void processResponse(org.jclouds.http.HttpResponse response, HttpFutureCommand command) throws IOException { - org.jclouds.http.HttpResponse response = HttpNioUtils - .convertToJavaCloudsResponse(apacheResponse); command.getResponseFuture().setResponse(response); logger.trace("submitting response task %1s", command .getResponseFuture()); diff --git a/extensions/s3nio/pom.xml b/extensions/s3nio/pom.xml deleted file mode 100644 index 2932160e30..0000000000 --- a/extensions/s3nio/pom.xml +++ /dev/null @@ -1,93 +0,0 @@ - - - - - org.jclouds - jclouds-project - 1.0-SNAPSHOT - ../../project/pom.xml - - 4.0.0 - org.jclouds - jclouds-s3nio - Connection pooling NIO connection for S3 - jar - Connection pooling NIO connection for S3 - - - scm:svn:http://jclouds.googlecode.com/svn/trunk/extensions/s3nio - scm:svn:https://jclouds.googlecode.com/svn/trunk/extensions/s3nio - http://jclouds.googlecode.com/svn/trunk/extensions/s3nio - - - - - - - - - - ${project.groupId} - jclouds-httpnio - ${project.version} - - - ${project.groupId} - jclouds-s3 - ${project.version} - - - ${project.groupId} - jclouds-s3 - ${project.version} - test-jar - test - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - jclouds.aws.accesskeyid - ${jclouds.aws.accesskeyid} - - - jclouds.aws.secretaccesskey - ${jclouds.aws.secretaccesskey} - - - - - - - diff --git a/extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandler.java b/extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandler.java deleted file mode 100644 index 12435ce0dd..0000000000 --- a/extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandler.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * - * Copyright (C) 2009 Adrian Cole - * - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - */ -package org.jclouds.aws.s3.nio; - -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; - -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpResponse; -import org.apache.http.nio.entity.NStringEntity; -import org.jclouds.http.HttpFutureCommand; -import org.jclouds.http.httpnio.pool.HttpNioFutureCommandExecutionHandler; - -import com.google.inject.Inject; -import com.google.inject.Singleton; - -/** - * // TODO: Adrian: Document this! - * - * @author Adrian Cole - */ -@Singleton -public class S3HttpNioFutureCommandExecutionHandler extends - HttpNioFutureCommandExecutionHandler { - - @Inject - public S3HttpNioFutureCommandExecutionHandler( - ConsumingNHttpEntityFactory entityFactory, - ExecutorService executor, - BlockingQueue> commandQueue) { - super(entityFactory, executor, commandQueue); - } - - @Override - protected boolean isRetryable(HttpResponse response) throws IOException { - if (super.isRetryable(response)) - return true; - int code = response.getStatusLine().getStatusCode(); - if (code == 409) { - return true; - } else if (code == 400) { - if (response.getEntity() != null) { - InputStream input = response.getEntity().getContent(); - if (input != null) { - String reason = null; - try { - reason = IOUtils.toString(input); - } finally { - IOUtils.closeQuietly(input); - } - if (reason != null) { - try { - if (reason.indexOf("RequestTime") >= 0) - return true; - } finally { - IOUtils.closeQuietly(input); - response.setEntity(new NStringEntity(reason)); - } - } - - } - - } - } - return false; - } - -} diff --git a/extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModule.java b/extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModule.java deleted file mode 100644 index cf67e3cbf3..0000000000 --- a/extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModule.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * - * Copyright (C) 2009 Adrian Cole - * - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - */ -package org.jclouds.aws.s3.nio.config; - -import org.apache.http.nio.protocol.NHttpRequestExecutionHandler; -import org.jclouds.aws.s3.nio.S3HttpNioFutureCommandExecutionHandler; -import org.jclouds.http.config.HttpFutureCommandClientModule; -import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule; - -import com.google.inject.AbstractModule; -import com.google.inject.util.Modules; - -/** - * This installs a {@link HttpNioConnectionPoolClientModule}, but overrides it - * binding {@link S3HttpNioFutureCommandExecutionHandler}. - * - * @author Adrian Cole - */ -@HttpFutureCommandClientModule -public class S3HttpNioConnectionPoolClientModule extends AbstractModule { - protected void configure() { - install(Modules.override(new HttpNioConnectionPoolClientModule()).with( - new AbstractModule() { - protected void configure() { - bind(NHttpRequestExecutionHandler.class).to( - S3HttpNioFutureCommandExecutionHandler.class); - } - })); - } -} diff --git a/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3InputStreamMapTest.java b/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3InputStreamMapTest.java deleted file mode 100644 index 0db0a9efba..0000000000 --- a/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3InputStreamMapTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * - * Copyright (C) 2009 Adrian Cole - * - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - */ -package org.jclouds.aws.s3.nio; - -import java.util.Properties; - -import org.jclouds.aws.s3.S3InputStreamMapTest; -import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule; -import org.testng.annotations.Test; - -import com.google.inject.Module; - -/** - * // TODO: Adrian: Document this! - * - * @author Adrian Cole - */ -@Test(groups = "unit", sequential = true, testName = "s3.NioS3InputStreamMapTest") -public class NioS3InputStreamMapTest extends S3InputStreamMapTest { - - @Override - protected Properties buildS3Properties(String AWSAccessKeyId, - String AWSSecretAccessKey) { - Properties properties = super.buildS3Properties(AWSAccessKeyId, - AWSSecretAccessKey); - properties.setProperty("jclouds.http.pool.max_connection_reuse", "75"); - properties.setProperty("jclouds.http.pool.max_session_failures", "2"); - properties - .setProperty("jclouds.http.pool.request_invoker_threads", "1"); - properties.setProperty("jclouds.http.pool.io_worker_threads", "2"); - properties.setProperty("jclouds.pool.max_connections", "12"); - return properties; - } - - @Override - protected Module createHttpModule() { - return new S3HttpNioConnectionPoolClientModule(); - } - -} diff --git a/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3ObjectMapTest.java b/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3ObjectMapTest.java deleted file mode 100644 index ea5905ac7e..0000000000 --- a/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3ObjectMapTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * - * Copyright (C) 2009 Adrian Cole - * - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - */ -package org.jclouds.aws.s3.nio; - -import java.util.Properties; - -import org.jclouds.aws.s3.S3ObjectMapTest; -import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule; -import org.testng.annotations.Test; - -import com.google.inject.Module; - -/** - * // TODO: Adrian: Document this! - * - * @author Adrian Cole - */ -@Test(groups = "unit", sequential = true, testName = "s3.NioS3ObjectMapTest") -public class NioS3ObjectMapTest extends S3ObjectMapTest { - - @Override - protected Properties buildS3Properties(String AWSAccessKeyId, - String AWSSecretAccessKey) { - Properties properties = super.buildS3Properties(AWSAccessKeyId, - AWSSecretAccessKey); - properties.setProperty("jclouds.http.pool.max_connection_reuse", "75"); - properties.setProperty("jclouds.http.pool.max_session_failures", "2"); - properties - .setProperty("jclouds.http.pool.request_invoker_threads", "1"); - properties.setProperty("jclouds.http.pool.io_worker_threads", "2"); - properties.setProperty("jclouds.pool.max_connections", "12"); - return properties; - } - - @Override - protected Module createHttpModule() { - return new S3HttpNioConnectionPoolClientModule(); - } - -} diff --git a/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandlerTest.java b/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandlerTest.java deleted file mode 100644 index b99fe71ae3..0000000000 --- a/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandlerTest.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * - * Copyright (C) 2009 Adrian Cole - * - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - */ -package org.jclouds.aws.s3.nio; - -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.isA; -import static org.easymock.classextension.EasyMock.createMock; -import static org.easymock.classextension.EasyMock.replay; -import static org.easymock.classextension.EasyMock.verify; - -import java.io.IOException; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutorService; - -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.StatusLine; -import org.apache.http.nio.entity.NStringEntity; -import org.jclouds.http.HttpFutureCommand; -import org.jclouds.http.httpnio.pool.HttpNioFutureCommandExecutionHandler; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -/** - * // TODO: Adrian: Document this! - * - * @author Adrian Cole - */ -@Test -public class S3HttpNioFutureCommandExecutionHandlerTest { - S3HttpNioFutureCommandExecutionHandler handler = null; - HttpResponse response = null; - StatusLine statusline = null; - - @BeforeMethod - public void createHandler() { - handler = new S3HttpNioFutureCommandExecutionHandler( - createMock(HttpNioFutureCommandExecutionHandler.ConsumingNHttpEntityFactory.class), - createMock(ExecutorService.class), - new ArrayBlockingQueue>(1)); - response = createMock(HttpResponse.class); - statusline = createMock(StatusLine.class); - expect(response.getStatusLine()).andReturn(statusline).atLeastOnce(); - } - - @Test - void test500isRetryable() throws IOException { - isRetryable(500); - } - - @Test - void test503isRetryable() throws IOException { - isRetryable(503); - } - - @Test - void test409isRetryable() throws IOException { - isRetryable(409); - } - - @Test - void test404NotRetryable() throws IOException { - expect(statusline.getStatusCode()).andReturn(404).atLeastOnce(); - - replay(statusline); - replay(response); - assert !handler.isRetryable(response); - verify(statusline); - verify(response); - } - - @Test - void test400WithNoEnitityNotRetryable() throws IOException { - expect(statusline.getStatusCode()).andReturn(400).atLeastOnce(); - expect(response.getEntity()).andReturn(null); - replay(statusline); - replay(response); - assert !handler.isRetryable(response); - verify(statusline); - verify(response); - } - - @Test - void test400WithIrrelevantEnitityNotRetryable() throws IOException { - expect(statusline.getStatusCode()).andReturn(400).atLeastOnce(); - HttpEntity entity = createMock(HttpEntity.class); - expect(response.getEntity()).andReturn(entity).atLeastOnce(); - expect(entity.getContent()).andReturn(IOUtils.toInputStream("hello")); - response.setEntity(isA(NStringEntity.class)); - replay(entity); - replay(statusline); - replay(response); - assert !handler.isRetryable(response); - verify(statusline); - verify(response); - verify(entity); - } - - @Test - void test400WithRequestTimeTooSkewedTimeEnitityRetryable() - throws IOException { - expect(statusline.getStatusCode()).andReturn(400).atLeastOnce(); - HttpEntity entity = createMock(HttpEntity.class); - expect(response.getEntity()).andReturn(entity).atLeastOnce(); - expect(entity.getContent()).andReturn( - IOUtils.toInputStream("RequestTimeTooSkewed")); - response.setEntity(isA(NStringEntity.class)); - replay(entity); - replay(statusline); - replay(response); - assert handler.isRetryable(response); - verify(statusline); - verify(response); - verify(entity); - } - - private void isRetryable(int code) throws IOException { - expect(statusline.getStatusCode()).andReturn(code).atLeastOnce(); - replay(statusline); - replay(response); - assert handler.isRetryable(response); - verify(statusline); - verify(response); - } - -} \ No newline at end of file diff --git a/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModuleTest.java b/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModuleTest.java deleted file mode 100644 index 1bb13af8ec..0000000000 --- a/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModuleTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * - * Copyright (C) 2009 Adrian Cole - * - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - */ -package org.jclouds.aws.s3.nio.config; - -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.name.Names; -import org.apache.http.nio.protocol.NHttpRequestExecutionHandler; -import org.jclouds.aws.s3.nio.S3HttpNioFutureCommandExecutionHandler; -import org.testng.annotations.Test; - -import java.util.Properties; - -/** - * // TODO: Adrian: Document this! - * - * @author Adrian Cole - */ -@Test -public class S3HttpNioConnectionPoolClientModuleTest { - - public void testConfigureBindsS3Handler() { - final Properties properties = new Properties(); - properties.put("jclouds.http.address", "localhost"); - properties.put("jclouds.http.port", "8088"); - properties.put("jclouds.http.secure", "false"); - properties.setProperty("jclouds.http.pool.max_connection_reuse", "75"); - properties.setProperty("jclouds.http.pool.max_session_failures", "2"); - properties.setProperty("jclouds.http.pool.request_invoker_threads", "1"); - properties.setProperty("jclouds.http.pool.io_worker_threads", "2"); - properties.setProperty("jclouds.pool.max_connections", "12"); - - Injector i = Guice.createInjector(new S3HttpNioConnectionPoolClientModule() { - @Override - protected void configure() { - Names.bindProperties(binder(), properties); - super.configure(); - } - }); - NHttpRequestExecutionHandler handler = i.getInstance(NHttpRequestExecutionHandler.class); - assert handler instanceof S3HttpNioFutureCommandExecutionHandler; - } -} diff --git a/gae/src/main/java/org/jclouds/gae/URLFetchServiceClient.java b/gae/src/main/java/org/jclouds/gae/URLFetchServiceClient.java index 0b6e69f453..0e1e110929 100644 --- a/gae/src/main/java/org/jclouds/gae/URLFetchServiceClient.java +++ b/gae/src/main/java/org/jclouds/gae/URLFetchServiceClient.java @@ -32,18 +32,15 @@ import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; -import java.util.Collections; import java.util.List; -import javax.annotation.Resource; - import org.apache.commons.io.IOUtils; +import org.jclouds.http.BaseHttpFutureCommandClient; import org.jclouds.http.HttpFutureCommand; import org.jclouds.http.HttpFutureCommandClient; import org.jclouds.http.HttpRequest; import org.jclouds.http.HttpRequestFilter; import org.jclouds.http.HttpResponse; -import org.jclouds.logging.Logger; import com.google.appengine.api.urlfetch.HTTPHeader; import com.google.appengine.api.urlfetch.HTTPMethod; @@ -58,67 +55,61 @@ import com.google.inject.Inject; * * @author Adrian Cole */ -public class URLFetchServiceClient implements HttpFutureCommandClient { - private final URL target; - private List requestFilters = Collections.emptyList(); - @Resource - private Logger logger = Logger.NULL; +public class URLFetchServiceClient extends BaseHttpFutureCommandClient { private final URLFetchService urlFetchService; - public List getRequestFilters() { - return requestFilters; - } - - @Inject(optional = true) - public void setRequestFilters(List requestFilters) { - this.requestFilters = requestFilters; - } - @Inject public URLFetchServiceClient(URL target, URLFetchService urlFetchService) throws MalformedURLException { + super(target); this.urlFetchService = urlFetchService; - this.target = target; - this.logger.info("configured to connect to target: %1s", target); } - public void submit(HttpFutureCommand operation) { - HttpRequest request = operation.getRequest(); + public void submit(HttpFutureCommand command) { + HttpRequest request = command.getRequest(); HTTPResponse gaeResponse = null; try { - for (HttpRequestFilter filter : getRequestFilters()) { + for (HttpRequestFilter filter : requestFilters) { filter.filter(request); } HttpResponse response = null; for (;;) { logger.trace("%1s - converting request %2s", target, request); HTTPRequest gaeRequest = convert(request); - logger - .trace("%1s - submitting request %2s", target, - gaeRequest); + if (logger.isTraceEnabled()) + logger.trace("%1s - submitting request %2s, headers: %3s", + target, gaeRequest.getURL(), + headersAsString(gaeRequest.getHeaders())); gaeResponse = this.urlFetchService.fetch(gaeRequest); - logger - .trace("%1s - received response %2s", target, - gaeResponse); + if (logger.isTraceEnabled()) + logger.trace( + "%1s - received response code %2s, headers: %3s", + target, gaeResponse.getResponseCode(), + headersAsString(gaeResponse.getHeaders())); response = convert(gaeResponse); - if (response.getStatusCode() >= 500) { + if (isRetryable(command, response)) continue; - } break; - } - operation.getResponseFuture().setResponse(response); - operation.getResponseFuture().run(); + handleResponse(command, response); } catch (Exception e) { if (gaeResponse != null && gaeResponse.getContent() != null) { logger.error(e, "error encountered during the execution: %1s%n%2s", gaeResponse, new String(gaeResponse.getContent())); } - operation.setException(e); + command.setException(e); } } + String headersAsString(List headers) { + StringBuilder builder = new StringBuilder(""); + for (HTTPHeader header : headers) + builder.append("[").append(header.getName()).append("=").append( + header.getValue()).append("],"); + return builder.toString(); + } + /** * byte [] content is replayable and the only content type supportable by * GAE. As such, we convert the original request content to a byte array. diff --git a/gae/src/test/java/org/jclouds/gae/URLFetchServiceClientTest.java b/gae/src/test/java/org/jclouds/gae/URLFetchServiceClientTest.java index 439dd10829..8b3229667b 100644 --- a/gae/src/test/java/org/jclouds/gae/URLFetchServiceClientTest.java +++ b/gae/src/test/java/org/jclouds/gae/URLFetchServiceClientTest.java @@ -41,7 +41,6 @@ import java.util.List; import org.apache.commons.io.IOUtils; import org.jclouds.http.HttpHeaders; import org.jclouds.http.HttpRequest; -import org.jclouds.http.HttpRequestFilter; import org.jclouds.http.HttpResponse; import org.testng.annotations.BeforeTest; import org.testng.annotations.Parameters; @@ -163,15 +162,6 @@ public class URLFetchServiceClientTest { } - @Test - void testRequestFilters() { - List filters = new ArrayList(); - filters.add(createNiceMock(HttpRequestFilter.class)); - assertEquals(client.getRequestFilters().size(), 0); - client.setRequestFilters(filters); - assertEquals(client.getRequestFilters(), filters); - } - @Test @Parameters("basedir") void testConvertRequestFileContent(String basedir) throws IOException { diff --git a/pom.xml b/pom.xml index 276fb13276..f2fcc56953 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,6 @@ project core extensions/httpnio - extensions/s3nio extensions/log4j gae s3 diff --git a/s3/perftest/pom.xml b/s3/perftest/pom.xml index 0fcd23dd76..9bf52b628e 100644 --- a/s3/perftest/pom.xml +++ b/s3/perftest/pom.xml @@ -74,7 +74,7 @@ ${project.groupId} - jclouds-s3nio + jclouds-httpnio ${project.version} diff --git a/s3/perftest/src/test/java/com/amazon/s3/AmazonPerformance.java b/s3/perftest/src/test/java/com/amazon/s3/AmazonPerformanceTest.java similarity index 94% rename from s3/perftest/src/test/java/com/amazon/s3/AmazonPerformance.java rename to s3/perftest/src/test/java/com/amazon/s3/AmazonPerformanceTest.java index a940b98553..b65d9d66fc 100644 --- a/s3/perftest/src/test/java/com/amazon/s3/AmazonPerformance.java +++ b/s3/perftest/src/test/java/com/amazon/s3/AmazonPerformanceTest.java @@ -42,8 +42,8 @@ import org.testng.annotations.Test; * * @author Adrian Cole */ -@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformance", groups = "performance") -public class AmazonPerformance extends BasePerformance { +@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformance") +public class AmazonPerformanceTest extends BasePerformance { private AWSAuthConnection amzClient; @Override @@ -58,33 +58,39 @@ public class AmazonPerformance extends BasePerformance { } @Override + @Test(enabled = false) public void testPutFileSerial() throws Exception { throw new UnsupportedOperationException(); } @Override + @Test(enabled = false) public void testPutFileParallel() throws InterruptedException, ExecutionException { throw new UnsupportedOperationException(); } @Override + @Test(enabled = false) public void testPutInputStreamSerial() throws Exception { throw new UnsupportedOperationException(); } @Override + @Test(enabled = false) public void testPutInputStreamParallel() throws InterruptedException, ExecutionException { throw new UnsupportedOperationException(); } @Override + @Test(enabled = false) public void testPutStringSerial() throws Exception { throw new UnsupportedOperationException(); } @Override + @Test(enabled = false) public void testPutStringParallel() throws InterruptedException, ExecutionException { throw new UnsupportedOperationException(); diff --git a/s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java b/s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java index f77da7e423..dd3df1f945 100644 --- a/s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java +++ b/s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java @@ -51,6 +51,7 @@ import com.google.inject.Provider; * * @author Adrian Cole */ +@Test public abstract class BasePerformance extends S3IntegrationTest { protected boolean debugEnabled() { return false; diff --git a/s3/perftest/src/test/java/com/amazon/s3/DateServiceTest.java b/s3/perftest/src/test/java/com/amazon/s3/DateServiceTest.java index 2f0fdd9c14..122bbc4193 100644 --- a/s3/perftest/src/test/java/com/amazon/s3/DateServiceTest.java +++ b/s3/perftest/src/test/java/com/amazon/s3/DateServiceTest.java @@ -39,7 +39,7 @@ import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; -@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.DateTest", groups = "performance") +@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.DateTest") public class DateServiceTest extends PerformanceTest { Injector i = Guice.createInjector(); diff --git a/s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformance.java b/s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformanceTest.java similarity index 85% rename from s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformance.java rename to s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformanceTest.java index 32cd2299fe..ba37b83893 100644 --- a/s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformance.java +++ b/s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformanceTest.java @@ -24,14 +24,15 @@ package com.amazon.s3; -import com.google.inject.Module; -import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule; -import org.testng.annotations.Test; - import java.util.Properties; -@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.JCloudsNioPerformance", groups = "performance") -public class JCloudsNioPerformance extends BaseJCloudsPerformance { +import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule; +import org.testng.annotations.Test; + +import com.google.inject.Module; + +@Test(sequential=true, testName = "s3.JCloudsNioPerformance") +public class JCloudsNioPerformanceTest extends BaseJCloudsPerformance { @Override protected Properties buildS3Properties(String AWSAccessKeyId, @@ -47,6 +48,6 @@ public class JCloudsNioPerformance extends BaseJCloudsPerformance { @Override protected Module createHttpModule() { - return new S3HttpNioConnectionPoolClientModule(); + return new HttpNioConnectionPoolClientModule(); } } \ No newline at end of file diff --git a/s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformance.java b/s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformanceTest.java similarity index 91% rename from s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformance.java rename to s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformanceTest.java index 277cb5bb96..e4c88f890c 100644 --- a/s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformance.java +++ b/s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformanceTest.java @@ -26,7 +26,7 @@ package com.amazon.s3; import org.testng.annotations.Test; -@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.JCloudsPerformance", groups = "performance") -public class JCloudsPerformance extends BaseJCloudsPerformance { +@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.JCloudsPerformance") +public class JCloudsPerformanceTest extends BaseJCloudsPerformance { } \ No newline at end of file diff --git a/s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformance.java b/s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformanceTest.java similarity index 85% rename from s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformance.java rename to s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformanceTest.java index 3c520c8404..ececbbad55 100644 --- a/s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformance.java +++ b/s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformanceTest.java @@ -37,8 +37,8 @@ import org.testng.annotations.Test; * * @author Adrian Cole */ -@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformance", groups = "performance") -public class Jets3tPerformance extends BasePerformance { +@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformance") +public class Jets3tPerformanceTest extends BasePerformance { private S3Service jetClient; @Override @@ -50,28 +50,46 @@ public class Jets3tPerformance extends BasePerformance { } @Override + @Test(enabled=false) public void testPutStringSerial() throws Exception { throw new UnsupportedOperationException(); } @Override + @Test(enabled=false) public void testPutStringParallel() throws InterruptedException, ExecutionException { throw new UnsupportedOperationException(); } @Override + @Test(enabled=false) public void testPutBytesSerial() throws Exception { throw new UnsupportedOperationException(); } @Override + @Test(enabled=false) public void testPutBytesParallel() throws InterruptedException, ExecutionException { throw new UnsupportedOperationException(); } @Override + @Test(enabled=false) + public void testPutInputStreamParallel() throws InterruptedException, + ExecutionException { + throw new UnsupportedOperationException(); + } + @Override + @Test(enabled=false) + public void testPutFileParallel() throws InterruptedException, + ExecutionException { + throw new UnsupportedOperationException(); + } + + @Override + @Test(enabled=false) protected boolean putByteArray(String bucket, String key, byte[] data, String contentType) throws Exception { throw new UnsupportedOperationException(); diff --git a/s3/perftest/src/test/java/com/amazon/s3/S3ParserTest.java b/s3/perftest/src/test/java/com/amazon/s3/S3ParserTest.java index d10dad021b..e4c94d4953 100644 --- a/s3/perftest/src/test/java/com/amazon/s3/S3ParserTest.java +++ b/s3/perftest/src/test/java/com/amazon/s3/S3ParserTest.java @@ -42,7 +42,7 @@ import java.util.concurrent.ExecutorCompletionService; * * @author Adrian Cole */ -@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3ParserTest", groups = "performance") +@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3ParserTest") public class S3ParserTest extends org.jclouds.aws.s3.commands.S3ParserTest { class MockHttpURLConnection extends HttpURLConnection { diff --git a/s3/perftest/src/test/java/com/amazon/s3/S3UtilsTest.java b/s3/perftest/src/test/java/com/amazon/s3/S3UtilsTest.java index e29d1ee2b2..5d1118db13 100644 --- a/s3/perftest/src/test/java/com/amazon/s3/S3UtilsTest.java +++ b/s3/perftest/src/test/java/com/amazon/s3/S3UtilsTest.java @@ -33,7 +33,7 @@ import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; -@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3UtilsTest", groups = "performance") +@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3UtilsTest") public class S3UtilsTest extends org.jclouds.aws.s3.S3UtilsTest { @Test(dataProvider = "hmacsha1") diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3Constants.java b/s3/src/main/java/org/jclouds/aws/s3/S3Constants.java index d845687999..692fac62e2 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/S3Constants.java +++ b/s3/src/main/java/org/jclouds/aws/s3/S3Constants.java @@ -35,6 +35,6 @@ public interface S3Constants extends HttpConstants, PoolConstants, S3Headers { public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey"; public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid"; - public static final String PROPERTY_AWS_MAP_TIMEOUT = "jclouds.aws.map.timeout"; + public static final String PROPERTY_S3_MAP_TIMEOUT = "jclouds.s3.map.timeout"; } diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java b/s3/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java index 9129818dd5..3327bbf779 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java +++ b/s3/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java @@ -39,8 +39,8 @@ import java.util.List; import java.util.Properties; import org.jclouds.aws.s3.config.S3ContextModule; -import org.jclouds.aws.s3.config.S3JavaUrlHttpFutureCommandClientModule; import org.jclouds.http.config.HttpFutureCommandClientModule; +import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule; import org.jclouds.logging.config.LoggingModule; import org.jclouds.logging.jdk.config.JDKLoggingModule; @@ -165,7 +165,7 @@ public class S3ContextFactory { /** * Bind the given properties and install the list of modules. If no modules * are specified, install the default {@link JDKLoggingModule} - * {@link S3JavaUrlHttpFutureCommandClientModule} + * {@link JavaUrlHttpFutureCommandClientModule} * * @param properties * - contains constants used by jclouds @@ -201,7 +201,7 @@ public class S3ContextFactory { } })) - modules.add(new S3JavaUrlHttpFutureCommandClientModule()); + modules.add(new JavaUrlHttpFutureCommandClientModule()); } @VisibleForTesting diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3ResponseException.java b/s3/src/main/java/org/jclouds/aws/s3/S3ResponseException.java index ea58972cdf..5d49621293 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/S3ResponseException.java +++ b/s3/src/main/java/org/jclouds/aws/s3/S3ResponseException.java @@ -24,19 +24,33 @@ package org.jclouds.aws.s3; import org.jclouds.aws.s3.domain.S3Error; -import org.jclouds.http.HttpMessage; +import org.jclouds.http.HttpFutureCommand; +import org.jclouds.http.HttpResponse; public class S3ResponseException extends RuntimeException { private static final long serialVersionUID = 1L; + private final HttpFutureCommand command; + private final HttpResponse response; private S3Error error; - private HttpMessage response; - - public S3ResponseException(S3Error error, HttpMessage response) { - super(error.toString()); + public S3ResponseException(HttpFutureCommand command, + HttpResponse response, S3Error error) { + super(String.format( + "command: %1s failed with response: %2s; error from s3: %3s", + command, response, error)); + this.command = command; + this.response = response; this.setError(error); - this.setResponse(response); + + } + + public S3ResponseException(HttpFutureCommand command, + HttpResponse response) { + super(String.format("command: %1s failed with response: %2s", command, + response)); + this.command = command; + this.response = response; } public void setError(S3Error error) { @@ -47,11 +61,12 @@ public class S3ResponseException extends RuntimeException { return error; } - public void setResponse(HttpMessage response) { - this.response = response; + public HttpFutureCommand getCommand() { + return command; } - public HttpMessage getResponse() { + public HttpResponse getResponse() { return response; } + } diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3Utils.java b/s3/src/main/java/org/jclouds/aws/s3/S3Utils.java index 6eefb05382..4fce2adfd9 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/S3Utils.java +++ b/s3/src/main/java/org/jclouds/aws/s3/S3Utils.java @@ -23,8 +23,10 @@ */ package org.jclouds.aws.s3; -import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.*; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; @@ -32,6 +34,8 @@ import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.NoSuchProviderException; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.ByteArrayOutputStream; import org.bouncycastle.crypto.digests.MD5Digest; import org.bouncycastle.crypto.digests.SHA1Digest; import org.bouncycastle.crypto.macs.HMac; @@ -60,6 +64,39 @@ public class S3Utils extends Utils { return new String(hex, "ASCII"); } + public static long calculateSize(Object data) { + long size = -1; + if (data instanceof byte[]) { + size = ((byte[]) data).length; + } else if (data instanceof String) { + size = ((String) data).length(); + } else if (data instanceof File) { + size = ((File) data).length(); + } + return size; + } + + /** + * + * @throws IOException + */ + public static byte[] md5(Object data) throws IOException { + checkNotNull(data, "data must be set before calling generateMd5()"); + byte[] md5 = null; + if (data == null || data instanceof byte[]) { + md5 = S3Utils.md5((byte[]) data); + } else if (data instanceof String) { + md5 = S3Utils.md5(((String) data).getBytes()); + } else if (data instanceof File) { + md5 = S3Utils.md5(((File) data)); + } else { + throw new UnsupportedOperationException("Content not supported " + + data.getClass()); + } + return md5; + + } + public static byte[] fromHexString(String hex) { byte[] bytes = new byte[hex.length() / 2]; for (int i = 0; i < bytes.length; i++) { @@ -107,22 +144,65 @@ public class S3Utils extends Utils { md5.doFinal(resBuf, 0); return resBuf; } - - public static byte[] md5(InputStream toEncode) throws IOException { + + public static byte[] md5(File toEncode) throws IOException { MD5Digest md5 = new MD5Digest(); byte[] resBuf = new byte[md5.getDigestSize()]; byte[] buffer = new byte[1024]; int numRead = -1; - do { - numRead = toEncode.read(buffer); - if (numRead > 0) { - md5.update(buffer, 0, numRead); - } - } while (numRead != -1); + InputStream i = new FileInputStream(toEncode); + try { + do { + numRead = i.read(buffer); + if (numRead > 0) { + md5.update(buffer, 0, numRead); + } + } while (numRead != -1); + } finally { + IOUtils.closeQuietly(i); + } md5.doFinal(resBuf, 0); return resBuf; } + public static Md5InputStreamResult generateMd5Result(InputStream toEncode) + throws IOException { + MD5Digest md5 = new MD5Digest(); + byte[] resBuf = new byte[md5.getDigestSize()]; + byte[] buffer = new byte[1024]; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + long length = 0; + int numRead = -1; + try { + do { + numRead = toEncode.read(buffer); + if (numRead > 0) { + length += numRead; + md5.update(buffer, 0, numRead); + out.write(buffer, 0, numRead); + } + } while (numRead != -1); + } finally { + IOUtils.closeQuietly(toEncode); + } + md5.doFinal(resBuf, 0); + return new Md5InputStreamResult(out.toByteArray(), resBuf, length); + } + + public static class Md5InputStreamResult { + public final byte[] data; + public final byte[] md5; + public final long length; + + Md5InputStreamResult(byte[] data, byte[] md5, long length) { + this.data = checkNotNull(data, "data"); + this.md5 = checkNotNull(md5, "md5"); + checkArgument(length >= 0, "length cannot me negative"); + this.length = length; + } + + } + public static String getContentAsStringAndClose(S3Object object) throws IOException { checkNotNull(object, "s3Object"); diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/BucketExists.java b/s3/src/main/java/org/jclouds/aws/s3/commands/BucketExists.java index e7ca1583da..9b3dd64a33 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/BucketExists.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/BucketExists.java @@ -24,8 +24,15 @@ package org.jclouds.aws.s3.commands; import static org.jclouds.aws.s3.commands.options.ListBucketOptions.Builder.maxResults; -import org.jclouds.http.commands.callables.ReturnTrueIf200; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.jclouds.aws.s3.S3ResponseException; +import org.jclouds.http.commands.callables.ReturnTrueIf2xx; + +import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import com.google.inject.name.Named; @@ -34,8 +41,39 @@ public class BucketExists extends S3FutureCommand { @Inject public BucketExists(@Named("jclouds.http.address") String amazonHost, - ReturnTrueIf200 callable, @Assisted String s3Bucket) { - super("HEAD", "/" + maxResults(0).buildQueryString(), callable, amazonHost, - s3Bucket); + ReturnTrueIf2xx callable, @Assisted String s3Bucket) { + super("HEAD", "/" + maxResults(0).buildQueryString(), callable, + amazonHost, s3Bucket); + } + + @Override + public Boolean get() throws InterruptedException, ExecutionException { + try { + return super.get(); + } catch (ExecutionException e) { + return attemptNotFound(e); + } + } + + @VisibleForTesting + Boolean attemptNotFound(ExecutionException e) throws ExecutionException { + if (e.getCause() != null && e.getCause() instanceof S3ResponseException) { + S3ResponseException responseException = (S3ResponseException) e + .getCause(); + if (responseException.getResponse().getStatusCode() == 404) { + return false; + } + } + throw e; + } + + @Override + public Boolean get(long l, TimeUnit timeUnit) throws InterruptedException, + ExecutionException, TimeoutException { + try { + return super.get(l, timeUnit); + } catch (ExecutionException e) { + return attemptNotFound(e); + } } } \ No newline at end of file diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/DeleteBucket.java b/s3/src/main/java/org/jclouds/aws/s3/commands/DeleteBucket.java index 83d9b362bd..0028edc246 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/DeleteBucket.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/DeleteBucket.java @@ -23,8 +23,14 @@ */ package org.jclouds.aws.s3.commands; -import org.jclouds.aws.s3.commands.callables.DeleteBucketCallable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.jclouds.aws.s3.S3ResponseException; +import org.jclouds.http.commands.callables.ReturnTrueIf2xx; + +import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import com.google.inject.name.Named; @@ -33,7 +39,41 @@ public class DeleteBucket extends S3FutureCommand { @Inject public DeleteBucket(@Named("jclouds.http.address") String amazonHost, - DeleteBucketCallable callable, @Assisted String s3Bucket) { + ReturnTrueIf2xx callable, @Assisted String s3Bucket) { super("DELETE", "/", callable, amazonHost, s3Bucket); } + + @Override + public Boolean get() throws InterruptedException, ExecutionException { + try { + return super.get(); + } catch (ExecutionException e) { + return attemptNotFound(e); + } + } + + @VisibleForTesting + Boolean attemptNotFound(ExecutionException e) throws ExecutionException { + if (e.getCause() != null && e.getCause() instanceof S3ResponseException) { + S3ResponseException responseException = (S3ResponseException) e + .getCause(); + if (responseException.getResponse().getStatusCode() == 404) { + return true; + } else if ("BucketNotEmpty".equals(responseException.getError() + .getCode())) { + return false; + } + } + throw e; + } + + @Override + public Boolean get(long l, TimeUnit timeUnit) throws InterruptedException, + ExecutionException, TimeoutException { + try { + return super.get(l, timeUnit); + } catch (ExecutionException e) { + return attemptNotFound(e); + } + } } \ No newline at end of file diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/DeleteObject.java b/s3/src/main/java/org/jclouds/aws/s3/commands/DeleteObject.java index 97831222c9..05bb3e3e93 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/DeleteObject.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/DeleteObject.java @@ -25,7 +25,7 @@ package org.jclouds.aws.s3.commands; import static com.google.common.base.Preconditions.checkNotNull; -import org.jclouds.aws.s3.commands.callables.DeleteCallable; +import org.jclouds.http.commands.callables.ReturnTrueIf2xx; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; @@ -35,7 +35,7 @@ public class DeleteObject extends S3FutureCommand { @Inject public DeleteObject(@Named("jclouds.http.address") String amazonHost, - DeleteCallable callable, @Assisted("bucketName") String bucket, + ReturnTrueIf2xx callable, @Assisted("bucketName") String bucket, @Assisted("key") String key) { super("DELETE", "/" + checkNotNull(key), callable, amazonHost, bucket); } diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/GetObject.java b/s3/src/main/java/org/jclouds/aws/s3/commands/GetObject.java index ba6cc3444c..cedd683ceb 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/GetObject.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/GetObject.java @@ -30,7 +30,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.jclouds.aws.s3.S3ResponseException; -import org.jclouds.aws.s3.commands.callables.GetObjectCallable; +import org.jclouds.aws.s3.commands.callables.ParseObjectFromHeadersAndHttpContent; import org.jclouds.aws.s3.commands.options.GetObjectOptions; import org.jclouds.aws.s3.domain.S3Object; @@ -50,7 +50,7 @@ public class GetObject extends S3FutureCommand { @Inject public GetObject(@Named("jclouds.http.address") String amazonHost, - GetObjectCallable callable, + ParseObjectFromHeadersAndHttpContent callable, @Assisted("bucketName") String s3Bucket, @Assisted("key") String key, @Assisted GetObjectOptions options) { super("GET", "/" + checkNotNull(key), callable, amazonHost, s3Bucket); diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/HeadMetaData.java b/s3/src/main/java/org/jclouds/aws/s3/commands/HeadMetaData.java index 6adfc414b0..0a26a5c887 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/HeadMetaData.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/HeadMetaData.java @@ -25,9 +25,15 @@ package org.jclouds.aws.s3.commands; import static com.google.common.base.Preconditions.checkNotNull; -import org.jclouds.aws.s3.commands.callables.HeadMetaDataCallable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.jclouds.aws.s3.S3ResponseException; +import org.jclouds.aws.s3.commands.callables.ParseMetadataFromHeaders; import org.jclouds.aws.s3.domain.S3Object; +import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import com.google.inject.name.Named; @@ -43,9 +49,42 @@ public class HeadMetaData extends S3FutureCommand { @Inject public HeadMetaData(@Named("jclouds.http.address") String amazonHost, - HeadMetaDataCallable callable, + ParseMetadataFromHeaders callable, @Assisted("bucketName") String bucket, @Assisted("key") String key) { super("HEAD", "/" + checkNotNull(key), callable, amazonHost, bucket); callable.setKey(key); } + + @Override + public S3Object.Metadata get() throws InterruptedException, + ExecutionException { + try { + return super.get(); + } catch (ExecutionException e) { + return attemptNotFound(e); + } + } + + @VisibleForTesting + S3Object.Metadata attemptNotFound(ExecutionException e) + throws ExecutionException { + if (e.getCause() != null && e.getCause() instanceof S3ResponseException) { + S3ResponseException responseException = (S3ResponseException) e + .getCause(); + if (responseException.getResponse().getStatusCode() == 404) { + return S3Object.Metadata.NOT_FOUND; + } + } + throw e; + } + + @Override + public S3Object.Metadata get(long l, TimeUnit timeUnit) + throws InterruptedException, ExecutionException, TimeoutException { + try { + return super.get(l, timeUnit); + } catch (ExecutionException e) { + return attemptNotFound(e); + } + } } \ No newline at end of file diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/PutBucket.java b/s3/src/main/java/org/jclouds/aws/s3/commands/PutBucket.java index 1f3f6140e9..e99b80830f 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/PutBucket.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/PutBucket.java @@ -23,9 +23,9 @@ */ package org.jclouds.aws.s3.commands; -import org.jclouds.aws.s3.commands.callables.PutBucketCallable; import org.jclouds.aws.s3.commands.options.PutBucketOptions; import org.jclouds.http.HttpHeaders; +import org.jclouds.http.commands.callables.ReturnTrueIf2xx; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; @@ -41,7 +41,7 @@ public class PutBucket extends S3FutureCommand { @Inject public PutBucket(@Named("jclouds.http.address") String amazonHost, - PutBucketCallable callable, @Assisted String s3Bucket, + ReturnTrueIf2xx callable, @Assisted String s3Bucket, @Assisted PutBucketOptions options) { super("PUT", "/", callable, amazonHost, s3Bucket); getRequest().getHeaders().putAll(options.buildRequestHeaders()); diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/PutObject.java b/s3/src/main/java/org/jclouds/aws/s3/commands/PutObject.java index 174b5ee93b..5b1f464c95 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/PutObject.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/PutObject.java @@ -23,10 +23,10 @@ */ package org.jclouds.aws.s3.commands; -import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.*; import org.jclouds.aws.s3.S3Utils; -import org.jclouds.aws.s3.commands.callables.PutObjectCallable; +import org.jclouds.aws.s3.commands.callables.ParseMd5FromETagHeader; import org.jclouds.aws.s3.commands.options.PutObjectOptions; import org.jclouds.aws.s3.domain.S3Object; import org.jclouds.http.HttpHeaders; @@ -39,10 +39,12 @@ public class PutObject extends S3FutureCommand { @Inject public PutObject(@Named("jclouds.http.address") String amazonHost, - PutObjectCallable callable, @Assisted String s3Bucket, + ParseMd5FromETagHeader callable, @Assisted String s3Bucket, @Assisted S3Object object, @Assisted PutObjectOptions options) { super("PUT", "/" + checkNotNull(object.getKey()), callable, amazonHost, s3Bucket); + checkArgument(object.getMetaData().getSize() >=0,"size must be set"); + getRequest().setPayload( checkNotNull(object.getData(), "object.getContent()")); @@ -50,6 +52,7 @@ public class PutObject extends S3FutureCommand { HttpHeaders.CONTENT_TYPE, checkNotNull(object.getMetaData().getContentType(), "object.metaData.contentType()")); + getRequest().getHeaders().put(HttpHeaders.CONTENT_LENGTH, object.getMetaData().getSize() + ""); @@ -75,5 +78,4 @@ public class PutObject extends S3FutureCommand { getRequest().getHeaders().putAll(options.buildRequestHeaders()); } - } \ No newline at end of file diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallable.java b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallable.java deleted file mode 100644 index e6276cb2de..0000000000 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallable.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * - * Copyright (C) 2009 Adrian Cole - * - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - */ -package org.jclouds.aws.s3.commands.callables; - -import java.io.IOException; - -import org.jclouds.aws.s3.S3Utils; -import org.jclouds.http.HttpException; - -/** - * // TODO: Adrian: Document this! - * - * @author Adrian Cole - */ -public class DeleteBucketCallable extends DeleteCallable { - - public Boolean call() throws HttpException { - if (getResponse().getStatusCode() == 404) { - return true; - } else if (getResponse().getStatusCode() == 409) { - try { - if (getResponse().getContent() == null) { - throw new HttpException( - "cannot determine error as there is no content"); - } - String reason = S3Utils.toStringAndClose(getResponse() - .getContent()); - if (reason.indexOf("BucketNotEmpty") >= 0) - return false; - else - throw new HttpException("Error deleting bucket.\n" + reason); - } catch (IOException e) { - throw new HttpException("Error deleting bucket", e); - } - } else { - return super.call(); - } - } -} \ No newline at end of file diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/PutBucketCallable.java b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseMd5FromETagHeader.java similarity index 67% rename from s3/src/main/java/org/jclouds/aws/s3/commands/callables/PutBucketCallable.java rename to s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseMd5FromETagHeader.java index aa3767cac9..3590dd8085 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/PutBucketCallable.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseMd5FromETagHeader.java @@ -23,24 +23,28 @@ */ package org.jclouds.aws.s3.commands.callables; +import org.apache.commons.io.IOUtils; +import org.jclouds.aws.s3.S3Headers; +import org.jclouds.aws.s3.S3Utils; import org.jclouds.http.HttpException; import org.jclouds.http.HttpFutureCommand; /** - * Returns true, if the bucket was created. - * + * Parses an MD5 checksum from the header {@link S3Headers#ETAG}. + * * @author Adrian Cole */ -public class PutBucketCallable extends - HttpFutureCommand.ResponseCallable { +public class ParseMd5FromETagHeader extends + HttpFutureCommand.ResponseCallable { - public Boolean call() throws HttpException { - if (getResponse().getStatusCode() == 200) { - return true; - } else if (getResponse().getStatusCode() == 404) { - return false; - } else { - throw new HttpException("Error checking bucket " + getResponse()); + public byte[] call() throws HttpException { + checkCode(); + IOUtils.closeQuietly(getResponse().getContent()); + + String eTag = getResponse().getFirstHeaderOrNull(S3Headers.ETAG); + if (eTag != null) { + return S3Utils.fromHexString(eTag.replaceAll("\"", "")); } + throw new HttpException("did not receive ETag"); } } \ No newline at end of file diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/HeadMetaDataCallable.java b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseMetadataFromHeaders.java similarity index 62% rename from s3/src/main/java/org/jclouds/aws/s3/commands/callables/HeadMetaDataCallable.java rename to s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseMetadataFromHeaders.java index 4b1d9b5032..2b3f2eadce 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/HeadMetaDataCallable.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseMetadataFromHeaders.java @@ -23,10 +23,8 @@ */ package org.jclouds.aws.s3.commands.callables; -import java.io.IOException; import java.util.Map.Entry; -import org.jclouds.Utils; import org.jclouds.aws.s3.DateService; import org.jclouds.aws.s3.S3Headers; import org.jclouds.aws.s3.S3Utils; @@ -38,58 +36,65 @@ import org.jclouds.http.HttpHeaders; import com.google.inject.Inject; /** - * This parses @{link {@link S3Object.Metadata} from http headers or returns - * {@link S3Object.Metadata#NOT_FOUND} on 404. + * This parses @{link {@link S3Object.Metadata} from http headers. * * @author Adrian Cole */ -public class HeadMetaDataCallable extends +public class ParseMetadataFromHeaders extends HttpFutureCommand.ResponseCallable { private final DateService dateParser; private String key; @Inject - public HeadMetaDataCallable(DateService dateParser) { + public ParseMetadataFromHeaders(DateService dateParser) { this.dateParser = dateParser; } /** - * @return S3Content.NOT_FOUND, if not found. - * @throws org.jclouds.http.HttpException + * parses the http response headers to create a new + * {@link S3Object.MetaData} object. */ public S3Object.Metadata call() throws HttpException { - if (getResponse().getStatusCode() == 200) { - S3Object.Metadata metaData = new S3Object.Metadata(key); + checkCode(); + + S3Object.Metadata metaData = new S3Object.Metadata(key); - extractUserMetadata(metaData); - addMd5(metaData); + extractUserMetadata(metaData); + addMd5(metaData); - metaData.setLastModified(dateParser - .dateTimeFromHeaderFormat(getResponse() - .getFirstHeaderOrNull(HttpHeaders.LAST_MODIFIED))); - metaData.setContentType(getResponse().getFirstHeaderOrNull( - HttpHeaders.CONTENT_TYPE)); - metaData.setSize(Long.parseLong(getResponse().getFirstHeaderOrNull( - HttpHeaders.CONTENT_LENGTH))); - metaData.setCacheControl(getResponse().getFirstHeaderOrNull( - HttpHeaders.CACHE_CONTROL)); - metaData.setContentDisposition(getResponse().getFirstHeaderOrNull( - HttpHeaders.CONTENT_DISPOSITION)); - metaData.setContentEncoding(getResponse().getFirstHeaderOrNull( - HttpHeaders.CONTENT_ENCODING)); - return metaData; - } else if (getResponse().getStatusCode() == 404) { - return S3Object.Metadata.NOT_FOUND; - } else { - String reason = null; - try { - reason = Utils.toStringAndClose(getResponse().getContent()); - } catch (IOException e) { - logger.error(e, "error parsing reason"); - } - throw new HttpException("Error parsing object " + getResponse() - + " reason: " + reason); - } + parseLastModifiedOrThrowException(metaData); + setContentTypeOrThrowException(metaData); + + metaData.setCacheControl(getResponse().getFirstHeaderOrNull( + HttpHeaders.CACHE_CONTROL)); + metaData.setContentDisposition(getResponse().getFirstHeaderOrNull( + HttpHeaders.CONTENT_DISPOSITION)); + metaData.setContentEncoding(getResponse().getFirstHeaderOrNull( + HttpHeaders.CONTENT_ENCODING)); + return metaData; + + } + + private void setContentTypeOrThrowException(S3Object.Metadata metaData) + throws HttpException { + String contentType = getResponse().getFirstHeaderOrNull( + HttpHeaders.CONTENT_TYPE); + if (contentType == null) + throw new HttpException(HttpHeaders.CONTENT_TYPE + + " not found in headers"); + else + metaData.setContentType(contentType); + } + + private void parseLastModifiedOrThrowException(S3Object.Metadata metaData) + throws HttpException { + String lastModified = getResponse().getFirstHeaderOrNull( + HttpHeaders.LAST_MODIFIED); + metaData.setLastModified(dateParser + .dateTimeFromHeaderFormat(lastModified)); + if (metaData.getLastModified() == null) + throw new HttpException("could not parse: " + + HttpHeaders.LAST_MODIFIED + ": " + lastModified); } private void addMd5(S3Object.Metadata metaData) { diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/GetObjectCallable.java b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseObjectFromHeadersAndHttpContent.java similarity index 59% rename from s3/src/main/java/org/jclouds/aws/s3/commands/callables/GetObjectCallable.java rename to s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseObjectFromHeadersAndHttpContent.java index c4fa649830..59ed37ab69 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/GetObjectCallable.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseObjectFromHeadersAndHttpContent.java @@ -23,51 +23,55 @@ */ package org.jclouds.aws.s3.commands.callables; -import java.io.IOException; - -import org.jclouds.Utils; import org.jclouds.aws.s3.domain.S3Object; +import org.jclouds.aws.s3.domain.S3Object.Metadata; import org.jclouds.http.HttpException; import org.jclouds.http.HttpFutureCommand; +import org.jclouds.http.HttpHeaders; import com.google.inject.Inject; /** - * // TODO: Adrian: Document this! + * Parses response headers and creates a new S3Object from them and the http + * content. * * @author Adrian Cole */ -public class GetObjectCallable extends +public class ParseObjectFromHeadersAndHttpContent extends HttpFutureCommand.ResponseCallable { - private final HeadMetaDataCallable metaDataParser; - private String key; + private final ParseMetadataFromHeaders metaDataParser; @Inject - public GetObjectCallable(HeadMetaDataCallable metaDataParser) { + public ParseObjectFromHeadersAndHttpContent(ParseMetadataFromHeaders metaDataParser) { this.metaDataParser = metaDataParser; } /** - * @return S3Content.NOT_FOUND, if not found. + * First, calls {@link ParseMetadataFromHeaders}. + * + * Then, sets the object size based on the Content-Length header and adds + * the content to the {@link S3Object} result. + * * @throws org.jclouds.http.HttpException */ public S3Object call() throws HttpException { + checkCode(); metaDataParser.setResponse(getResponse()); S3Object.Metadata metaData = metaDataParser.call(); - if (metaData == S3Object.Metadata.NOT_FOUND) - return S3Object.NOT_FOUND; - if (getResponse().getContent() != null) { - return new S3Object(metaData, getResponse().getContent()); - } else { - String reason = null; - try { - reason = Utils.toStringAndClose(getResponse().getContent()); - } catch (IOException e) { - logger.error(e, "error parsing reason"); - } - throw new HttpException("No content retrieving object " + key + ":" - + getResponse() + " reason: " + reason); - } + + parseContentLengthOrThrowException(metaData); + return new S3Object(metaData, getResponse().getContent()); + } + + private void parseContentLengthOrThrowException(Metadata metaData) + throws HttpException { + String contentLength = getResponse().getFirstHeaderOrNull( + HttpHeaders.CONTENT_LENGTH); + if (contentLength == null) + throw new HttpException(HttpHeaders.CONTENT_LENGTH + + " header not present in headers: " + + getResponse().getHeaders()); + metaData.setSize(Long.parseLong(contentLength)); } public void setKey(String key) { diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/PutObjectCallable.java b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/PutObjectCallable.java deleted file mode 100644 index 3acaea2c79..0000000000 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/PutObjectCallable.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * - * Copyright (C) 2009 Adrian Cole - * - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - */ -package org.jclouds.aws.s3.commands.callables; - -import java.io.IOException; - -import org.jclouds.aws.s3.S3Utils; -import org.jclouds.http.HttpException; -import org.jclouds.http.HttpFutureCommand; - -/** - * // TODO: Adrian: Document this! - * - * @author Adrian Cole - */ -public class PutObjectCallable extends - HttpFutureCommand.ResponseCallable { - - public byte [] call() throws HttpException { - if (getResponse().getStatusCode() == 200) { - try { - getResponse().getContent().close(); - } catch (IOException e) { - logger.error(e, "error consuming content"); - } - String eTag = getResponse().getFirstHeaderOrNull("ETag"); - if (eTag != null) { - return S3Utils - .fromHexString(eTag.replaceAll("\"", "")); - } - throw new HttpException("did not receive ETag"); - } else { - try { - String reason = S3Utils.toStringAndClose(getResponse() - .getContent()); - throw new HttpException(getResponse().getStatusCode() - + ": Problem uploading content.\n" + reason); - } catch (IOException e) { - throw new HttpException(getResponse().getStatusCode() - + ": Problem uploading content", e); - } - } - } -} \ No newline at end of file diff --git a/s3/src/main/java/org/jclouds/aws/s3/config/S3ContextModule.java b/s3/src/main/java/org/jclouds/aws/s3/config/S3ContextModule.java index c5e097f2d7..ebffe43428 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/config/S3ContextModule.java +++ b/s3/src/main/java/org/jclouds/aws/s3/config/S3ContextModule.java @@ -26,22 +26,33 @@ package org.jclouds.aws.s3.config; import java.util.ArrayList; import java.util.List; +import javax.annotation.Resource; + import org.jclouds.aws.s3.S3Connection; import org.jclouds.aws.s3.S3Context; import org.jclouds.aws.s3.commands.config.S3CommandsModule; +import org.jclouds.aws.s3.filters.ParseS3ErrorFromXmlContent; import org.jclouds.aws.s3.filters.RemoveTransferEncodingHeader; import org.jclouds.aws.s3.filters.RequestAuthorizeSignature; import org.jclouds.aws.s3.internal.GuiceS3Context; import org.jclouds.aws.s3.internal.LiveS3Connection; import org.jclouds.aws.s3.internal.LiveS3InputStreamMap; import org.jclouds.aws.s3.internal.LiveS3ObjectMap; +import org.jclouds.http.HttpConstants; import org.jclouds.http.HttpRequestFilter; +import org.jclouds.http.HttpResponseHandler; +import org.jclouds.http.annotation.ClientErrorHandler; +import org.jclouds.http.annotation.RedirectHandler; +import org.jclouds.http.annotation.ServerErrorHandler; +import org.jclouds.logging.Logger; import com.google.inject.AbstractModule; +import com.google.inject.Inject; import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.Singleton; import com.google.inject.assistedinject.FactoryProvider; +import com.google.inject.name.Named; /** * // TODO: Adrian: Document this! @@ -49,6 +60,18 @@ import com.google.inject.assistedinject.FactoryProvider; * @author Adrian Cole */ public class S3ContextModule extends AbstractModule { + @Resource + protected Logger logger = Logger.NULL; + + @Inject + @Named(HttpConstants.PROPERTY_HTTP_ADDRESS) + String address; + @Inject + @Named(HttpConstants.PROPERTY_HTTP_PORT) + int port; + @Inject + @Named(HttpConstants.PROPERTY_HTTP_SECURE) + boolean isSecure; @Override protected void configure() { @@ -64,6 +87,15 @@ public class S3ContextModule extends AbstractModule { GuiceS3Context.S3InputStreamMapFactory.class, LiveS3InputStreamMap.class)); bind(S3Context.class).to(GuiceS3Context.class); + bind(HttpResponseHandler.class).annotatedWith(RedirectHandler.class) + .to(ParseS3ErrorFromXmlContent.class).in(Scopes.SINGLETON); + bind(HttpResponseHandler.class).annotatedWith(ClientErrorHandler.class) + .to(ParseS3ErrorFromXmlContent.class).in(Scopes.SINGLETON); + bind(HttpResponseHandler.class).annotatedWith(ServerErrorHandler.class) + .to(ParseS3ErrorFromXmlContent.class).in(Scopes.SINGLETON); + requestInjection(this); + logger.info("S3 Context = %1s://%2s:%3s", + (isSecure ? "https" : "http"), address, port); } @Provides @@ -76,4 +108,5 @@ public class S3ContextModule extends AbstractModule { filters.add(requestAuthorizeSignature); return filters; } + } diff --git a/s3/src/main/java/org/jclouds/aws/s3/config/S3JavaUrlHttpFutureCommandClientModule.java b/s3/src/main/java/org/jclouds/aws/s3/config/S3JavaUrlHttpFutureCommandClientModule.java deleted file mode 100644 index 05b54c3b05..0000000000 --- a/s3/src/main/java/org/jclouds/aws/s3/config/S3JavaUrlHttpFutureCommandClientModule.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * - * Copyright (C) 2009 Adrian Cole - * - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - */ -package org.jclouds.aws.s3.config; - -import org.jclouds.aws.s3.internal.S3JavaUrlHttpFutureCommandClient; -import org.jclouds.http.HttpFutureCommandClient; -import org.jclouds.http.config.HttpFutureCommandClientModule; -import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule; - -/** - * Configures {@link S3JavaUrlHttpFutureCommandClient}. - * - * @author Adrian Cole - */ -@HttpFutureCommandClientModule -public class S3JavaUrlHttpFutureCommandClientModule extends - JavaUrlHttpFutureCommandClientModule { - - @Override - protected void bindClient() { - // note this is not threadsafe, so it cannot be singleton - bind(HttpFutureCommandClient.class).to( - S3JavaUrlHttpFutureCommandClient.class); - } -} diff --git a/s3/src/main/java/org/jclouds/aws/s3/domain/S3Object.java b/s3/src/main/java/org/jclouds/aws/s3/domain/S3Object.java index 52988df01d..1e3c2e238e 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/domain/S3Object.java +++ b/s3/src/main/java/org/jclouds/aws/s3/domain/S3Object.java @@ -27,14 +27,12 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; -import org.apache.commons.io.IOUtils; import org.jclouds.aws.s3.S3Utils; +import org.jclouds.aws.s3.S3Utils.Md5InputStreamResult; import org.jclouds.http.ContentTypes; import org.joda.time.DateTime; @@ -62,12 +60,12 @@ public class S3Object { public S3Object(Metadata metaData, Object data) { this(metaData); - this.data = data; + setData(data); } public S3Object(String key, Object data) { this(key); - this.data = data; + setData(data); } public static class Metadata { @@ -76,7 +74,7 @@ public class S3Object { // parsed during list, head, or get private final String key; private byte[] md5; - private long size = -1; + private volatile long size = -1; // only parsed during head or get private Multimap userMetadata = HashMultimap.create(); @@ -238,44 +236,22 @@ public class S3Object { } public void setData(Object data) { - this.data = data; + this.data = checkNotNull(data, "data"); + if (getMetaData().getSize() == -1) + this.getMetaData().setSize(S3Utils.calculateSize(data)); } - /** - * converts String or InputStreams to byte [] type before generating an md5; - * - * @throws IOException - */ public void generateMd5() throws IOException { - checkState(data != null, - "data must be set before calling generateMd5()"); - byte[] md5; - if (data == null || data instanceof byte[]) { - md5 = S3Utils.md5((byte[]) data); - } else if (data instanceof String) { - this.data = ((String) data).getBytes(); - md5 = S3Utils.md5((byte[]) data); - } else if (data instanceof File) { - InputStream i = new FileInputStream((File) data); - try { - md5 = S3Utils.md5(i); - } finally { - IOUtils.closeQuietly(i); - } - } else if (data instanceof InputStream) { - InputStream i = (InputStream) data; - try { - this.data = (IOUtils.toByteArray(i)); - md5 = S3Utils.md5(i); - } finally { - IOUtils.closeQuietly(i); - - } + checkState(data != null, "data"); + if (data instanceof InputStream) { + Md5InputStreamResult result = S3Utils + .generateMd5Result((InputStream) data); + getMetaData().setSize(result.length); + getMetaData().setMd5(result.md5); + setData(result.data); } else { - throw new UnsupportedOperationException("Content not supported " - + data.getClass()); + getMetaData().setMd5(S3Utils.md5(data)); } - getMetaData().setMd5(md5); } public Object getData() { diff --git a/s3/src/main/java/org/jclouds/aws/s3/internal/S3JavaUrlHttpFutureCommandClient.java b/s3/src/main/java/org/jclouds/aws/s3/filters/ParseS3ErrorFromXmlContent.java similarity index 65% rename from s3/src/main/java/org/jclouds/aws/s3/internal/S3JavaUrlHttpFutureCommandClient.java rename to s3/src/main/java/org/jclouds/aws/s3/filters/ParseS3ErrorFromXmlContent.java index 987f651070..3634defed6 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/internal/S3JavaUrlHttpFutureCommandClient.java +++ b/s3/src/main/java/org/jclouds/aws/s3/filters/ParseS3ErrorFromXmlContent.java @@ -21,40 +21,42 @@ * under the License. * ==================================================================== */ -package org.jclouds.aws.s3.internal; +package org.jclouds.aws.s3.filters; -import java.io.IOException; import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; + +import javax.annotation.Resource; import org.apache.commons.io.IOUtils; import org.jclouds.aws.s3.S3ResponseException; import org.jclouds.aws.s3.domain.S3Error; import org.jclouds.aws.s3.xml.S3ParserFactory; import org.jclouds.http.HttpException; +import org.jclouds.http.HttpFutureCommand; import org.jclouds.http.HttpResponse; -import org.jclouds.http.JavaUrlHttpFutureCommandClient; +import org.jclouds.http.HttpResponseHandler; +import org.jclouds.logging.Logger; import com.google.inject.Inject; -public class S3JavaUrlHttpFutureCommandClient extends - JavaUrlHttpFutureCommandClient { +/** + * This will parse and set an appropriate exception on the command object. + * + * @author Adrian Cole + * + */ +public class ParseS3ErrorFromXmlContent implements HttpResponseHandler { + @Resource + protected Logger logger = Logger.NULL; - private S3ParserFactory parserFactory; + private final S3ParserFactory parserFactory; @Inject - public S3JavaUrlHttpFutureCommandClient(S3ParserFactory parserFactory, - URL target) throws MalformedURLException { - super(target); + public ParseS3ErrorFromXmlContent(S3ParserFactory parserFactory) { this.parserFactory = parserFactory; } - @Override - protected HttpResponse getResponse(HttpURLConnection connection) - throws IOException { - HttpResponse response = super.getResponse(connection); + public void handle(HttpFutureCommand command, HttpResponse response) { int code = response.getStatusCode(); if (code >= 300) { InputStream errorStream = response.getContent(); @@ -62,16 +64,20 @@ public class S3JavaUrlHttpFutureCommandClient extends try { S3Error error = parserFactory.createErrorParser().parse( errorStream); - logger.error("received the following error from s3: %1s", + logger.trace("received the following error from s3: %1s", error); - throw new S3ResponseException(error, response); + command.setException(new S3ResponseException(command, + response, error)); } catch (HttpException he) { - logger.error(he, "error parsing response"); + logger.error(he, "error parsing response %1s", response); } finally { IOUtils.closeQuietly(errorStream); } + } else { + command + .setException(new S3ResponseException(command, response)); } } - return response; } -} + +} \ No newline at end of file diff --git a/s3/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java b/s3/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java index 2ab4eca3a6..1111053034 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java +++ b/s3/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java @@ -24,8 +24,6 @@ package org.jclouds.aws.s3.filters; import java.util.Collection; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -36,20 +34,17 @@ import org.jclouds.http.HttpException; import org.jclouds.http.HttpHeaders; import org.jclouds.http.HttpRequest; import org.jclouds.http.HttpRequestFilter; -import org.joda.time.DateTime; import com.google.inject.Inject; import com.google.inject.name.Named; public class RequestAuthorizeSignature implements HttpRequestFilter { private static final String[] firstHeadersToSign = new String[] { - HttpHeaders.CONTENT_MD5, HttpHeaders.CONTENT_TYPE, - HttpHeaders.DATE }; + HttpHeaders.CONTENT_MD5, HttpHeaders.CONTENT_TYPE, HttpHeaders.DATE }; private final String accessKey; private final String secretKey; private final DateService dateService; - private Map dateCache = new ConcurrentHashMap(); public static final long BILLION = 1000000000; private final AtomicReference timeStamp; @@ -80,24 +75,6 @@ public class RequestAuthorizeSignature implements HttpRequestFilter { return timeStamp.get(); } - public DateTime dateTimeFromXMLFormat(String toParse) { - DateTime time = dateCache.get(toParse); - if (time == null) { - time = dateService.dateTimeFromXMLFormat(toParse); - dateCache.put(toParse, time); - } - return time; - } - - public DateTime dateTimeFromHeaderFormat(String toParse) { - DateTime time = dateCache.get(toParse); - if (time == null) { - time = dateService.dateTimeFromHeaderFormat(toParse); - dateCache.put(toParse, time); - } - return time; - } - @Inject public RequestAuthorizeSignature( @Named(S3Constants.PROPERTY_AWS_ACCESSKEYID) String accessKey, @@ -149,8 +126,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter { } private void addDateHeader(HttpRequest request) { - request.getHeaders().put(HttpHeaders.DATE, - dateService.timestampAsHeaderString()); + request.getHeaders().put(HttpHeaders.DATE, timestampAsHeaderString()); } private void appendAmzHeaders(HttpRequest request, StringBuilder toSign) { diff --git a/s3/src/main/java/org/jclouds/aws/s3/internal/BaseS3Map.java b/s3/src/main/java/org/jclouds/aws/s3/internal/BaseS3Map.java index 224c0fe26b..54a3ab18b0 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/internal/BaseS3Map.java +++ b/s3/src/main/java/org/jclouds/aws/s3/internal/BaseS3Map.java @@ -25,11 +25,8 @@ package org.jclouds.aws.s3.internal; import static com.google.common.base.Preconditions.checkNotNull; -import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -45,7 +42,6 @@ import org.jclouds.Utils; import org.jclouds.aws.s3.S3Connection; import org.jclouds.aws.s3.S3Constants; import org.jclouds.aws.s3.S3Map; -import org.jclouds.aws.s3.S3Utils; import org.jclouds.aws.s3.domain.S3Bucket; import org.jclouds.aws.s3.domain.S3Object; @@ -62,7 +58,7 @@ public abstract class BaseS3Map implements Map, S3Map { * maximum duration of an S3 Request */ @Inject(optional = true) - @Named(S3Constants.PROPERTY_AWS_MAP_TIMEOUT) + @Named(S3Constants.PROPERTY_S3_MAP_TIMEOUT) protected long requestTimeoutMilliseconds = 10000; @Inject @@ -71,6 +67,13 @@ public abstract class BaseS3Map implements Map, S3Map { this.bucket = checkNotNull(bucket, "bucket"); } + /** + * {@inheritDoc} + *

+ * This returns the number of keys in the {@link S3Bucket} + * + * @see S3Bucket#getContents() + */ public int size() { try { S3Bucket bucket = refreshBucket(); @@ -95,28 +98,15 @@ public abstract class BaseS3Map implements Map, S3Map { protected byte[] getMd5(Object value) throws IOException, FileNotFoundException, InterruptedException, ExecutionException, TimeoutException { - byte[] md5; - - if (value instanceof InputStream) { - md5 = S3Utils.md5((InputStream) value); - } else if (value instanceof byte[]) { - md5 = S3Utils.md5((byte[]) value); - } else if (value instanceof String) { - md5 = S3Utils.md5(((String) value).getBytes()); - } else if (value instanceof File) { - md5 = S3Utils.md5(new FileInputStream((File) value)); - } else if (value instanceof S3Object) { - S3Object object = (S3Object) value; - object = connection.getObject(bucket, object.getKey()).get( - requestTimeoutMilliseconds, TimeUnit.MILLISECONDS); - if (S3Object.NOT_FOUND.equals(object)) - throw new FileNotFoundException("not found: " + object.getKey()); - md5 = object.getMetaData().getMd5(); + S3Object object = null; + if (value instanceof S3Object) { + object = (S3Object) value; } else { - throw new IllegalArgumentException("unsupported value type: " - + value.getClass()); + object = new S3Object("dummy", value); } - return md5; + if (object.getMetaData().getMd5() == null) + object.generateMd5(); + return object.getMetaData().getMd5(); } protected Set getAllObjects() { @@ -141,13 +131,14 @@ public abstract class BaseS3Map implements Map, S3Map { return objects; } - /* - * (non-Javadoc) + /** + * {@inheritDoc} * - * @see org.jclouds.aws.s3.S3ObjectMapi#containsValue(java.lang.Object) + * Note that if value is an instance of InputStream, it will be read and + * closed following this method. To reuse data from InputStreams, pass + * {@link InputStream}s inside {@link S3Object}s */ public boolean containsValue(Object value) { - try { byte[] md5 = getMd5(value); return containsMd5(md5); diff --git a/s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3InputStreamMap.java b/s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3InputStreamMap.java index f2733f77df..b904a25925 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3InputStreamMap.java +++ b/s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3InputStreamMap.java @@ -23,9 +23,7 @@ */ package org.jclouds.aws.s3.internal; -import java.io.ByteArrayInputStream; import java.io.File; -import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; @@ -37,7 +35,6 @@ import java.util.Set; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.apache.commons.io.IOUtils; import org.jclouds.Utils; import org.jclouds.aws.s3.S3Connection; import org.jclouds.aws.s3.S3InputStreamMap; @@ -151,7 +148,7 @@ public class LiveS3InputStreamMap extends BaseS3Map implements try { InputStream returnVal = containsKey(s) ? get(s) : null; object.setData(o); - setSizeIfContentIsInputStream(object); + object.generateMd5(); connection.putObject(bucket, object).get( requestTimeoutMilliseconds, TimeUnit.MILLISECONDS); return returnVal; @@ -189,7 +186,7 @@ public class LiveS3InputStreamMap extends BaseS3Map implements for (String key : map.keySet()) { S3Object object = new S3Object(key); object.setData(map.get(key)); - setSizeIfContentIsInputStream(object); + object.generateMd5(); puts.add(connection.putObject(bucket, object)); } for (Future put : puts) @@ -202,15 +199,6 @@ public class LiveS3InputStreamMap extends BaseS3Map implements } } - private void setSizeIfContentIsInputStream(S3Object object) - throws IOException { - if (object.getData() instanceof InputStream) { - byte[] buffer = IOUtils.toByteArray((InputStream) object.getData()); - object.getMetaData().setSize(buffer.length); - object.setData(new ByteArrayInputStream(buffer)); - } - } - /* * (non-Javadoc) * diff --git a/s3/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java b/s3/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java index b6f1e1fc36..23d993c218 100644 --- a/s3/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java +++ b/s3/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java @@ -44,10 +44,10 @@ import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; -import org.jclouds.aws.s3.config.S3JavaUrlHttpFutureCommandClientModule; import org.jclouds.aws.s3.domain.S3Bucket; import org.jclouds.aws.s3.domain.S3Object; import org.jclouds.http.HttpConstants; +import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule; import org.testng.annotations.AfterTest; import org.testng.annotations.BeforeTest; import org.testng.annotations.Optional; @@ -90,6 +90,7 @@ public class S3IntegrationTest { .getContents().size(), 1); S3Object newObject = client.getObject(sourceBucket, key).get(10, TimeUnit.SECONDS); + assert newObject != S3Object.NOT_FOUND; assertEquals(S3Utils.getContentAsStringAndClose(newObject), TEST_STRING); return newObject; } @@ -147,7 +148,7 @@ public class S3IntegrationTest { } protected boolean debugEnabled() { - return true; + return false; } protected S3Context createS3Context(String AWSAccessKeyId, @@ -171,7 +172,7 @@ public class S3IntegrationTest { } protected Module createHttpModule() { - return new S3JavaUrlHttpFutureCommandClientModule(); + return new JavaUrlHttpFutureCommandClientModule(); } protected void deleteEverything() throws Exception { diff --git a/s3/src/test/java/org/jclouds/aws/s3/S3ObjectMapTest.java b/s3/src/test/java/org/jclouds/aws/s3/S3ObjectMapTest.java index 39fe83f1b2..84608d29d6 100644 --- a/s3/src/test/java/org/jclouds/aws/s3/S3ObjectMapTest.java +++ b/s3/src/test/java/org/jclouds/aws/s3/S3ObjectMapTest.java @@ -126,11 +126,11 @@ public class S3ObjectMapTest extends BaseS3MapTest { public void testPut() throws IOException { S3Object object = new S3Object("one"); object.setData(IOUtils.toInputStream("apple")); - object.getMetaData().setSize("apple".getBytes().length); + object.generateMd5(); S3Object old = map.put(object.getKey(), object); getOneReturnsAppleAndOldValueIsNull(old); object.setData(IOUtils.toInputStream("bear")); - object.getMetaData().setSize("bear".getBytes().length); + object.generateMd5(); S3Object apple = map.put(object.getKey(), object); getOneReturnsBearAndOldValueIsApple(apple); } diff --git a/s3/src/test/java/org/jclouds/aws/s3/commands/PutObjectIntegrationTest.java b/s3/src/test/java/org/jclouds/aws/s3/commands/PutObjectIntegrationTest.java index 45b83b9500..2a6f23e529 100644 --- a/s3/src/test/java/org/jclouds/aws/s3/commands/PutObjectIntegrationTest.java +++ b/s3/src/test/java/org/jclouds/aws/s3/commands/PutObjectIntegrationTest.java @@ -30,6 +30,7 @@ import static org.testng.Assert.assertNotNull; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.net.URL; import java.util.concurrent.TimeUnit; @@ -77,6 +78,9 @@ public class PutObjectIntegrationTest extends S3IntegrationTest { S3Object object = new S3Object(key); object.getMetaData().setContentType(type); object.setData(content); + if (content instanceof InputStream) { + object.generateMd5(); + } assertNotNull(client.putObject(bucketName, object).get(10, TimeUnit.SECONDS)); object = client.getObject(bucketName, object.getKey()).get(10, @@ -105,19 +109,19 @@ public class PutObjectIntegrationTest extends S3IntegrationTest { object.getMetaData().setMd5(S3Utils.md5(TEST_STRING.getBytes())); addObjectToBucket(bucketName, object); - object = validateContent(bucketName, key); + S3Object newObject = validateContent(bucketName, key); // TODO.. why does this come back as binary/octetstring - assertEquals(object.getMetaData().getContentType(), + assertEquals(newObject.getMetaData().getContentType(), "binary/octet-stream"); - assertEquals(object.getMetaData().getContentEncoding(), "x-compress"); - assertEquals(object.getMetaData().getContentDisposition(), + assertEquals(newObject.getMetaData().getContentEncoding(), "x-compress"); + assertEquals(newObject.getMetaData().getContentDisposition(), "attachment; filename=hello.txt"); - assertEquals(object.getMetaData().getCacheControl(), "no-cache"); - assertEquals(object.getMetaData().getSize(), TEST_STRING.length()); - assertEquals(object.getMetaData().getUserMetadata().values().iterator() - .next(), "powderpuff"); - assertEquals(object.getMetaData().getMd5(), S3Utils.md5(TEST_STRING + assertEquals(newObject.getMetaData().getCacheControl(), "no-cache"); + assertEquals(newObject.getMetaData().getSize(), TEST_STRING.length()); + assertEquals(newObject.getMetaData().getUserMetadata().values() + .iterator().next(), "powderpuff"); + assertEquals(newObject.getMetaData().getMd5(), S3Utils.md5(TEST_STRING .getBytes())); } diff --git a/s3/src/test/java/org/jclouds/aws/s3/commands/S3CommandFactoryTest.java b/s3/src/test/java/org/jclouds/aws/s3/commands/S3CommandFactoryTest.java index 4de653f1aa..54a4a461b5 100644 --- a/s3/src/test/java/org/jclouds/aws/s3/commands/S3CommandFactoryTest.java +++ b/s3/src/test/java/org/jclouds/aws/s3/commands/S3CommandFactoryTest.java @@ -118,19 +118,21 @@ public class S3CommandFactoryTest { void testCreatePutObject() { S3Object.Metadata metaData = createMock(S3Object.Metadata.class); S3Object object = new S3Object(metaData); - object.setData(""); + expect(metaData.getSize()).andReturn(4L).atLeastOnce(); expect(metaData.getKey()).andReturn("rawr"); expect(metaData.getContentType()).andReturn("text/xml").atLeastOnce(); - expect(metaData.getSize()).andReturn(4L); expect(metaData.getCacheControl()).andReturn("no-cache").atLeastOnce(); - expect(metaData.getContentDisposition()).andReturn("disposition").atLeastOnce(); - expect(metaData.getContentEncoding()).andReturn("encoding").atLeastOnce(); - expect(metaData.getMd5()).andReturn("encoding".getBytes()).atLeastOnce(); - Multimap userMdata = HashMultimap.create(); + expect(metaData.getContentDisposition()).andReturn("disposition") + .atLeastOnce(); + expect(metaData.getContentEncoding()).andReturn("encoding") + .atLeastOnce(); + expect(metaData.getMd5()).andReturn("encoding".getBytes()) + .atLeastOnce(); + Multimap userMdata = HashMultimap.create(); expect(metaData.getUserMetadata()).andReturn(userMdata).atLeastOnce(); - replay(metaData); + object.setData(""); assert commandFactory.createPutObject("test", object, PutObjectOptions.NONE) != null; diff --git a/s3/src/test/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallableTest.java b/s3/src/test/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallableTest.java index 0783fe58b1..e95b3c56a8 100644 --- a/s3/src/test/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallableTest.java +++ b/s3/src/test/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallableTest.java @@ -23,85 +23,70 @@ */ package org.jclouds.aws.s3.commands.callables; -import static org.easymock.EasyMock.expect; -import static org.easymock.classextension.EasyMock.createMock; -import static org.easymock.classextension.EasyMock.replay; -import static org.easymock.classextension.EasyMock.verify; -import static org.testng.Assert.assertEquals; - -import java.io.IOException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; - -import org.apache.commons.io.IOUtils; -import org.jclouds.http.HttpException; -import org.jclouds.http.HttpFutureCommand; -import org.jclouds.http.HttpResponse; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @Test public class DeleteBucketCallableTest { - private HttpFutureCommand.ResponseCallable callable = null; - - @BeforeMethod - void setUp() { - callable = new DeleteBucketCallable(); - } - - @AfterMethod - void tearDown() { - callable = null; - } - - @Test(expectedExceptions = HttpException.class) - public void testExceptionWhenNoContentOn409() throws Exception { - HttpResponse response = createMock(HttpResponse.class); - expect(response.getStatusCode()).andReturn(409).atLeastOnce(); - expect(response.getContent()).andReturn(null); - replay(response); - callable.setResponse(response); - callable.call(); - } - - @Test - public void testExceptionWhenIOExceptionOn409() throws ExecutionException, - InterruptedException, TimeoutException, IOException { - HttpResponse response = createMock(HttpResponse.class); - expect(response.getStatusCode()).andReturn(409).atLeastOnce(); - RuntimeException exception = new RuntimeException("bad"); - expect(response.getContent()).andThrow(exception); - replay(response); - callable.setResponse(response); - try { - callable.call(); - } catch (Exception e) { - assert e.equals(exception); - } - verify(response); - } - - @Test - public void testFalseWhenBucketNotEmptyOn409() throws Exception { - HttpResponse response = createMock(HttpResponse.class); - expect(response.getStatusCode()).andReturn(409).atLeastOnce(); - expect(response.getContent()).andReturn( - IOUtils.toInputStream("BucketNotEmpty")).atLeastOnce(); - replay(response); - callable.setResponse(response); - assert !callable.call().booleanValue(); - verify(response); - } - - @Test - public void testResponseOk() throws Exception { - HttpResponse response = createMock(HttpResponse.class); - expect(response.getStatusCode()).andReturn(204).atLeastOnce(); - replay(response); - callable.setResponse(response); - assertEquals(callable.call(), new Boolean(true)); - verify(response); - } + // private HttpFutureCommand.ResponseCallable callable = null; + // + // @BeforeMethod + // void setUp() { + // callable = new DeleteBucketCallable(); + // } + // + // @AfterMethod + // void tearDown() { + // callable = null; + // } + // + // @Test(expectedExceptions = HttpException.class) + // public void testExceptionWhenNoContentOn409() throws Exception { + // HttpResponse response = createMock(HttpResponse.class); + // expect(response.getStatusCode()).andReturn(409).atLeastOnce(); + // expect(response.getContent()).andReturn(null); + // replay(response); + // callable.setResponse(response); + // callable.call(); + // } + // + // @Test + // public void testExceptionWhenIOExceptionOn409() throws + // ExecutionException, + // InterruptedException, TimeoutException, IOException { + // HttpResponse response = createMock(HttpResponse.class); + // expect(response.getStatusCode()).andReturn(409).atLeastOnce(); + // RuntimeException exception = new RuntimeException("bad"); + // expect(response.getContent()).andThrow(exception); + // replay(response); + // callable.setResponse(response); + // try { + // callable.call(); + // } catch (Exception e) { + // assert e.equals(exception); + // } + // verify(response); + // } + // + // @Test + // public void testFalseWhenBucketNotEmptyOn409() throws Exception { + // HttpResponse response = createMock(HttpResponse.class); + // expect(response.getStatusCode()).andReturn(409).atLeastOnce(); + // expect(response.getContent()).andReturn( + // IOUtils.toInputStream("BucketNotEmpty")).atLeastOnce(); + // replay(response); + // callable.setResponse(response); + // assert !callable.call().booleanValue(); + // verify(response); + // } + // + // @Test + // public void testResponseOk() throws Exception { + // HttpResponse response = createMock(HttpResponse.class); + // expect(response.getStatusCode()).andReturn(204).atLeastOnce(); + // replay(response); + // callable.setResponse(response); + // assertEquals(callable.call(), new Boolean(true)); + // verify(response); + // } } \ No newline at end of file diff --git a/samples/googleappengine/src/it/java/org/jclouds/samples/googleappengine/functest/GoogleAppEngineTest.java b/samples/googleappengine/src/it/java/org/jclouds/samples/googleappengine/functest/GoogleAppEngineTest.java index d613bddee1..00cbd0a577 100644 --- a/samples/googleappengine/src/it/java/org/jclouds/samples/googleappengine/functest/GoogleAppEngineTest.java +++ b/samples/googleappengine/src/it/java/org/jclouds/samples/googleappengine/functest/GoogleAppEngineTest.java @@ -80,8 +80,17 @@ public class GoogleAppEngineTest extends BaseGoogleAppEngineTest { assert string.indexOf("Hello World!") >= 0 : string; } - @Test(invocationCount = 50, enabled = true, threadPoolSize = 10) - public void testGuiceJCloudsServed() throws InterruptedException, + @Test(invocationCount = 5, enabled = true) + public void testGuiceJCloudsSerial() throws InterruptedException, + IOException { + URL gurl = new URL(url, "/guice/listbuckets.s3"); + InputStream i = gurl.openStream(); + String string = IOUtils.toString(i); + assert string.indexOf("List") >= 0 : string; + } + + @Test(invocationCount = 50, enabled = false, threadPoolSize = 10) + public void testGuiceJCloudsParallel() throws InterruptedException, IOException { URL gurl = new URL(url, "/guice/listbuckets.s3"); InputStream i = gurl.openStream(); diff --git a/samples/googleappengine/src/main/java/org/jclouds/samples/googleappengine/config/GuiceServletConfig.java b/samples/googleappengine/src/main/java/org/jclouds/samples/googleappengine/config/GuiceServletConfig.java index d44a6c0599..b3f0afdf85 100644 --- a/samples/googleappengine/src/main/java/org/jclouds/samples/googleappengine/config/GuiceServletConfig.java +++ b/samples/googleappengine/src/main/java/org/jclouds/samples/googleappengine/config/GuiceServletConfig.java @@ -75,7 +75,6 @@ public class GuiceServletConfig extends GuiceServletContextListener { } finally { IOUtils.closeQuietly(input); } - props.setProperty("jclouds.http.sax.debug", "true"); return props; }