From 25395596405b1666aa9d8634eb7d64efb45ef23e Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Thu, 24 Feb 2011 11:17:00 -0800 Subject: [PATCH 1/3] carved out a place to add aws-s3 specific functionality --- core/src/main/resources/rest.properties | 2 +- .../org/jclouds/aws/s3/AWSS3AsyncClient.java | 57 ++++++++++++++++++ .../java/org/jclouds/aws/s3/AWSS3Client.java | 36 +++++++++++ .../jclouds/aws/s3/AWSS3ContextBuilder.java | 45 ++++++++++++++ .../aws/s3/config/AWSS3RestClientModule.java | 59 +++++++++++++++++++ 5 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java create mode 100644 providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java create mode 100644 providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ContextBuilder.java create mode 100644 providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java diff --git a/core/src/main/resources/rest.properties b/core/src/main/resources/rest.properties index eb210a198e..d4d3cbdc7a 100644 --- a/core/src/main/resources/rest.properties +++ b/core/src/main/resources/rest.properties @@ -44,7 +44,7 @@ aws-elb.propertiesbuilder=org.jclouds.aws.elb.AWSELBPropertiesBuilder cloudwatch.contextbuilder=org.jclouds.cloudwatch.CloudWatchContextBuilder cloudwatch.propertiesbuilder=org.jclouds.cloudwatch.CloudWatchPropertiesBuilder -aws-s3.contextbuilder=org.jclouds.s3.S3ContextBuilder +aws-s3.contextbuilder=org.jclouds.aws.s3.AWSS3ContextBuilder aws-s3.propertiesbuilder=org.jclouds.aws.s3.AWSS3PropertiesBuilder aws-ec2.contextbuilder=org.jclouds.aws.ec2.AWSEC2ContextBuilder diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java new file mode 100644 index 0000000000..5513f295eb --- /dev/null +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java @@ -0,0 +1,57 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3; +/** + * + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER; + +import org.jclouds.blobstore.attr.BlobScope; +import org.jclouds.rest.annotations.RequestFilters; +import org.jclouds.rest.annotations.SkipEncoding; +import org.jclouds.s3.S3AsyncClient; +import org.jclouds.s3.filters.RequestAuthorizeSignature; + +/** + * Provides access to amazon-specific S3 features + * + * @author Adrian Cole + */ +@SkipEncoding('/') +@RequestFilters(RequestAuthorizeSignature.class) +@BlobScope(CONTAINER) +public interface AWSS3AsyncClient extends S3AsyncClient { + +} diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java new file mode 100644 index 0000000000..b464041786 --- /dev/null +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java @@ -0,0 +1,36 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3; + +import java.util.concurrent.TimeUnit; + +import org.jclouds.concurrent.Timeout; +import org.jclouds.s3.S3Client; + +/** + * Provides access to amazon-specific S3 features + * + * @author Adrian Cole + * @see AWSS3AsyncClient + */ +@Timeout(duration = 90, timeUnit = TimeUnit.SECONDS) +public interface AWSS3Client extends S3Client { + +} diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ContextBuilder.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ContextBuilder.java new file mode 100644 index 0000000000..2a717f8eaf --- /dev/null +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ContextBuilder.java @@ -0,0 +1,45 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3; + +import java.util.List; +import java.util.Properties; + +import org.jclouds.aws.s3.config.AWSS3RestClientModule; +import org.jclouds.s3.S3ContextBuilder; + +import com.google.inject.Module; + +/** + * + * @author Adrian Cole + */ +public class AWSS3ContextBuilder extends S3ContextBuilder { + + public AWSS3ContextBuilder(Properties props) { + super(props); + } + + @Override + protected void addClientModule(List modules) { + modules.add(new AWSS3RestClientModule()); + } + +} diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java new file mode 100644 index 0000000000..7dc6e9236d --- /dev/null +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java @@ -0,0 +1,59 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3.config; + +import javax.inject.Singleton; + +import org.jclouds.aws.s3.AWSS3AsyncClient; +import org.jclouds.aws.s3.AWSS3Client; +import org.jclouds.http.RequiresHttp; +import org.jclouds.rest.ConfiguresRestClient; +import org.jclouds.s3.S3AsyncClient; +import org.jclouds.s3.S3Client; +import org.jclouds.s3.config.S3RestClientModule; + +import com.google.inject.Provides; + +/** + * Configures the S3 connection. + * + * @author Adrian Cole + */ +@RequiresHttp +@ConfiguresRestClient +public class AWSS3RestClientModule extends S3RestClientModule { + + public AWSS3RestClientModule() { + super(AWSS3Client.class, AWSS3AsyncClient.class); + } + + @Singleton + @Provides + S3Client provide(AWSS3Client in) { + return in; + } + + @Singleton + @Provides + S3AsyncClient provide(AWSS3AsyncClient in) { + return in; + } + +} From 84004b89866ed2c4c355450f6748e8234c406a7a Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Thu, 24 Feb 2011 21:45:04 -0800 Subject: [PATCH 2/3] added debug to http executor --- .../BaseHttpCommandExecutorService.java | 2 +- .../JavaUrlHttpCommandExecutorService.java | 49 ++++++++++++------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java b/core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java index 1073b01561..aeecab7497 100644 --- a/core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java +++ b/core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java @@ -153,8 +153,8 @@ public abstract class BaseHttpCommandExecutorService implements HttpCommandEx "After filtering, the request has niether chunked encoding nor content length: " + request); logger.debug("Sending request %s: %s", request.hashCode(), request.getRequestLine()); wirePayloadIfEnabled(wire, request); - nativeRequest = convert(request); utils.logRequest(headerLog, request, ">>"); + nativeRequest = convert(request); response = invoke(nativeRequest); logger.debug("Receiving response %s: %s", request.hashCode(), response.getStatusLine()); diff --git a/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java b/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java index debfa9d54f..f04b3befae 100644 --- a/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java +++ b/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java @@ -19,6 +19,7 @@ package org.jclouds.http.internal; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Throwables.propagate; import static com.google.common.collect.Iterables.getLast; @@ -66,8 +67,9 @@ import org.jclouds.rest.internal.RestAnnotationProcessor; import com.google.common.base.Supplier; import com.google.common.base.Throwables; -import com.google.common.collect.LinkedHashMultimap; -import com.google.common.collect.Multimap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableMultimap.Builder; +import com.google.common.io.CountingOutputStream; /** * Basic implementation of a {@link HttpCommandExecutorService}. @@ -86,11 +88,11 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe @Inject public JavaUrlHttpCommandExecutorService(HttpUtils utils, - @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor, - DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler, - DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier, - @Named("untrusted") Supplier untrustedSSLContextProvider) throws SecurityException, - NoSuchFieldException { + @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor, + DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler, + DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier, + @Named("untrusted") Supplier untrustedSSLContextProvider) throws SecurityException, + NoSuchFieldException { super(utils, ioWorkerExecutor, retryHandler, ioRetryHandler, errorHandler, wire); if (utils.getMaxConnections() > 0) System.setProperty("http.maxConnections", String.valueOf(checkNotNull(utils, "utils").getMaxConnections())); @@ -102,6 +104,7 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe @Override protected HttpResponse invoke(HttpURLConnection connection) throws IOException, InterruptedException { + HttpResponse.Builder builder = HttpResponse.builder(); InputStream in = null; try { in = consumeOnClose(connection.getInputStream()); @@ -113,19 +116,28 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe assert false : "should have propagated exception"; } - if (connection.getResponseCode() == 204) { + int responseCode = connection.getResponseCode(); + if (responseCode == 204) { closeQuietly(in); in = null; } - Multimap headers = LinkedHashMultimap.create(); + builder.statusCode(responseCode); + builder.message(connection.getResponseMessage()); + + Builder headerBuilder = ImmutableMultimap. builder(); for (String header : connection.getHeaderFields().keySet()) { - headers.putAll(header, connection.getHeaderFields().get(header)); + // HTTP message comes back as a header without a key + if (header != null) + headerBuilder.putAll(header, connection.getHeaderFields().get(header)); } + ImmutableMultimap headers = headerBuilder.build(); Payload payload = in != null ? Payloads.newInputStreamPayload(in) : null; - if (payload != null) + if (payload != null) { payload.getContentMetadata().setPropertiesFromHttpHeaders(headers); - return new HttpResponse(connection.getResponseCode(), connection.getResponseMessage(), payload, - RestAnnotationProcessor.filterOutContentHeaders(headers)); + builder.payload(payload); + } + builder.headers(RestAnnotationProcessor.filterOutContentHeaders(headers)); + return builder.build(); } private InputStream bufferAndCloseStream(InputStream inputStream) throws IOException { @@ -220,14 +232,17 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe } else { Long length = checkNotNull(md.getContentLength(), "payload.getContentLength"); connection.setRequestProperty(HttpHeaders.CONTENT_LENGTH, length.toString()); + // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6755625 + checkArgument(length < Integer.MAX_VALUE, + "JDK 1.6 does not support >2GB chunks. Use chunked encoding, if possible."); connection.setFixedLengthStreamingMode(length.intValue()); } - // writeTo will close the output stream + CountingOutputStream out = new CountingOutputStream(connection.getOutputStream()); try { - request.getPayload().writeTo(connection.getOutputStream()); + request.getPayload().writeTo(out); } catch (IOException e) { - e.printStackTrace(); - throw e; + throw new RuntimeException(String.format("error after writing %d/%s bytes to %s", out.getCount(), md + .getContentLength(), request.getRequestLine()), e); } } else { connection.setRequestProperty(HttpHeaders.CONTENT_LENGTH, "0"); From 95a19b99fd0045ea7b3cf24a56fe42397f95b83d Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Thu, 24 Feb 2011 21:46:14 -0800 Subject: [PATCH 3/3] Issue 430:large blob support on aws-s3 --- .../DelegatingMutableContentMetadata.java | 6 + .../filesystem/FilesystemAsyncBlobStore.java | 89 +++++---- .../main/java/org/jclouds/s3/S3Client.java | 3 +- .../BindS3ObjectMetadataToRequest.java | 16 +- .../s3/domain/MutableObjectMetadata.java | 3 +- .../org/jclouds/s3/domain/ObjectMetadata.java | 2 +- .../s3/domain/ObjectMetadataBuilder.java | 114 +++++++++++ .../s3/filters/RequestAuthorizeSignature.java | 34 ++-- .../org/jclouds/s3/BaseS3AsyncClientTest.java | 10 +- .../org/jclouds/s3/S3AsyncClientTest.java | 4 +- .../BindAsHostPrefixIfConfiguredTest.java | 18 +- .../BindNoBucketLoggingToXmlPayloadTest.java | 16 +- .../BindS3ObjectMetadataToRequestTest.java | 28 ++- .../s3/blobstore/S3BlobRequestSignerTest.java | 28 ++- .../RequestAuthorizeSignatureTest.java | 40 ++-- .../walrus/WalrusAsyncClientTestDisabled.java | 13 +- .../handlers/ParseAWSErrorFromXmlContent.java | 5 +- .../java/org/jclouds/io/ContentMetadata.java | 4 +- .../jclouds/io/ContentMetadataBuilder.java | 186 ++++++++++++++++++ .../BaseImmutableContentMetadata.java | 6 + .../payloads/BaseMutableContentMetadata.java | 129 ++---------- .../jclouds/io/payloads/PhantomPayload.java | 5 + providers/aws-s3/pom.xml | 12 ++ .../org/jclouds/aws/s3/AWSS3AsyncClient.java | 76 ++++++- .../java/org/jclouds/aws/s3/AWSS3Client.java | 109 ++++++++++ .../binders/BindObjectMetadataToRequest.java | 83 ++++++++ .../binders/BindPartIdsAndETagsToRequest.java | 65 ++++++ .../ETagFromHttpResponseViaRegex.java | 60 ++++++ .../aws/s3/functions/ObjectMetadataKey.java | 39 ++++ .../UploadIdFromHttpResponseViaRegex.java | 62 ++++++ .../jclouds/aws/s3/AWSS3AsyncClientTest.java | 134 ++++++++++++- .../jclouds/aws/s3/AWSS3ClientLiveTest.java | 74 +++++++ .../BindObjectMetadataToRequestTest.java | 104 ++++++++++ .../BindPartIdsAndETagsToRequestTest.java | 72 +++++++ .../ETagFromHttpResponseViaRegexTest.java | 49 +++++ .../UploadIdFromHttpResponseViaRegexTest.java | 49 +++++ .../resources/complete-multipart-upload.xml | 7 + .../resources/initiate-multipart-upload.xml | 6 + providers/aws-s3/src/test/resources/log4j.xml | 124 ++++++++++++ ...dGoogleStorageObjectMetadataToRequest.java | 15 +- .../GoogleStorageAsyncClientTestDisabled.java | 12 +- ...gleStorageObjectMetadataToRequestTest.java | 30 ++- ...ScaleUpStorageAsyncClientTestDisabled.java | 13 +- .../VirtualMachineClientLiveTest.java | 23 ++- 44 files changed, 1707 insertions(+), 270 deletions(-) create mode 100644 apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadataBuilder.java create mode 100644 core/src/main/java/org/jclouds/io/ContentMetadataBuilder.java create mode 100644 providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequest.java create mode 100644 providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequest.java create mode 100644 providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegex.java create mode 100644 providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ObjectMetadataKey.java create mode 100644 providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegex.java create mode 100644 providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequestTest.java create mode 100644 providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequestTest.java create mode 100644 providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegexTest.java create mode 100644 providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegexTest.java create mode 100644 providers/aws-s3/src/test/resources/complete-multipart-upload.xml create mode 100644 providers/aws-s3/src/test/resources/initiate-multipart-upload.xml create mode 100644 providers/aws-s3/src/test/resources/log4j.xml diff --git a/apis/atmos/src/main/java/org/jclouds/atmos/domain/internal/DelegatingMutableContentMetadata.java b/apis/atmos/src/main/java/org/jclouds/atmos/domain/internal/DelegatingMutableContentMetadata.java index de0110bd0e..96afe78dd1 100644 --- a/apis/atmos/src/main/java/org/jclouds/atmos/domain/internal/DelegatingMutableContentMetadata.java +++ b/apis/atmos/src/main/java/org/jclouds/atmos/domain/internal/DelegatingMutableContentMetadata.java @@ -20,6 +20,7 @@ package org.jclouds.atmos.domain.internal; import org.jclouds.atmos.domain.MutableContentMetadata; +import org.jclouds.io.ContentMetadataBuilder; import org.jclouds.io.payloads.BaseMutableContentMetadata; import com.google.common.collect.Multimap; @@ -157,4 +158,9 @@ public class DelegatingMutableContentMetadata implements MutableContentMetadata delegate.setPropertiesFromHttpHeaders(headers); } + @Override + public ContentMetadataBuilder toBuilder() { + return delegate.toBuilder(); + } + } diff --git a/apis/filesystem/src/main/java/org/jclouds/filesystem/FilesystemAsyncBlobStore.java b/apis/filesystem/src/main/java/org/jclouds/filesystem/FilesystemAsyncBlobStore.java index 0603b8211c..694e884d65 100644 --- a/apis/filesystem/src/main/java/org/jclouds/filesystem/FilesystemAsyncBlobStore.java +++ b/apis/filesystem/src/main/java/org/jclouds/filesystem/FilesystemAsyncBlobStore.java @@ -49,10 +49,10 @@ import java.util.Collection; import java.util.Date; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import java.util.Map.Entry; import java.util.concurrent.ExecutorService; import javax.annotation.Nullable; @@ -67,13 +67,13 @@ import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.domain.Blob; -import org.jclouds.blobstore.domain.Blob.Factory; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.MutableBlobMetadata; import org.jclouds.blobstore.domain.MutableStorageMetadata; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.domain.StorageType; +import org.jclouds.blobstore.domain.Blob.Factory; import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl; import org.jclouds.blobstore.domain.internal.MutableStorageMetadataImpl; import org.jclouds.blobstore.domain.internal.PageSetImpl; @@ -97,6 +97,7 @@ import org.jclouds.http.HttpResponseException; import org.jclouds.http.options.HttpRequestOptions; import org.jclouds.io.Payload; import org.jclouds.io.Payloads; +import org.jclouds.io.payloads.BaseMutableContentMetadata; import org.jclouds.io.payloads.FilePayload; import org.jclouds.logging.Logger; import org.jclouds.rest.annotations.ParamValidators; @@ -129,10 +130,10 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore { @Inject protected FilesystemAsyncBlobStore(BlobStoreContext context, DateService dateService, Crypto crypto, - HttpGetOptionsListToGetOptions httpGetOptionsConverter, IfDirectoryReturnNameStrategy ifDirectoryReturnName, - Factory blobFactory, BlobUtils blobUtils, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, - Supplier defaultLocation, @Memoized Supplier> locations, - FilesystemStorageStrategy storageStrategy) { + HttpGetOptionsListToGetOptions httpGetOptionsConverter, + IfDirectoryReturnNameStrategy ifDirectoryReturnName, Factory blobFactory, BlobUtils blobUtils, + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier defaultLocation, + @Memoized Supplier> locations, FilesystemStorageStrategy storageStrategy) { super(context, blobUtils, service, defaultLocation, locations); // super(context, blobUtils, service, null, null); this.blobFactory = blobFactory; @@ -164,22 +165,22 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore { } SortedSet contents = newTreeSet(transform(blobBelongingToContainer, - new Function() { - public StorageMetadata apply(String key) { - Blob oldBlob = loadFileBlob(container, key); + new Function() { + public StorageMetadata apply(String key) { + Blob oldBlob = loadFileBlob(container, key); - checkState(oldBlob != null, "blob " + key + " is not present although it was in the list of " - + container); - checkState(oldBlob.getMetadata() != null, "blob " + container + "/" + key + " has no metadata"); - MutableBlobMetadata md = copy(oldBlob.getMetadata()); - String directoryName = ifDirectoryReturnName.execute(md); - if (directoryName != null) { - md.setName(directoryName); - md.setType(StorageType.RELATIVE_PATH); + checkState(oldBlob != null, "blob " + key + " is not present although it was in the list of " + + container); + checkState(oldBlob.getMetadata() != null, "blob " + container + "/" + key + " has no metadata"); + MutableBlobMetadata md = copy(oldBlob.getMetadata()); + String directoryName = ifDirectoryReturnName.execute(md); + if (directoryName != null) { + md.setName(directoryName); + md.setType(StorageType.RELATIVE_PATH); + } + return md; } - return md; - } - })); + })); String marker = null; if (options != null) { @@ -219,21 +220,21 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore { if (delimiter != null) { SortedSet commonPrefixes = null; Iterable iterable = transform(contents, new CommonPrefixes(prefix != null ? prefix : null, - delimiter)); + delimiter)); commonPrefixes = iterable != null ? newTreeSet(iterable) : new TreeSet(); commonPrefixes.remove(CommonPrefixes.NO_PREFIX); contents = newTreeSet(filter(contents, new DelimiterFilter(prefix != null ? prefix : null, delimiter))); - Iterables. addAll(contents, - transform(commonPrefixes, new Function() { - public StorageMetadata apply(String o) { - MutableStorageMetadata md = new MutableStorageMetadataImpl(); - md.setType(StorageType.RELATIVE_PATH); - md.setName(o); - return md; - } - })); + Iterables. addAll(contents, transform(commonPrefixes, + new Function() { + public StorageMetadata apply(String o) { + MutableStorageMetadata md = new MutableStorageMetadataImpl(); + md.setType(StorageType.RELATIVE_PATH); + md.setName(o); + return md; + } + })); } // trim metadata, if the response isn't supposed to be detailed. @@ -245,7 +246,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore { } return Futures.> immediateFuture(new PageSetImpl(contents, - marker)); + marker)); } @@ -262,6 +263,8 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore { ObjectInput is = new ObjectInputStream(new ByteArrayInputStream(bout.toByteArray())); MutableBlobMetadata metadata = (MutableBlobMetadata) is.readObject(); convertUserMetadataKeysToLowercase(metadata); + metadata.setContentMetadata(BaseMutableContentMetadata.fromContentMetadata(in.getContentMetadata().toBuilder() + .build())); return metadata; } catch (Exception e) { propagate(e); @@ -310,14 +313,14 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore { Iterable containers = storageStrategy.getAllContainerNames(); return Futures.> immediateFuture(new PageSetImpl(transform( - containers, new Function() { - public StorageMetadata apply(String name) { - MutableStorageMetadata cmd = create(); - cmd.setName(name); - cmd.setType(StorageType.CONTAINER); - return cmd; - } - }), null)); + containers, new Function() { + public StorageMetadata apply(String name) { + MutableStorageMetadata cmd = create(); + cmd.setName(name); + cmd.setType(StorageType.CONTAINER); + return cmd; + } + }), null)); } protected MutableStorageMetadata create() { @@ -330,7 +333,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore { @Path("{container}") @Override public ListenableFuture createContainerInLocation(final Location location, - @PathParam("container") @ParamValidators({ FilesystemContainerNameValidator.class }) String name) { + @PathParam("container") @ParamValidators( { FilesystemContainerNameValidator.class }) String name) { boolean result = storageStrategy.createContainer(name); return immediateFuture(result); } @@ -500,7 +503,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore { storageStrategy.writePayloadOnFile(containerName, blobKey, object.getPayload()); } catch (IOException e) { logger.error(e, "An error occurred storing the new object with name [%s] to container [%s].", blobKey, - containerName); + containerName); Throwables.propagate(e); } return immediateFuture(eTag); @@ -547,7 +550,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore { if (blob.getMetadata().getLastModified().before(modifiedSince)) { HttpResponse response = new HttpResponse(304, null, null); return immediateFailedFuture(new HttpResponseException(String.format("%1$s is before %2$s", blob - .getMetadata().getLastModified(), modifiedSince), null, response)); + .getMetadata().getLastModified(), modifiedSince), null, response)); } } @@ -556,7 +559,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore { if (blob.getMetadata().getLastModified().after(unmodifiedSince)) { HttpResponse response = new HttpResponse(412, null, null); return immediateFailedFuture(new HttpResponseException(String.format("%1$s is after %2$s", blob - .getMetadata().getLastModified(), unmodifiedSince), null, response)); + .getMetadata().getLastModified(), unmodifiedSince), null, response)); } } diff --git a/apis/s3/src/main/java/org/jclouds/s3/S3Client.java b/apis/s3/src/main/java/org/jclouds/s3/S3Client.java index ea61ce78e9..f9fe89a657 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/S3Client.java +++ b/apis/s3/src/main/java/org/jclouds/s3/S3Client.java @@ -137,7 +137,6 @@ public interface S3Client { * namespace of the object you are deleting * @param key * unique key in the s3Bucket identifying the object - * @return true if deleted * @throws org.jclouds.http.HttpResponseException * if the bucket is not available * @see { public enum StorageClass { - STANDARD + STANDARD, REDUCED_REDUNDANCY } /** diff --git a/apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadataBuilder.java b/apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadataBuilder.java new file mode 100644 index 0000000000..0701ea9529 --- /dev/null +++ b/apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadataBuilder.java @@ -0,0 +1,114 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.s3.domain; + +import java.util.Map; + +import org.jclouds.io.ContentMetadataBuilder; +import org.jclouds.io.payloads.BaseMutableContentMetadata; +import org.jclouds.s3.domain.ObjectMetadata.StorageClass; +import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl; + +import com.google.common.collect.ImmutableMap; + +/** + * Allows you to create {@link ObjectMetadata} objects. + * + * @author Adrian Cole + */ +public class ObjectMetadataBuilder { + public static ObjectMetadataBuilder create() { + return new ObjectMetadataBuilder(); + } + + private final ContentMetadataBuilder contentMetadataBuilder = new ContentMetadataBuilder().contentType("binary/octet-stream"); + + private String key; + private StorageClass storageClass; + private String cacheControl; + private Map userMetadata = ImmutableMap.of(); + + + public ObjectMetadataBuilder key(String key) { + this.key = key; + return this; + } + + public ObjectMetadataBuilder storageClass(StorageClass storageClass) { + this.storageClass = storageClass; + return this; + } + + public ObjectMetadataBuilder cacheControl(String cacheControl) { + this.cacheControl = cacheControl; + return this; + } + + public ObjectMetadataBuilder userMetadata(Map userMetadata) { + this.userMetadata = ImmutableMap.copyOf(userMetadata); + return this; + } + + public ObjectMetadataBuilder contentDisposition(String contentDisposition) { + contentMetadataBuilder.contentDisposition(contentDisposition); + return this; + } + + public ObjectMetadataBuilder contentEncoding(String contentEncoding) { + contentMetadataBuilder.contentEncoding(contentEncoding); + return this; + + } + + public ObjectMetadataBuilder contentLanguage(String contentLanguage) { + contentMetadataBuilder.contentLanguage(contentLanguage); + return this; + + } + + public ObjectMetadataBuilder contentLength(Long contentLength) { + contentMetadataBuilder.contentLength(contentLength); + return this; + + } + + public ObjectMetadataBuilder contentMD5(byte[] md5) { + contentMetadataBuilder.contentMD5(md5); + return this; + + } + + public ObjectMetadataBuilder contentType(String contentType) { + contentMetadataBuilder.contentType(contentType); + return this; + + } + + public ObjectMetadata build() { + MutableObjectMetadataImpl toReturn = new MutableObjectMetadataImpl(); + toReturn.setContentMetadata(BaseMutableContentMetadata.fromContentMetadata(contentMetadataBuilder.build())); + toReturn.setCacheControl(cacheControl); + toReturn.setKey(key); + toReturn.setStorageClass(storageClass); + toReturn.setUserMetadata(userMetadata); + return toReturn; + } + +} diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java b/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java index de28941deb..7dce8a597b 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java +++ b/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java @@ -78,7 +78,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign private final String[] firstHeadersToSign = new String[] { HttpHeaders.DATE }; public static Set SPECIAL_QUERIES = ImmutableSet.of("acl", "torrent", "logging", "location", - "requestPayment"); + "requestPayment", "uploads"); private final SignatureWire signatureWire; private final String accessKey; private final String secretKey; @@ -97,10 +97,10 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign @Inject public RequestAuthorizeSignature(SignatureWire signatureWire, @Named(PROPERTY_AUTH_TAG) String authTag, - @Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle, - @Named(PROPERTY_S3_SERVICE_PATH) String servicePath, @Named(PROPERTY_HEADER_TAG) String headerTag, - @Named(PROPERTY_IDENTITY) String accessKey, @Named(PROPERTY_CREDENTIAL) String secretKey, - @TimeStamp Provider timeStampProvider, Crypto crypto, HttpUtils utils) { + @Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle, + @Named(PROPERTY_S3_SERVICE_PATH) String servicePath, @Named(PROPERTY_HEADER_TAG) String headerTag, + @Named(PROPERTY_IDENTITY) String accessKey, @Named(PROPERTY_CREDENTIAL) String secretKey, + @TimeStamp Provider timeStampProvider, Crypto crypto, HttpUtils utils) { this.isVhostStyle = isVhostStyle; this.servicePath = servicePath; this.headerTag = headerTag; @@ -123,7 +123,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign HttpRequest replaceAuthorizationHeader(HttpRequest request, String signature) { request = ModifyRequest.replaceHeader(request, HttpHeaders.AUTHORIZATION, authTag + " " + accessKey + ":" - + signature); + + signature); return request; } @@ -161,8 +161,8 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign public String sign(String toSign) { String signature; try { - signature = CryptoStreams.base64(CryptoStreams.mac(InputSuppliers.of(toSign), - crypto.hmacSHA1(secretKey.getBytes()))); + signature = CryptoStreams.base64(CryptoStreams.mac(InputSuppliers.of(toSign), crypto.hmacSHA1(secretKey + .getBytes()))); } catch (Exception e) { throw new HttpException("error signing request", e); } @@ -189,11 +189,11 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) { buffer.append( - utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() - .getContentMD5())).append("\n"); + utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() + .getContentMD5())).append("\n"); buffer.append( - utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() - .getContentType())).append("\n"); + utils.valueOrEmpty(request.getPayload() == null ? request.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE) + : request.getPayload().getContentMetadata().getContentType())).append("\n"); } void appendHttpHeaders(HttpRequest request, StringBuilder toSign) { @@ -210,11 +210,11 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign for (int i = 0; i < request.getJavaMethod().getParameterAnnotations().length; i++) { if (Iterables.any(Arrays.asList(request.getJavaMethod().getParameterAnnotations()[i]), - new Predicate() { - public boolean apply(Annotation input) { - return input.annotationType().equals(Bucket.class); - } - })) { + new Predicate() { + public boolean apply(Annotation input) { + return input.annotationType().equals(Bucket.class); + } + })) { bucketName = (String) request.getArgs().get(i); break; } diff --git a/apis/s3/src/test/java/org/jclouds/s3/BaseS3AsyncClientTest.java b/apis/s3/src/test/java/org/jclouds/s3/BaseS3AsyncClientTest.java index 26333fbe10..6f741c8303 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/BaseS3AsyncClientTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/BaseS3AsyncClientTest.java @@ -28,20 +28,17 @@ import org.jclouds.http.HttpRequest; import org.jclouds.rest.RestClientTest; import org.jclouds.rest.RestContextFactory; import org.jclouds.rest.RestContextSpec; -import org.jclouds.rest.internal.RestAnnotationProcessor; import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.filters.RequestAuthorizeSignature; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import com.google.inject.TypeLiteral; - /** * * @author Adrian Cole */ @Test(groups = "unit") -public abstract class BaseS3AsyncClientTest extends RestClientTest { +public abstract class BaseS3AsyncClientTest extends RestClientTest { protected BlobToObject blobToS3Object; protected RequestAuthorizeSignature filter; @@ -52,11 +49,6 @@ public abstract class BaseS3AsyncClientTest extends RestClientTest> createTypeLiteral() { - return new TypeLiteral>() { - }; - } @BeforeClass @Override diff --git a/apis/s3/src/test/java/org/jclouds/s3/S3AsyncClientTest.java b/apis/s3/src/test/java/org/jclouds/s3/S3AsyncClientTest.java index 8e0c75475c..7a1d0b15d4 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/S3AsyncClientTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/S3AsyncClientTest.java @@ -80,7 +80,7 @@ import com.google.inject.Module; */ // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(groups = "unit", testName = "S3AsyncClientTest") -public class S3AsyncClientTest extends BaseS3AsyncClientTest { +public abstract class S3AsyncClientTest extends BaseS3AsyncClientTest { protected String url = "s3.amazonaws.com"; @@ -388,7 +388,7 @@ public class S3AsyncClientTest extends BaseS3AsyncClientTest { checkFilters(request); } - + public void testPutObject() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException, NoSuchMethodException, IOException { diff --git a/apis/s3/src/test/java/org/jclouds/s3/binders/BindAsHostPrefixIfConfiguredTest.java b/apis/s3/src/test/java/org/jclouds/s3/binders/BindAsHostPrefixIfConfiguredTest.java index 40b4001602..8279310e06 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/binders/BindAsHostPrefixIfConfiguredTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/binders/BindAsHostPrefixIfConfiguredTest.java @@ -27,11 +27,15 @@ import java.io.IOException; import java.net.URI; import java.util.Properties; -import org.jclouds.s3.BaseS3AsyncClientTest; import org.jclouds.http.HttpRequest; +import org.jclouds.rest.internal.RestAnnotationProcessor; +import org.jclouds.s3.BaseS3AsyncClientTest; +import org.jclouds.s3.S3AsyncClient; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import com.google.inject.TypeLiteral; + /** * Tests behavior of {@code BindAsHostPrefixIfConfigured} * @@ -39,7 +43,13 @@ import org.testng.annotations.Test; */ // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(groups = "unit", testName = "BindAsHostPrefixIfConfiguredTest") -public class BindAsHostPrefixIfConfiguredTest extends BaseS3AsyncClientTest { +public class BindAsHostPrefixIfConfiguredTest extends BaseS3AsyncClientTest { + + @Override + protected TypeLiteral> createTypeLiteral() { + return new TypeLiteral>() { + }; + } public void testBucket() throws IOException { @@ -64,8 +74,8 @@ public class BindAsHostPrefixIfConfiguredTest extends BaseS3AsyncClientTest { @DataProvider(name = "objects") public Object[][] createData() { - return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" }, - { "colon:" }, { "asteri*k" }, { "quote\"" }, { "{greaten" }, { "p|pe" } }; + return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" }, { "colon:" }, + { "asteri*k" }, { "quote\"" }, { "{greaten" }, { "p|pe" } }; } @Override diff --git a/apis/s3/src/test/java/org/jclouds/s3/binders/BindNoBucketLoggingToXmlPayloadTest.java b/apis/s3/src/test/java/org/jclouds/s3/binders/BindNoBucketLoggingToXmlPayloadTest.java index 7626a173d8..67ae409673 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/binders/BindNoBucketLoggingToXmlPayloadTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/binders/BindNoBucketLoggingToXmlPayloadTest.java @@ -24,10 +24,14 @@ import static org.testng.Assert.assertEquals; import java.io.IOException; import java.net.URI; -import org.jclouds.s3.BaseS3AsyncClientTest; import org.jclouds.http.HttpRequest; +import org.jclouds.rest.internal.RestAnnotationProcessor; +import org.jclouds.s3.BaseS3AsyncClientTest; +import org.jclouds.s3.S3AsyncClient; import org.testng.annotations.Test; +import com.google.inject.TypeLiteral; + /** * Tests behavior of {@code BindNoBucketLoggingToXmlPayload} * @@ -35,7 +39,13 @@ import org.testng.annotations.Test; */ // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(groups = "unit", testName = "BindNoBucketLoggingToXmlPayloadTest") -public class BindNoBucketLoggingToXmlPayloadTest extends BaseS3AsyncClientTest { +public class BindNoBucketLoggingToXmlPayloadTest extends BaseS3AsyncClientTest { + + @Override + protected TypeLiteral> createTypeLiteral() { + return new TypeLiteral>() { + }; + } public void testApplyInputStream() throws IOException { @@ -44,7 +54,7 @@ public class BindNoBucketLoggingToXmlPayloadTest extends BaseS3AsyncClientTest { request = binder.bindToRequest(request, "bucket"); assertEquals(request.getPayload().getRawContent(), - ""); + ""); } } diff --git a/apis/s3/src/test/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequestTest.java b/apis/s3/src/test/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequestTest.java index 0d35725916..e6a28e409d 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequestTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequestTest.java @@ -26,15 +26,19 @@ import java.net.URI; import javax.ws.rs.HttpMethod; -import org.jclouds.s3.BaseS3AsyncClientTest; -import org.jclouds.s3.domain.S3Object; import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix; import org.jclouds.http.HttpRequest; import org.jclouds.io.Payload; import org.jclouds.io.Payloads; +import org.jclouds.rest.internal.RestAnnotationProcessor; +import org.jclouds.s3.BaseS3AsyncClientTest; +import org.jclouds.s3.S3AsyncClient; +import org.jclouds.s3.domain.S3Object; import org.testng.annotations.Test; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; +import com.google.inject.TypeLiteral; /** * Tests behavior of {@code BindS3ObjectMetadataToRequest} @@ -43,7 +47,13 @@ import com.google.common.collect.ImmutableMultimap; */ // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(groups = "unit", testName = "BindS3ObjectMetadataToRequestTest") -public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest { +public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest { + + @Override + protected TypeLiteral> createTypeLiteral() { + return new TypeLiteral>() { + }; + } @Test public void testPassWithMinimumDetailsAndPayload5GB() { @@ -56,8 +66,8 @@ public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest { HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build(); BindS3ObjectMetadataToRequest binder = injector.getInstance(BindS3ObjectMetadataToRequest.class); - assertEquals(binder.bindToRequest(request, object), - HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build()); + assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint( + URI.create("http://localhost")).build()); } @Test @@ -68,14 +78,14 @@ public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest { object.setPayload(payload); object.getMetadata().setKey("foo"); object.getMetadata().setCacheControl("no-cache"); + object.getMetadata().setUserMetadata(ImmutableMap.of("foo", "bar")); HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build(); BindS3ObjectMetadataToRequest binder = injector.getInstance(BindS3ObjectMetadataToRequest.class); - assertEquals( - binder.bindToRequest(request, object), - HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")) - .headers(ImmutableMultimap.of("Cache-Control", "no-cache")).build()); + assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint( + URI.create("http://localhost")).headers( + ImmutableMultimap.of("Cache-Control", "no-cache", "x-amz-meta-foo", "bar")).build()); } @Test(expectedExceptions = IllegalArgumentException.class) diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobRequestSignerTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobRequestSignerTest.java index a0eb3c9064..046836d6ee 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobRequestSignerTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobRequestSignerTest.java @@ -31,6 +31,7 @@ import org.jclouds.http.HttpRequest; import org.jclouds.http.RequiresHttp; import org.jclouds.io.payloads.PhantomPayload; import org.jclouds.rest.ConfiguresRestClient; +import org.jclouds.rest.internal.RestAnnotationProcessor; import org.jclouds.s3.BaseS3AsyncClientTest; import org.jclouds.s3.S3AsyncClient; import org.jclouds.s3.S3Client; @@ -40,6 +41,7 @@ import org.testng.annotations.Test; import com.google.common.base.Supplier; import com.google.inject.Module; +import com.google.inject.TypeLiteral; /** * Tests behavior of {@code S3BlobRequestSigner} @@ -48,39 +50,45 @@ import com.google.inject.Module; */ // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(groups = "unit", testName = "S3BlobRequestSignerTest") -public class S3BlobRequestSignerTest extends BaseS3AsyncClientTest { +public class S3BlobRequestSignerTest extends BaseS3AsyncClientTest { + + @Override + protected TypeLiteral> createTypeLiteral() { + return new TypeLiteral>() { + }; + } private BlobRequestSigner signer; private Factory blobFactory; public void testSignGetBlob() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException, - NoSuchMethodException, IOException { + NoSuchMethodException, IOException { HttpRequest request = signer.signGetBlob("container", "name"); assertRequestLineEquals(request, "GET https://container.s3.amazonaws.com/name HTTP/1.1"); assertNonPayloadHeadersEqual( - request, - "Authorization: AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n"); + request, + "Authorization: AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n"); assertPayloadEquals(request, null, null, false); assertEquals(request.getFilters().size(), 0); } public void testSignRemoveBlob() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException, - NoSuchMethodException, IOException { + NoSuchMethodException, IOException { HttpRequest request = signer.signRemoveBlob("container", "name"); assertRequestLineEquals(request, "DELETE https://container.s3.amazonaws.com/name HTTP/1.1"); assertNonPayloadHeadersEqual( - request, - "Authorization: AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n"); + request, + "Authorization: AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n"); assertPayloadEquals(request, null, null, false); assertEquals(request.getFilters().size(), 0); } public void testSignPutBlob() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException, - NoSuchMethodException, IOException { + NoSuchMethodException, IOException { Blob blob = blobFactory.create(null); blob.getMetadata().setName("name"); blob.setPayload(new PhantomPayload()); @@ -92,8 +100,8 @@ public class S3BlobRequestSignerTest extends BaseS3AsyncClientTest { assertRequestLineEquals(request, "PUT https://container.s3.amazonaws.com/name HTTP/1.1"); assertNonPayloadHeadersEqual( - request, - "Authorization: AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n"); + request, + "Authorization: AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n"); assertContentHeadersEqual(request, "text/plain", null, null, null, (long) 2l, new byte[] { 0, 2, 4, 8 }); diff --git a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java index 2d0d2fee19..07084726a3 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java @@ -26,18 +26,21 @@ import java.util.Properties; import javax.ws.rs.core.HttpHeaders; +import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest; +import org.jclouds.http.HttpRequest; +import org.jclouds.rest.internal.GeneratedHttpRequest; +import org.jclouds.rest.internal.RestAnnotationProcessor; import org.jclouds.s3.BaseS3AsyncClientTest; import org.jclouds.s3.S3AsyncClient; import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.CannedAccessPolicy; import org.jclouds.s3.domain.S3Object; import org.jclouds.s3.options.PutObjectOptions; -import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest; -import org.jclouds.http.HttpRequest; -import org.jclouds.rest.internal.GeneratedHttpRequest; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import com.google.inject.TypeLiteral; + /** * Tests behavior of {@code RequestAuthorizeSignature} * @@ -45,7 +48,13 @@ import org.testng.annotations.Test; */ // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(groups = "unit", testName = "RequestAuthorizeSignatureTest") -public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest { +public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest { + + @Override + protected TypeLiteral> createTypeLiteral() { + return new TypeLiteral>() { + }; + } @DataProvider(parallel = true) public Object[][] dataProvider() throws NoSuchMethodException { @@ -70,20 +79,20 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest { filter.filter(request); if (request.getFirstHeaderOrNull(HttpHeaders.DATE).equals(date)) assert signature.equals(request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION)) : String.format( - "sig: %s != %s on attempt %s", signature, request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION), - iterations); + "sig: %s != %s on attempt %s", signature, request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION), + iterations); else iterations++; } System.out.printf("%s: %d iterations before the timestamp updated %n", Thread.currentThread().getName(), - iterations); + iterations); } @Test void testAppendBucketNameHostHeader() throws SecurityException, NoSuchMethodException { HttpRequest request = processor.createRequest(S3AsyncClient.class.getMethod("getBucketLocation", String.class), - "bucket"); + "bucket"); StringBuilder builder = new StringBuilder(); filter.appendBucketName(request, builder); assertEquals(builder.toString(), "/bucket"); @@ -98,12 +107,12 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest { } private GeneratedHttpRequest putBucketAcl() throws NoSuchMethodException { - return processor.createRequest( - S3AsyncClient.class.getMethod("putBucketACL", String.class, AccessControlList.class), "bucket", - AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE, "1234")); + return processor.createRequest(S3AsyncClient.class.getMethod("putBucketACL", String.class, + AccessControlList.class), "bucket", AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE, + "1234")); } - // "?acl", "?location", "?logging", or "?torrent" + // "?acl", "?location", "?logging", "?uploads", or "?torrent" @Test void testAppendBucketNameHostHeaderService() throws SecurityException, NoSuchMethodException { @@ -129,16 +138,15 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest { S3Object object = blobToS3Object.apply(BindBlobToMultipartFormTest.TEST_BLOB); object.getMetadata().getUserMetadata().put("x-amz-Adrian", "foo"); - HttpRequest request = processor.createRequest( - S3AsyncClient.class.getMethod("putObject", String.class, S3Object.class, PutObjectOptions[].class), - "bucket", object); + HttpRequest request = processor.createRequest(S3AsyncClient.class.getMethod("putObject", String.class, + S3Object.class, PutObjectOptions[].class), "bucket", object); return request; } @Test void testAppendBucketNameURIHost() throws SecurityException, NoSuchMethodException { HttpRequest request = processor.createRequest(S3AsyncClient.class.getMethod("getBucketLocation", String.class), - "bucket"); + "bucket"); assertEquals(request.getEndpoint().getHost(), "bucket.s3.amazonaws.com"); } diff --git a/apis/walrus/src/test/java/org/jclouds/walrus/WalrusAsyncClientTestDisabled.java b/apis/walrus/src/test/java/org/jclouds/walrus/WalrusAsyncClientTestDisabled.java index bab7574ead..d17d7b6c2c 100644 --- a/apis/walrus/src/test/java/org/jclouds/walrus/WalrusAsyncClientTestDisabled.java +++ b/apis/walrus/src/test/java/org/jclouds/walrus/WalrusAsyncClientTestDisabled.java @@ -19,14 +19,25 @@ package org.jclouds.walrus; +import org.jclouds.rest.internal.RestAnnotationProcessor; +import org.jclouds.s3.S3AsyncClient; +import org.jclouds.s3.S3AsyncClientTest; import org.testng.annotations.Test; +import com.google.inject.TypeLiteral; + /** * @author Adrian Cole */ // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(enabled = false, groups = "unit", testName = "WalrusAsyncClientTest") -public class WalrusAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest { +public class WalrusAsyncClientTestDisabled extends S3AsyncClientTest { + + @Override + protected TypeLiteral> createTypeLiteral() { + return new TypeLiteral>() { + }; + } public WalrusAsyncClientTestDisabled() { this.provider = "walrus"; diff --git a/common/aws/src/main/java/org/jclouds/aws/handlers/ParseAWSErrorFromXmlContent.java b/common/aws/src/main/java/org/jclouds/aws/handlers/ParseAWSErrorFromXmlContent.java index 76c06390d7..b5cd52b1ea 100755 --- a/common/aws/src/main/java/org/jclouds/aws/handlers/ParseAWSErrorFromXmlContent.java +++ b/common/aws/src/main/java/org/jclouds/aws/handlers/ParseAWSErrorFromXmlContent.java @@ -27,6 +27,7 @@ import javax.annotation.Resource; import javax.inject.Inject; import javax.inject.Singleton; +import org.jclouds.aws.AWSResponseException; import org.jclouds.aws.domain.AWSError; import org.jclouds.aws.util.AWSUtils; import org.jclouds.http.HttpCommand; @@ -71,8 +72,10 @@ public class ParseAWSErrorFromXmlContent implements HttpErrorHandler { error = utils.parseAWSErrorFromContent(command.getCurrentRequest(), response); if (error != null) { message = error.getMessage(); + exception = new AWSResponseException(command, response, error); + } else { + exception = new HttpResponseException(command, response, message); } - exception = new HttpResponseException(command, response, message); } else { try { message = Strings2.toStringAndClose(response.getPayload().getInput()); diff --git a/core/src/main/java/org/jclouds/io/ContentMetadata.java b/core/src/main/java/org/jclouds/io/ContentMetadata.java index a73168564c..ac72cc6b6a 100644 --- a/core/src/main/java/org/jclouds/io/ContentMetadata.java +++ b/core/src/main/java/org/jclouds/io/ContentMetadata.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableSet; */ public interface ContentMetadata { public static final Set HTTP_HEADERS = ImmutableSet.of(CONTENT_LENGTH, "Content-MD5", CONTENT_TYPE, - "Content-Disposition", "Content-Encoding", "Content-Language"); + "Content-Disposition", "Content-Encoding", "Content-Language"); /** * Returns the total size of the payload, or the chunk that's available. @@ -87,4 +87,6 @@ public interface ContentMetadata { @Nullable String getContentLanguage(); + ContentMetadataBuilder toBuilder(); + } \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/io/ContentMetadataBuilder.java b/core/src/main/java/org/jclouds/io/ContentMetadataBuilder.java new file mode 100644 index 0000000000..fa14eea645 --- /dev/null +++ b/core/src/main/java/org/jclouds/io/ContentMetadataBuilder.java @@ -0,0 +1,186 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.io; + +import static com.google.common.collect.Iterables.any; +import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH; +import static javax.ws.rs.core.HttpHeaders.CONTENT_TYPE; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Map.Entry; + +import javax.annotation.Nullable; + +import org.jclouds.crypto.CryptoStreams; +import org.jclouds.io.payloads.BaseImmutableContentMetadata; + +import com.google.common.base.Predicate; +import com.google.common.collect.Multimap; + +/** + * @author Adrian Cole + */ +public class ContentMetadataBuilder implements Serializable { + /** The serialVersionUID */ + private static final long serialVersionUID = -5279643002875371558L; + + public static ContentMetadataBuilder create() { + return new ContentMetadataBuilder(); + } + + protected String contentType = "application/unknown"; + protected Long contentLength; + protected byte[] contentMD5; + protected String contentDisposition; + protected String contentLanguage; + protected String contentEncoding; + + public ContentMetadataBuilder fromHttpHeaders(Multimap headers) { + boolean chunked = any(headers.entries(), new Predicate>() { + @Override + public boolean apply(Entry input) { + return "Transfer-Encoding".equalsIgnoreCase(input.getKey()) && "chunked".equalsIgnoreCase(input.getValue()); + } + }); + for (Entry header : headers.entries()) { + if (!chunked && CONTENT_LENGTH.equalsIgnoreCase(header.getKey())) { + contentLength(new Long(header.getValue())); + } else if ("Content-MD5".equalsIgnoreCase(header.getKey())) { + contentMD5(CryptoStreams.base64(header.getValue())); + } else if (CONTENT_TYPE.equalsIgnoreCase(header.getKey())) { + contentType(header.getValue()); + } else if ("Content-Disposition".equalsIgnoreCase(header.getKey())) { + contentDisposition(header.getValue()); + } else if ("Content-Encoding".equalsIgnoreCase(header.getKey())) { + contentEncoding(header.getValue()); + } else if ("Content-Language".equalsIgnoreCase(header.getKey())) { + contentLanguage(header.getValue()); + } + } + return this; + } + + public ContentMetadataBuilder contentLength(@Nullable Long contentLength) { + this.contentLength = contentLength; + return this; + } + + public ContentMetadataBuilder contentMD5(byte[] md5) { + if (md5 != null) { + byte[] retval = new byte[md5.length]; + System.arraycopy(md5, 0, retval, 0, md5.length); + this.contentMD5 = md5; + } + return this; + + } + + public ContentMetadataBuilder contentType(@Nullable String contentType) { + this.contentType = contentType; + return this; + } + + public ContentMetadataBuilder contentDisposition(@Nullable String contentDisposition) { + this.contentDisposition = contentDisposition; + return this; + + } + + public ContentMetadataBuilder contentLanguage(@Nullable String contentLanguage) { + this.contentLanguage = contentLanguage; + return this; + } + + public ContentMetadataBuilder contentEncoding(@Nullable String contentEncoding) { + this.contentEncoding = contentEncoding; + return this; + } + + public ContentMetadata build() { + return new BaseImmutableContentMetadata(contentType, contentLength, contentMD5, contentDisposition, + contentLanguage, contentEncoding); + } + + public static ContentMetadataBuilder fromContentMetadata(ContentMetadata in) { + return new ContentMetadataBuilder().contentType(in.getContentType()).contentLength(in.getContentLength()) + .contentMD5(in.getContentMD5()).contentDisposition(in.getContentDisposition()).contentLanguage( + in.getContentLanguage()).contentEncoding(in.getContentEncoding()); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((contentDisposition == null) ? 0 : contentDisposition.hashCode()); + result = prime * result + ((contentEncoding == null) ? 0 : contentEncoding.hashCode()); + result = prime * result + ((contentLanguage == null) ? 0 : contentLanguage.hashCode()); + result = prime * result + ((contentLength == null) ? 0 : contentLength.hashCode()); + result = prime * result + Arrays.hashCode(contentMD5); + result = prime * result + ((contentType == null) ? 0 : contentType.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ContentMetadataBuilder other = (ContentMetadataBuilder) obj; + if (contentDisposition == null) { + if (other.contentDisposition != null) + return false; + } else if (!contentDisposition.equals(other.contentDisposition)) + return false; + if (contentEncoding == null) { + if (other.contentEncoding != null) + return false; + } else if (!contentEncoding.equals(other.contentEncoding)) + return false; + if (contentLanguage == null) { + if (other.contentLanguage != null) + return false; + } else if (!contentLanguage.equals(other.contentLanguage)) + return false; + if (contentLength == null) { + if (other.contentLength != null) + return false; + } else if (!contentLength.equals(other.contentLength)) + return false; + if (!Arrays.equals(contentMD5, other.contentMD5)) + return false; + if (contentType == null) { + if (other.contentType != null) + return false; + } else if (!contentType.equals(other.contentType)) + return false; + return true; + } + + @Override + public String toString() { + return "[contentDisposition=" + contentDisposition + ", contentEncoding=" + contentEncoding + + ", contentLanguage=" + contentLanguage + ", contentLength=" + contentLength + ", contentMD5=" + + Arrays.toString(contentMD5) + ", contentType=" + contentType + "]"; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/io/payloads/BaseImmutableContentMetadata.java b/core/src/main/java/org/jclouds/io/payloads/BaseImmutableContentMetadata.java index 36919dce27..cb7b274bb9 100644 --- a/core/src/main/java/org/jclouds/io/payloads/BaseImmutableContentMetadata.java +++ b/core/src/main/java/org/jclouds/io/payloads/BaseImmutableContentMetadata.java @@ -23,6 +23,7 @@ import java.io.Serializable; import java.util.Arrays; import org.jclouds.io.ContentMetadata; +import org.jclouds.io.ContentMetadataBuilder; /** * @author Adrian Cole @@ -161,4 +162,9 @@ public class BaseImmutableContentMetadata implements ContentMetadata, Serializab return true; } + @Override + public ContentMetadataBuilder toBuilder() { + return ContentMetadataBuilder.fromContentMetadata(this); + } + } \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/io/payloads/BaseMutableContentMetadata.java b/core/src/main/java/org/jclouds/io/payloads/BaseMutableContentMetadata.java index 57f0ba7a08..e1b77fe3d8 100644 --- a/core/src/main/java/org/jclouds/io/payloads/BaseMutableContentMetadata.java +++ b/core/src/main/java/org/jclouds/io/payloads/BaseMutableContentMetadata.java @@ -19,63 +19,26 @@ package org.jclouds.io.payloads; -import static com.google.common.collect.Iterables.any; -import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH; -import static javax.ws.rs.core.HttpHeaders.CONTENT_TYPE; - import java.io.Serializable; -import java.util.Arrays; -import java.util.Map.Entry; import javax.annotation.Nullable; -import org.jclouds.crypto.CryptoStreams; +import org.jclouds.io.ContentMetadata; +import org.jclouds.io.ContentMetadataBuilder; import org.jclouds.io.MutableContentMetadata; -import com.google.common.base.Predicate; import com.google.common.collect.Multimap; /** * @author Adrian Cole */ -public class BaseMutableContentMetadata implements MutableContentMetadata, Serializable { +public class BaseMutableContentMetadata extends ContentMetadataBuilder implements MutableContentMetadata, Serializable { + /** The serialVersionUID */ + private static final long serialVersionUID = 8364286391963469370L; + @Override public void setPropertiesFromHttpHeaders(Multimap headers) { - boolean chunked = any(headers.entries(), new Predicate>() { - @Override - public boolean apply(Entry input) { - return "Transfer-Encoding".equalsIgnoreCase(input.getKey()) && "chunked".equalsIgnoreCase(input.getValue()); - } - }); - for (Entry header : headers.entries()) { - if (!chunked && CONTENT_LENGTH.equalsIgnoreCase(header.getKey())) { - setContentLength(new Long(header.getValue())); - } else if ("Content-MD5".equalsIgnoreCase(header.getKey())) { - setContentMD5(CryptoStreams.base64(header.getValue())); - } else if (CONTENT_TYPE.equalsIgnoreCase(header.getKey())) { - setContentType(header.getValue()); - } else if ("Content-Disposition".equalsIgnoreCase(header.getKey())) { - setContentDisposition(header.getValue()); - } else if ("Content-Encoding".equalsIgnoreCase(header.getKey())) { - setContentEncoding(header.getValue()); - } else if ("Content-Language".equalsIgnoreCase(header.getKey())) { - setContentLanguage(header.getValue()); - } - } - } - - /** The serialVersionUID */ - private static final long serialVersionUID = 4572381435863125873L; - - protected String contentType = "application/unknown"; - protected Long contentLength; - protected byte[] contentMD5; - protected String contentDisposition; - protected String contentLanguage; - protected String contentEncoding; - - public BaseMutableContentMetadata() { - super(); + fromHttpHeaders(headers); } /** @@ -91,7 +54,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria */ @Override public void setContentLength(@Nullable Long contentLength) { - this.contentLength = contentLength; + contentLength(contentLength); } /** @@ -113,11 +76,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria */ @Override public void setContentMD5(byte[] md5) { - if (md5 != null) { - byte[] retval = new byte[md5.length]; - System.arraycopy(md5, 0, retval, 0, md5.length); - this.contentMD5 = md5; - } + contentMD5(md5); } /** @@ -133,7 +92,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria */ @Override public void setContentType(@Nullable String contentType) { - this.contentType = contentType; + contentType(contentType); } /** @@ -141,7 +100,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria */ @Override public void setContentDisposition(@Nullable String contentDisposition) { - this.contentDisposition = contentDisposition; + contentDisposition(contentDisposition); } /** @@ -157,7 +116,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria */ @Override public void setContentLanguage(@Nullable String contentLanguage) { - this.contentLanguage = contentLanguage; + contentLanguage(contentLanguage); } /** @@ -173,7 +132,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria */ @Override public void setContentEncoding(@Nullable String contentEncoding) { - this.contentEncoding = contentEncoding; + contentEncoding(contentEncoding); } /** @@ -185,62 +144,14 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria } @Override - public String toString() { - return "[contentType=" + contentType + ", contentLength=" + contentLength + ", contentDisposition=" - + contentDisposition + ", contentEncoding=" + contentEncoding + ", contentLanguage=" + contentLanguage - + ", contentMD5=" + Arrays.toString(contentMD5) + "]"; + public BaseMutableContentMetadata toBuilder() { + return BaseMutableContentMetadata.fromContentMetadata(this); } - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((contentDisposition == null) ? 0 : contentDisposition.hashCode()); - result = prime * result + ((contentEncoding == null) ? 0 : contentEncoding.hashCode()); - result = prime * result + ((contentLanguage == null) ? 0 : contentLanguage.hashCode()); - result = prime * result + ((contentLength == null) ? 0 : contentLength.hashCode()); - result = prime * result + Arrays.hashCode(contentMD5); - result = prime * result + ((contentType == null) ? 0 : contentType.hashCode()); - return result; + public static BaseMutableContentMetadata fromContentMetadata(ContentMetadata in) { + return (BaseMutableContentMetadata) new BaseMutableContentMetadata().contentType(in.getContentType()) + .contentLength(in.getContentLength()).contentMD5(in.getContentMD5()).contentDisposition( + in.getContentDisposition()).contentLanguage(in.getContentLanguage()).contentEncoding( + in.getContentEncoding()); } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - BaseMutableContentMetadata other = (BaseMutableContentMetadata) obj; - if (contentDisposition == null) { - if (other.contentDisposition != null) - return false; - } else if (!contentDisposition.equals(other.contentDisposition)) - return false; - if (contentEncoding == null) { - if (other.contentEncoding != null) - return false; - } else if (!contentEncoding.equals(other.contentEncoding)) - return false; - if (contentLanguage == null) { - if (other.contentLanguage != null) - return false; - } else if (!contentLanguage.equals(other.contentLanguage)) - return false; - if (contentLength == null) { - if (other.contentLength != null) - return false; - } else if (!contentLength.equals(other.contentLength)) - return false; - if (!Arrays.equals(contentMD5, other.contentMD5)) - return false; - if (contentType == null) { - if (other.contentType != null) - return false; - } else if (!contentType.equals(other.contentType)) - return false; - return true; - } - } \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/io/payloads/PhantomPayload.java b/core/src/main/java/org/jclouds/io/payloads/PhantomPayload.java index a0d0299235..c2c799580d 100644 --- a/core/src/main/java/org/jclouds/io/payloads/PhantomPayload.java +++ b/core/src/main/java/org/jclouds/io/payloads/PhantomPayload.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import org.jclouds.io.ContentMetadata; import org.jclouds.io.MutableContentMetadata; /** @@ -36,6 +37,10 @@ public class PhantomPayload extends BasePayload { super(Object.class); } + public PhantomPayload(ContentMetadata contentMetadata) { + this(BaseMutableContentMetadata.fromContentMetadata(checkNotNull(contentMetadata, "contentMetadata"))); + } + public PhantomPayload(MutableContentMetadata contentMetadata) { super(Object.class, checkNotNull(contentMetadata, "contentMetadata")); } diff --git a/providers/aws-s3/pom.xml b/providers/aws-s3/pom.xml index 152cb8b1a6..9fea31babb 100644 --- a/providers/aws-s3/pom.xml +++ b/providers/aws-s3/pom.xml @@ -70,6 +70,18 @@ test-jar test + + org.jclouds.driver + jclouds-apachehc + ${project.version} + test + + + org.apache.httpcomponents + httpclient + 4.1 + test + org.jclouds.driver jclouds-log4j diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java index 5513f295eb..986679d800 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java @@ -18,6 +18,7 @@ */ package org.jclouds.aws.s3; + /** * * @@ -38,11 +39,41 @@ package org.jclouds.aws.s3; import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER; +import java.util.Map; + +import javax.ws.rs.DELETE; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; + +import org.jclouds.aws.s3.binders.BindObjectMetadataToRequest; +import org.jclouds.aws.s3.binders.BindPartIdsAndETagsToRequest; +import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex; +import org.jclouds.aws.s3.functions.ObjectMetadataKey; +import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex; import org.jclouds.blobstore.attr.BlobScope; +import org.jclouds.http.functions.ParseETagHeader; +import org.jclouds.io.Payload; +import org.jclouds.rest.annotations.BinderParam; +import org.jclouds.rest.annotations.ExceptionParser; +import org.jclouds.rest.annotations.ParamParser; +import org.jclouds.rest.annotations.ParamValidators; +import org.jclouds.rest.annotations.QueryParams; import org.jclouds.rest.annotations.RequestFilters; +import org.jclouds.rest.annotations.ResponseParser; import org.jclouds.rest.annotations.SkipEncoding; +import org.jclouds.rest.functions.ReturnVoidOnNotFoundOr404; +import org.jclouds.s3.Bucket; import org.jclouds.s3.S3AsyncClient; +import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured; +import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.filters.RequestAuthorizeSignature; +import org.jclouds.s3.options.PutObjectOptions; +import org.jclouds.s3.predicates.validators.BucketNameValidator; + +import com.google.common.util.concurrent.ListenableFuture; /** * Provides access to amazon-specific S3 features @@ -53,5 +84,48 @@ import org.jclouds.s3.filters.RequestAuthorizeSignature; @RequestFilters(RequestAuthorizeSignature.class) @BlobScope(CONTAINER) public interface AWSS3AsyncClient extends S3AsyncClient { - + /** + * @see AWSS3Client#initiateMultipartUpload + */ + @POST + @QueryParams(keys = "uploads") + @Path("/{key}") + @ResponseParser(UploadIdFromHttpResponseViaRegex.class) + ListenableFuture initiateMultipartUpload( + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName, + @PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class) ObjectMetadata objectMetadata, + PutObjectOptions... options); + + /** + * @see AWSS3Client#abortMultipartUpload + */ + @DELETE + @Path("/{key}") + @ExceptionParser(ReturnVoidOnNotFoundOr404.class) + ListenableFuture abortMultipartUpload( + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName, + @PathParam("key") String key, @QueryParam("uploadId") String uploadId); + + /** + * @see AWSS3Client#uploadPart + */ + @PUT + @Path("/{key}") + @ResponseParser(ParseETagHeader.class) + ListenableFuture uploadPart( + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName, + @PathParam("key") String key, @QueryParam("partNumber") int partNumber, + @QueryParam("uploadId") String uploadId, Payload part); + + /** + * @see AWSS3Client#completeMultipartUpload + */ + @POST + @Path("/{key}") + @ResponseParser(ETagFromHttpResponseViaRegex.class) + ListenableFuture completeMultipartUpload( + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName, + @PathParam("key") String key, @QueryParam("uploadId") String uploadId, + @BinderParam(BindPartIdsAndETagsToRequest.class) Map parts); + } diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java index b464041786..c1a1c75a0e 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java @@ -19,10 +19,14 @@ package org.jclouds.aws.s3; +import java.util.Map; import java.util.concurrent.TimeUnit; import org.jclouds.concurrent.Timeout; +import org.jclouds.io.Payload; import org.jclouds.s3.S3Client; +import org.jclouds.s3.domain.ObjectMetadata; +import org.jclouds.s3.options.PutObjectOptions; /** * Provides access to amazon-specific S3 features @@ -32,5 +36,110 @@ import org.jclouds.s3.S3Client; */ @Timeout(duration = 90, timeUnit = TimeUnit.SECONDS) public interface AWSS3Client extends S3Client { + /** + * This operation initiates a multipart upload and returns an upload ID. This upload ID is used + * to associate all the parts in the specific multipart upload. You specify this upload ID in + * each of your subsequent upload part requests (see Upload Part). You also include this upload + * ID in the final request to either complete or abort the multipart upload request. + * + *

Note

If you create an object using the multipart upload APIs, currently you cannot + * copy the object between regions. + * + * + * @param bucketName + * namespace of the object you are to upload + * @param objectMetadata + * metadata around the object you wish to upload + * @param options + * controls optional parameters such as canned ACL + * @return ID for the initiated multipart upload. + */ + String initiateMultipartUpload(String bucketName, ObjectMetadata objectMetadata, PutObjectOptions... options); + /** + * This operation aborts a multipart upload. After a multipart upload is aborted, no additional + * parts can be uploaded using that upload ID. The storage consumed by any previously uploaded + * parts will be freed. However, if any part uploads are currently in progress, those part + * uploads might or might not succeed. As a result, it might be necessary to abort a given + * multipart upload multiple times in order to completely free all storage consumed by all parts. + * + * + * @param bucketName + * namespace of the object you are deleting + * @param key + * unique key in the s3Bucket identifying the object + * @param uploadId + * id of the multipart upload in progress. + */ + void abortMultipartUpload(String bucketName, String key, String uploadId); + + /** + * This operation uploads a part in a multipart upload. You must initiate a multipart upload (see + * Initiate Multipart Upload) before you can upload any part. In response to your initiate + * request. Amazon S3 returns an upload ID, a unique identifier, that you must include in your + * upload part request. + * + *

+ * Part numbers can be any number from 1 to 10,000, inclusive. A part number uniquely identifies + * a part and also defines its position within the object being created. If you upload a new part + * using the same part number that was used with a previous part, the previously uploaded part is + * overwritten. Each part must be at least 5 MB in size, except the last part. There is no size + * limit on the last part of your multipart upload. + * + *

+ * To ensure that data is not corrupted when traversing the network, specify the Content-MD5 + * header in the upload part request. Amazon S3 checks the part data against the provided MD5 + * value. If they do not match, Amazon S3 returns an error. + * + * + * @param bucketName + * namespace of the object you are storing + * @param key + * unique key in the s3Bucket identifying the object + * @param partNumber + * which part is this. + * @param uploadId + * id of the multipart upload in progress. + * @param part + * contains the data to create or overwrite + * @return ETag of the content uploaded + * @see + */ + @Timeout(duration = 5 * 1024 * 1024 / 128, timeUnit = TimeUnit.SECONDS) + String uploadPart(String bucketName, String key, int partNumber, String uploadId, Payload part); + + /** + * + This operation completes a multipart upload by assembling previously uploaded parts. + *

+ * You first initiate the multipart upload and then upload all parts using the Upload Parts + * operation (see Upload Part). After successfully uploading all relevant parts of an upload, you + * call this operation to complete the upload. Upon receiving this request, Amazon S3 + * concatenates all the parts in ascending order by part number to create a new object. In the + * Complete Multipart Upload request, you must provide the parts list. For each part in the list, + * you must provide the part number and the ETag header value, returned after that part was + * uploaded. + *

+ * Processing of a Complete Multipart Upload request could take several minutes to complete. + * After Amazon S3 begins processing the request, it sends an HTTP response header that specifies + * a 200 OK response. While processing is in progress, Amazon S3 periodically sends whitespace + * characters to keep the connection from timing out. Because a request could fail after the + * initial 200 OK response has been sent, it is important that you check the response body to + * determine whether the request succeeded. + *

+ * Note that if Complete Multipart Upload fails, applications should be prepared to retry the + * failed requests. + * + * @param bucketName + * namespace of the object you are deleting + * @param key + * unique key in the s3Bucket identifying the object + * @param uploadId + * id of the multipart upload in progress. + * @param parts + * a map of part id to eTag from the {@link #uploadPart} command. + * @return ETag of the content uploaded + */ + String completeMultipartUpload(String bucketName, String key, String uploadId, Map parts); } diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequest.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequest.java new file mode 100644 index 0000000000..d85f7876eb --- /dev/null +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequest.java @@ -0,0 +1,83 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3.binders; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.ws.rs.core.HttpHeaders; + +import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix; +import org.jclouds.http.HttpRequest; +import org.jclouds.http.utils.ModifyRequest; +import org.jclouds.rest.Binder; +import org.jclouds.s3.domain.ObjectMetadata; + +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableMultimap.Builder; + +/** + * + * @author Adrian Cole + */ +@Singleton +public class BindObjectMetadataToRequest implements Binder { + protected final BindMapToHeadersWithPrefix metadataPrefixer; + + @Inject + public BindObjectMetadataToRequest(BindMapToHeadersWithPrefix metadataPrefixer) { + this.metadataPrefixer = checkNotNull(metadataPrefixer, "metadataPrefixer"); + } + + @Override + public R bindToRequest(R request, Object input) { + checkArgument(checkNotNull(input, "input") instanceof ObjectMetadata, + "this binder is only valid for ObjectMetadata!"); + checkNotNull(request, "request"); + + ObjectMetadata md = ObjectMetadata.class.cast(input); + checkArgument(md.getKey() != null, "objectMetadata.getKey() must be set!"); + + request = metadataPrefixer.bindToRequest(request, md.getUserMetadata()); + + Builder headers = ImmutableMultimap. builder(); + if (md.getCacheControl() != null) { + headers.put(HttpHeaders.CACHE_CONTROL, md.getCacheControl()); + } + + if (md.getContentMetadata().getContentDisposition() != null) { + headers.put("Content-Disposition", md.getContentMetadata().getContentDisposition()); + } + + if (md.getContentMetadata().getContentEncoding() != null) { + headers.put("Content-Encoding", md.getContentMetadata().getContentEncoding()); + } + if (md.getContentMetadata().getContentType() != null) { + headers.put(HttpHeaders.CONTENT_TYPE, md.getContentMetadata().getContentType()); + } else { + headers.put(HttpHeaders.CONTENT_TYPE, "binary/octet-stream"); + } + + request = ModifyRequest.replaceHeaders(request, headers.build()); + return request; + } +} diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequest.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequest.java new file mode 100644 index 0000000000..72d90fbb2a --- /dev/null +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequest.java @@ -0,0 +1,65 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3.binders; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.Map; +import java.util.Map.Entry; + +import javax.inject.Singleton; +import javax.ws.rs.core.MediaType; + +import org.jclouds.http.HttpRequest; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; +import org.jclouds.rest.Binder; + +/** + * + * @author Adrian Cole + */ +@Singleton +public class BindPartIdsAndETagsToRequest implements Binder { + + @SuppressWarnings("unchecked") + @Override + public R bindToRequest(R request, Object input) { + checkArgument(checkNotNull(input, "input") instanceof Map, "this binder is only valid for Map!"); + checkNotNull(request, "request"); + + Map map = (Map) input; + checkArgument(map.size() != 0, "Please send parts"); + StringBuilder content = new StringBuilder(); + content.append(""); + for (Entry entry : map.entrySet()) { + content.append(""); + content.append("").append(entry.getKey()).append(""); + content.append("").append(entry.getValue()).append(""); + content.append(""); + } + content.append(""); + Payload payload = Payloads.newStringPayload(content.toString()); + payload.getContentMetadata().setContentType(MediaType.TEXT_XML); + request.setPayload(payload); + return request; + } +} diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegex.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegex.java new file mode 100644 index 0000000000..f030bef210 --- /dev/null +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegex.java @@ -0,0 +1,60 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3.functions; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import org.jclouds.http.HttpResponse; +import org.jclouds.http.functions.ReturnStringIf2xx; + +import com.google.common.base.Function; + +/** + * @see + * @author Adrian Cole + */ +@Singleton +public class ETagFromHttpResponseViaRegex implements Function { + Pattern pattern = Pattern.compile("([\\S&&[^<]]+)"); + private final ReturnStringIf2xx returnStringIf200; + + @Inject + ETagFromHttpResponseViaRegex(ReturnStringIf2xx returnStringIf200) { + this.returnStringIf200 = returnStringIf200; + } + + @Override + public String apply(HttpResponse response) { + String value = null; + String content = returnStringIf200.apply(response); + if (content != null) { + Matcher matcher = pattern.matcher(content); + if (matcher.find()) { + value = matcher.group(1); + } + } + return value; + } + +} diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ObjectMetadataKey.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ObjectMetadataKey.java new file mode 100644 index 0000000000..548307d19b --- /dev/null +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ObjectMetadataKey.java @@ -0,0 +1,39 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3.functions; + +import javax.inject.Singleton; + +import org.jclouds.s3.domain.ObjectMetadata; + +import com.google.common.base.Function; + +/** + * + * @author Adrian Cole + */ +@Singleton +public class ObjectMetadataKey implements Function { + + public String apply(Object from) { + return ((ObjectMetadata) from).getKey(); + } + +} diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegex.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegex.java new file mode 100644 index 0000000000..1474d2c587 --- /dev/null +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegex.java @@ -0,0 +1,62 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3.functions; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import org.jclouds.http.HttpResponse; +import org.jclouds.http.functions.ReturnStringIf2xx; + +import com.google.common.base.Function; + +/** + * @see + * @author Adrian Cole + */ +@Singleton +public class UploadIdFromHttpResponseViaRegex implements Function { + Pattern pattern = Pattern.compile("([\\S&&[^<]]+)"); + private final ReturnStringIf2xx returnStringIf200; + + @Inject + UploadIdFromHttpResponseViaRegex(ReturnStringIf2xx returnStringIf200) { + this.returnStringIf200 = returnStringIf200; + } + + @Override + public String apply(HttpResponse response) { + String value = null; + String content = returnStringIf200.apply(response); + if (content != null) { + Matcher matcher = pattern.matcher(content); + if (matcher.find()) { + value = matcher.group(1); + } + } + return value; + } + +} diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java index 0cea0d22c4..9fbaf68391 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java @@ -22,43 +22,143 @@ package org.jclouds.aws.s3; import java.io.IOException; import java.lang.reflect.Array; import java.lang.reflect.Method; +import java.util.Map; import java.util.Properties; +import org.jclouds.aws.s3.config.AWSS3RestClientModule; +import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex; +import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex; +import org.jclouds.date.TimeStamp; import org.jclouds.http.HttpRequest; +import org.jclouds.http.RequiresHttp; +import org.jclouds.http.functions.ParseETagHeader; +import org.jclouds.http.functions.ReleasePayloadAndReturn; import org.jclouds.http.functions.ReturnTrueIf2xx; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; +import org.jclouds.rest.ConfiguresRestClient; import org.jclouds.rest.RestContextFactory; -import org.jclouds.s3.S3AsyncClient; +import org.jclouds.rest.functions.MapHttp4xxCodesToExceptions; +import org.jclouds.rest.functions.ReturnVoidOnNotFoundOr404; +import org.jclouds.rest.internal.RestAnnotationProcessor; +import org.jclouds.s3.domain.ObjectMetadata; +import org.jclouds.s3.domain.ObjectMetadataBuilder; import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState; import org.jclouds.s3.options.PutBucketOptions; +import org.jclouds.s3.options.PutObjectOptions; import org.testng.annotations.Test; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Module; +import com.google.inject.TypeLiteral; + /** * @author Adrian Cole */ // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(groups = "unit", testName = "AWSS3AsyncClientTest") -public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest { +public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest { public AWSS3AsyncClientTest() { this.provider = "aws-s3"; } + @Override + protected TypeLiteral> createTypeLiteral() { + return new TypeLiteral>() { + }; + } + @Override protected Properties getProperties() { return RestContextFactory.getPropertiesFromResource("/rest.properties"); } + public void testInitiateMultipartUpload() throws SecurityException, NegativeArraySizeException, + NoSuchMethodException { + Method method = AWSS3AsyncClient.class.getMethod("initiateMultipartUpload", String.class, ObjectMetadata.class, + PutObjectOptions[].class); + HttpRequest request = processor + .createRequest(method, "bucket", ObjectMetadataBuilder.create().key("foo").build()); + + assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1"); + assertNonPayloadHeadersEqual(request, "Content-Type: binary/octet-stream\nHost: bucket." + url + "\n"); + assertPayloadEquals(request, null, null, false); + + assertResponseParserClassEquals(method, request, UploadIdFromHttpResponseViaRegex.class); + assertSaxResponseParserClassEquals(method, null); + assertExceptionParserClassEquals(method, MapHttp4xxCodesToExceptions.class); + + checkFilters(request); + } + + public void testAbortMultipartUpload() throws SecurityException, NegativeArraySizeException, NoSuchMethodException { + Method method = AWSS3AsyncClient.class + .getMethod("abortMultipartUpload", String.class, String.class, String.class); + HttpRequest request = processor.createRequest(method, "bucket", "foo", "asdsadasdas", 1, Payloads + .newStringPayload("")); + + assertRequestLineEquals(request, "DELETE https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1"); + assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n"); + assertPayloadEquals(request, "", "application/unknown", false); + + assertResponseParserClassEquals(method, request, ReleasePayloadAndReturn.class); + assertSaxResponseParserClassEquals(method, null); + assertExceptionParserClassEquals(method, ReturnVoidOnNotFoundOr404.class); + + checkFilters(request); + } + + public void testUploadPart() throws SecurityException, NegativeArraySizeException, NoSuchMethodException { + Method method = AWSS3AsyncClient.class.getMethod("uploadPart", String.class, String.class, int.class, + String.class, Payload.class); + HttpRequest request = processor.createRequest(method, "bucket", "foo", 1, "asdsadasdas", Payloads + .newStringPayload("")); + + assertRequestLineEquals(request, "PUT https://bucket." + url + "/foo?partNumber=1&uploadId=asdsadasdas HTTP/1.1"); + assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n"); + assertPayloadEquals(request, "", "application/unknown", false); + + assertResponseParserClassEquals(method, request, ParseETagHeader.class); + assertSaxResponseParserClassEquals(method, null); + assertExceptionParserClassEquals(method, MapHttp4xxCodesToExceptions.class); + + checkFilters(request); + } + + public void testCompleteMultipartUpload() throws SecurityException, NegativeArraySizeException, + NoSuchMethodException { + Method method = AWSS3AsyncClient.class.getMethod("completeMultipartUpload", String.class, String.class, + String.class, Map.class); + HttpRequest request = processor.createRequest(method, "bucket", "foo", "asdsadasdas", ImmutableMap + . of(1, "\"a54357aff0632cce46d942af68356b38\"")); + + assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1"); + assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n"); + assertPayloadEquals( + request, + "1\"a54357aff0632cce46d942af68356b38\"", + "text/xml", false); + + assertResponseParserClassEquals(method, request, ETagFromHttpResponseViaRegex.class); + assertSaxResponseParserClassEquals(method, null); + assertExceptionParserClassEquals(method, MapHttp4xxCodesToExceptions.class); + + checkFilters(request); + } + public void testPutBucketEu() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException, - NoSuchMethodException, IOException { - Method method = S3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class, - Array.newInstance(PutBucketOptions.class, 0).getClass()); + NoSuchMethodException, IOException { + Method method = AWSS3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class, Array + .newInstance(PutBucketOptions.class, 0).getClass()); HttpRequest request = processor.createRequest(method, "EU", "bucket"); assertRequestLineEquals(request, "PUT https://bucket." + url + "/ HTTP/1.1"); assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n"); assertPayloadEquals(request, - "EU", - "text/xml", false); + "EU", + "text/xml", false); assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class); assertSaxResponseParserClassEquals(method, null); @@ -66,4 +166,24 @@ public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest { checkFilters(request); } + + @RequiresHttp + @ConfiguresRestClient + private static final class TestAWSS3RestClientModule extends AWSS3RestClientModule { + + public TestAWSS3RestClientModule() { + super(); + } + + @Override + protected String provideTimeStamp(@TimeStamp Supplier cache) { + return "2009-11-08T15:54:08.897Z"; + } + } + + @Override + protected Module createModule() { + return new TestAWSS3RestClientModule(); + } + } diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java index 94a69e6c54..013775c81a 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java @@ -19,9 +19,29 @@ package org.jclouds.aws.s3; +import static org.testng.Assert.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.GZIPInputStream; + +import org.jclouds.crypto.CryptoStreams; +import org.jclouds.http.BaseJettyTest; +import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; import org.jclouds.s3.S3ClientLiveTest; +import org.jclouds.s3.domain.ObjectMetadataBuilder; +import org.testng.ITestContext; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.ByteStreams; +import com.google.common.io.InputSupplier; +import com.google.inject.Module; + /** * Tests behavior of {@code S3Client} * @@ -29,5 +49,59 @@ import org.testng.annotations.Test; */ @Test(groups = "live", sequential = true, testName = "AWSS3ClientLiveTest") public class AWSS3ClientLiveTest extends S3ClientLiveTest { + private InputSupplier oneHundredOneConstitutions; + private byte[] oneHundredOneConstitutionsMD5; + private static long oneHundredOneConstitutionsLength; + + @Override + public AWSS3Client getApi() { + return (AWSS3Client) context.getProviderSpecificContext().getApi(); + } + + @BeforeClass(groups = { "integration", "live" }) + @Override + public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { + super.setUpResourcesOnThisThread(testContext); + oneHundredOneConstitutions = getTestDataSupplier(); + oneHundredOneConstitutionsMD5 = CryptoStreams.md5(oneHundredOneConstitutions); + } + + @SuppressWarnings("unchecked") + public static InputSupplier getTestDataSupplier() throws IOException { + byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class + .getResourceAsStream("/const.txt.gz"))); + InputSupplier constitutionSupplier = ByteStreams.newInputStreamSupplier(oneConstitution); + + InputSupplier temp = ByteStreams.join(constitutionSupplier); + // we have to go beyond 5MB per part + for (oneHundredOneConstitutionsLength = oneConstitution.length; oneHundredOneConstitutionsLength < 5 * 1024 * 1024; oneHundredOneConstitutionsLength += oneConstitution.length) { + temp = ByteStreams.join(temp, constitutionSupplier); + } + return temp; + } + + public void testMultipartSynchronously() throws InterruptedException, IOException { + String containerName = getContainerName(); + try { + String key = "constitution.txt"; + String uploadId = getApi().initiateMultipartUpload(containerName, + ObjectMetadataBuilder.create().key(key).build()); + + byte[] buffer = ByteStreams.toByteArray(oneHundredOneConstitutions.getInput()); + Payload part1 = Payloads.newByteArrayPayload(buffer); + part1.getContentMetadata().setContentLength((long) buffer.length); + part1.getContentMetadata().setContentMD5(oneHundredOneConstitutionsMD5); + + // failure here looks very similar to http://java.net/jira/browse/GLASSFISH-15773 + String eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1); + + String eTag = getApi().completeMultipartUpload(containerName, key, uploadId, ImmutableMap.of(1, eTagOf1)); + + assertEquals(eTagOf1, eTag); + + } finally { + returnContainer(containerName); + } + } } diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequestTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequestTest.java new file mode 100644 index 0000000000..2606b6ea43 --- /dev/null +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequestTest.java @@ -0,0 +1,104 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3.binders; + +import static org.testng.Assert.assertEquals; + +import java.io.File; +import java.net.URI; + +import javax.ws.rs.HttpMethod; + +import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix; +import org.jclouds.http.HttpRequest; +import org.jclouds.rest.internal.RestAnnotationProcessor; +import org.jclouds.s3.BaseS3AsyncClientTest; +import org.jclouds.s3.S3AsyncClient; +import org.jclouds.s3.domain.ObjectMetadata; +import org.jclouds.s3.domain.ObjectMetadataBuilder; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.inject.TypeLiteral; + +/** + * Tests behavior of {@code BindObjectMetadataToRequest} + * + * @author Adrian Cole + */ +// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire +@Test(groups = "unit", testName = "BindObjectMetadataToRequestTest") +public class BindObjectMetadataToRequestTest extends BaseS3AsyncClientTest { + + @Override + protected TypeLiteral> createTypeLiteral() { + return new TypeLiteral>() { + }; + } + + @Test + public void testPassWithMinimumDetailsAndPayload5GB() { + ObjectMetadata md = ObjectMetadataBuilder.create().key("foo").build(); + + HttpRequest request = HttpRequest.builder().method("POST").endpoint(URI.create("http://localhost")).build(); + BindObjectMetadataToRequest binder = injector.getInstance(BindObjectMetadataToRequest.class); + + assertEquals(binder.bindToRequest(request, md), HttpRequest.builder().method("POST").endpoint( + URI.create("http://localhost")).headers(ImmutableMultimap.of("Content-Type", "binary/octet-stream")) + .build()); + } + + @Test + public void testExtendedPropertiesBind() { + ObjectMetadata md = ObjectMetadataBuilder.create().key("foo").cacheControl("no-cache").userMetadata( + ImmutableMap.of("foo", "bar")).build(); + + HttpRequest request = HttpRequest.builder().method("POST").endpoint(URI.create("http://localhost")).build(); + BindObjectMetadataToRequest binder = injector.getInstance(BindObjectMetadataToRequest.class); + + assertEquals(binder.bindToRequest(request, md), HttpRequest.builder().method("POST").endpoint( + URI.create("http://localhost")).headers( + ImmutableMultimap.of("Cache-Control", "no-cache", "x-amz-meta-foo", "bar", "Content-Type", + "binary/octet-stream")).build()); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNoKeyIsBad() { + ObjectMetadata md = ObjectMetadataBuilder.create().build(); + + HttpRequest request = HttpRequest.builder().method("POST").endpoint(URI.create("http://localhost")).build(); + BindObjectMetadataToRequest binder = injector.getInstance(BindObjectMetadataToRequest.class); + binder.bindToRequest(request, md); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testMustBeObjectMetadata() { + HttpRequest request = new HttpRequest(HttpMethod.POST, URI.create("http://localhost")); + injector.getInstance(BindObjectMetadataToRequest.class).bindToRequest(request, new File("foo")); + } + + @Test(expectedExceptions = { NullPointerException.class, IllegalStateException.class }) + public void testNullIsBad() { + BindMapToHeadersWithPrefix binder = new BindMapToHeadersWithPrefix("prefix:"); + HttpRequest request = HttpRequest.builder().method("GET").endpoint(URI.create("http://momma")).build(); + binder.bindToRequest(request, null); + } +} diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequestTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequestTest.java new file mode 100644 index 0000000000..f898bf2b7a --- /dev/null +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequestTest.java @@ -0,0 +1,72 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3.binders; + +import static org.testng.Assert.assertEquals; + +import java.net.URI; + +import javax.ws.rs.core.MediaType; + +import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix; +import org.jclouds.http.HttpRequest; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableMap; + +/** + * Tests behavior of {@code BindPartIdsAndETagsToRequest} + * + * @author Adrian Cole + */ +// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire +@Test(groups = "unit", testName = "BindPartIdsAndETagsToRequestTest") +public class BindPartIdsAndETagsToRequestTest { + BindPartIdsAndETagsToRequest binder = new BindPartIdsAndETagsToRequest(); + + @Test + public void testPassWithMinimumDetailsAndPayload5GB() { + HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build(); + Payload payload = Payloads + .newStringPayload("1\"a54357aff0632cce46d942af68356b38\""); + payload.getContentMetadata().setContentType(MediaType.TEXT_XML); + request = binder.bindToRequest(request, ImmutableMap. of(1, + "\"a54357aff0632cce46d942af68356b38\"")); + assertEquals(request.getPayload().getRawContent(), payload.getRawContent()); + assertEquals(request, HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).payload( + payload).build()); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testEmptyIsBad() { + + HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build(); + binder.bindToRequest(request, ImmutableMap. of()); + } + + @Test(expectedExceptions = { NullPointerException.class, IllegalStateException.class }) + public void testNullIsBad() { + BindMapToHeadersWithPrefix binder = new BindMapToHeadersWithPrefix("prefix:"); + HttpRequest request = HttpRequest.builder().method("GET").endpoint(URI.create("http://momma")).build(); + binder.bindToRequest(request, null); + } +} diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegexTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegexTest.java new file mode 100644 index 0000000000..f79a22bf2d --- /dev/null +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegexTest.java @@ -0,0 +1,49 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3.functions; + +import static org.testng.Assert.assertEquals; + +import org.jclouds.http.HttpResponse; +import org.jclouds.http.functions.ReturnStringIf2xx; +import org.jclouds.io.Payloads; +import org.testng.annotations.Test; + +/** + * Tests behavior of {@code ETagFromHttpResponseViaRegex} + * + * @author Adrian Cole + */ +// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire +@Test(groups = "unit", testName = "ETagFromHttpResponseViaRegexTest") +public class ETagFromHttpResponseViaRegexTest { + + @Test + public void test() { + + HttpResponse response = HttpResponse.builder().statusCode(200).payload( + Payloads.newInputStreamPayload(getClass().getResourceAsStream("/complete-multipart-upload.xml"))) + .build(); + ETagFromHttpResponseViaRegex parser = new ETagFromHttpResponseViaRegex(new ReturnStringIf2xx()); + + assertEquals(parser.apply(response), "\"3858f62230ac3c915f300c664312c11f-9\""); + } + +} diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegexTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegexTest.java new file mode 100644 index 0000000000..c6f84e08fd --- /dev/null +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegexTest.java @@ -0,0 +1,49 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.aws.s3.functions; + +import static org.testng.Assert.assertEquals; + +import org.jclouds.http.HttpResponse; +import org.jclouds.http.functions.ReturnStringIf2xx; +import org.jclouds.io.Payloads; +import org.testng.annotations.Test; + +/** + * Tests behavior of {@code UploadIdFromHttpResponseViaRegex} + * + * @author Adrian Cole + */ +// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire +@Test(groups = "unit", testName = "UploadIdFromHttpResponseViaRegexTest") +public class UploadIdFromHttpResponseViaRegexTest { + + @Test + public void test() { + + HttpResponse response = HttpResponse.builder().statusCode(200).payload( + Payloads.newInputStreamPayload(getClass().getResourceAsStream("/initiate-multipart-upload.xml"))) + .build(); + UploadIdFromHttpResponseViaRegex parser = new UploadIdFromHttpResponseViaRegex(new ReturnStringIf2xx()); + + assertEquals(parser.apply(response), "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"); + } + +} diff --git a/providers/aws-s3/src/test/resources/complete-multipart-upload.xml b/providers/aws-s3/src/test/resources/complete-multipart-upload.xml new file mode 100644 index 0000000000..d24e65ca5a --- /dev/null +++ b/providers/aws-s3/src/test/resources/complete-multipart-upload.xml @@ -0,0 +1,7 @@ + + + http://Example-Bucket.s3.amazonaws.com/Example-Object + Example-Bucket + Example-Object + "3858f62230ac3c915f300c664312c11f-9" + \ No newline at end of file diff --git a/providers/aws-s3/src/test/resources/initiate-multipart-upload.xml b/providers/aws-s3/src/test/resources/initiate-multipart-upload.xml new file mode 100644 index 0000000000..7b9a0aa09b --- /dev/null +++ b/providers/aws-s3/src/test/resources/initiate-multipart-upload.xml @@ -0,0 +1,6 @@ + + + example-bucket + example-object + VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA + \ No newline at end of file diff --git a/providers/aws-s3/src/test/resources/log4j.xml b/providers/aws-s3/src/test/resources/log4j.xml new file mode 100644 index 0000000000..04c2a146d0 --- /dev/null +++ b/providers/aws-s3/src/test/resources/log4j.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/providers/googlestorage/src/main/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequest.java b/providers/googlestorage/src/main/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequest.java index 4273e546ca..b9b4ea51d7 100644 --- a/providers/googlestorage/src/main/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequest.java +++ b/providers/googlestorage/src/main/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequest.java @@ -26,11 +26,10 @@ import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.core.HttpHeaders; -import org.jclouds.blobstore.binders.BindUserMetadataToHeadersWithPrefix; +import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix; import org.jclouds.http.HttpRequest; import org.jclouds.http.utils.ModifyRequest; import org.jclouds.s3.binders.BindS3ObjectMetadataToRequest; -import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.domain.S3Object; /** @@ -39,15 +38,9 @@ import org.jclouds.s3.domain.S3Object; */ @Singleton public class BindGoogleStorageObjectMetadataToRequest extends BindS3ObjectMetadataToRequest { - private final BindUserMetadataToHeadersWithPrefix blobBinder; - private final ObjectToBlob object2Blob; - @Inject - public BindGoogleStorageObjectMetadataToRequest(ObjectToBlob object2Blob, - BindUserMetadataToHeadersWithPrefix blobBinder) { - super(object2Blob, blobBinder); - this.blobBinder = checkNotNull(blobBinder, "blobBinder"); - this.object2Blob = checkNotNull(object2Blob, "object2Blob"); + public BindGoogleStorageObjectMetadataToRequest(BindMapToHeadersWithPrefix metadataPrefixer) { + super(metadataPrefixer); } @Override @@ -67,7 +60,7 @@ public class BindGoogleStorageObjectMetadataToRequest extends BindS3ObjectMetada request = ModifyRequest.replaceHeader(request, "Transfer-Encoding", "chunked"); } - request = blobBinder.bindToRequest(request, object2Blob.apply(s3Object)); + request = metadataPrefixer.bindToRequest(request, s3Object.getMetadata().getUserMetadata()); if (s3Object.getMetadata().getCacheControl() != null) { request = ModifyRequest.replaceHeader(request, HttpHeaders.CACHE_CONTROL, s3Object.getMetadata() diff --git a/providers/googlestorage/src/test/java/org/jclouds/googlestorage/GoogleStorageAsyncClientTestDisabled.java b/providers/googlestorage/src/test/java/org/jclouds/googlestorage/GoogleStorageAsyncClientTestDisabled.java index 93de2e29a8..75ac09f42d 100644 --- a/providers/googlestorage/src/test/java/org/jclouds/googlestorage/GoogleStorageAsyncClientTestDisabled.java +++ b/providers/googlestorage/src/test/java/org/jclouds/googlestorage/GoogleStorageAsyncClientTestDisabled.java @@ -19,19 +19,29 @@ package org.jclouds.googlestorage; +import org.jclouds.rest.internal.RestAnnotationProcessor; +import org.jclouds.s3.S3AsyncClient; import org.testng.annotations.Test; +import com.google.inject.TypeLiteral; + /** * @author Adrian Cole */ // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(enabled = false, groups = "unit", testName = "GoogleStorageAsyncClientTest") -public class GoogleStorageAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest { +public class GoogleStorageAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest { public GoogleStorageAsyncClientTestDisabled() { this.provider = "googlestorage"; this.url = "commondatastorage.googleapis.com"; } + @Override + protected TypeLiteral> createTypeLiteral() { + return new TypeLiteral>() { + }; + } + // TODO parameterize this test so that it can pass } diff --git a/providers/googlestorage/src/test/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequestTest.java b/providers/googlestorage/src/test/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequestTest.java index 90d2673220..a90580fd75 100644 --- a/providers/googlestorage/src/test/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequestTest.java +++ b/providers/googlestorage/src/test/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequestTest.java @@ -10,12 +10,15 @@ import javax.ws.rs.HttpMethod; import org.jclouds.http.HttpRequest; import org.jclouds.io.Payload; import org.jclouds.io.Payloads; +import org.jclouds.rest.internal.RestAnnotationProcessor; import org.jclouds.s3.BaseS3AsyncClientTest; +import org.jclouds.s3.S3AsyncClient; import org.jclouds.s3.domain.S3Object; import org.testng.annotations.Test; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; +import com.google.inject.TypeLiteral; /** * Tests behavior of {@code BindGoogleStorageObjectMetadataToRequest} @@ -24,8 +27,13 @@ import com.google.common.collect.ImmutableMultimap; */ // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(groups = "unit", testName = "BindGoogleStorageObjectMetadataToRequestTest") -public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncClientTest { - +public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncClientTest { + + @Override + protected TypeLiteral> createTypeLiteral() { + return new TypeLiteral>() { + }; + } @Test public void testPassWithMinimumDetailsAndPayload5GB() { @@ -36,7 +44,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli object.getMetadata().setKey("foo"); HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build(); - BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class); + BindGoogleStorageObjectMetadataToRequest binder = injector + .getInstance(BindGoogleStorageObjectMetadataToRequest.class); assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint( URI.create("http://localhost")).build()); @@ -52,7 +61,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli object.getMetadata().getUserMetadata().putAll(ImmutableMap.of("foo", "bar")); HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build(); - BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class); + BindGoogleStorageObjectMetadataToRequest binder = injector + .getInstance(BindGoogleStorageObjectMetadataToRequest.class); assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint( URI.create("http://localhost")).headers(ImmutableMultimap.of("x-amz-meta-foo", "bar")).build()); @@ -66,7 +76,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli object.getMetadata().setKey("foo"); HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build(); - BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class); + BindGoogleStorageObjectMetadataToRequest binder = injector + .getInstance(BindGoogleStorageObjectMetadataToRequest.class); assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint( URI.create("http://localhost")).headers(ImmutableMultimap.of("Transfer-Encoding", "chunked")).build()); @@ -80,7 +91,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli object.setPayload(payload); HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build(); - BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class); + BindGoogleStorageObjectMetadataToRequest binder = injector + .getInstance(BindGoogleStorageObjectMetadataToRequest.class); binder.bindToRequest(request, object); } @@ -93,7 +105,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli object.getMetadata().setKey("foo"); HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build(); - BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class); + BindGoogleStorageObjectMetadataToRequest binder = injector + .getInstance(BindGoogleStorageObjectMetadataToRequest.class); binder.bindToRequest(request, object); } @@ -105,7 +118,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli @Test(expectedExceptions = { NullPointerException.class, IllegalStateException.class }) public void testNullIsBad() { - BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class); + BindGoogleStorageObjectMetadataToRequest binder = injector + .getInstance(BindGoogleStorageObjectMetadataToRequest.class); HttpRequest request = HttpRequest.builder().method("GET").endpoint(URI.create("http://momma")).build(); binder.bindToRequest(request, null); } diff --git a/providers/scaleup-storage/src/test/java/org/jclouds/scaleup/storage/ScaleUpStorageAsyncClientTestDisabled.java b/providers/scaleup-storage/src/test/java/org/jclouds/scaleup/storage/ScaleUpStorageAsyncClientTestDisabled.java index d70a578b5e..e9e693728f 100644 --- a/providers/scaleup-storage/src/test/java/org/jclouds/scaleup/storage/ScaleUpStorageAsyncClientTestDisabled.java +++ b/providers/scaleup-storage/src/test/java/org/jclouds/scaleup/storage/ScaleUpStorageAsyncClientTestDisabled.java @@ -19,14 +19,25 @@ package org.jclouds.scaleup.storage; +import org.jclouds.rest.internal.RestAnnotationProcessor; +import org.jclouds.s3.S3AsyncClient; +import org.jclouds.s3.S3AsyncClientTest; import org.testng.annotations.Test; +import com.google.inject.TypeLiteral; + /** * @author Adrian Cole */ // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(enabled = false, groups = "unit", testName = "ScaleUpStorageAsyncClientTest") -public class ScaleUpStorageAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest { +public class ScaleUpStorageAsyncClientTestDisabled extends S3AsyncClientTest { + + @Override + protected TypeLiteral> createTypeLiteral() { + return new TypeLiteral>() { + }; + } public ScaleUpStorageAsyncClientTestDisabled() { this.provider = "scaleup-storage"; diff --git a/sandbox-apis/cloudstack/src/test/java/org/jclouds/cloudstack/features/VirtualMachineClientLiveTest.java b/sandbox-apis/cloudstack/src/test/java/org/jclouds/cloudstack/features/VirtualMachineClientLiveTest.java index 0e4b20f02f..eac269d799 100644 --- a/sandbox-apis/cloudstack/src/test/java/org/jclouds/cloudstack/features/VirtualMachineClientLiveTest.java +++ b/sandbox-apis/cloudstack/src/test/java/org/jclouds/cloudstack/features/VirtualMachineClientLiveTest.java @@ -21,12 +21,14 @@ package org.jclouds.cloudstack.features; import static com.google.common.base.Predicates.equalTo; import static com.google.common.base.Predicates.or; +import static com.google.common.collect.Iterables.filter; import static com.google.common.collect.Iterables.find; import static com.google.common.collect.Iterables.get; import static com.google.common.collect.Iterables.getOnlyElement; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.util.NoSuchElementException; import java.util.Set; import org.jclouds.cloudstack.CloudStackClient; @@ -72,15 +74,30 @@ public class VirtualMachineClientLiveTest extends BaseCloudStackClientLiveTest { long serviceOfferingId = DEFAULT_SIZE_ORDERING.min(client.getOfferingClient().listServiceOfferings()).getId(); - long templateId = find(client.getTemplateClient().listTemplates(), new Predicate