From 74b404eab8a2dac84d80dfedf4b985fec5c80f87 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Fri, 6 Jul 2012 22:54:15 -0700 Subject: [PATCH] Issue 1008:aws-s3 could not get location for region eu-west-1 error --- .../java/org/jclouds/s3/S3AsyncClient.java | 87 +++++++------ .../main/java/org/jclouds/s3/S3Client.java | 2 +- .../s3/blobstore/S3AsyncBlobStore.java | 22 ++-- .../org/jclouds/s3/blobstore/S3BlobStore.java | 34 +++--- .../config/S3BlobStoreContextModule.java | 11 +- .../functions/BucketToResourceMetadata.java | 6 +- .../functions/BucketsToStorageMetadata.java | 86 +++++++++++++ .../functions/LocationFromBucketLocation.java | 87 ------------- .../functions/LocationFromBucketName.java | 75 ++++++++++++ .../functions/ObjectToBlobMetadata.java | 6 +- .../jclouds/s3/config/S3RestClientModule.java | 91 +++++++++++++- .../org/jclouds/s3/domain/BucketMetadata.java | 2 +- .../s3/filters/RequestAuthorizeSignature.java | 31 +---- .../AssignCorrectHostnameForBucket.java | 64 ++++++++++ ... DefaultEndpointThenInvalidateRegion.java} | 37 ++++-- .../s3/functions/GetRegionForBucket.java | 65 ++++++++++ ...YouOrOperationAbortedWhenBucketExists.java | 70 +++++++++++ ...rueOn404OrNotFoundFalseOnIllegalState.java | 1 + .../java/org/jclouds/s3/util/S3Utils.java | 31 +++++ .../s3/xml/LocationConstraintHandler.java | 13 +- .../org/jclouds/s3/S3AsyncClientTest.java | 6 +- .../BucketsToStorageMetadataTest.java | 70 +++++++++++ .../functions/LocationFromBucketNameTest.java | 89 ++++++++++++++ .../AssignCorrectHostnameForBucketTest.java | 86 +++++++++++++ ...faultEndpointThenInvalidateRegionTest.java | 56 +++++++++ .../s3/functions/GetRegionForBucketTest.java | 81 +++++++++++++ ...etAlreadyOwnedByYouOrIllegalStateTest.java | 54 --------- ...rOperationAbortedWhenBucketExistsTest.java | 114 ++++++++++++++++++ .../jclouds/s3/services/BucketsLiveTest.java | 51 +------- .../org/jclouds/walrus/WalrusAsyncClient.java | 4 +- .../internal/DeleteAllKeysInList.java | 12 +- .../internal/BaseBlobIntegrationTest.java | 18 +++ .../BaseBlobStoreIntegrationTest.java | 80 +++++++++--- .../internal/BaseContainerLiveTest.java | 84 ++++++++++++- .../java/org/jclouds/aws/domain/Region.java | 6 +- .../handlers/AWSClientErrorRetryHandler.java | 31 ++--- .../AWSClientErrorRetryHandlerTest.java | 64 ++++++++-- .../org/jclouds/aws/s3/AWSS3AsyncClient.java | 58 +++------ .../jclouds/aws/s3/AWSS3ProviderMetadata.java | 3 +- ...stnameAndBindAsHostPrefixIfConfigured.java | 71 ----------- .../aws/s3/blobstore/AWSS3AsyncBlobStore.java | 16 ++- .../aws/s3/blobstore/AWSS3BlobStore.java | 20 +-- .../aws/s3/config/AWSS3RestClientModule.java | 17 +-- .../jclouds/aws/s3/AWSS3AsyncClientTest.java | 61 ++++++---- .../jclouds/aws/s3/AWSS3ClientExpectTest.java | 28 ++++- ...meAndBindAsHostPrefixIfConfiguredTest.java | 92 -------------- .../integration/AWSS3ContainerLiveTest.java | 79 +----------- .../aws/s3/services/AWSBucketsLiveTest.java | 81 +++++++++++++ ...HPCloudObjectStorageContainerLiveTest.java | 9 -- 49 files changed, 1525 insertions(+), 737 deletions(-) create mode 100644 apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BucketsToStorageMetadata.java delete mode 100644 apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketLocation.java create mode 100644 apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketName.java create mode 100644 apis/s3/src/main/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucket.java rename apis/s3/src/main/java/org/jclouds/s3/functions/{ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.java => DefaultEndpointThenInvalidateRegion.java} (50%) create mode 100644 apis/s3/src/main/java/org/jclouds/s3/functions/GetRegionForBucket.java create mode 100644 apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.java create mode 100644 apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/BucketsToStorageMetadataTest.java create mode 100644 apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/LocationFromBucketNameTest.java create mode 100644 apis/s3/src/test/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucketTest.java create mode 100644 apis/s3/src/test/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegionTest.java create mode 100644 apis/s3/src/test/java/org/jclouds/s3/functions/GetRegionForBucketTest.java delete mode 100644 apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest.java create mode 100644 apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest.java delete mode 100644 providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfigured.java delete mode 100644 providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest.java create mode 100644 providers/aws-s3/src/test/java/org/jclouds/aws/s3/services/AWSBucketsLiveTest.java diff --git a/apis/s3/src/main/java/org/jclouds/s3/S3AsyncClient.java b/apis/s3/src/main/java/org/jclouds/s3/S3AsyncClient.java index b29b572747..749d28c80d 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/S3AsyncClient.java +++ b/apis/s3/src/main/java/org/jclouds/s3/S3AsyncClient.java @@ -41,8 +41,8 @@ import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404; import org.jclouds.http.functions.ParseETagHeader; import org.jclouds.http.options.GetOptions; import org.jclouds.javax.annotation.Nullable; -import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; import org.jclouds.rest.annotations.BinderParam; +import org.jclouds.rest.annotations.Endpoint; import org.jclouds.rest.annotations.EndpointParam; import org.jclouds.rest.annotations.ExceptionParser; import org.jclouds.rest.annotations.Headers; @@ -69,11 +69,13 @@ import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.domain.Payer; import org.jclouds.s3.domain.S3Object; import org.jclouds.s3.filters.RequestAuthorizeSignature; +import org.jclouds.s3.functions.AssignCorrectHostnameForBucket; import org.jclouds.s3.functions.BindRegionToXmlPayload; +import org.jclouds.s3.functions.DefaultEndpointThenInvalidateRegion; import org.jclouds.s3.functions.ObjectKey; import org.jclouds.s3.functions.ParseObjectFromHeadersAndHttpContent; import org.jclouds.s3.functions.ParseObjectMetadataFromHeaders; -import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState; +import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists; import org.jclouds.s3.functions.ReturnTrueOn404OrNotFoundFalseOnIllegalState; import org.jclouds.s3.options.CopyObjectOptions; import org.jclouds.s3.options.ListBucketOptions; @@ -123,8 +125,8 @@ public interface S3AsyncClient { @ExceptionParser(ReturnNullOnKeyNotFound.class) @ResponseParser(ParseObjectFromHeadersAndHttpContent.class) ListenableFuture getObject( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key, GetOptions... options); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, GetOptions... options); /** * @see S3Client#headObject @@ -134,8 +136,8 @@ public interface S3AsyncClient { @ExceptionParser(ReturnNullOnKeyNotFound.class) @ResponseParser(ParseObjectMetadataFromHeaders.class) ListenableFuture headObject( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key); /** * @see S3Client#objectExists @@ -144,8 +146,8 @@ public interface S3AsyncClient { @Path("/{key}") @ExceptionParser(ReturnFalseOnKeyNotFound.class) ListenableFuture objectExists( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key); /** * @see S3Client#deleteObject @@ -154,8 +156,8 @@ public interface S3AsyncClient { @Path("/{key}") @ExceptionParser(ReturnVoidOnNotFoundOr404.class) ListenableFuture deleteObject( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key); /** * @see S3Client#putObject @@ -164,39 +166,43 @@ public interface S3AsyncClient { @Path("/{key}") @ResponseParser(ParseETagHeader.class) ListenableFuture putObject( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") @ParamParser(ObjectKey.class) @BinderParam(BindS3ObjectMetadataToRequest.class) S3Object object, - PutObjectOptions... options); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") @ParamParser(ObjectKey.class) @BinderParam(BindS3ObjectMetadataToRequest.class) S3Object object, + PutObjectOptions... options); /** * @see S3Client#putBucketInRegion */ @PUT @Path("/") - @ExceptionParser(ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class) + @Endpoint(Bucket.class) + @ExceptionParser(ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.class) ListenableFuture putBucketInRegion( - @EndpointParam(parser = RegionToEndpointOrProviderIfNull.class) @BinderParam(BindRegionToXmlPayload.class) @Nullable String region, - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - PutBucketOptions... options); + @BinderParam(BindRegionToXmlPayload.class) @Nullable String region, + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + PutBucketOptions... options); /** * @see S3Client#deleteBucketIfEmpty */ @DELETE @Path("/") + @Endpoint(Bucket.class) @ExceptionParser(ReturnTrueOn404OrNotFoundFalseOnIllegalState.class) ListenableFuture deleteBucketIfEmpty( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @EndpointParam(parser = DefaultEndpointThenInvalidateRegion.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); /** * @see S3Client#bucketExists */ @HEAD @Path("/") + @Endpoint(Bucket.class) @QueryParams(keys = "max-keys", values = "0") @ExceptionParser(ReturnFalseOnContainerNotFound.class) ListenableFuture bucketExists( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); + /** * @see S3Client#getBucketLocation @@ -204,9 +210,10 @@ public interface S3AsyncClient { @GET @QueryParams(keys = "location") @Path("/") + @Endpoint(Bucket.class) @XMLResponseParser(LocationConstraintHandler.class) ListenableFuture getBucketLocation( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); /** * @see S3Client#getBucketPayer @@ -216,7 +223,7 @@ public interface S3AsyncClient { @Path("/") @XMLResponseParser(PayerHandler.class) ListenableFuture getBucketPayer( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); /** * @see S3Client#setBucketPayer @@ -225,8 +232,8 @@ public interface S3AsyncClient { @QueryParams(keys = "requestPayment") @Path("/") ListenableFuture setBucketPayer( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @BinderParam(BindPayerToXmlPayload.class) Payer payer); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @BinderParam(BindPayerToXmlPayload.class) Payer payer); /** * @see S3Client#listBucket @@ -235,8 +242,8 @@ public interface S3AsyncClient { @Path("/") @XMLResponseParser(ListBucketHandler.class) ListenableFuture listBucket( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - ListBucketOptions... options); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + ListBucketOptions... options); /** * @see S3Client#listOwnedBuckets @@ -255,10 +262,10 @@ public interface S3AsyncClient { @Headers(keys = "x-amz-copy-source", values = "/{sourceBucket}/{sourceObject}") @XMLResponseParser(CopyObjectHandler.class) ListenableFuture copyObject( - @PathParam("sourceBucket") String sourceBucket, - @PathParam("sourceObject") String sourceObject, - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String destinationBucket, - @PathParam("destinationObject") String destinationObject, CopyObjectOptions... options); + @PathParam("sourceBucket") String sourceBucket, + @PathParam("sourceObject") String sourceObject, + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String destinationBucket, + @PathParam("destinationObject") String destinationObject, CopyObjectOptions... options); /** * @see S3Client#getBucketACL @@ -269,7 +276,7 @@ public interface S3AsyncClient { @ExceptionParser(ThrowContainerNotFoundOn404.class) @Path("/") ListenableFuture getBucketACL( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); /** * @see S3Client#putBucketACL @@ -278,8 +285,8 @@ public interface S3AsyncClient { @Path("/") @QueryParams(keys = "acl") ListenableFuture putBucketACL( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @BinderParam(BindACLToXMLPayload.class) AccessControlList acl); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @BinderParam(BindACLToXMLPayload.class) AccessControlList acl); /** * @see S3Client#getObjectACL @@ -290,8 +297,8 @@ public interface S3AsyncClient { @XMLResponseParser(AccessControlListHandler.class) @ExceptionParser(ThrowKeyNotFoundOn404.class) ListenableFuture getObjectACL( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key); /** * @see S3Client#putObjectACL @@ -300,8 +307,8 @@ public interface S3AsyncClient { @QueryParams(keys = "acl") @Path("/{key}") ListenableFuture putObjectACL( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key, @BinderParam(BindACLToXMLPayload.class) AccessControlList acl); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, @BinderParam(BindACLToXMLPayload.class) AccessControlList acl); /** * @see S3Client#getBucketLogging @@ -312,7 +319,7 @@ public interface S3AsyncClient { @ExceptionParser(ThrowContainerNotFoundOn404.class) @Path("/") ListenableFuture getBucketLogging( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); /** * @see S3Client#enableBucketLogging @@ -321,8 +328,8 @@ public interface S3AsyncClient { @Path("/") @QueryParams(keys = "logging") ListenableFuture enableBucketLogging( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @BinderParam(BindBucketLoggingToXmlPayload.class) BucketLogging logging); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @BinderParam(BindBucketLoggingToXmlPayload.class) BucketLogging logging); /** * @see S3Client#putBucketLogging @@ -332,6 +339,6 @@ public interface S3AsyncClient { @QueryParams(keys = "logging") @Produces(MediaType.TEXT_XML) ListenableFuture disableBucketLogging( - @Bucket @BinderParam(BindNoBucketLoggingToXmlPayload.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindNoBucketLoggingToXmlPayload.class) @ParamValidators(BucketNameValidator.class) String bucketName); } diff --git a/apis/s3/src/main/java/org/jclouds/s3/S3Client.java b/apis/s3/src/main/java/org/jclouds/s3/S3Client.java index 84dd24a71c..aff4cc5f40 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/S3Client.java +++ b/apis/s3/src/main/java/org/jclouds/s3/S3Client.java @@ -196,7 +196,7 @@ public interface S3Client { * */ @Timeout(duration = 90, timeUnit = TimeUnit.SECONDS) - boolean putBucketInRegion(@Nullable String region, String bucketName, PutBucketOptions... options); + boolean putBucketInRegion(@Nullable String region, @Bucket String bucketName, PutBucketOptions... options); /** * Deletes the bucket, if it is empty. diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3AsyncBlobStore.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3AsyncBlobStore.java index 3bded56016..bf77d74e49 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3AsyncBlobStore.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3AsyncBlobStore.java @@ -34,7 +34,6 @@ import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; -import org.jclouds.blobstore.domain.internal.PageSetImpl; import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.internal.BaseAsyncBlobStore; import org.jclouds.blobstore.options.CreateContainerOptions; @@ -50,7 +49,6 @@ import org.jclouds.s3.S3AsyncClient; import org.jclouds.s3.S3Client; import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BucketToResourceList; -import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; @@ -70,7 +68,6 @@ import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.google.common.collect.Iterables; import com.google.common.util.concurrent.ListenableFuture; /** @@ -82,7 +79,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore { private final S3AsyncClient async; private final S3Client sync; - private final BucketToResourceMetadata bucket2ResourceMd; + private final Function, PageSet> convertBucketsToStorageMetadata; private final ContainerToBucketListOptions container2BucketListOptions; private final BlobToHttpGetOptions blob2ObjectGetOptions; private final BucketToResourceList bucket2ResourceList; @@ -94,17 +91,18 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore { @Inject protected S3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier defaultLocation, - @Memoized Supplier> locations, S3AsyncClient async, S3Client sync, - BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, - BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, - BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, - Provider fetchBlobMetadataProvider, LoadingCache bucketAcls) { + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier defaultLocation, + @Memoized Supplier> locations, S3AsyncClient async, S3Client sync, + Function, PageSet> convertBucketsToStorageMetadata, + ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList, + ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, + ObjectToBlobMetadata object2BlobMd, Provider fetchBlobMetadataProvider, + LoadingCache bucketAcls) { super(context, blobUtils, service, defaultLocation, locations); this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions"); this.async = checkNotNull(async, "async"); this.sync = checkNotNull(sync, "sync"); - this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd"); + this.convertBucketsToStorageMetadata = checkNotNull(convertBucketsToStorageMetadata, "convertBucketsToStorageMetadata"); this.container2BucketListOptions = checkNotNull(container2BucketListOptions, "container2BucketListOptions"); this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList"); this.object2Blob = checkNotNull(object2Blob, "object2Blob"); @@ -122,7 +120,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore { return Futures.compose(async.listOwnedBuckets(), new Function, org.jclouds.blobstore.domain.PageSet>() { public org.jclouds.blobstore.domain.PageSet apply(Set from) { - return new PageSetImpl(Iterables.transform(from, bucket2ResourceMd), null); + return convertBucketsToStorageMetadata.apply(from); } }, service); } diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java index 3c75d3d3c2..4d6b295efa 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java @@ -27,11 +27,11 @@ import javax.inject.Provider; import javax.inject.Singleton; import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; -import org.jclouds.blobstore.domain.internal.PageSetImpl; import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.internal.BaseBlobStore; import org.jclouds.blobstore.options.CreateContainerOptions; @@ -45,7 +45,6 @@ import org.jclouds.http.options.GetOptions; import org.jclouds.s3.S3Client; import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BucketToResourceList; -import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; @@ -64,7 +63,6 @@ import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.google.common.collect.Iterables; /** * @@ -73,7 +71,7 @@ import com.google.common.collect.Iterables; @Singleton public class S3BlobStore extends BaseBlobStore { private final S3Client sync; - private final BucketToResourceMetadata bucket2ResourceMd; + private final Function, PageSet> convertBucketsToStorageMetadata; private final ContainerToBucketListOptions container2BucketListOptions; private final BucketToResourceList bucket2ResourceList; private final ObjectToBlob object2Blob; @@ -85,15 +83,16 @@ public class S3BlobStore extends BaseBlobStore { @Inject protected S3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier defaultLocation, - @Memoized Supplier> locations, S3Client sync, - BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, - BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, - BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, - Provider fetchBlobMetadataProvider, LoadingCache bucketAcls) { + @Memoized Supplier> locations, S3Client sync, + Function, PageSet> convertBucketsToStorageMetadata, + ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList, + ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, + ObjectToBlobMetadata object2BlobMd, Provider fetchBlobMetadataProvider, + LoadingCache bucketAcls) { super(context, blobUtils, defaultLocation, locations); this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions"); this.sync = checkNotNull(sync, "sync"); - this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd"); + this.convertBucketsToStorageMetadata = checkNotNull(convertBucketsToStorageMetadata, "convertBucketsToStorageMetadata"); this.container2BucketListOptions = checkNotNull(container2BucketListOptions, "container2BucketListOptions"); this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList"); this.object2Blob = checkNotNull(object2Blob, "object2Blob"); @@ -108,11 +107,7 @@ public class S3BlobStore extends BaseBlobStore { */ @Override public PageSet list() { - return new Function, org.jclouds.blobstore.domain.PageSet>() { - public org.jclouds.blobstore.domain.PageSet apply(Set from) { - return new PageSetImpl(Iterables.transform(from, bucket2ResourceMd), null); - } - }.apply(sync.listOwnedBuckets()); + return convertBucketsToStorageMetadata.apply(sync.listOwnedBuckets()); } /** @@ -169,10 +164,15 @@ public class S3BlobStore extends BaseBlobStore { */ public void clearAndDeleteContainer(final String container) { try { + //TODO: probably it is better to use a retryable predicate if (!Assertions.eventuallyTrue(new Supplier() { public Boolean get() { - clearContainer(container); - return sync.deleteBucketIfEmpty(container); + try { + clearContainer(container); + return sync.deleteBucketIfEmpty(container); + } catch (ContainerNotFoundException e) { + return true; + } } }, 30000)) { throw new IllegalStateException(container + " still exists after deleting!"); diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java index 165257678e..d89d98a392 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java @@ -33,9 +33,8 @@ import org.jclouds.s3.blobstore.S3AsyncBlobStore; import org.jclouds.s3.blobstore.S3BlobRequestSigner; import org.jclouds.s3.blobstore.S3BlobStore; import org.jclouds.s3.blobstore.S3BlobStoreContext; -import org.jclouds.s3.blobstore.functions.LocationFromBucketLocation; +import org.jclouds.s3.blobstore.functions.LocationFromBucketName; import org.jclouds.s3.domain.AccessControlList; -import org.jclouds.s3.domain.BucketMetadata; import com.google.common.base.Function; import com.google.common.cache.CacheBuilder; @@ -60,12 +59,8 @@ public class S3BlobStoreContextModule extends AbstractModule { bind(AsyncBlobStore.class).to(S3AsyncBlobStore.class).in(Scopes.SINGLETON); bind(BlobStore.class).to(S3BlobStore.class).in(Scopes.SINGLETON); bind(BlobRequestSigner.class).to(S3BlobRequestSigner.class); - bindBucketLocationStrategy(); - } - - protected void bindBucketLocationStrategy() { - bind(new TypeLiteral>() { - }).to(LocationFromBucketLocation.class); + bind(new TypeLiteral>() { + }).to(LocationFromBucketName.class); } @Provides diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BucketToResourceMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BucketToResourceMetadata.java index 8fb168c150..0ae946fb81 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BucketToResourceMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BucketToResourceMetadata.java @@ -36,10 +36,10 @@ import com.google.common.base.Function; */ @Singleton public class BucketToResourceMetadata implements Function { - private final Function locationOfBucket; + private final Function locationOfBucket; @Inject - BucketToResourceMetadata(Function locationOfBucket) { + BucketToResourceMetadata(Function locationOfBucket) { this.locationOfBucket = locationOfBucket; } @@ -47,7 +47,7 @@ public class BucketToResourceMetadata implements Function, PageSet> { + + @Resource + protected Logger logger = Logger.NULL; + + private final ExecutorService userExecutor; + private final BucketToResourceMetadata bucket2ResourceMd; + + @Inject + public BucketsToStorageMetadata(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, BucketToResourceMetadata bucket2ResourceMd) { + this.userExecutor = checkNotNull(userExecutor, "userExecutor"); + this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd"); + } + + + @Override + public PageSet apply(Set input) { + // parallel as listing buckets is slow when looking up regions + Iterable buckets = FutureIterables + . transformParallel(input, + new Function>() { + @Override + public Future apply(final BucketMetadata from) { + return userExecutor.submit(new Callable() { + + @Override + public StorageMetadata call() throws Exception { + return bucket2ResourceMd.apply(from); + } + + @Override + public String toString() { + return "bucket2ResourceMd.apply(" + from + ")"; + } + }); + } + + }, userExecutor, null, logger, "my buckets"); + return new PageSetImpl(buckets, null); + } + +} \ No newline at end of file diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketLocation.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketLocation.java deleted file mode 100644 index 7793cb6b83..0000000000 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketLocation.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Licensed to jclouds, Inc. (jclouds) under one or more - * contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. jclouds licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.jclouds.s3.blobstore.functions; - -import java.util.NoSuchElementException; -import java.util.Set; - -import javax.annotation.Resource; -import javax.inject.Inject; -import javax.inject.Singleton; - -import org.jclouds.blobstore.ContainerNotFoundException; -import org.jclouds.collect.Memoized; -import org.jclouds.domain.Location; -import org.jclouds.logging.Logger; -import org.jclouds.s3.S3Client; -import org.jclouds.s3.domain.BucketMetadata; - -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.base.Supplier; -import com.google.common.collect.Iterables; - -/** - * @author Adrian Cole - */ -@Singleton -public class LocationFromBucketLocation implements Function { - private final Location onlyLocation; - private final Supplier> locations; - private final S3Client client; - - @Resource - protected Logger logger = Logger.NULL; - - @Inject - LocationFromBucketLocation(S3Client client, @Memoized Supplier> locations) { - this.client = client; - this.onlyLocation = locations.get().size() == 1 ? Iterables.get(locations.get(), 0) : null; - this.locations = locations; - } - - public Location apply(BucketMetadata from) { - if (onlyLocation != null) - return onlyLocation; - try { - Set locations = this.locations.get(); - final String region = client.getBucketLocation(from.getName()); - assert region != null : String.format("could not get region for %s", from.getName()); - if (region != null) { - try { - return Iterables.find(locations, new Predicate() { - - @Override - public boolean apply(Location input) { - return input.getId().equalsIgnoreCase(region.toString()); - } - - }); - } catch (NoSuchElementException e) { - logger.error("could not get location for region %s in %s", region, locations); - } - } else { - logger.error("could not get region for %s", from.getName()); - } - } catch (ContainerNotFoundException e) { - logger.error(e, "could not get region for %s, as service suggests the bucket doesn't exist", from.getName()); - } - return null; - } -} diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketName.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketName.java new file mode 100644 index 0000000000..f4ec53e5e9 --- /dev/null +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketName.java @@ -0,0 +1,75 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.blobstore.functions; + +import static com.google.common.collect.Iterables.find; +import static com.google.common.collect.Iterables.get; +import static org.jclouds.location.predicates.LocationPredicates.idEquals; + +import java.util.NoSuchElementException; +import java.util.Set; + +import javax.annotation.Resource; +import javax.inject.Inject; +import javax.inject.Singleton; + +import org.jclouds.collect.Memoized; +import org.jclouds.domain.Location; +import org.jclouds.logging.Logger; +import org.jclouds.s3.Bucket; + +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.base.Supplier; + +/** + * @author Adrian Cole + */ +@Singleton +public class LocationFromBucketName implements Function { + private final Supplier> locations; + private final Function> bucketToRegion; + + @Resource + protected Logger logger = Logger.NULL; + + @Inject + LocationFromBucketName(@Bucket Function> bucketToRegion, + @Memoized Supplier> locations) { + this.bucketToRegion = bucketToRegion; + this.locations = locations; + } + + public Location apply(String bucket) { + Set locations = this.locations.get(); + if (locations.size() == 1) + return get(locations, 0); + final Optional region = bucketToRegion.apply(bucket); + if (region.isPresent()) { + try { + return find(locations, idEquals(region.get())); + } catch (NoSuchElementException e) { + logger.debug("could not get location for region %s in %s", region, locations); + } + } else { + logger.debug("could not get region for %s", bucket); + } + return null; + } +} diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java index 30154408e4..3eebc5c3e1 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java @@ -25,6 +25,7 @@ import org.jclouds.blobstore.domain.MutableBlobMetadata; import org.jclouds.blobstore.domain.StorageType; import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl; import org.jclouds.blobstore.strategy.IfDirectoryReturnNameStrategy; +import org.jclouds.domain.Location; import org.jclouds.http.HttpUtils; import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI; @@ -42,12 +43,14 @@ import com.google.common.cache.LoadingCache; public class ObjectToBlobMetadata implements Function { private final IfDirectoryReturnNameStrategy ifDirectoryReturnName; private final LoadingCache bucketAcls; + private final Function locationOfBucket; @Inject public ObjectToBlobMetadata(IfDirectoryReturnNameStrategy ifDirectoryReturnName, - LoadingCache bucketAcls) { + LoadingCache bucketAcls,Function locationOfBucket) { this.ifDirectoryReturnName = ifDirectoryReturnName; this.bucketAcls = bucketAcls; + this.locationOfBucket = locationOfBucket; } public MutableBlobMetadata apply(ObjectMetadata from) { @@ -68,6 +71,7 @@ public class ObjectToBlobMetadata implements Function ext @SuppressWarnings("unchecked") public S3RestClientModule() { - this((TypeToken) TypeToken.of(S3Client.class), (TypeToken) TypeToken.of(S3AsyncClient.class)); + this(TypeToken.class.cast(TypeToken.of(S3Client.class)), TypeToken.class.cast(TypeToken.of(S3AsyncClient.class))); } protected S3RestClientModule(TypeToken syncClientType, TypeToken asyncClientType) { @@ -70,8 +85,61 @@ public class S3RestClientModule ext @Provides @Bucket @Singleton - protected Map bucketToRegion() { - return Maps.newConcurrentMap(); + protected CacheLoader> bucketToRegion(@Region Supplier> regionSupplier, + final S3Client client) { + Set regions = regionSupplier.get(); + if (regions.size() == 0) { + return new CacheLoader>() { + + @Override + public Optional load(String bucket) { + return Optional.absent(); + } + + @Override + public String toString() { + return "noRegions()"; + } + }; + } else if (regions.size() == 1) { + final String onlyRegion = Iterables.getOnlyElement(regions); + return new CacheLoader>() { + Optional onlyRegionOption = Optional.of(onlyRegion); + + @Override + public Optional load(String bucket) { + return onlyRegionOption; + } + + @Override + public String toString() { + return "onlyRegion(" + onlyRegion + ")"; + } + }; + } else { + return new CacheLoader>() { + @Override + public Optional load(String bucket) { + try { + return Optional.fromNullable(client.getBucketLocation(bucket)); + } catch (ContainerNotFoundException e) { + return null; + } + } + + @Override + public String toString() { + return "bucketToRegion()"; + } + }; + } + } + + @Provides + @Bucket + @Singleton + protected LoadingCache> bucketToRegion(@Bucket CacheLoader> loader) { + return CacheBuilder.newBuilder().build(loader); } @Provides @@ -81,12 +149,24 @@ public class S3RestClientModule ext return defaultRegion; } + @Provides + @Singleton + @Bucket + protected Supplier provideBucketURI(@Bucket Supplier defaultRegion, + RegionToEndpointOrProviderIfNull regionToEndpoint) { + return Suppliers.compose(regionToEndpoint, defaultRegion); + } + @Override protected void configure() { super.configure(); install(new S3ObjectModule()); install(new S3ParserModule()); bind(RequestAuthorizeSignature.class).in(Scopes.SINGLETON); + bind(new TypeLiteral>>() { + }).annotatedWith(Bucket.class).to(GetRegionForBucket.class); + bind(new TypeLiteral, PageSet>>() { + }).to(BucketsToStorageMetadata.class); } @Override @@ -105,6 +185,7 @@ public class S3RestClientModule ext @Override protected void bindRetryHandlers() { bind(HttpRetryHandler.class).annotatedWith(Redirection.class).to(S3RedirectionRetryHandler.class); + bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(AWSClientErrorRetryHandler.class); } @Provides diff --git a/apis/s3/src/main/java/org/jclouds/s3/domain/BucketMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/domain/BucketMetadata.java index 27be757a28..9723487824 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/domain/BucketMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/domain/BucketMetadata.java @@ -27,8 +27,8 @@ import java.util.Date; */ public class BucketMetadata implements Comparable { private final Date creationDate; - private final CanonicalUser owner; private final String name; + private final CanonicalUser owner; public BucketMetadata(String name, Date creationDate, CanonicalUser owner) { this.name = name; diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java b/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java index f6a2d20a0d..4d7e86ccf0 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java +++ b/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java @@ -18,8 +18,6 @@ */ package org.jclouds.s3.filters; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.Iterables.any; import static com.google.common.collect.Iterables.get; import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AUTH_TAG; import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG; @@ -29,8 +27,6 @@ import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH; import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS; import static org.jclouds.util.Strings2.toInputStream; -import java.lang.annotation.Annotation; -import java.util.Arrays; import java.util.Collection; import java.util.Locale; import java.util.Map.Entry; @@ -57,11 +53,9 @@ import org.jclouds.logging.Logger; import org.jclouds.rest.RequestSigner; import org.jclouds.rest.annotations.Credential; import org.jclouds.rest.annotations.Identity; -import org.jclouds.rest.internal.GeneratedHttpRequest; -import org.jclouds.s3.Bucket; +import org.jclouds.s3.util.S3Utils; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Predicate; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -81,11 +75,6 @@ import com.google.common.collect.TreeMultimap; */ @Singleton public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSigner { - private static final Predicate ANNOTATIONTYPE_BUCKET = new Predicate() { - public boolean apply(Annotation input) { - return input.annotationType().equals(Bucket.class); - } - }; private static final Collection FIRST_HEADERS_TO_SIGN = ImmutableList.of(HttpHeaders.DATE); @@ -231,7 +220,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign @VisibleForTesting void appendBucketName(HttpRequest req, StringBuilder toSign) { - String bucketName = getBucketName(req); + String bucketName = S3Utils.getBucketName(req); // If we have a payload/bucket/container that is not all lowercase, vhost-style URLs are not an option and must be // automatically converted to their path-based equivalent. This should only be possible for AWS-S3 since it is @@ -266,20 +255,4 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign } } - private String getBucketName(HttpRequest req) { - checkArgument(req instanceof GeneratedHttpRequest, "this should be a generated http request"); - GeneratedHttpRequest request = GeneratedHttpRequest.class.cast(req); - - String bucketName = null; - - for (int i = 0; i < request.getJavaMethod().getParameterAnnotations().length; i++) { - if (any(Arrays.asList(request.getJavaMethod().getParameterAnnotations()[i]), ANNOTATIONTYPE_BUCKET)) { - bucketName = (String) request.getArgs().get(i); - break; - } - } - - return bucketName; - } - } diff --git a/apis/s3/src/main/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucket.java b/apis/s3/src/main/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucket.java new file mode 100644 index 0000000000..e1adae7be4 --- /dev/null +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucket.java @@ -0,0 +1,64 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import java.net.URI; + +import javax.annotation.Resource; +import javax.inject.Inject; +import javax.inject.Singleton; + +import org.jclouds.javax.annotation.Nullable; +import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; +import org.jclouds.logging.Logger; +import org.jclouds.s3.Bucket; + +import com.google.common.base.Function; +import com.google.common.base.Optional; + +/** + * + * @author Adrian Cole + */ +@Singleton +public class AssignCorrectHostnameForBucket implements Function { + @Resource + protected Logger logger = Logger.NULL; + + protected final RegionToEndpointOrProviderIfNull r2; + protected final Function> bucketToRegion; + + @Inject + public AssignCorrectHostnameForBucket(RegionToEndpointOrProviderIfNull r2, + @Bucket Function> bucketToRegion) { + this.bucketToRegion = bucketToRegion; + this.r2 = r2; + } + + @Override + public URI apply(@Nullable Object from) { + String bucket = from.toString(); + Optional region = bucketToRegion.apply(bucket); + if (region.isPresent()) { + return r2.apply(region.get()); + } + return r2.apply(null); + } + +} diff --git a/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.java b/apis/s3/src/main/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegion.java similarity index 50% rename from apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.java rename to apis/s3/src/main/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegion.java index 7f10f2fd73..680f16500b 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.java +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegion.java @@ -18,29 +18,42 @@ */ package org.jclouds.s3.functions; +import java.net.URI; + +import javax.inject.Inject; import javax.inject.Singleton; -import org.jclouds.aws.AWSResponseException; -import org.jclouds.util.Throwables2; +import org.jclouds.javax.annotation.Nullable; +import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; +import org.jclouds.s3.Bucket; import com.google.common.base.Function; -import com.google.common.base.Throwables; +import com.google.common.base.Optional; +import com.google.common.cache.LoadingCache; /** * * @author Adrian Cole */ @Singleton -public class ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState implements Function { +public class DefaultEndpointThenInvalidateRegion implements Function { - public Boolean apply(Exception from) { - AWSResponseException exception = Throwables2.getFirstThrowableOfType(from, AWSResponseException.class); - if (exception != null && exception.getError() != null - && exception.getError().getCode().equals("BucketAlreadyOwnedByYou")) { - return false; - } else if (Throwables2.getFirstThrowableOfType(from, IllegalStateException.class) != null) { - return false; + private final LoadingCache> bucketToRegionCache; + private final RegionToEndpointOrProviderIfNull r2; + + @Inject + public DefaultEndpointThenInvalidateRegion(RegionToEndpointOrProviderIfNull r2, + @Bucket LoadingCache> bucketToRegionCache) { + this.r2 = r2; + this.bucketToRegionCache = bucketToRegionCache; + } + + @Override + public URI apply(@Nullable Object from) { + try { + return r2.apply(null); + } finally { + bucketToRegionCache.invalidate(from.toString()); } - throw Throwables.propagate(from); } } diff --git a/apis/s3/src/main/java/org/jclouds/s3/functions/GetRegionForBucket.java b/apis/s3/src/main/java/org/jclouds/s3/functions/GetRegionForBucket.java new file mode 100644 index 0000000000..fb5544cee7 --- /dev/null +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/GetRegionForBucket.java @@ -0,0 +1,65 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import java.util.concurrent.ExecutionException; + +import javax.annotation.Resource; +import javax.inject.Inject; +import javax.inject.Singleton; + +import org.jclouds.logging.Logger; +import org.jclouds.s3.Bucket; + +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.cache.CacheLoader.InvalidCacheLoadException; +import com.google.common.cache.LoadingCache; +import com.google.common.util.concurrent.UncheckedExecutionException; + +/** + * + * @author Adrian Cole + */ +@Singleton +public class GetRegionForBucket implements Function> { + @Resource + protected Logger logger = Logger.NULL; + + protected final LoadingCache> bucketToRegion; + + @Inject + public GetRegionForBucket(@Bucket LoadingCache> bucketToRegion) { + this.bucketToRegion = bucketToRegion; + } + + @Override + public Optional apply(String bucket) { + try { + return bucketToRegion.get(bucket); + } catch (ExecutionException e) { + logger.debug("error looking up region for bucket %s: %s", bucket, e); + } catch (UncheckedExecutionException e) { + logger.debug("error looking up region for bucket %s: %s", bucket, e); + } catch (InvalidCacheLoadException e) { + logger.trace("bucket %s not found: %s", bucket, e); + } + return Optional.absent(); + } +} diff --git a/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.java b/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.java new file mode 100644 index 0000000000..3e3f6588e3 --- /dev/null +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.java @@ -0,0 +1,70 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import javax.inject.Inject; + +import org.jclouds.aws.AWSResponseException; +import org.jclouds.http.HttpRequest; +import org.jclouds.javax.annotation.Nullable; +import org.jclouds.rest.InvocationContext; +import org.jclouds.s3.S3Client; +import org.jclouds.s3.util.S3Utils; +import org.jclouds.util.Throwables2; + +import com.google.common.base.Function; +import com.google.common.base.Throwables; + +/** + * + * @author Adrian Cole + */ +public class ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists implements + Function, + InvocationContext { + + private final S3Client client; + private String bucket; + + @Inject + ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(S3Client client) { + this.client = client; + } + + public Boolean apply(Exception from) { + AWSResponseException exception = Throwables2.getFirstThrowableOfType(from, AWSResponseException.class); + if (exception != null && exception.getError() != null && exception.getError().getCode() != null) { + String code = exception.getError().getCode(); + if (code.equals("BucketAlreadyOwnedByYou")) + return false; + else if (code.equals("OperationAborted") && bucket != null && client.bucketExists(bucket)) + return false; + } + throw Throwables.propagate(from); + } + + @Override + public ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists setContext( + @Nullable HttpRequest request) { + if (request != null) + this.bucket = S3Utils.getBucketName(request); + return this; + } + +} diff --git a/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnTrueOn404OrNotFoundFalseOnIllegalState.java b/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnTrueOn404OrNotFoundFalseOnIllegalState.java index fc05c9670e..021cf13c33 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnTrueOn404OrNotFoundFalseOnIllegalState.java +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnTrueOn404OrNotFoundFalseOnIllegalState.java @@ -36,6 +36,7 @@ import com.google.common.base.Throwables; @Singleton public class ReturnTrueOn404OrNotFoundFalseOnIllegalState implements Function { + //TODO: invalidate public Boolean apply(Exception from) { if (getFirstThrowableOfType(from, IllegalStateException.class) != null) { return false; diff --git a/apis/s3/src/main/java/org/jclouds/s3/util/S3Utils.java b/apis/s3/src/main/java/org/jclouds/s3/util/S3Utils.java index 3d71cffcab..954bdb8163 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/util/S3Utils.java +++ b/apis/s3/src/main/java/org/jclouds/s3/util/S3Utils.java @@ -20,12 +20,20 @@ package org.jclouds.s3.util; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.Iterables.any; +import java.lang.annotation.Annotation; +import java.util.Arrays; import java.util.regex.Pattern; +import org.jclouds.http.HttpRequest; +import org.jclouds.rest.internal.GeneratedHttpRequest; +import org.jclouds.s3.Bucket; import org.jclouds.s3.S3Client; import org.jclouds.util.Patterns; +import com.google.common.base.Predicate; + /** * Encryption, Hashing, and IO Utilities needed to sign and verify S3 requests and responses. * @@ -56,4 +64,27 @@ public class S3Utils { sync.deleteBucketIfEmpty(container); return sync.bucketExists(container); } + + private static final Predicate ANNOTATIONTYPE_BUCKET = new Predicate() { + public boolean apply(Annotation input) { + return input.annotationType().equals(Bucket.class); + } + }; + + public static String getBucketName(HttpRequest req) { + checkArgument(req instanceof GeneratedHttpRequest, "this should be a generated http request"); + GeneratedHttpRequest request = GeneratedHttpRequest.class.cast(req); + + String bucketName = null; + + for (int i = 0; i < request.getJavaMethod().getParameterAnnotations().length; i++) { + if (any(Arrays.asList(request.getJavaMethod().getParameterAnnotations()[i]), ANNOTATIONTYPE_BUCKET)) { + bucketName = (String) request.getArgs().get(i); + break; + } + } + + return bucketName; + } + } diff --git a/apis/s3/src/main/java/org/jclouds/s3/xml/LocationConstraintHandler.java b/apis/s3/src/main/java/org/jclouds/s3/xml/LocationConstraintHandler.java index df75a85ebd..49d106ac5e 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/xml/LocationConstraintHandler.java +++ b/apis/s3/src/main/java/org/jclouds/s3/xml/LocationConstraintHandler.java @@ -20,8 +20,6 @@ package org.jclouds.s3.xml; import static org.jclouds.util.SaxUtils.currentOrNull; -import java.util.Map; - import javax.inject.Inject; import org.jclouds.aws.domain.Region; @@ -30,6 +28,9 @@ import org.jclouds.http.functions.ParseSax; import org.jclouds.rest.internal.GeneratedHttpRequest; import org.jclouds.s3.Bucket; +import com.google.common.base.Optional; +import com.google.common.cache.LoadingCache; + /** * Parses the response from Amazon S3 GET Bucket Location *

@@ -41,13 +42,13 @@ import org.jclouds.s3.Bucket; * @author Adrian Cole */ public class LocationConstraintHandler extends ParseSax.HandlerWithResult { - private final Map bucketToRegion; + private final LoadingCache> bucketToRegion; private StringBuilder currentText = new StringBuilder(); private String region; private String bucket; @Inject - public LocationConstraintHandler(@Bucket Map bucketToRegion) { + public LocationConstraintHandler(@Bucket LoadingCache> bucketToRegion) { this.bucketToRegion = bucketToRegion; } @@ -57,7 +58,7 @@ public class LocationConstraintHandler extends ParseSax.HandlerWithResult extends BaseS3A Method method = S3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class, Array.newInstance( PutBucketOptions.class, 0).getClass()); for (String region : Region.DEFAULT_S3) { - processor.createRequest(method, region, "bucket-name"); + processor.createRequest(method, region, "bucket-" + region); } } @@ -383,7 +383,7 @@ public abstract class S3AsyncClientTest extends BaseS3A assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class); assertSaxResponseParserClassEquals(method, null); - assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class); + assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.class); checkFilters(request); } diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/BucketsToStorageMetadataTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/BucketsToStorageMetadataTest.java new file mode 100644 index 0000000000..ebdb2a1b06 --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/BucketsToStorageMetadataTest.java @@ -0,0 +1,70 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.blobstore.functions; + +import static org.testng.Assert.assertEquals; + +import org.jclouds.blobstore.domain.MutableStorageMetadata; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.domain.StorageType; +import org.jclouds.blobstore.domain.internal.MutableStorageMetadataImpl; +import org.jclouds.blobstore.domain.internal.PageSetImpl; +import org.jclouds.concurrent.MoreExecutors; +import org.jclouds.domain.Location; +import org.jclouds.domain.LocationBuilder; +import org.jclouds.domain.LocationScope; +import org.jclouds.s3.domain.BucketMetadata; +import org.testng.annotations.Test; + +import com.google.common.base.Functions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** + * Tests behavior of {@code BucketsToStorageMetadata} + * + * @author Adrian Cole + */ +// NOTE:without testName, this will not call @Before* and fail w/NPE during +// surefire +@Test(groups = "unit", testName = "BucketsToStorageMetadataTest") +public class BucketsToStorageMetadataTest { + protected Location provider = new LocationBuilder().scope(LocationScope.PROVIDER).id("aws-ec2") + .description("aws-ec2").build(); + + protected Location region = new LocationBuilder().scope(LocationScope.REGION).id("us-east-1") + .description("us-east-1").parent(provider).build(); + + public void test() { + + BucketsToStorageMetadata fn = new BucketsToStorageMetadata( + MoreExecutors.sameThreadExecutor(), + new BucketToResourceMetadata(Functions.forMap(ImmutableMap. of("mycontainer", region)))); + + MutableStorageMetadata expected = new MutableStorageMetadataImpl(); + expected.setName("mycontainer"); + expected.setType(StorageType.CONTAINER); + expected.setLocation(region); + + assertEquals( + fn.apply(ImmutableSet.of(new BucketMetadata("mycontainer", null, null))).toString(), + new PageSetImpl(ImmutableSet. of(expected), null).toString()); + + } +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/LocationFromBucketNameTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/LocationFromBucketNameTest.java new file mode 100644 index 0000000000..f4f13e7c20 --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/LocationFromBucketNameTest.java @@ -0,0 +1,89 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.blobstore.functions; + +import static org.testng.Assert.assertEquals; + +import java.util.Set; + +import org.jclouds.domain.Location; +import org.jclouds.domain.LocationBuilder; +import org.jclouds.domain.LocationScope; +import org.testng.annotations.Test; + +import com.google.common.base.Functions; +import com.google.common.base.Optional; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** + * Tests behavior of {@code LocationFromBucketName} + * + * @author Adrian Cole + */ +// NOTE:without testName, this will not call @Before* and fail w/NPE during +// surefire +@Test(groups = "unit", testName = "LocationFromBucketNameTest") +public class LocationFromBucketNameTest { + protected Location provider = new LocationBuilder().scope(LocationScope.PROVIDER).id("aws-ec2") + .description("aws-ec2").build(); + + protected Location region = new LocationBuilder().scope(LocationScope.REGION).id("us-east-1") + .description("us-east-1").parent(provider).build(); + + protected Location region2 = new LocationBuilder().scope(LocationScope.REGION).id("eu-west-1") + .description("eu-west-1").parent(provider).build(); + + public void testOnlyLocationDoesntNeedMapping() { + LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap + .> of()), Suppliers.> ofInstance(ImmutableSet + .of(provider))); + + assertEquals(fn.apply("mybucket"), provider); + + } + + public void testMapsToCorrectRegion() { + LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap + .> of("mybucket", Optional.of("eu-west-1"))), + Suppliers.> ofInstance(ImmutableSet.of(region, region2))); + + assertEquals(fn.apply("mybucket"), region2); + + } + + public void testNullOnUnmatchedRegion() { + LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap + .> of("mybucket", Optional.of("eu-west-2"))), + Suppliers.> ofInstance(ImmutableSet.of(region, region2))); + + assertEquals(fn.apply("mybucket"), null); + + } + + public void testNullOnAbsentData() { + LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap + .> of("mybucket", Optional. absent())), + Suppliers.> ofInstance(ImmutableSet.of(region, region2))); + + assertEquals(fn.apply("mybucket"), null); + + } +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucketTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucketTest.java new file mode 100644 index 0000000000..dbf54c1b24 --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucketTest.java @@ -0,0 +1,86 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import static org.testng.Assert.assertEquals; + +import java.net.URI; +import java.util.Map; + +import javax.inject.Provider; +import javax.ws.rs.core.UriBuilder; + +import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; +import org.testng.annotations.Test; + +import com.google.common.base.Functions; +import com.google.common.base.Optional; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableMap; +import com.sun.jersey.api.uri.UriBuilderImpl; + +/** + * Tests behavior of {@code AssignCorrectHostnameForBucket} + * + * @author Adrian Cole + */ +// NOTE:without testName, this will not call @Before* and fail w/NPE during +// surefire +@Test(groups = "unit", testName = "AssignCorrectHostnameForBucketTest") +public class AssignCorrectHostnameForBucketTest { + Provider uriBuilderProvider = new Provider() { + + @Override + public UriBuilder get() { + return new UriBuilderImpl(); + } + + }; + + public void testWhenNoBucketRegionMappingInCache() { + + AssignCorrectHostnameForBucket fn = new AssignCorrectHostnameForBucket(new RegionToEndpointOrProviderIfNull( + "aws-s3", Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")), + + Suppliers.>> ofInstance(ImmutableMap.of("us-standard", + Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1", + Suppliers.ofInstance(URI.create("https://s3-us-west-1.amazonaws.com"))))), + + Functions.forMap(ImmutableMap.> of("bucket", Optional. absent()))); + + assertEquals(fn.apply("bucket"), URI.create("https://s3.amazonaws.com")); + + } + + public void testWhenBucketRegionMappingInCache() { + + AssignCorrectHostnameForBucket fn = new AssignCorrectHostnameForBucket(new RegionToEndpointOrProviderIfNull( + "aws-s3", Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")), + + Suppliers.>> ofInstance(ImmutableMap.of("us-standard", + Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1", + Suppliers.ofInstance(URI.create("https://s3-us-west-1.amazonaws.com"))))), + + Functions.forMap(ImmutableMap.> of("bucket", Optional.of("us-west-1")))); + + assertEquals(fn.apply("bucket"), URI.create("https://s3-us-west-1.amazonaws.com")); + + } +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegionTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegionTest.java new file mode 100644 index 0000000000..514ee8c9ec --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegionTest.java @@ -0,0 +1,56 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +import java.net.URI; + +import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; +import org.testng.annotations.Test; + +import com.google.common.base.Optional; +import com.google.common.cache.LoadingCache; + +/** + * @author Adrian Cole + */ +@Test(testName = "DefaultEndpointThenInvalidateRegionTest") +public class DefaultEndpointThenInvalidateRegionTest { + + @SuppressWarnings("unchecked") + @Test + void testInvalidate() throws Exception { + RegionToEndpointOrProviderIfNull r2 = createMock(RegionToEndpointOrProviderIfNull.class); + LoadingCache> bucketToRegionCache = createMock(LoadingCache.class); + + expect(r2.apply(null)).andReturn(URI.create("http://east-url")); + bucketToRegionCache.invalidate("mybucket"); + + replay(r2, bucketToRegionCache); + + new DefaultEndpointThenInvalidateRegion(r2, bucketToRegionCache).apply("mybucket"); + verify(r2, bucketToRegionCache); + + } + +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/functions/GetRegionForBucketTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/GetRegionForBucketTest.java new file mode 100644 index 0000000000..2eaf7dc6a8 --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/functions/GetRegionForBucketTest.java @@ -0,0 +1,81 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.testng.Assert.assertEquals; + +import java.util.concurrent.ExecutionException; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import com.google.common.base.Optional; +import com.google.common.cache.CacheLoader.InvalidCacheLoadException; +import com.google.common.cache.LoadingCache; +import com.google.common.util.concurrent.UncheckedExecutionException; + +/** + * @author Adrian Cole + */ +@Test(testName = "GetRegionForBucketTest") +public class GetRegionForBucketTest { + + @SuppressWarnings("unchecked") + @Test + void test() throws Exception { + LoadingCache> bucketToRegionCache = createMock(LoadingCache.class); + + expect(bucketToRegionCache.get("bucket")).andReturn(Optional.of("us-east-1")); + + replay(bucketToRegionCache); + GetRegionForBucket fn = new GetRegionForBucket(bucketToRegionCache); + + assertEquals(fn.apply("bucket"), Optional.of("us-east-1")); + + verify(bucketToRegionCache); + + } + + @SuppressWarnings("serial") + @DataProvider(name = "exceptions") + public Object[][] createExceptions() { + return new Object[][] { { new ExecutionException() { + } }, { new UncheckedExecutionException() { + } }, { new InvalidCacheLoadException("foo") } }; + } + + @SuppressWarnings("unchecked") + @Test(dataProvider = "exceptions") + void testGracefulOnException(Exception exception) throws Exception { + LoadingCache> bucketToRegionCache = createMock(LoadingCache.class); + + expect(bucketToRegionCache.get("bucket")).andThrow(exception); + + replay(bucketToRegionCache); + GetRegionForBucket fn = new GetRegionForBucket(bucketToRegionCache); + + assertEquals(fn.apply("bucket"), Optional.absent()); + + verify(bucketToRegionCache); + } +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest.java deleted file mode 100644 index bb21e5c0bb..0000000000 --- a/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to jclouds, Inc. (jclouds) under one or more - * contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. jclouds licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.jclouds.s3.functions; - -import org.jclouds.aws.AWSResponseException; -import org.jclouds.aws.domain.AWSError; -import org.testng.annotations.Test; - -/** - * @author Adrian Cole - */ -@Test(testName = "ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest") -public class ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest { - - @Test - void testBucketAlreadyOwnedByYouIsOk() throws Exception { - Exception e = getErrorWithCode("BucketAlreadyOwnedByYou"); - assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState().apply(e); - } - - @Test - void testIllegalStateIsOk() throws Exception { - Exception e = new IllegalStateException(); - assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState().apply(e); - } - - @Test(expectedExceptions = AWSResponseException.class) - void testBlahIsNotOk() throws Exception { - Exception e = getErrorWithCode("blah"); - new ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState().apply(e); - } - - private Exception getErrorWithCode(String code) { - AWSError error = new AWSError(); - error.setCode(code); - return new AWSResponseException(null, null, null, error); - } -} diff --git a/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest.java new file mode 100644 index 0000000000..c35979306d --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest.java @@ -0,0 +1,114 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +import java.net.URI; + +import org.jclouds.aws.AWSResponseException; +import org.jclouds.aws.domain.AWSError; +import org.jclouds.rest.internal.GeneratedHttpRequest; +import org.jclouds.s3.S3Client; +import org.jclouds.s3.options.PutBucketOptions; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * @author Adrian Cole + */ +@Test(testName = "ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest") +public class ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest { + + GeneratedHttpRequest putBucket; + + @BeforeClass + void setUp() throws SecurityException, NoSuchMethodException { + putBucket = GeneratedHttpRequest + . requestBuilder() + .method("PUT") + .endpoint(URI.create("https://adriancole-blobstore113.s3.amazonaws.com/")) + .declaring(S3Client.class) + .javaMethod( + S3Client.class.getMethod("putBucketInRegion", String.class, String.class, + PutBucketOptions[].class)).args(new Object[] { null, "bucket" }).build(); + } + + @Test + void testBucketAlreadyOwnedByYouIsOk() throws Exception { + S3Client client = createMock(S3Client.class); + replay(client); + + Exception e = getErrorWithCode("BucketAlreadyOwnedByYou"); + assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).setContext(putBucket) + .apply(e); + verify(client); + } + + @Test + void testOperationAbortedIsOkWhenBucketExists() throws Exception { + S3Client client = createMock(S3Client.class); + expect(client.bucketExists("bucket")).andReturn(true); + replay(client); + Exception e = getErrorWithCode("OperationAborted"); + assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).setContext(putBucket) + .apply(e); + verify(client); + } + + @Test(expectedExceptions = Exception.class) + void testOperationAbortedNotOkWhenBucketDoesntExist() throws Exception { + S3Client client = createMock(S3Client.class); + expect(client.bucketExists("bucket")).andReturn(false); + replay(client); + Exception e = getErrorWithCode("OperationAborted"); + new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).setContext(putBucket).apply(e); + Assert.fail(); + } + + @Test(expectedExceptions = IllegalStateException.class) + void testIllegalStateIsNotOk() throws Exception { + S3Client client = createMock(S3Client.class); + replay(client); + + Exception e = new IllegalStateException(); + new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).apply(e); + Assert.fail(); + + } + + @Test(expectedExceptions = AWSResponseException.class) + void testBlahIsNotOk() throws Exception { + S3Client client = createMock(S3Client.class); + replay(client); + Exception e = getErrorWithCode("blah"); + new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).apply(e); + Assert.fail(); + } + + private Exception getErrorWithCode(String code) { + AWSError error = new AWSError(); + error.setCode(code); + return new AWSResponseException(null, null, null, error); + } +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/services/BucketsLiveTest.java b/apis/s3/src/test/java/org/jclouds/s3/services/BucketsLiveTest.java index ea22c1f3ff..b2bfab9768 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/services/BucketsLiveTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/services/BucketsLiveTest.java @@ -34,22 +34,21 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; -import org.jclouds.aws.domain.Region; import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest; import org.jclouds.s3.S3ApiMetadata; import org.jclouds.s3.S3Client; import org.jclouds.s3.domain.AccessControlList; +import org.jclouds.s3.domain.AccessControlList.CanonicalUserGrantee; +import org.jclouds.s3.domain.AccessControlList.EmailAddressGrantee; +import org.jclouds.s3.domain.AccessControlList.Grant; +import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI; +import org.jclouds.s3.domain.AccessControlList.Permission; import org.jclouds.s3.domain.BucketLogging; import org.jclouds.s3.domain.BucketMetadata; import org.jclouds.s3.domain.CannedAccessPolicy; import org.jclouds.s3.domain.ListBucketResponse; import org.jclouds.s3.domain.Payer; import org.jclouds.s3.domain.S3Object; -import org.jclouds.s3.domain.AccessControlList.CanonicalUserGrantee; -import org.jclouds.s3.domain.AccessControlList.EmailAddressGrantee; -import org.jclouds.s3.domain.AccessControlList.Grant; -import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI; -import org.jclouds.s3.domain.AccessControlList.Permission; import org.jclouds.s3.internal.StubS3AsyncClient; import org.jclouds.util.Strings2; import org.testng.annotations.Test; @@ -71,7 +70,7 @@ public class BucketsLiveTest extends BaseBlobStoreIntegrationTest { } public S3Client getApi() { - return (S3Client) view.unwrap(S3ApiMetadata.CONTEXT_TOKEN).getApi(); + return view.unwrap(S3ApiMetadata.CONTEXT_TOKEN).getApi(); } /** @@ -177,17 +176,6 @@ public class BucketsLiveTest extends BaseBlobStoreIntegrationTest { } - public void testDefaultBucketLocation() throws Exception { - - String bucketName = getContainerName(); - try { - String location = getApi().getBucketLocation(bucketName); - assert location.equals(Region.US_STANDARD) : "bucket: " + bucketName + " location: " + location; - } finally { - returnContainer(bucketName); - } - } - public void testBucketPayer() throws Exception { final String bucketName = getContainerName(); try { @@ -276,33 +264,6 @@ public class BucketsLiveTest extends BaseBlobStoreIntegrationTest { assertTrue(getApi().putBucketACL(targetBucket, acl)); } - /** - * using scratch bucketName as we are changing location - */ - public void testEu() throws Exception { - final String bucketName = getScratchContainerName(); - try { - getApi().putBucketInRegion(Region.EU, bucketName + "eu", withBucketAcl(CannedAccessPolicy.PUBLIC_READ)); - assertConsistencyAware(new Runnable() { - public void run() { - try { - AccessControlList acl = getApi().getBucketACL(bucketName + "eu"); - assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ), acl.toString()); - } catch (Exception e) { - Throwables.propagateIfPossible(e); - } - } - }); - assertEquals(Region.EU, getApi().getBucketLocation(bucketName + "eu")); - // TODO: I believe that the following should work based on the above acl assertion passing. - // However, it fails on 403 - // URL url = new URL(String.format("http://%s.s3.amazonaws.com", bucketName)); - // Utils.toStringAndClose(url.openStream()); - } finally { - destroyContainer(bucketName + "eu"); - } - } - void bucketExists() throws Exception { String bucketName = getContainerName(); try { diff --git a/apis/walrus/src/main/java/org/jclouds/walrus/WalrusAsyncClient.java b/apis/walrus/src/main/java/org/jclouds/walrus/WalrusAsyncClient.java index 5dfa1c31b7..8b90f49cd5 100644 --- a/apis/walrus/src/main/java/org/jclouds/walrus/WalrusAsyncClient.java +++ b/apis/walrus/src/main/java/org/jclouds/walrus/WalrusAsyncClient.java @@ -26,6 +26,7 @@ import javax.ws.rs.Path; import org.jclouds.blobstore.attr.BlobScope; import org.jclouds.blobstore.functions.ReturnFalseOnContainerNotFound; import org.jclouds.rest.annotations.BinderParam; +import org.jclouds.rest.annotations.Endpoint; import org.jclouds.rest.annotations.ExceptionParser; import org.jclouds.rest.annotations.ParamValidators; import org.jclouds.rest.annotations.QueryParams; @@ -55,9 +56,10 @@ public interface WalrusAsyncClient extends S3AsyncClient { @Override @GET @Path("/") + @Endpoint(Bucket.class) @QueryParams(keys = "max-keys", values = "0") @ExceptionParser(ReturnFalseOnContainerNotFound.class) ListenableFuture bucketExists( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName); + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); } diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java index e028975226..87d58ae43d 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java @@ -22,7 +22,6 @@ import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursi import static org.jclouds.concurrent.FutureIterables.awaitCompletion; import java.util.Map; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -42,10 +41,9 @@ import org.jclouds.blobstore.strategy.ClearListStrategy; import org.jclouds.http.handlers.BackoffLimitedRetryHandler; import org.jclouds.logging.Logger; -import com.google.common.base.Predicate; import com.google.common.base.Throwables; -import com.google.common.collect.Iterables; import com.google.common.collect.Maps; +import com.google.common.util.concurrent.Futures; import com.google.inject.Inject; /** @@ -97,16 +95,14 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr for (int i = 0; i < maxErrors; ) { // fetch partial directory listing try { - listing = connection.list(containerName, options).get(); - } catch (ExecutionException ee) { + listing = Futures.getUnchecked(connection.list(containerName, options)); + } catch (RuntimeException ee) { ++i; if (i == maxErrors) { - throw new BlobRuntimeException("list error", ee.getCause()); + throw Throwables.propagate(ee.getCause()); } retryHandler.imposeBackoffExponentialDelay(i, message); continue; - } catch (InterruptedException ie) { - throw Throwables.propagate(ie); } // recurse on subdirectories diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java index 753d27bb2f..20616a16b0 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java @@ -48,6 +48,7 @@ import java.util.zip.GZIPInputStream; import javax.ws.rs.core.MediaType; +import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobBuilder.PayloadBlobBuilder; @@ -239,6 +240,23 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { } + @Test(groups = { "integration", "live" }) + public void testCreateBlobWithExpiry() throws InterruptedException { + String container = getContainerName(); + BlobStore blobStore = view.getBlobStore(); + try { + final String blobName = "hello"; + final Date expires = new Date((System.currentTimeMillis() / 1000) * 1000 + 60 * 1000); + + blobStore.putBlob(container, blobStore.blobBuilder(blobName).payload(TEST_STRING).expires(expires).build()); + + assertConsistencyAwareBlobExpiryMetadata(container, blobName, expires); + + } finally { + returnContainer(container); + } + } + @Test(groups = { "integration", "live" }) public void testGetIfUnmodifiedSince() throws InterruptedException { String container = getContainerName(); diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobStoreIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobStoreIntegrationTest.java index 5af7ef47fb..15a6209703 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobStoreIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobStoreIntegrationTest.java @@ -23,6 +23,7 @@ import static org.jclouds.blobstore.util.BlobStoreUtils.getContentAsStringOrNull import static org.testng.Assert.assertEquals; import java.io.IOException; +import java.security.SecureRandom; import java.util.Date; import java.util.Map; import java.util.Map.Entry; @@ -35,6 +36,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import javax.annotation.Nullable; import javax.ws.rs.core.MediaType; import org.jclouds.apis.BaseViewLiveTest; @@ -80,7 +82,7 @@ public class BaseBlobStoreIntegrationTest extends BaseViewLiveTest() { public boolean apply(StorageMetadata input) { return (input.getType() == StorageType.CONTAINER || input.getType() == StorageType.FOLDER) - && input.getName().startsWith(CONTAINER_PREFIX.toLowerCase()); + && input.getName().startsWith(CONTAINER_PREFIX); } }); for (StorageMetadata container : testContainers) { @@ -339,20 +346,57 @@ public class BaseBlobStoreIntegrationTest extends BaseViewLiveTest() { + + @Override + public boolean apply(@Nullable StorageMetadata input) { + return input.getName().equals(containerName); + } + + }); + Location actualLoc = container.getLocation(); + + assert loc.equals(actualLoc) : String.format("blob %s, in location %s instead of %s", containerName, + actualLoc, loc); + } catch (Exception e) { + Throwables.propagate(e); + } + } + }); + } + + protected void assertConsistencyAwareBlobExpiryMetadata(final String containerName, final String blobName, + final Date expectedExpires) throws InterruptedException { + assertConsistencyAware(new Runnable() { + public void run() { + try { + Blob blob = view.getBlobStore().getBlob(containerName, blobName); + Date actualExpires = blob.getPayload().getContentMetadata().getExpires(); + assert expectedExpires.equals(actualExpires) : "expires=" + actualExpires + "; expected=" + + expectedExpires; + } catch (Exception e) { + Throwables.propagateIfPossible(e); + } + } + }); + } protected void assertConsistencyAwareBlobInLocation(final String containerName, final String blobName, final Location loc) throws InterruptedException { @@ -452,7 +496,7 @@ public class BaseBlobStoreIntegrationTest extends BaseViewLiveTest() { + + @Override + public boolean apply(@Nullable StorageMetadata input) { + return input.getName().equals(containerName); + } + + }).getLocation(); view.getBlobStore().putBlob(containerName, view.getBlobStore().blobBuilder("hello").payload(TEST_STRING).build()); @@ -56,4 +82,58 @@ public class BaseContainerLiveTest extends BaseBlobStoreIntegrationTest { recycleContainer(containerName); } } + + static Location findNonDefaultLocationOrSkip(BlobStore blobStore, Location defaultLocation) { + List locs = Lists.newArrayList(Iterables.filter(blobStore.listAssignableLocations(), + Predicates.not(Predicates.equalTo(defaultLocation)))); + if (locs.size() == 0) + throw new SkipException("No non-default location found in " + locs); + // try to use a diverse location + Collections.shuffle(locs); + return locs.get(0); + } + + @Test(groups = "live", dependsOnMethods = "testPublicAccess") + public void testPublicAccessInNonDefaultLocation() throws InterruptedException, MalformedURLException, IOException { + Location nonDefault = findNonDefaultLocationOrSkip(view.getBlobStore(), defaultLocation); + + String payload = "my data"; + runCreateContainerInLocation(payload, nonDefault); + } + + @Test(groups = "live", dependsOnMethods = "testPublicAccess") + public void testPublicAccessInNonDefaultLocationWithBigBlob() throws InterruptedException, MalformedURLException, + IOException { + Location nonDefault = findNonDefaultLocationOrSkip(view.getBlobStore(), defaultLocation); + String payload = Strings.repeat("a", 1024 * 1024); // 1MB + runCreateContainerInLocation(payload, nonDefault); + } + + private void runCreateContainerInLocation(String payload, Location nonDefault) throws InterruptedException, + IOException { + String blobName = "hello"; + BlobStore blobStore = view.getBlobStore(); + final String containerName = getScratchContainerName(); + try { + Logger.getAnonymousLogger().info( + String.format("creating public container %s in location %s", containerName, nonDefault.getId())); + blobStore.createContainerInLocation(nonDefault, containerName, publicRead()); + assertConsistencyAwareContainerExists(containerName); + assertConsistencyAwareContainerInLocation(containerName, nonDefault); + + blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload(payload).build()); + + assertConsistencyAwareContainerSize(containerName, 1); + + BlobMetadata metadata = view.getBlobStore().blobMetadata(containerName, blobName); + assertEquals(Strings2.toStringAndClose(view.utils().http().get(metadata.getPublicUri())), payload); + + assertConsistencyAwareBlobInLocation(containerName, blobName, nonDefault); + + } finally { + // this container is now public, so we can't reuse it directly + recycleContainer(containerName); + } + } + } \ No newline at end of file diff --git a/common/aws/src/main/java/org/jclouds/aws/domain/Region.java b/common/aws/src/main/java/org/jclouds/aws/domain/Region.java index e7ca0fd228..3c8c50745f 100644 --- a/common/aws/src/main/java/org/jclouds/aws/domain/Region.java +++ b/common/aws/src/main/java/org/jclouds/aws/domain/Region.java @@ -48,8 +48,6 @@ public class Region { * In Amazon S3, the EU (Ireland) Region provides read-after-write consistency for PUTS of new * objects in your Amazon S3 bucket and eventual consistency for overwrite PUTS and DELETES. */ - public static final String EU = "EU"; - public static final String EU_WEST_1 = "eu-west-1"; /** @@ -106,7 +104,7 @@ public class Region { */ public static final String AP_NORTHEAST_1 = "ap-northeast-1"; - public static Set DEFAULT_S3 = ImmutableSet.of(EU, US_STANDARD, US_WEST_1, US_WEST_2, SA_EAST_1, AP_SOUTHEAST_1, + public static Set DEFAULT_S3 = ImmutableSet.of(US_STANDARD, US_WEST_1, US_WEST_2, EU_WEST_1, SA_EAST_1, AP_SOUTHEAST_1, AP_NORTHEAST_1); public static Set DEFAULT_REGIONS = ImmutableSet.of(US_EAST_1, US_WEST_1, US_WEST_2, SA_EAST_1, EU_WEST_1, @@ -119,7 +117,7 @@ public class Region { // note that due to US_STANDARD the codes include US instead of US-VA properties.setProperty(PROPERTY_ISO3166_CODES, "US,US-CA,US-OR,BR-SP,IE,SG,JP-13"); properties.setProperty(PROPERTY_REGION + "." + US_STANDARD + "." + ISO3166_CODES, "US"); - properties.setProperty(PROPERTY_REGION + "." + EU + "." + ISO3166_CODES, "IE"); + properties.setProperty(PROPERTY_REGION + "." + EU_WEST_1 + "." + ISO3166_CODES, "IE"); return properties; } diff --git a/common/aws/src/main/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandler.java b/common/aws/src/main/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandler.java index e17c77f438..f074f99a0f 100644 --- a/common/aws/src/main/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandler.java +++ b/common/aws/src/main/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandler.java @@ -20,17 +20,14 @@ package org.jclouds.aws.handlers; import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream; -import javax.annotation.Resource; -import javax.inject.Named; - -import org.jclouds.Constants; import org.jclouds.aws.domain.AWSError; import org.jclouds.aws.util.AWSUtils; import org.jclouds.http.HttpCommand; import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpRetryHandler; -import org.jclouds.logging.Logger; +import org.jclouds.http.handlers.BackoffLimitedRetryHandler; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; /** @@ -40,35 +37,25 @@ import com.google.inject.Inject; */ public class AWSClientErrorRetryHandler implements HttpRetryHandler { - @Inject(optional = true) - @Named(Constants.PROPERTY_MAX_RETRIES) - private int retryCountLimit = 5; - private final AWSUtils utils; - - @Resource - protected Logger logger = Logger.NULL; + private final BackoffLimitedRetryHandler backoffLimitedRetryHandler; @Inject - public AWSClientErrorRetryHandler(AWSUtils utils) { + public AWSClientErrorRetryHandler(AWSUtils utils, BackoffLimitedRetryHandler backoffLimitedRetryHandler) { this.utils = utils; + this.backoffLimitedRetryHandler = backoffLimitedRetryHandler; } public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) { - if (command.getFailureCount() > retryCountLimit) - return false; - if (response.getStatusCode() == 400 || response.getStatusCode() == 403 - || response.getStatusCode() == 409) { - command.incrementFailureCount(); + if (response.getStatusCode() == 400 || response.getStatusCode() == 403 || response.getStatusCode() == 409) { // Content can be null in the case of HEAD requests if (response.getPayload() != null) { closeClientButKeepContentStream(response); AWSError error = utils.parseAWSErrorFromContent(command.getCurrentRequest(), response); if (error != null - && ("RequestTimeout".equals(error.getCode()) - || "OperationAborted".equals(error.getCode()) || "SignatureDoesNotMatch" - .equals(error.getCode()))) { - return true; + && ImmutableSet.of("RequestTimeout", "OperationAborted", "SignatureDoesNotMatch").contains( + error.getCode())) { + return backoffLimitedRetryHandler.shouldRetryRequest(command, response); } } } diff --git a/common/aws/src/test/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandlerTest.java b/common/aws/src/test/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandlerTest.java index edcdebed0e..86f85b0489 100644 --- a/common/aws/src/test/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandlerTest.java +++ b/common/aws/src/test/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandlerTest.java @@ -23,9 +23,16 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; +import java.net.URI; + +import org.jclouds.aws.domain.AWSError; import org.jclouds.aws.util.AWSUtils; import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpRequest; import org.jclouds.http.HttpResponse; +import org.jclouds.http.handlers.BackoffLimitedRetryHandler; +import org.jclouds.io.Payloads; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; /** @@ -33,29 +40,60 @@ import org.testng.annotations.Test; * * @author Adrian Cole */ -@Test(groups = "unit") +@Test(groups = "unit", testName = "AWSClientErrorRetryHandlerTest") public class AWSClientErrorRetryHandlerTest { @Test public void test401DoesNotRetry() { AWSUtils utils = createMock(AWSUtils.class); + BackoffLimitedRetryHandler backoffLimitedRetryHandler = createMock(BackoffLimitedRetryHandler.class); HttpCommand command = createMock(HttpCommand.class); - HttpResponse response = createMock(HttpResponse.class); - expect(command.getFailureCount()).andReturn(0); - expect(response.getStatusCode()).andReturn(401).atLeastOnce(); + replay(utils, backoffLimitedRetryHandler, command); - replay(utils); - replay(command); - replay(response); + AWSClientErrorRetryHandler retry = new AWSClientErrorRetryHandler(utils, backoffLimitedRetryHandler); - AWSClientErrorRetryHandler retry = new AWSClientErrorRetryHandler(utils); + assert !retry.shouldRetryRequest(command, HttpResponse.builder().statusCode(401).build()); - assert !retry.shouldRetryRequest(command, response); - - verify(utils); - verify(command); - verify(response); + verify(utils, backoffLimitedRetryHandler, command); } + + @DataProvider(name = "codes") + public Object[][] createData() { + return new Object[][] { { "RequestTimeout" }, { "OperationAborted" }, { "SignatureDoesNotMatch" } }; + } + + @Test(dataProvider = "codes") + public void test409DoesBackoffAndRetryForCode(String code) { + + AWSUtils utils = createMock(AWSUtils.class); + BackoffLimitedRetryHandler backoffLimitedRetryHandler = createMock(BackoffLimitedRetryHandler.class); + HttpCommand command = createMock(HttpCommand.class); + + HttpRequest putBucket = HttpRequest.builder().method("PUT") + .endpoint(URI.create("https://adriancole-blobstore113.s3.amazonaws.com/")).build(); + + HttpResponse operationAborted = HttpResponse.builder().statusCode(409) + .payload(Payloads.newStringPayload(String.format("%s", code))).build(); + + expect(command.getCurrentRequest()).andReturn(putBucket); + + AWSError error = new AWSError(); + error.setCode(code); + + expect(utils.parseAWSErrorFromContent(putBucket, operationAborted)).andReturn(error); + + expect(backoffLimitedRetryHandler.shouldRetryRequest(command, operationAborted)).andReturn(Boolean.TRUE); + + replay(utils, backoffLimitedRetryHandler, command); + + AWSClientErrorRetryHandler retry = new AWSClientErrorRetryHandler(utils, backoffLimitedRetryHandler); + + assert retry.shouldRetryRequest(command, operationAborted); + + verify(utils, backoffLimitedRetryHandler, command); + + } + } diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java index ac0ec71c0b..179796769b 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java @@ -21,8 +21,10 @@ package org.jclouds.aws.s3; import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER; import java.util.Map; + import javax.ws.rs.DELETE; import javax.ws.rs.GET; +import javax.ws.rs.HEAD; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; @@ -35,11 +37,13 @@ import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex; import org.jclouds.aws.s3.functions.ObjectMetadataKey; import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex; import org.jclouds.blobstore.attr.BlobScope; +import org.jclouds.blobstore.functions.ReturnFalseOnContainerNotFound; import org.jclouds.http.functions.ParseETagHeader; import org.jclouds.io.Payload; import org.jclouds.javax.annotation.Nullable; import org.jclouds.rest.annotations.BinderParam; import org.jclouds.rest.annotations.Endpoint; +import org.jclouds.rest.annotations.EndpointParam; import org.jclouds.rest.annotations.ExceptionParser; import org.jclouds.rest.annotations.ParamParser; import org.jclouds.rest.annotations.ParamValidators; @@ -55,8 +59,11 @@ import org.jclouds.s3.S3Client; import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured; import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.filters.RequestAuthorizeSignature; +import org.jclouds.s3.functions.AssignCorrectHostnameForBucket; import org.jclouds.s3.functions.BindRegionToXmlPayload; -import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState; +import org.jclouds.s3.functions.DefaultEndpointThenInvalidateRegion; +import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists; +import org.jclouds.s3.functions.ReturnTrueOn404OrNotFoundFalseOnIllegalState; import org.jclouds.s3.options.PutBucketOptions; import org.jclouds.s3.options.PutObjectOptions; import org.jclouds.s3.predicates.validators.BucketNameValidator; @@ -73,32 +80,7 @@ import com.google.common.util.concurrent.ListenableFuture; @RequestFilters(RequestAuthorizeSignature.class) @BlobScope(CONTAINER) public interface AWSS3AsyncClient extends S3AsyncClient { - - /** - * @see S3Client#putBucketInRegion - */ - @Override - @PUT - @Path("/") - @Endpoint(Bucket.class) - @ExceptionParser(ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class) - ListenableFuture putBucketInRegion( - @BinderParam(BindRegionToXmlPayload.class) @Nullable String region, - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - PutBucketOptions... options); - - /** - * @see S3Client#getBucketLocation - */ - @Override - @GET - @QueryParams(keys = "location") - @Path("/") - @Endpoint(Bucket.class) - @XMLResponseParser(LocationConstraintHandler.class) - ListenableFuture getBucketLocation( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); - + /** * @see AWSS3Client#initiateMultipartUpload */ @@ -107,9 +89,9 @@ public interface AWSS3AsyncClient extends S3AsyncClient { @Path("/{key}") @ResponseParser(UploadIdFromHttpResponseViaRegex.class) ListenableFuture initiateMultipartUpload( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class) ObjectMetadata objectMetadata, - PutObjectOptions... options); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class) ObjectMetadata objectMetadata, + PutObjectOptions... options); /** * @see AWSS3Client#abortMultipartUpload @@ -118,8 +100,8 @@ public interface AWSS3AsyncClient extends S3AsyncClient { @Path("/{key}") @ExceptionParser(ReturnVoidOnNotFoundOr404.class) ListenableFuture abortMultipartUpload( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key, @QueryParam("uploadId") String uploadId); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, @QueryParam("uploadId") String uploadId); /** * @see AWSS3Client#uploadPart @@ -128,9 +110,9 @@ public interface AWSS3AsyncClient extends S3AsyncClient { @Path("/{key}") @ResponseParser(ParseETagHeader.class) ListenableFuture uploadPart( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key, @QueryParam("partNumber") int partNumber, - @QueryParam("uploadId") String uploadId, Payload part); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, @QueryParam("partNumber") int partNumber, + @QueryParam("uploadId") String uploadId, Payload part); /** * @see AWSS3Client#completeMultipartUpload @@ -139,8 +121,8 @@ public interface AWSS3AsyncClient extends S3AsyncClient { @Path("/{key}") @ResponseParser(ETagFromHttpResponseViaRegex.class) ListenableFuture completeMultipartUpload( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key, @QueryParam("uploadId") String uploadId, - @BinderParam(BindPartIdsAndETagsToRequest.class) Map parts); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, @QueryParam("uploadId") String uploadId, + @BinderParam(BindPartIdsAndETagsToRequest.class) Map parts); } diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ProviderMetadata.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ProviderMetadata.java index 24e9d6234c..4f774e1fe6 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ProviderMetadata.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ProviderMetadata.java @@ -21,6 +21,7 @@ package org.jclouds.aws.s3; import static org.jclouds.Constants.PROPERTY_ENDPOINT; import static org.jclouds.aws.domain.Region.AP_NORTHEAST_1; import static org.jclouds.aws.domain.Region.AP_SOUTHEAST_1; +import static org.jclouds.aws.domain.Region.EU_WEST_1; import static org.jclouds.aws.domain.Region.SA_EAST_1; import static org.jclouds.aws.domain.Region.US_STANDARD; import static org.jclouds.aws.domain.Region.US_WEST_1; @@ -71,7 +72,7 @@ public class AWSS3ProviderMetadata extends BaseProviderMetadata { properties.setProperty(PROPERTY_REGION + "." + US_WEST_1 + "." + ENDPOINT, "https://s3-us-west-1.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + US_WEST_2 + "." + ENDPOINT, "https://s3-us-west-2.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + SA_EAST_1 + "." + ENDPOINT, "https://s3-sa-east-1.amazonaws.com"); - properties.setProperty(PROPERTY_REGION + "." + "EU" + "." + ENDPOINT, "https://s3-eu-west-1.amazonaws.com"); + properties.setProperty(PROPERTY_REGION + "." + EU_WEST_1 + "." + ENDPOINT, "https://s3-eu-west-1.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + AP_SOUTHEAST_1 + "." + ENDPOINT, "https://s3-ap-southeast-1.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + AP_NORTHEAST_1 + "." + ENDPOINT, diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfigured.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfigured.java deleted file mode 100644 index aa8795472c..0000000000 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfigured.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to jclouds, Inc. (jclouds) under one or more - * contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. jclouds licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.jclouds.aws.s3.binders; - -import static org.jclouds.http.utils.ModifyRequest.endpoint; -import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH; -import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS; - -import java.net.URI; -import java.util.Map; - -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Provider; -import javax.inject.Singleton; -import javax.ws.rs.core.UriBuilder; - -import org.jclouds.http.HttpRequest; -import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; -import org.jclouds.rest.binders.BindAsHostPrefix; -import org.jclouds.s3.Bucket; -import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured; - -/** - * - * @author Adrian Cole - */ -@Singleton -public class AssignCorrectHostnameAndBindAsHostPrefixIfConfigured extends BindAsHostPrefixIfConfigured { - private final Map bucketToRegion; - private final RegionToEndpointOrProviderIfNull r2; - - @Inject - public AssignCorrectHostnameAndBindAsHostPrefixIfConfigured(BindAsHostPrefix bindAsHostPrefix, - @Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle, - @Named(PROPERTY_S3_SERVICE_PATH) String servicePath, - RegionToEndpointOrProviderIfNull r2, Provider uriBuilderProvider, - @Bucket Map bucketToRegion) { - super(bindAsHostPrefix, isVhostStyle, servicePath, uriBuilderProvider); - this.bucketToRegion = bucketToRegion; - this.r2 = r2; - } - - @Override - public R bindToRequest(R request, Object payload) { - String bucket = payload.toString(); - String region = bucketToRegion.get(bucket); - if (region != null) { - URI endpoint = r2.apply(region); - request = endpoint(request, uriBuilderProvider.get().uri(endpoint).path(request.getEndpoint().getPath()) - .replaceQuery(request.getEndpoint().getQuery()).build()); - } - return super.bindToRequest(request, payload); - } -} diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3AsyncBlobStore.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3AsyncBlobStore.java index 6d54d71953..784d57ec0c 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3AsyncBlobStore.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3AsyncBlobStore.java @@ -36,6 +36,8 @@ import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions; import org.jclouds.aws.s3.blobstore.strategy.AsyncMultipartUploadStrategy; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.options.PutOptions; import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata; @@ -45,14 +47,15 @@ import org.jclouds.domain.Location; import org.jclouds.s3.blobstore.S3AsyncBlobStore; import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BucketToResourceList; -import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; import org.jclouds.s3.domain.AccessControlList; +import org.jclouds.s3.domain.BucketMetadata; import org.jclouds.s3.domain.CannedAccessPolicy; import org.jclouds.s3.domain.ObjectMetadata; +import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -72,12 +75,13 @@ public class AWSS3AsyncBlobStore extends S3AsyncBlobStore { public AWSS3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier defaultLocation, @Memoized Supplier> locations, AWSS3AsyncClient async, AWSS3Client sync, - BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, - BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, - BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, - Provider fetchBlobMetadataProvider, LoadingCache bucketAcls, + Function, PageSet> convertBucketsToStorageMetadata, + ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList, + ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, + ObjectToBlobMetadata object2BlobMd, Provider fetchBlobMetadataProvider, + LoadingCache bucketAcls, Provider multipartUploadStrategy) { - super(context, blobUtils, service, defaultLocation, locations, async, sync, bucket2ResourceMd, + super(context, blobUtils, service, defaultLocation, locations, async, sync, convertBucketsToStorageMetadata, container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object, object2BlobMd, fetchBlobMetadataProvider, bucketAcls); this.multipartUploadStrategy = multipartUploadStrategy; diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java index 89cc7e5ac1..3c610d5ebe 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java @@ -32,6 +32,8 @@ import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions; import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.options.PutOptions; import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata; @@ -41,14 +43,15 @@ import org.jclouds.domain.Location; import org.jclouds.s3.blobstore.S3BlobStore; import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BucketToResourceList; -import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; import org.jclouds.s3.domain.AccessControlList; +import org.jclouds.s3.domain.BucketMetadata; import org.jclouds.s3.domain.CannedAccessPolicy; import org.jclouds.s3.domain.ObjectMetadata; +import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -67,14 +70,15 @@ public class AWSS3BlobStore extends S3BlobStore { @Inject AWSS3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier defaultLocation, @Memoized Supplier> locations, AWSS3Client sync, - BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, - BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, - BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, - Provider fetchBlobMetadataProvider, LoadingCache bucketAcls, + Function, PageSet> convertBucketsToStorageMetadata, + ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList, + ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, + ObjectToBlobMetadata object2BlobMd, Provider fetchBlobMetadataProvider, + LoadingCache bucketAcls, Provider multipartUploadStrategy) { - super(context, blobUtils, defaultLocation, locations, sync, bucket2ResourceMd, container2BucketListOptions, - bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object, object2BlobMd, - fetchBlobMetadataProvider, bucketAcls); + super(context, blobUtils, defaultLocation, locations, sync, convertBucketsToStorageMetadata, + container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object, + object2BlobMd, fetchBlobMetadataProvider, bucketAcls); this.multipartUploadStrategy = multipartUploadStrategy; this.bucketAcls = bucketAcls; this.blob2Object = blob2Object; diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java index aab19e953d..cb5118181c 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java @@ -19,24 +19,17 @@ package org.jclouds.aws.s3.config; import static org.jclouds.aws.domain.Region.US_STANDARD; -import static org.jclouds.location.reference.LocationConstants.ENDPOINT; -import static org.jclouds.location.reference.LocationConstants.PROPERTY_REGION; -import java.net.URI; -import javax.inject.Named; import javax.inject.Singleton; import org.jclouds.aws.s3.AWSS3AsyncClient; import org.jclouds.aws.s3.AWSS3Client; -import org.jclouds.aws.s3.binders.AssignCorrectHostnameAndBindAsHostPrefixIfConfigured; import org.jclouds.aws.s3.predicates.validators.AWSS3BucketNameValidator; import org.jclouds.location.Region; import org.jclouds.rest.ConfiguresRestClient; import org.jclouds.rest.RestContext; -import org.jclouds.s3.Bucket; import org.jclouds.s3.S3AsyncClient; import org.jclouds.s3.S3Client; -import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured; import org.jclouds.s3.config.S3RestClientModule; import org.jclouds.s3.predicates.validators.BucketNameValidator; @@ -61,20 +54,12 @@ public class AWSS3RestClientModule extends S3RestClientModule defaultRegionForBucket(@Region Supplier defaultRegion) { return Suppliers.ofInstance(US_STANDARD); } - + @Override protected void configure() { - bind(BindAsHostPrefixIfConfigured.class).to(AssignCorrectHostnameAndBindAsHostPrefixIfConfigured.class); bind(BucketNameValidator.class).to(AWSS3BucketNameValidator.class); super.configure(); } - - @Provides - @Singleton - @Bucket - protected Supplier provideBucketURI(@Named(PROPERTY_REGION + "." + US_STANDARD + "." + ENDPOINT) String endpoint){ - return Suppliers.ofInstance(URI.create(endpoint)); - } @Singleton @Provides diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java index 57c8407c38..4210dcaa03 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java @@ -22,13 +22,10 @@ import java.io.IOException; import java.lang.reflect.Array; import java.lang.reflect.Method; import java.util.Map; -import java.util.concurrent.ConcurrentMap; +import java.util.Set; + +import javax.inject.Named; -import com.google.common.base.Supplier; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import com.google.inject.Module; -import com.google.inject.TypeLiteral; import org.jclouds.aws.s3.config.AWSS3RestClientModule; import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex; import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex; @@ -41,22 +38,34 @@ import org.jclouds.http.functions.ReleasePayloadAndReturn; import org.jclouds.http.functions.ReturnTrueIf2xx; import org.jclouds.io.Payload; import org.jclouds.io.Payloads; +import org.jclouds.location.Region; +import org.jclouds.location.reference.LocationConstants; import org.jclouds.rest.ConfiguresRestClient; import org.jclouds.rest.functions.MapHttp4xxCodesToExceptions; import org.jclouds.rest.functions.ReturnVoidOnNotFoundOr404; import org.jclouds.rest.internal.RestAnnotationProcessor; import org.jclouds.s3.S3AsyncClient; import org.jclouds.s3.S3AsyncClientTest; +import org.jclouds.s3.S3Client; import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.domain.ObjectMetadataBuilder; import org.jclouds.s3.domain.S3Object; -import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState; +import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists; import org.jclouds.s3.options.CopyObjectOptions; import org.jclouds.s3.options.PutBucketOptions; import org.jclouds.s3.options.PutObjectOptions; import org.jclouds.s3.xml.LocationConstraintHandler; import org.testng.annotations.Test; +import com.google.common.base.Functions; +import com.google.common.base.Optional; +import com.google.common.base.Supplier; +import com.google.common.cache.CacheLoader; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.TypeLiteral; + /** * @author Adrian Cole */ @@ -72,15 +81,15 @@ public class AWSS3AsyncClientTest extends S3AsyncClientTest { Method method = S3AsyncClient.class.getMethod("copyObject", String.class, String.class, String.class, String.class, Array.newInstance(CopyObjectOptions.class, 0).getClass()); - processor.createRequest(method, "sourceBucket", "sourceObject", "destinationBucket", "destinationObject"); + processor.createRequest(method, "sourceBucket", "sourceObject", "destinationbucket", "destinationObject"); } - public void testGetBucketLocationEU() throws SecurityException, NoSuchMethodException, IOException { + public void testGetBucketLocationEUIsStillDefault() throws SecurityException, NoSuchMethodException, IOException { Method method = AWSS3AsyncClient.class.getMethod("getBucketLocation", String.class); - HttpRequest request = processor.createRequest(method, "eubucket"); + HttpRequest request = processor.createRequest(method, "bucket-eu-west-1"); - assertRequestLineEquals(request, "GET https://eubucket.s3-eu-west-1.amazonaws.com/?location HTTP/1.1"); - assertNonPayloadHeadersEqual(request, "Host: eubucket.s3-eu-west-1.amazonaws.com\n"); + assertRequestLineEquals(request, "GET https://bucket-eu-west-1.s3.amazonaws.com/?location HTTP/1.1"); + assertNonPayloadHeadersEqual(request, "Host: bucket-eu-west-1.s3.amazonaws.com\n"); assertPayloadEquals(request, null, null, false); assertResponseParserClassEquals(method, request, ParseSax.class); @@ -147,7 +156,7 @@ public class AWSS3AsyncClientTest extends S3AsyncClientTest { assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class); assertSaxResponseParserClassEquals(method, null); - assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class); + assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.class); checkFilters(request); } @@ -257,23 +266,29 @@ public class AWSS3AsyncClientTest extends S3AsyncClientTest { assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class); assertSaxResponseParserClassEquals(method, null); - assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class); + assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.class); checkFilters(request); } - @ConfiguresRestClient + @ConfiguresRestClient private static final class TestAWSS3RestClientModule extends AWSS3RestClientModule { - public TestAWSS3RestClientModule() { - super(); - } - @Override - protected ConcurrentMap bucketToRegion() { - ConcurrentMap returnVal = Maps.newConcurrentMap(); - returnVal.put("eubucket", "EU"); - return returnVal; + protected CacheLoader> bucketToRegion(@Region Supplier> regionSupplier, + final S3Client client) { + return CacheLoader.> from(Functions.forMap(ImmutableMap + .> builder() + .put("bucket", Optional. absent()) + .put("destinationbucket", Optional. absent()) + .put("bucket-us-standard", Optional.of("us-standard")) + .put("bucket-us-west-1", Optional.of("us-west-1")) + .put("bucket-us-west-2", Optional.of("us-west-2")) + .put("bucket-eu-west-1", Optional.of("eu-west-1")) + .put("bucket-sa-east-1", Optional.of("sa-east-1")) + .put("bucket-ap-southeast-1", Optional.of("ap-southeast-1")) + .put("bucket-ap-northeast-1", Optional.of("ap-northeast-1")) + .build())); } @Override diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientExpectTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientExpectTest.java index 020ba759ff..5e64501bc4 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientExpectTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientExpectTest.java @@ -42,7 +42,27 @@ import com.google.inject.Injector; */ @Test public class AWSS3ClientExpectTest extends BaseAWSS3ClientExpectTest { - + HttpRequest bucketLocationRequest = HttpRequest.builder() + .method("GET") + .endpoint(URI.create("https://test.s3.amazonaws.com/?location")) + .headers(ImmutableMultimap.of( + "Host", "test.s3.amazonaws.com", + "Date", CONSTANT_DATE, + "Authorization", "AWS identity:D1rymKrEdvzvhmZXeg+Z0R+tiug=" + )) + .build(); + + HttpResponse bucketLocationResponse = HttpResponse.builder() + .statusCode(200) + .payload( + payloadFromStringWithContentType("eu-west-1", "application/xml")) + .headers(ImmutableMultimap.of( + "x-amz-id-2", "BtioT9wIK04YkE2DPgWUrQFiAbjwJVP8cLyfOkJ1FHMbn2hVjBZvkMMuXPDHfGVw", + "x-amz-request-id", "51BF4F45D49B1B34", + "Date", CONSTANT_DATE, + "Server", "AmazonS3" + )) + .build(); @Test public void testPutWithReducedRedundancy() { Injector injector = createInjector(Functions.forMap(ImmutableMap.of()), createModule(), setupProperties()); @@ -50,13 +70,13 @@ public class AWSS3ClientExpectTest extends BaseAWSS3ClientExpectTest { Blob blob = injector.getInstance(BlobBuilder.class).name("test").payload("content").build(); BlobToObject blobToObject = injector.getInstance(BlobToObject.class); - AWSS3Client client = requestSendsResponse( + AWSS3Client client = requestsSendResponses(bucketLocationRequest, bucketLocationResponse, HttpRequest.builder() .method("PUT") - .endpoint(URI.create("https://test.s3.amazonaws.com/test")) + .endpoint(URI.create("https://test.s3-eu-west-1.amazonaws.com/test")) .headers(ImmutableMultimap.of( "x-amz-storage-class", "REDUCED_REDUNDANCY", - "Host", "test.s3.amazonaws.com", + "Host", "test.s3-eu-west-1.amazonaws.com", "Date", CONSTANT_DATE, "Authorization", "AWS identity:1mJrW85/mqZpYTFIK5Ebtt2MM6E=" )) diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest.java deleted file mode 100644 index aae81e2686..0000000000 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to jclouds, Inc. (jclouds) under one or more - * contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. jclouds licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.jclouds.aws.s3.binders; - -import static org.testng.Assert.assertEquals; - -import java.net.URI; -import java.util.Map; - -import javax.inject.Provider; -import javax.ws.rs.core.UriBuilder; - -import org.jclouds.http.HttpRequest; -import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; -import org.jclouds.rest.binders.BindAsHostPrefix; -import org.testng.annotations.Test; - -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import com.google.common.collect.ImmutableMap; -import com.sun.jersey.api.uri.UriBuilderImpl; - -/** - * Tests behavior of {@code AssignCorrectHostnameAndBindAsHostPrefixIfConfigured} - * - * @author Adrian Cole - */ -// NOTE:without testName, this will not call @Before* and fail w/NPE during -// surefire -@Test(groups = "unit", testName = "AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest") -public class AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest { - Provider uriBuilderProvider = new Provider() { - - @Override - public UriBuilder get() { - return new UriBuilderImpl(); - } - - }; - - public void testWhenNoBucketRegionMappingInCache() { - HttpRequest request = new HttpRequest("GET", URI.create("https://s3.amazonaws.com")); - - AssignCorrectHostnameAndBindAsHostPrefixIfConfigured binder = new AssignCorrectHostnameAndBindAsHostPrefixIfConfigured( - new BindAsHostPrefix(uriBuilderProvider), true, "/", new RegionToEndpointOrProviderIfNull("aws-s3", Suppliers - .ofInstance(URI.create("https://s3.amazonaws.com")), - - Suppliers.>> ofInstance(ImmutableMap.of("us-standard", Suppliers - .ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1", Suppliers.ofInstance(URI - .create("https://s3-us-west-1.amazonaws.com"))))), - - uriBuilderProvider, ImmutableMap. of()); - - request = binder.bindToRequest(request, "bucket"); - assertEquals(request.getRequestLine(), "GET https://bucket.s3.amazonaws.com HTTP/1.1"); - - } - - public void testWhenBucketRegionMappingInCache() { - HttpRequest request = new HttpRequest("GET", URI.create("https://s3.amazonaws.com")); - - AssignCorrectHostnameAndBindAsHostPrefixIfConfigured binder = new AssignCorrectHostnameAndBindAsHostPrefixIfConfigured( - new BindAsHostPrefix(uriBuilderProvider), true, "/", new RegionToEndpointOrProviderIfNull("aws-s3", Suppliers - .ofInstance(URI.create("https://s3.amazonaws.com")), - - Suppliers.>> ofInstance(ImmutableMap.of("us-standard", Suppliers - .ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1", Suppliers.ofInstance(URI - .create("https://s3-us-west-1.amazonaws.com"))))), - - uriBuilderProvider, ImmutableMap. of("bucket", "us-west-1")); - - request = binder.bindToRequest(request, "bucket"); - assertEquals(request.getRequestLine(), "GET https://bucket.s3-us-west-1.amazonaws.com HTTP/1.1"); - - } -} diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3ContainerLiveTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3ContainerLiveTest.java index f2cf6b9771..cceb5ea776 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3ContainerLiveTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3ContainerLiveTest.java @@ -18,93 +18,16 @@ */ package org.jclouds.aws.s3.blobstore.integration; -import static org.jclouds.blobstore.options.CreateContainerOptions.Builder.publicRead; -import static org.testng.Assert.assertEquals; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.Date; -import java.util.NoSuchElementException; -import java.util.Set; - -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.domain.BlobMetadata; -import org.jclouds.domain.Location; import org.jclouds.s3.blobstore.integration.S3ContainerLiveTest; -import org.jclouds.util.Strings2; import org.testng.annotations.Test; -import com.google.common.base.Strings; - /** * @author Adrian Cole */ -@Test(groups = "live", testName = "AWSS3ContainerLiveTest") +@Test(groups = "live", singleThreaded = true, testName = "AWSS3ContainerLiveTest") public class AWSS3ContainerLiveTest extends S3ContainerLiveTest { public AWSS3ContainerLiveTest() { provider = "aws-s3"; } - @Test(groups = { "live" }) - public void testCreateBlobWithExpiry() throws InterruptedException, MalformedURLException, IOException { - final String containerName = getScratchContainerName(); - BlobStore blobStore = view.getBlobStore(); - try { - final String blobName = "hello"; - final Date expires = new Date( (System.currentTimeMillis() / 1000) * 1000 + 60*1000); - - blobStore.createContainerInLocation(null, containerName, publicRead()); - blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload(TEST_STRING).expires(expires).build()); - - assertConsistencyAwareBlobExpiryMetadata(containerName, blobName, expires); - - } finally { - recycleContainer(containerName); - } - } - - @Test(groups = { "live" }) - public void testCreateBlobInLocation() throws InterruptedException, MalformedURLException, IOException { - String payload = "my data"; - runCreateContainerInLocation(payload); - } - - @Test(groups = { "live" }) - public void testCreateBigBlobInLocation() throws InterruptedException, MalformedURLException, IOException { - String payload = Strings.repeat("a", 1024*1024); // 1MB - runCreateContainerInLocation(payload); - } - - private void runCreateContainerInLocation(String payload) throws InterruptedException, MalformedURLException, IOException { - String blobName = "hello"; - BlobStore blobStore = view.getBlobStore(); - final String containerName = getScratchContainerName(); - try { - String locationId = "EU"; - Location location = findLocation(blobStore, locationId); - blobStore.createContainerInLocation(location, containerName, publicRead()); - blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload(payload).build()); - - assertConsistencyAwareContainerSize(containerName, 1); - - BlobMetadata metadata = view.getBlobStore().blobMetadata(containerName, blobName); - assertEquals(Strings2.toStringAndClose(view.utils().http().get(metadata.getPublicUri())), payload); - - assertConsistencyAwareBlobInLocation(containerName, blobName, location); - - } finally { - // this container is now public, so we can't reuse it directly - recycleContainer(containerName); - } - } - - private Location findLocation(BlobStore blobStore, String id) { - Set locs = blobStore.listAssignableLocations(); - for (Location loc : locs) { - if (loc.getId().equals(id)) { - return loc; - } - } - throw new NoSuchElementException("No location found with id '"+id+"'; contenders were "+locs); - } } diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/services/AWSBucketsLiveTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/services/AWSBucketsLiveTest.java new file mode 100644 index 0000000000..b23bd977aa --- /dev/null +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/services/AWSBucketsLiveTest.java @@ -0,0 +1,81 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.aws.s3.services; + +import static org.jclouds.s3.options.PutBucketOptions.Builder.withBucketAcl; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import org.jclouds.aws.domain.Region; +import org.jclouds.s3.domain.AccessControlList; +import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI; +import org.jclouds.s3.domain.AccessControlList.Permission; +import org.jclouds.s3.domain.CannedAccessPolicy; +import org.jclouds.s3.services.BucketsLiveTest; +import org.testng.annotations.Test; + +import com.google.common.base.Throwables; + +/** + * @author Adrian Cole + */ +@Test(groups = "live", testName = "AWSBucketsLiveTest") +public class AWSBucketsLiveTest extends BucketsLiveTest { + public AWSBucketsLiveTest() { + provider = "aws-s3"; + } + + public void testDefaultBucketLocation() throws Exception { + + String bucketName = getContainerName(); + try { + String location = getApi().getBucketLocation(bucketName); + assert location.equals(Region.US_STANDARD) : "bucket: " + bucketName + " location: " + location; + } finally { + returnContainer(bucketName); + } + } + + /** + * using scratch bucketName as we are changing location + */ + public void testEu() throws Exception { + final String bucketName = getScratchContainerName(); + try { + getApi().putBucketInRegion(Region.EU_WEST_1, bucketName + "eu", withBucketAcl(CannedAccessPolicy.PUBLIC_READ)); + assertConsistencyAware(new Runnable() { + public void run() { + try { + AccessControlList acl = getApi().getBucketACL(bucketName + "eu"); + assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ), acl.toString()); + } catch (Exception e) { + Throwables.propagateIfPossible(e); + } + } + }); + assertEquals(Region.EU_WEST_1, getApi().getBucketLocation(bucketName + "eu")); + // TODO: I believe that the following should work based on the above acl assertion passing. + // However, it fails on 403 + // URL url = new URL(String.format("http://%s.s3.amazonaws.com", bucketName)); + // Utils.toStringAndClose(url.openStream()); + } finally { + destroyContainer(bucketName + "eu"); + } + } +} diff --git a/providers/hpcloud-objectstorage/src/test/java/org/jclouds/hpcloud/objectstorage/blobstore/integration/HPCloudObjectStorageContainerLiveTest.java b/providers/hpcloud-objectstorage/src/test/java/org/jclouds/hpcloud/objectstorage/blobstore/integration/HPCloudObjectStorageContainerLiveTest.java index 91d500d9b8..d5457a3217 100644 --- a/providers/hpcloud-objectstorage/src/test/java/org/jclouds/hpcloud/objectstorage/blobstore/integration/HPCloudObjectStorageContainerLiveTest.java +++ b/providers/hpcloud-objectstorage/src/test/java/org/jclouds/hpcloud/objectstorage/blobstore/integration/HPCloudObjectStorageContainerLiveTest.java @@ -18,9 +18,6 @@ */ package org.jclouds.hpcloud.objectstorage.blobstore.integration; -import java.io.IOException; -import java.net.MalformedURLException; - import org.jclouds.blobstore.integration.internal.BaseContainerLiveTest; import org.testng.annotations.Test; @@ -31,11 +28,5 @@ import org.testng.annotations.Test; public class HPCloudObjectStorageContainerLiveTest extends BaseContainerLiveTest { public HPCloudObjectStorageContainerLiveTest() { provider = "hpcloud-objectstorage"; - } - - @Test(enabled = false) - //@Test(expectedExceptions=UnsupportedOperationException.class) - public void testPublicAccess() throws MalformedURLException, InterruptedException, IOException { - super.testPublicAccess(); } }