diff --git a/apis/s3/src/main/java/org/jclouds/s3/S3AsyncClient.java b/apis/s3/src/main/java/org/jclouds/s3/S3AsyncClient.java index 3bd477de58..749d28c80d 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/S3AsyncClient.java +++ b/apis/s3/src/main/java/org/jclouds/s3/S3AsyncClient.java @@ -23,7 +23,6 @@ import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER; import java.util.Set; import java.util.concurrent.ExecutionException; -import org.jclouds.javax.annotation.Nullable; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.HEAD; @@ -41,8 +40,9 @@ import org.jclouds.blobstore.functions.ThrowContainerNotFoundOn404; import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404; import org.jclouds.http.functions.ParseETagHeader; import org.jclouds.http.options.GetOptions; -import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; +import org.jclouds.javax.annotation.Nullable; import org.jclouds.rest.annotations.BinderParam; +import org.jclouds.rest.annotations.Endpoint; import org.jclouds.rest.annotations.EndpointParam; import org.jclouds.rest.annotations.ExceptionParser; import org.jclouds.rest.annotations.Headers; @@ -69,11 +69,13 @@ import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.domain.Payer; import org.jclouds.s3.domain.S3Object; import org.jclouds.s3.filters.RequestAuthorizeSignature; +import org.jclouds.s3.functions.AssignCorrectHostnameForBucket; import org.jclouds.s3.functions.BindRegionToXmlPayload; +import org.jclouds.s3.functions.DefaultEndpointThenInvalidateRegion; import org.jclouds.s3.functions.ObjectKey; import org.jclouds.s3.functions.ParseObjectFromHeadersAndHttpContent; import org.jclouds.s3.functions.ParseObjectMetadataFromHeaders; -import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState; +import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists; import org.jclouds.s3.functions.ReturnTrueOn404OrNotFoundFalseOnIllegalState; import org.jclouds.s3.options.CopyObjectOptions; import org.jclouds.s3.options.ListBucketOptions; @@ -95,7 +97,7 @@ import com.google.inject.Provides; * Provides asynchronous access to S3 via their REST API. *

* All commands return a ListenableFuture of the result from S3. Any exceptions incurred during - * processing will be wrapped in an {@link ExecutionException} as documented in + * processing will be backend in an {@link ExecutionException} as documented in * {@link ListenableFuture#get()}. * * @author Adrian Cole @@ -123,8 +125,8 @@ public interface S3AsyncClient { @ExceptionParser(ReturnNullOnKeyNotFound.class) @ResponseParser(ParseObjectFromHeadersAndHttpContent.class) ListenableFuture getObject( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key, GetOptions... options); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, GetOptions... options); /** * @see S3Client#headObject @@ -134,8 +136,8 @@ public interface S3AsyncClient { @ExceptionParser(ReturnNullOnKeyNotFound.class) @ResponseParser(ParseObjectMetadataFromHeaders.class) ListenableFuture headObject( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key); /** * @see S3Client#objectExists @@ -144,8 +146,8 @@ public interface S3AsyncClient { @Path("/{key}") @ExceptionParser(ReturnFalseOnKeyNotFound.class) ListenableFuture objectExists( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key); /** * @see S3Client#deleteObject @@ -154,8 +156,8 @@ public interface S3AsyncClient { @Path("/{key}") @ExceptionParser(ReturnVoidOnNotFoundOr404.class) ListenableFuture deleteObject( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key); /** * @see S3Client#putObject @@ -164,39 +166,43 @@ public interface S3AsyncClient { @Path("/{key}") @ResponseParser(ParseETagHeader.class) ListenableFuture putObject( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") @ParamParser(ObjectKey.class) @BinderParam(BindS3ObjectMetadataToRequest.class) S3Object object, - PutObjectOptions... options); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") @ParamParser(ObjectKey.class) @BinderParam(BindS3ObjectMetadataToRequest.class) S3Object object, + PutObjectOptions... options); /** * @see S3Client#putBucketInRegion */ @PUT @Path("/") - @ExceptionParser(ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class) + @Endpoint(Bucket.class) + @ExceptionParser(ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.class) ListenableFuture putBucketInRegion( - @EndpointParam(parser = RegionToEndpointOrProviderIfNull.class) @BinderParam(BindRegionToXmlPayload.class) @Nullable String region, - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - PutBucketOptions... options); + @BinderParam(BindRegionToXmlPayload.class) @Nullable String region, + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + PutBucketOptions... options); /** * @see S3Client#deleteBucketIfEmpty */ @DELETE @Path("/") + @Endpoint(Bucket.class) @ExceptionParser(ReturnTrueOn404OrNotFoundFalseOnIllegalState.class) ListenableFuture deleteBucketIfEmpty( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @EndpointParam(parser = DefaultEndpointThenInvalidateRegion.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); /** * @see S3Client#bucketExists */ @HEAD @Path("/") + @Endpoint(Bucket.class) @QueryParams(keys = "max-keys", values = "0") @ExceptionParser(ReturnFalseOnContainerNotFound.class) ListenableFuture bucketExists( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); + /** * @see S3Client#getBucketLocation @@ -204,9 +210,10 @@ public interface S3AsyncClient { @GET @QueryParams(keys = "location") @Path("/") + @Endpoint(Bucket.class) @XMLResponseParser(LocationConstraintHandler.class) ListenableFuture getBucketLocation( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); /** * @see S3Client#getBucketPayer @@ -216,7 +223,7 @@ public interface S3AsyncClient { @Path("/") @XMLResponseParser(PayerHandler.class) ListenableFuture getBucketPayer( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); /** * @see S3Client#setBucketPayer @@ -225,8 +232,8 @@ public interface S3AsyncClient { @QueryParams(keys = "requestPayment") @Path("/") ListenableFuture setBucketPayer( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @BinderParam(BindPayerToXmlPayload.class) Payer payer); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @BinderParam(BindPayerToXmlPayload.class) Payer payer); /** * @see S3Client#listBucket @@ -235,8 +242,8 @@ public interface S3AsyncClient { @Path("/") @XMLResponseParser(ListBucketHandler.class) ListenableFuture listBucket( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - ListBucketOptions... options); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + ListBucketOptions... options); /** * @see S3Client#listOwnedBuckets @@ -255,10 +262,10 @@ public interface S3AsyncClient { @Headers(keys = "x-amz-copy-source", values = "/{sourceBucket}/{sourceObject}") @XMLResponseParser(CopyObjectHandler.class) ListenableFuture copyObject( - @PathParam("sourceBucket") String sourceBucket, - @PathParam("sourceObject") String sourceObject, - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String destinationBucket, - @PathParam("destinationObject") String destinationObject, CopyObjectOptions... options); + @PathParam("sourceBucket") String sourceBucket, + @PathParam("sourceObject") String sourceObject, + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String destinationBucket, + @PathParam("destinationObject") String destinationObject, CopyObjectOptions... options); /** * @see S3Client#getBucketACL @@ -269,7 +276,7 @@ public interface S3AsyncClient { @ExceptionParser(ThrowContainerNotFoundOn404.class) @Path("/") ListenableFuture getBucketACL( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); /** * @see S3Client#putBucketACL @@ -278,8 +285,8 @@ public interface S3AsyncClient { @Path("/") @QueryParams(keys = "acl") ListenableFuture putBucketACL( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @BinderParam(BindACLToXMLPayload.class) AccessControlList acl); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @BinderParam(BindACLToXMLPayload.class) AccessControlList acl); /** * @see S3Client#getObjectACL @@ -290,8 +297,8 @@ public interface S3AsyncClient { @XMLResponseParser(AccessControlListHandler.class) @ExceptionParser(ThrowKeyNotFoundOn404.class) ListenableFuture getObjectACL( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key); /** * @see S3Client#putObjectACL @@ -300,8 +307,8 @@ public interface S3AsyncClient { @QueryParams(keys = "acl") @Path("/{key}") ListenableFuture putObjectACL( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key, @BinderParam(BindACLToXMLPayload.class) AccessControlList acl); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, @BinderParam(BindACLToXMLPayload.class) AccessControlList acl); /** * @see S3Client#getBucketLogging @@ -312,7 +319,7 @@ public interface S3AsyncClient { @ExceptionParser(ThrowContainerNotFoundOn404.class) @Path("/") ListenableFuture getBucketLogging( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); /** * @see S3Client#enableBucketLogging @@ -321,8 +328,8 @@ public interface S3AsyncClient { @Path("/") @QueryParams(keys = "logging") ListenableFuture enableBucketLogging( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @BinderParam(BindBucketLoggingToXmlPayload.class) BucketLogging logging); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @BinderParam(BindBucketLoggingToXmlPayload.class) BucketLogging logging); /** * @see S3Client#putBucketLogging @@ -332,6 +339,6 @@ public interface S3AsyncClient { @QueryParams(keys = "logging") @Produces(MediaType.TEXT_XML) ListenableFuture disableBucketLogging( - @Bucket @BinderParam(BindNoBucketLoggingToXmlPayload.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindNoBucketLoggingToXmlPayload.class) @ParamValidators(BucketNameValidator.class) String bucketName); } diff --git a/apis/s3/src/main/java/org/jclouds/s3/S3Client.java b/apis/s3/src/main/java/org/jclouds/s3/S3Client.java index 68133874e3..c974ca29b0 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/S3Client.java +++ b/apis/s3/src/main/java/org/jclouds/s3/S3Client.java @@ -198,7 +198,7 @@ public interface S3Client { * */ @Timeout(duration = 90, timeUnit = TimeUnit.SECONDS) - boolean putBucketInRegion(@Nullable String region, String bucketName, PutBucketOptions... options); + boolean putBucketInRegion(@Nullable String region, @Bucket String bucketName, PutBucketOptions... options); /** * Deletes the bucket, if it is empty. diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3AsyncBlobStore.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3AsyncBlobStore.java index 7f2a5b6053..fe071f0626 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3AsyncBlobStore.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3AsyncBlobStore.java @@ -34,7 +34,6 @@ import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; -import org.jclouds.blobstore.domain.internal.PageSetImpl; import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.internal.BaseAsyncBlobStore; import org.jclouds.blobstore.options.CreateContainerOptions; @@ -50,7 +49,6 @@ import org.jclouds.s3.S3AsyncClient; import org.jclouds.s3.S3Client; import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BucketToResourceList; -import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; @@ -70,7 +68,6 @@ import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.google.common.collect.Iterables; import com.google.common.util.concurrent.ListenableFuture; /** @@ -82,7 +79,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore { private final S3AsyncClient async; private final S3Client sync; - private final BucketToResourceMetadata bucket2ResourceMd; + private final Function, PageSet> convertBucketsToStorageMetadata; private final ContainerToBucketListOptions container2BucketListOptions; private final BlobToHttpGetOptions blob2ObjectGetOptions; private final BucketToResourceList bucket2ResourceList; @@ -94,17 +91,18 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore { @Inject protected S3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier defaultLocation, - @Memoized Supplier> locations, S3AsyncClient async, S3Client sync, - BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, - BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, - BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, - Provider fetchBlobMetadataProvider, LoadingCache bucketAcls) { + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier defaultLocation, + @Memoized Supplier> locations, S3AsyncClient async, S3Client sync, + Function, PageSet> convertBucketsToStorageMetadata, + ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList, + ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, + ObjectToBlobMetadata object2BlobMd, Provider fetchBlobMetadataProvider, + LoadingCache bucketAcls) { super(context, blobUtils, service, defaultLocation, locations); this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions"); this.async = checkNotNull(async, "async"); this.sync = checkNotNull(sync, "sync"); - this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd"); + this.convertBucketsToStorageMetadata = checkNotNull(convertBucketsToStorageMetadata, "convertBucketsToStorageMetadata"); this.container2BucketListOptions = checkNotNull(container2BucketListOptions, "container2BucketListOptions"); this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList"); this.object2Blob = checkNotNull(object2Blob, "object2Blob"); @@ -122,7 +120,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore { return Futures.compose(async.listOwnedBuckets(), new Function, org.jclouds.blobstore.domain.PageSet>() { public org.jclouds.blobstore.domain.PageSet apply(Set from) { - return new PageSetImpl(Iterables.transform(from, bucket2ResourceMd), null); + return convertBucketsToStorageMetadata.apply(from); } }, service); } diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java index 6f8d01a925..41c6b4999a 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java @@ -27,11 +27,11 @@ import javax.inject.Provider; import javax.inject.Singleton; import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; -import org.jclouds.blobstore.domain.internal.PageSetImpl; import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.internal.BaseBlobStore; import org.jclouds.blobstore.options.CreateContainerOptions; @@ -45,7 +45,6 @@ import org.jclouds.http.options.GetOptions; import org.jclouds.s3.S3Client; import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BucketToResourceList; -import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; @@ -64,7 +63,6 @@ import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.google.common.collect.Iterables; /** * @@ -73,7 +71,7 @@ import com.google.common.collect.Iterables; @Singleton public class S3BlobStore extends BaseBlobStore { private final S3Client sync; - private final BucketToResourceMetadata bucket2ResourceMd; + private final Function, PageSet> convertBucketsToStorageMetadata; private final ContainerToBucketListOptions container2BucketListOptions; private final BucketToResourceList bucket2ResourceList; private final ObjectToBlob object2Blob; @@ -85,15 +83,16 @@ public class S3BlobStore extends BaseBlobStore { @Inject protected S3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier defaultLocation, - @Memoized Supplier> locations, S3Client sync, - BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, - BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, - BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, - Provider fetchBlobMetadataProvider, LoadingCache bucketAcls) { + @Memoized Supplier> locations, S3Client sync, + Function, PageSet> convertBucketsToStorageMetadata, + ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList, + ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, + ObjectToBlobMetadata object2BlobMd, Provider fetchBlobMetadataProvider, + LoadingCache bucketAcls) { super(context, blobUtils, defaultLocation, locations); this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions"); this.sync = checkNotNull(sync, "sync"); - this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd"); + this.convertBucketsToStorageMetadata = checkNotNull(convertBucketsToStorageMetadata, "convertBucketsToStorageMetadata"); this.container2BucketListOptions = checkNotNull(container2BucketListOptions, "container2BucketListOptions"); this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList"); this.object2Blob = checkNotNull(object2Blob, "object2Blob"); @@ -108,11 +107,7 @@ public class S3BlobStore extends BaseBlobStore { */ @Override public PageSet list() { - return new Function, org.jclouds.blobstore.domain.PageSet>() { - public org.jclouds.blobstore.domain.PageSet apply(Set from) { - return new PageSetImpl(Iterables.transform(from, bucket2ResourceMd), null); - } - }.apply(sync.listOwnedBuckets()); + return convertBucketsToStorageMetadata.apply(sync.listOwnedBuckets()); } /** @@ -169,10 +164,15 @@ public class S3BlobStore extends BaseBlobStore { */ public void clearAndDeleteContainer(final String container) { try { + //TODO: probably it is better to use a retryable predicate if (!Assertions.eventuallyTrue(new Supplier() { public Boolean get() { - clearContainer(container); - return sync.deleteBucketIfEmpty(container); + try { + clearContainer(container); + return sync.deleteBucketIfEmpty(container); + } catch (ContainerNotFoundException e) { + return true; + } } }, 30000)) { throw new IllegalStateException(container + " still exists after deleting!"); diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java index 709333e7df..c05acb54d7 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java @@ -35,7 +35,7 @@ import org.jclouds.s3.S3Client; import org.jclouds.s3.blobstore.S3AsyncBlobStore; import org.jclouds.s3.blobstore.S3BlobRequestSigner; import org.jclouds.s3.blobstore.S3BlobStore; -import org.jclouds.s3.blobstore.functions.LocationFromBucketLocation; +import org.jclouds.s3.blobstore.functions.LocationFromBucketName; import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.BucketMetadata; @@ -63,7 +63,8 @@ public class S3BlobStoreContextModule extends AbstractModule { bind(BlobStore.class).to(S3BlobStore.class).in(Scopes.SINGLETON); bindContext(); bind(BlobRequestSigner.class).to(S3BlobRequestSigner.class); - bindBucketLocationStrategy(); + bind(new TypeLiteral>() { + }).to(LocationFromBucketName.class); } protected void bindContext() { @@ -71,11 +72,6 @@ public class S3BlobStoreContextModule extends AbstractModule { }).in(Scopes.SINGLETON); } - protected void bindBucketLocationStrategy() { - bind(new TypeLiteral>() { - }).to(LocationFromBucketLocation.class); - } - @Provides @Singleton protected LoadingCache bucketAcls(final S3Client client) { diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BucketToResourceMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BucketToResourceMetadata.java index 8fc3212750..b7aa78b328 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BucketToResourceMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BucketToResourceMetadata.java @@ -36,10 +36,10 @@ import com.google.common.base.Function; */ @Singleton public class BucketToResourceMetadata implements Function { - private final Function locationOfBucket; + private final Function locationOfBucket; @Inject - BucketToResourceMetadata(Function locationOfBucket) { + BucketToResourceMetadata(Function locationOfBucket) { this.locationOfBucket = locationOfBucket; } @@ -47,7 +47,7 @@ public class BucketToResourceMetadata implements Function, PageSet> { + + @Resource + protected Logger logger = Logger.NULL; + + private final ExecutorService userExecutor; + private final BucketToResourceMetadata bucket2ResourceMd; + + @Inject + public BucketsToStorageMetadata(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, BucketToResourceMetadata bucket2ResourceMd) { + this.userExecutor = checkNotNull(userExecutor, "userExecutor"); + this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd"); + } + + + @Override + public PageSet apply(Set input) { + // parallel as listing buckets is slow when looking up regions + Iterable buckets = FutureIterables + . transformParallel(input, + new Function>() { + @Override + public Future apply(final BucketMetadata from) { + return userExecutor.submit(new Callable() { + + @Override + public StorageMetadata call() throws Exception { + return bucket2ResourceMd.apply(from); + } + + @Override + public String toString() { + return "bucket2ResourceMd.apply(" + from + ")"; + } + }); + } + + }, userExecutor, null, logger, "my buckets"); + return new PageSetImpl(buckets, null); + } + +} diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketLocation.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketLocation.java deleted file mode 100644 index 49c765f74b..0000000000 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketLocation.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Licensed to jclouds, Inc. (jclouds) under one or more - * contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. jclouds licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.jclouds.s3.blobstore.functions; - -import java.util.NoSuchElementException; -import java.util.Set; - -import javax.annotation.Resource; -import javax.inject.Inject; -import javax.inject.Singleton; - -import org.jclouds.s3.S3Client; -import org.jclouds.s3.domain.BucketMetadata; -import org.jclouds.blobstore.ContainerNotFoundException; -import org.jclouds.collect.Memoized; -import org.jclouds.domain.Location; -import org.jclouds.logging.Logger; - -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.base.Supplier; -import com.google.common.collect.Iterables; - -/** - * @author Adrian Cole - */ -@Singleton -public class LocationFromBucketLocation implements Function { - private final Location onlyLocation; - private final Supplier> locations; - private final S3Client client; - - @Resource - protected Logger logger = Logger.NULL; - - @Inject - LocationFromBucketLocation(S3Client client, @Memoized Supplier> locations) { - this.client = client; - this.onlyLocation = locations.get().size() == 1 ? Iterables.get(locations.get(), 0) : null; - this.locations = locations; - } - - public Location apply(BucketMetadata from) { - if (onlyLocation != null) - return onlyLocation; - try { - Set locations = this.locations.get(); - final String region = client.getBucketLocation(from.getName()); - assert region != null : String.format("could not get region for %s", from.getName()); - if (region != null) { - try { - return Iterables.find(locations, new Predicate() { - - @Override - public boolean apply(Location input) { - return input.getId().equalsIgnoreCase(region.toString()); - } - - }); - } catch (NoSuchElementException e) { - logger.error("could not get location for region %s in %s", region, locations); - } - } else { - logger.error("could not get region for %s", from.getName()); - } - } catch (ContainerNotFoundException e) { - logger.error(e, "could not get region for %s, as service suggests the bucket doesn't exist", from.getName()); - } - return null; - } -} diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketName.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketName.java new file mode 100644 index 0000000000..f4ec53e5e9 --- /dev/null +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketName.java @@ -0,0 +1,75 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.blobstore.functions; + +import static com.google.common.collect.Iterables.find; +import static com.google.common.collect.Iterables.get; +import static org.jclouds.location.predicates.LocationPredicates.idEquals; + +import java.util.NoSuchElementException; +import java.util.Set; + +import javax.annotation.Resource; +import javax.inject.Inject; +import javax.inject.Singleton; + +import org.jclouds.collect.Memoized; +import org.jclouds.domain.Location; +import org.jclouds.logging.Logger; +import org.jclouds.s3.Bucket; + +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.base.Supplier; + +/** + * @author Adrian Cole + */ +@Singleton +public class LocationFromBucketName implements Function { + private final Supplier> locations; + private final Function> bucketToRegion; + + @Resource + protected Logger logger = Logger.NULL; + + @Inject + LocationFromBucketName(@Bucket Function> bucketToRegion, + @Memoized Supplier> locations) { + this.bucketToRegion = bucketToRegion; + this.locations = locations; + } + + public Location apply(String bucket) { + Set locations = this.locations.get(); + if (locations.size() == 1) + return get(locations, 0); + final Optional region = bucketToRegion.apply(bucket); + if (region.isPresent()) { + try { + return find(locations, idEquals(region.get())); + } catch (NoSuchElementException e) { + logger.debug("could not get location for region %s in %s", region, locations); + } + } else { + logger.debug("could not get region for %s", bucket); + } + return null; + } +} diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java index 3fe757a005..64d11a6c5a 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java @@ -25,6 +25,7 @@ import org.jclouds.blobstore.domain.MutableBlobMetadata; import org.jclouds.blobstore.domain.StorageType; import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl; import org.jclouds.blobstore.strategy.IfDirectoryReturnNameStrategy; +import org.jclouds.domain.Location; import org.jclouds.http.HttpUtils; import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.ObjectMetadata; @@ -42,12 +43,14 @@ import com.google.common.cache.LoadingCache; public class ObjectToBlobMetadata implements Function { private final IfDirectoryReturnNameStrategy ifDirectoryReturnName; private final LoadingCache bucketAcls; + private final Function locationOfBucket; @Inject public ObjectToBlobMetadata(IfDirectoryReturnNameStrategy ifDirectoryReturnName, - LoadingCache bucketAcls) { + LoadingCache bucketAcls,Function locationOfBucket) { this.ifDirectoryReturnName = ifDirectoryReturnName; this.bucketAcls = bucketAcls; + this.locationOfBucket = locationOfBucket; } public MutableBlobMetadata apply(ObjectMetadata from) { @@ -68,6 +71,7 @@ public class ObjectToBlobMetadata implements Function ext @Provides @Bucket @Singleton - protected Map bucketToRegion() { - return Maps.newConcurrentMap(); + protected CacheLoader> bucketToRegion(@Region Supplier> regionSupplier, + final S3Client client) { + Set regions = regionSupplier.get(); + if (regions.size() == 0) { + return new CacheLoader>() { + + @Override + public Optional load(String bucket) { + return Optional.absent(); + } + + @Override + public String toString() { + return "noRegions()"; + } + }; + } else if (regions.size() == 1) { + final String onlyRegion = Iterables.getOnlyElement(regions); + return new CacheLoader>() { + Optional onlyRegionOption = Optional.of(onlyRegion); + + @Override + public Optional load(String bucket) { + return onlyRegionOption; + } + + @Override + public String toString() { + return "onlyRegion(" + onlyRegion + ")"; + } + }; + } else { + return new CacheLoader>() { + @Override + public Optional load(String bucket) { + try { + return Optional.fromNullable(client.getBucketLocation(bucket)); + } catch (ContainerNotFoundException e) { + return null; + } + } + + @Override + public String toString() { + return "bucketToRegion()"; + } + }; + } + } + + @Provides + @Bucket + @Singleton + protected LoadingCache> bucketToRegion(@Bucket CacheLoader> loader) { + return CacheBuilder.newBuilder().build(loader); } @Provides @@ -80,12 +149,24 @@ public class S3RestClientModule ext return defaultRegion; } + @Provides + @Singleton + @Bucket + protected Supplier provideBucketURI(@Bucket Supplier defaultRegion, + RegionToEndpointOrProviderIfNull regionToEndpoint) { + return Suppliers.compose(regionToEndpoint, defaultRegion); + } + @Override protected void configure() { + super.configure(); install(new S3ObjectModule()); install(new S3ParserModule()); bind(RequestAuthorizeSignature.class).in(Scopes.SINGLETON); - super.configure(); + bind(new TypeLiteral>>() { + }).annotatedWith(Bucket.class).to(GetRegionForBucket.class); + bind(new TypeLiteral, PageSet>>() { + }).to(BucketsToStorageMetadata.class); } @Override @@ -104,6 +185,7 @@ public class S3RestClientModule ext @Override protected void bindRetryHandlers() { bind(HttpRetryHandler.class).annotatedWith(Redirection.class).to(S3RedirectionRetryHandler.class); + bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(AWSClientErrorRetryHandler.class); } @Provides diff --git a/apis/s3/src/main/java/org/jclouds/s3/domain/BucketMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/domain/BucketMetadata.java index 27be757a28..9723487824 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/domain/BucketMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/domain/BucketMetadata.java @@ -27,8 +27,8 @@ import java.util.Date; */ public class BucketMetadata implements Comparable { private final Date creationDate; - private final CanonicalUser owner; private final String name; + private final CanonicalUser owner; public BucketMetadata(String name, Date creationDate, CanonicalUser owner) { this.name = name; diff --git a/apis/s3/src/main/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucket.java b/apis/s3/src/main/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucket.java new file mode 100644 index 0000000000..e1adae7be4 --- /dev/null +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucket.java @@ -0,0 +1,64 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import java.net.URI; + +import javax.annotation.Resource; +import javax.inject.Inject; +import javax.inject.Singleton; + +import org.jclouds.javax.annotation.Nullable; +import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; +import org.jclouds.logging.Logger; +import org.jclouds.s3.Bucket; + +import com.google.common.base.Function; +import com.google.common.base.Optional; + +/** + * + * @author Adrian Cole + */ +@Singleton +public class AssignCorrectHostnameForBucket implements Function { + @Resource + protected Logger logger = Logger.NULL; + + protected final RegionToEndpointOrProviderIfNull r2; + protected final Function> bucketToRegion; + + @Inject + public AssignCorrectHostnameForBucket(RegionToEndpointOrProviderIfNull r2, + @Bucket Function> bucketToRegion) { + this.bucketToRegion = bucketToRegion; + this.r2 = r2; + } + + @Override + public URI apply(@Nullable Object from) { + String bucket = from.toString(); + Optional region = bucketToRegion.apply(bucket); + if (region.isPresent()) { + return r2.apply(region.get()); + } + return r2.apply(null); + } + +} diff --git a/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.java b/apis/s3/src/main/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegion.java similarity index 50% rename from apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.java rename to apis/s3/src/main/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegion.java index dd94e9c027..680f16500b 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.java +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegion.java @@ -18,30 +18,42 @@ */ package org.jclouds.s3.functions; -import static org.jclouds.util.Throwables2.propagateOrNull; +import java.net.URI; +import javax.inject.Inject; import javax.inject.Singleton; -import org.jclouds.aws.AWSResponseException; -import org.jclouds.util.Throwables2; +import org.jclouds.javax.annotation.Nullable; +import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; +import org.jclouds.s3.Bucket; import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.cache.LoadingCache; /** * * @author Adrian Cole */ @Singleton -public class ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState implements Function { +public class DefaultEndpointThenInvalidateRegion implements Function { - public Boolean apply(Exception from) { - AWSResponseException exception = Throwables2.getFirstThrowableOfType(from, AWSResponseException.class); - if (exception != null && exception.getError() != null - && exception.getError().getCode().equals("BucketAlreadyOwnedByYou")) { - return false; - } else if (Throwables2.getFirstThrowableOfType(from, IllegalStateException.class) != null) { - return false; + private final LoadingCache> bucketToRegionCache; + private final RegionToEndpointOrProviderIfNull r2; + + @Inject + public DefaultEndpointThenInvalidateRegion(RegionToEndpointOrProviderIfNull r2, + @Bucket LoadingCache> bucketToRegionCache) { + this.r2 = r2; + this.bucketToRegionCache = bucketToRegionCache; + } + + @Override + public URI apply(@Nullable Object from) { + try { + return r2.apply(null); + } finally { + bucketToRegionCache.invalidate(from.toString()); } - return Boolean.class.cast(propagateOrNull(from)); } } diff --git a/apis/s3/src/main/java/org/jclouds/s3/functions/GetRegionForBucket.java b/apis/s3/src/main/java/org/jclouds/s3/functions/GetRegionForBucket.java new file mode 100644 index 0000000000..fb5544cee7 --- /dev/null +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/GetRegionForBucket.java @@ -0,0 +1,65 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import java.util.concurrent.ExecutionException; + +import javax.annotation.Resource; +import javax.inject.Inject; +import javax.inject.Singleton; + +import org.jclouds.logging.Logger; +import org.jclouds.s3.Bucket; + +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.cache.CacheLoader.InvalidCacheLoadException; +import com.google.common.cache.LoadingCache; +import com.google.common.util.concurrent.UncheckedExecutionException; + +/** + * + * @author Adrian Cole + */ +@Singleton +public class GetRegionForBucket implements Function> { + @Resource + protected Logger logger = Logger.NULL; + + protected final LoadingCache> bucketToRegion; + + @Inject + public GetRegionForBucket(@Bucket LoadingCache> bucketToRegion) { + this.bucketToRegion = bucketToRegion; + } + + @Override + public Optional apply(String bucket) { + try { + return bucketToRegion.get(bucket); + } catch (ExecutionException e) { + logger.debug("error looking up region for bucket %s: %s", bucket, e); + } catch (UncheckedExecutionException e) { + logger.debug("error looking up region for bucket %s: %s", bucket, e); + } catch (InvalidCacheLoadException e) { + logger.trace("bucket %s not found: %s", bucket, e); + } + return Optional.absent(); + } +} diff --git a/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.java b/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.java new file mode 100644 index 0000000000..3e3f6588e3 --- /dev/null +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.java @@ -0,0 +1,70 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import javax.inject.Inject; + +import org.jclouds.aws.AWSResponseException; +import org.jclouds.http.HttpRequest; +import org.jclouds.javax.annotation.Nullable; +import org.jclouds.rest.InvocationContext; +import org.jclouds.s3.S3Client; +import org.jclouds.s3.util.S3Utils; +import org.jclouds.util.Throwables2; + +import com.google.common.base.Function; +import com.google.common.base.Throwables; + +/** + * + * @author Adrian Cole + */ +public class ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists implements + Function, + InvocationContext { + + private final S3Client client; + private String bucket; + + @Inject + ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(S3Client client) { + this.client = client; + } + + public Boolean apply(Exception from) { + AWSResponseException exception = Throwables2.getFirstThrowableOfType(from, AWSResponseException.class); + if (exception != null && exception.getError() != null && exception.getError().getCode() != null) { + String code = exception.getError().getCode(); + if (code.equals("BucketAlreadyOwnedByYou")) + return false; + else if (code.equals("OperationAborted") && bucket != null && client.bucketExists(bucket)) + return false; + } + throw Throwables.propagate(from); + } + + @Override + public ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists setContext( + @Nullable HttpRequest request) { + if (request != null) + this.bucket = S3Utils.getBucketName(request); + return this; + } + +} diff --git a/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnTrueOn404OrNotFoundFalseOnIllegalState.java b/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnTrueOn404OrNotFoundFalseOnIllegalState.java index b5f232ec6d..28e54e60f7 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnTrueOn404OrNotFoundFalseOnIllegalState.java +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/ReturnTrueOn404OrNotFoundFalseOnIllegalState.java @@ -36,6 +36,7 @@ import com.google.common.base.Function; @Singleton public class ReturnTrueOn404OrNotFoundFalseOnIllegalState implements Function { + //TODO: invalidate public Boolean apply(Exception from) { if (getFirstThrowableOfType(from, IllegalStateException.class) != null) { return false; diff --git a/apis/s3/src/main/java/org/jclouds/s3/util/S3Utils.java b/apis/s3/src/main/java/org/jclouds/s3/util/S3Utils.java index 3d71cffcab..954bdb8163 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/util/S3Utils.java +++ b/apis/s3/src/main/java/org/jclouds/s3/util/S3Utils.java @@ -20,12 +20,20 @@ package org.jclouds.s3.util; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.Iterables.any; +import java.lang.annotation.Annotation; +import java.util.Arrays; import java.util.regex.Pattern; +import org.jclouds.http.HttpRequest; +import org.jclouds.rest.internal.GeneratedHttpRequest; +import org.jclouds.s3.Bucket; import org.jclouds.s3.S3Client; import org.jclouds.util.Patterns; +import com.google.common.base.Predicate; + /** * Encryption, Hashing, and IO Utilities needed to sign and verify S3 requests and responses. * @@ -56,4 +64,27 @@ public class S3Utils { sync.deleteBucketIfEmpty(container); return sync.bucketExists(container); } + + private static final Predicate ANNOTATIONTYPE_BUCKET = new Predicate() { + public boolean apply(Annotation input) { + return input.annotationType().equals(Bucket.class); + } + }; + + public static String getBucketName(HttpRequest req) { + checkArgument(req instanceof GeneratedHttpRequest, "this should be a generated http request"); + GeneratedHttpRequest request = GeneratedHttpRequest.class.cast(req); + + String bucketName = null; + + for (int i = 0; i < request.getJavaMethod().getParameterAnnotations().length; i++) { + if (any(Arrays.asList(request.getJavaMethod().getParameterAnnotations()[i]), ANNOTATIONTYPE_BUCKET)) { + bucketName = (String) request.getArgs().get(i); + break; + } + } + + return bucketName; + } + } diff --git a/apis/s3/src/main/java/org/jclouds/s3/xml/LocationConstraintHandler.java b/apis/s3/src/main/java/org/jclouds/s3/xml/LocationConstraintHandler.java index df75a85ebd..49d106ac5e 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/xml/LocationConstraintHandler.java +++ b/apis/s3/src/main/java/org/jclouds/s3/xml/LocationConstraintHandler.java @@ -20,8 +20,6 @@ package org.jclouds.s3.xml; import static org.jclouds.util.SaxUtils.currentOrNull; -import java.util.Map; - import javax.inject.Inject; import org.jclouds.aws.domain.Region; @@ -30,6 +28,9 @@ import org.jclouds.http.functions.ParseSax; import org.jclouds.rest.internal.GeneratedHttpRequest; import org.jclouds.s3.Bucket; +import com.google.common.base.Optional; +import com.google.common.cache.LoadingCache; + /** * Parses the response from Amazon S3 GET Bucket Location *

@@ -41,13 +42,13 @@ import org.jclouds.s3.Bucket; * @author Adrian Cole */ public class LocationConstraintHandler extends ParseSax.HandlerWithResult { - private final Map bucketToRegion; + private final LoadingCache> bucketToRegion; private StringBuilder currentText = new StringBuilder(); private String region; private String bucket; @Inject - public LocationConstraintHandler(@Bucket Map bucketToRegion) { + public LocationConstraintHandler(@Bucket LoadingCache> bucketToRegion) { this.bucketToRegion = bucketToRegion; } @@ -57,7 +58,7 @@ public class LocationConstraintHandler extends ParseSax.HandlerWithResult extends BaseS3A Method method = S3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class, Array.newInstance( PutBucketOptions.class, 0).getClass()); for (String region : Region.DEFAULT_S3) { - processor.createRequest(method, region, "bucket-name"); + processor.createRequest(method, region, "bucket-" + region); } } @@ -384,7 +384,7 @@ public abstract class S3AsyncClientTest extends BaseS3A assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class); assertSaxResponseParserClassEquals(method, null); - assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class); + assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.class); checkFilters(request); } diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/BucketsToStorageMetadataTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/BucketsToStorageMetadataTest.java new file mode 100644 index 0000000000..ebdb2a1b06 --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/BucketsToStorageMetadataTest.java @@ -0,0 +1,70 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.blobstore.functions; + +import static org.testng.Assert.assertEquals; + +import org.jclouds.blobstore.domain.MutableStorageMetadata; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.domain.StorageType; +import org.jclouds.blobstore.domain.internal.MutableStorageMetadataImpl; +import org.jclouds.blobstore.domain.internal.PageSetImpl; +import org.jclouds.concurrent.MoreExecutors; +import org.jclouds.domain.Location; +import org.jclouds.domain.LocationBuilder; +import org.jclouds.domain.LocationScope; +import org.jclouds.s3.domain.BucketMetadata; +import org.testng.annotations.Test; + +import com.google.common.base.Functions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** + * Tests behavior of {@code BucketsToStorageMetadata} + * + * @author Adrian Cole + */ +// NOTE:without testName, this will not call @Before* and fail w/NPE during +// surefire +@Test(groups = "unit", testName = "BucketsToStorageMetadataTest") +public class BucketsToStorageMetadataTest { + protected Location provider = new LocationBuilder().scope(LocationScope.PROVIDER).id("aws-ec2") + .description("aws-ec2").build(); + + protected Location region = new LocationBuilder().scope(LocationScope.REGION).id("us-east-1") + .description("us-east-1").parent(provider).build(); + + public void test() { + + BucketsToStorageMetadata fn = new BucketsToStorageMetadata( + MoreExecutors.sameThreadExecutor(), + new BucketToResourceMetadata(Functions.forMap(ImmutableMap. of("mycontainer", region)))); + + MutableStorageMetadata expected = new MutableStorageMetadataImpl(); + expected.setName("mycontainer"); + expected.setType(StorageType.CONTAINER); + expected.setLocation(region); + + assertEquals( + fn.apply(ImmutableSet.of(new BucketMetadata("mycontainer", null, null))).toString(), + new PageSetImpl(ImmutableSet. of(expected), null).toString()); + + } +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/LocationFromBucketNameTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/LocationFromBucketNameTest.java new file mode 100644 index 0000000000..f4f13e7c20 --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/LocationFromBucketNameTest.java @@ -0,0 +1,89 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.blobstore.functions; + +import static org.testng.Assert.assertEquals; + +import java.util.Set; + +import org.jclouds.domain.Location; +import org.jclouds.domain.LocationBuilder; +import org.jclouds.domain.LocationScope; +import org.testng.annotations.Test; + +import com.google.common.base.Functions; +import com.google.common.base.Optional; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** + * Tests behavior of {@code LocationFromBucketName} + * + * @author Adrian Cole + */ +// NOTE:without testName, this will not call @Before* and fail w/NPE during +// surefire +@Test(groups = "unit", testName = "LocationFromBucketNameTest") +public class LocationFromBucketNameTest { + protected Location provider = new LocationBuilder().scope(LocationScope.PROVIDER).id("aws-ec2") + .description("aws-ec2").build(); + + protected Location region = new LocationBuilder().scope(LocationScope.REGION).id("us-east-1") + .description("us-east-1").parent(provider).build(); + + protected Location region2 = new LocationBuilder().scope(LocationScope.REGION).id("eu-west-1") + .description("eu-west-1").parent(provider).build(); + + public void testOnlyLocationDoesntNeedMapping() { + LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap + .> of()), Suppliers.> ofInstance(ImmutableSet + .of(provider))); + + assertEquals(fn.apply("mybucket"), provider); + + } + + public void testMapsToCorrectRegion() { + LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap + .> of("mybucket", Optional.of("eu-west-1"))), + Suppliers.> ofInstance(ImmutableSet.of(region, region2))); + + assertEquals(fn.apply("mybucket"), region2); + + } + + public void testNullOnUnmatchedRegion() { + LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap + .> of("mybucket", Optional.of("eu-west-2"))), + Suppliers.> ofInstance(ImmutableSet.of(region, region2))); + + assertEquals(fn.apply("mybucket"), null); + + } + + public void testNullOnAbsentData() { + LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap + .> of("mybucket", Optional. absent())), + Suppliers.> ofInstance(ImmutableSet.of(region, region2))); + + assertEquals(fn.apply("mybucket"), null); + + } +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucketTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucketTest.java new file mode 100644 index 0000000000..dbf54c1b24 --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucketTest.java @@ -0,0 +1,86 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import static org.testng.Assert.assertEquals; + +import java.net.URI; +import java.util.Map; + +import javax.inject.Provider; +import javax.ws.rs.core.UriBuilder; + +import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; +import org.testng.annotations.Test; + +import com.google.common.base.Functions; +import com.google.common.base.Optional; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableMap; +import com.sun.jersey.api.uri.UriBuilderImpl; + +/** + * Tests behavior of {@code AssignCorrectHostnameForBucket} + * + * @author Adrian Cole + */ +// NOTE:without testName, this will not call @Before* and fail w/NPE during +// surefire +@Test(groups = "unit", testName = "AssignCorrectHostnameForBucketTest") +public class AssignCorrectHostnameForBucketTest { + Provider uriBuilderProvider = new Provider() { + + @Override + public UriBuilder get() { + return new UriBuilderImpl(); + } + + }; + + public void testWhenNoBucketRegionMappingInCache() { + + AssignCorrectHostnameForBucket fn = new AssignCorrectHostnameForBucket(new RegionToEndpointOrProviderIfNull( + "aws-s3", Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")), + + Suppliers.>> ofInstance(ImmutableMap.of("us-standard", + Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1", + Suppliers.ofInstance(URI.create("https://s3-us-west-1.amazonaws.com"))))), + + Functions.forMap(ImmutableMap.> of("bucket", Optional. absent()))); + + assertEquals(fn.apply("bucket"), URI.create("https://s3.amazonaws.com")); + + } + + public void testWhenBucketRegionMappingInCache() { + + AssignCorrectHostnameForBucket fn = new AssignCorrectHostnameForBucket(new RegionToEndpointOrProviderIfNull( + "aws-s3", Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")), + + Suppliers.>> ofInstance(ImmutableMap.of("us-standard", + Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1", + Suppliers.ofInstance(URI.create("https://s3-us-west-1.amazonaws.com"))))), + + Functions.forMap(ImmutableMap.> of("bucket", Optional.of("us-west-1")))); + + assertEquals(fn.apply("bucket"), URI.create("https://s3-us-west-1.amazonaws.com")); + + } +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegionTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegionTest.java new file mode 100644 index 0000000000..514ee8c9ec --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegionTest.java @@ -0,0 +1,56 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +import java.net.URI; + +import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; +import org.testng.annotations.Test; + +import com.google.common.base.Optional; +import com.google.common.cache.LoadingCache; + +/** + * @author Adrian Cole + */ +@Test(testName = "DefaultEndpointThenInvalidateRegionTest") +public class DefaultEndpointThenInvalidateRegionTest { + + @SuppressWarnings("unchecked") + @Test + void testInvalidate() throws Exception { + RegionToEndpointOrProviderIfNull r2 = createMock(RegionToEndpointOrProviderIfNull.class); + LoadingCache> bucketToRegionCache = createMock(LoadingCache.class); + + expect(r2.apply(null)).andReturn(URI.create("http://east-url")); + bucketToRegionCache.invalidate("mybucket"); + + replay(r2, bucketToRegionCache); + + new DefaultEndpointThenInvalidateRegion(r2, bucketToRegionCache).apply("mybucket"); + verify(r2, bucketToRegionCache); + + } + +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/functions/GetRegionForBucketTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/GetRegionForBucketTest.java new file mode 100644 index 0000000000..2eaf7dc6a8 --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/functions/GetRegionForBucketTest.java @@ -0,0 +1,81 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.testng.Assert.assertEquals; + +import java.util.concurrent.ExecutionException; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import com.google.common.base.Optional; +import com.google.common.cache.CacheLoader.InvalidCacheLoadException; +import com.google.common.cache.LoadingCache; +import com.google.common.util.concurrent.UncheckedExecutionException; + +/** + * @author Adrian Cole + */ +@Test(testName = "GetRegionForBucketTest") +public class GetRegionForBucketTest { + + @SuppressWarnings("unchecked") + @Test + void test() throws Exception { + LoadingCache> bucketToRegionCache = createMock(LoadingCache.class); + + expect(bucketToRegionCache.get("bucket")).andReturn(Optional.of("us-east-1")); + + replay(bucketToRegionCache); + GetRegionForBucket fn = new GetRegionForBucket(bucketToRegionCache); + + assertEquals(fn.apply("bucket"), Optional.of("us-east-1")); + + verify(bucketToRegionCache); + + } + + @SuppressWarnings("serial") + @DataProvider(name = "exceptions") + public Object[][] createExceptions() { + return new Object[][] { { new ExecutionException() { + } }, { new UncheckedExecutionException() { + } }, { new InvalidCacheLoadException("foo") } }; + } + + @SuppressWarnings("unchecked") + @Test(dataProvider = "exceptions") + void testGracefulOnException(Exception exception) throws Exception { + LoadingCache> bucketToRegionCache = createMock(LoadingCache.class); + + expect(bucketToRegionCache.get("bucket")).andThrow(exception); + + replay(bucketToRegionCache); + GetRegionForBucket fn = new GetRegionForBucket(bucketToRegionCache); + + assertEquals(fn.apply("bucket"), Optional.absent()); + + verify(bucketToRegionCache); + } +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest.java deleted file mode 100644 index bb21e5c0bb..0000000000 --- a/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to jclouds, Inc. (jclouds) under one or more - * contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. jclouds licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.jclouds.s3.functions; - -import org.jclouds.aws.AWSResponseException; -import org.jclouds.aws.domain.AWSError; -import org.testng.annotations.Test; - -/** - * @author Adrian Cole - */ -@Test(testName = "ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest") -public class ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest { - - @Test - void testBucketAlreadyOwnedByYouIsOk() throws Exception { - Exception e = getErrorWithCode("BucketAlreadyOwnedByYou"); - assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState().apply(e); - } - - @Test - void testIllegalStateIsOk() throws Exception { - Exception e = new IllegalStateException(); - assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState().apply(e); - } - - @Test(expectedExceptions = AWSResponseException.class) - void testBlahIsNotOk() throws Exception { - Exception e = getErrorWithCode("blah"); - new ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState().apply(e); - } - - private Exception getErrorWithCode(String code) { - AWSError error = new AWSError(); - error.setCode(code); - return new AWSResponseException(null, null, null, error); - } -} diff --git a/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest.java new file mode 100644 index 0000000000..c35979306d --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest.java @@ -0,0 +1,114 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3.functions; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +import java.net.URI; + +import org.jclouds.aws.AWSResponseException; +import org.jclouds.aws.domain.AWSError; +import org.jclouds.rest.internal.GeneratedHttpRequest; +import org.jclouds.s3.S3Client; +import org.jclouds.s3.options.PutBucketOptions; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * @author Adrian Cole + */ +@Test(testName = "ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest") +public class ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest { + + GeneratedHttpRequest putBucket; + + @BeforeClass + void setUp() throws SecurityException, NoSuchMethodException { + putBucket = GeneratedHttpRequest + . requestBuilder() + .method("PUT") + .endpoint(URI.create("https://adriancole-blobstore113.s3.amazonaws.com/")) + .declaring(S3Client.class) + .javaMethod( + S3Client.class.getMethod("putBucketInRegion", String.class, String.class, + PutBucketOptions[].class)).args(new Object[] { null, "bucket" }).build(); + } + + @Test + void testBucketAlreadyOwnedByYouIsOk() throws Exception { + S3Client client = createMock(S3Client.class); + replay(client); + + Exception e = getErrorWithCode("BucketAlreadyOwnedByYou"); + assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).setContext(putBucket) + .apply(e); + verify(client); + } + + @Test + void testOperationAbortedIsOkWhenBucketExists() throws Exception { + S3Client client = createMock(S3Client.class); + expect(client.bucketExists("bucket")).andReturn(true); + replay(client); + Exception e = getErrorWithCode("OperationAborted"); + assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).setContext(putBucket) + .apply(e); + verify(client); + } + + @Test(expectedExceptions = Exception.class) + void testOperationAbortedNotOkWhenBucketDoesntExist() throws Exception { + S3Client client = createMock(S3Client.class); + expect(client.bucketExists("bucket")).andReturn(false); + replay(client); + Exception e = getErrorWithCode("OperationAborted"); + new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).setContext(putBucket).apply(e); + Assert.fail(); + } + + @Test(expectedExceptions = IllegalStateException.class) + void testIllegalStateIsNotOk() throws Exception { + S3Client client = createMock(S3Client.class); + replay(client); + + Exception e = new IllegalStateException(); + new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).apply(e); + Assert.fail(); + + } + + @Test(expectedExceptions = AWSResponseException.class) + void testBlahIsNotOk() throws Exception { + S3Client client = createMock(S3Client.class); + replay(client); + Exception e = getErrorWithCode("blah"); + new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).apply(e); + Assert.fail(); + } + + private Exception getErrorWithCode(String code) { + AWSError error = new AWSError(); + error.setCode(code); + return new AWSResponseException(null, null, null, error); + } +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/services/BucketsLiveTest.java b/apis/s3/src/test/java/org/jclouds/s3/services/BucketsLiveTest.java index d1b65f5969..1751a3cd94 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/services/BucketsLiveTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/services/BucketsLiveTest.java @@ -34,7 +34,6 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; -import org.jclouds.aws.domain.Region; import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest; import org.jclouds.s3.S3Client; import org.jclouds.s3.domain.AccessControlList; @@ -171,17 +170,6 @@ public class BucketsLiveTest extends BaseBlobStoreIntegrationTest { } - public void testDefaultBucketLocation() throws Exception { - - String bucketName = getContainerName(); - try { - String location = getApi().getBucketLocation(bucketName); - assert location.equals(Region.US_STANDARD) : "bucket: " + bucketName + " location: " + location; - } finally { - returnContainer(bucketName); - } - } - public void testBucketPayer() throws Exception { final String bucketName = getContainerName(); try { @@ -270,33 +258,6 @@ public class BucketsLiveTest extends BaseBlobStoreIntegrationTest { assertTrue(getApi().putBucketACL(targetBucket, acl)); } - /** - * using scratch bucketName as we are changing location - */ - public void testEu() throws Exception { - final String bucketName = getScratchContainerName(); - try { - getApi().putBucketInRegion(Region.EU, bucketName + "eu", withBucketAcl(CannedAccessPolicy.PUBLIC_READ)); - assertConsistencyAware(new Runnable() { - public void run() { - try { - AccessControlList acl = getApi().getBucketACL(bucketName + "eu"); - assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ), acl.toString()); - } catch (Exception e) { - Throwables.propagateIfPossible(e); - } - } - }); - assertEquals(Region.EU, getApi().getBucketLocation(bucketName + "eu")); - // TODO: I believe that the following should work based on the above acl assertion passing. - // However, it fails on 403 - // URL url = new URL(String.format("http://%s.s3.amazonaws.com", bucketName)); - // Utils.toStringAndClose(url.openStream()); - } finally { - destroyContainer(bucketName + "eu"); - } - } - void bucketExists() throws Exception { String bucketName = getContainerName(); try { diff --git a/apis/walrus/src/main/java/org/jclouds/walrus/WalrusAsyncClient.java b/apis/walrus/src/main/java/org/jclouds/walrus/WalrusAsyncClient.java index 5dfa1c31b7..8b90f49cd5 100644 --- a/apis/walrus/src/main/java/org/jclouds/walrus/WalrusAsyncClient.java +++ b/apis/walrus/src/main/java/org/jclouds/walrus/WalrusAsyncClient.java @@ -26,6 +26,7 @@ import javax.ws.rs.Path; import org.jclouds.blobstore.attr.BlobScope; import org.jclouds.blobstore.functions.ReturnFalseOnContainerNotFound; import org.jclouds.rest.annotations.BinderParam; +import org.jclouds.rest.annotations.Endpoint; import org.jclouds.rest.annotations.ExceptionParser; import org.jclouds.rest.annotations.ParamValidators; import org.jclouds.rest.annotations.QueryParams; @@ -55,9 +56,10 @@ public interface WalrusAsyncClient extends S3AsyncClient { @Override @GET @Path("/") + @Endpoint(Bucket.class) @QueryParams(keys = "max-keys", values = "0") @ExceptionParser(ReturnFalseOnContainerNotFound.class) ListenableFuture bucketExists( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName); + @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName); } diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java index bb7ef36e6d..87d58ae43d 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java @@ -22,8 +22,8 @@ import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursi import static org.jclouds.concurrent.FutureIterables.awaitCompletion; import java.util.Map; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import javax.annotation.Resource; import javax.inject.Named; @@ -41,11 +41,9 @@ import org.jclouds.blobstore.strategy.ClearListStrategy; import org.jclouds.http.handlers.BackoffLimitedRetryHandler; import org.jclouds.logging.Logger; -import com.google.common.base.Predicate; import com.google.common.base.Throwables; -import com.google.common.collect.Iterables; import com.google.common.collect.Maps; -import java.util.concurrent.Future; +import com.google.common.util.concurrent.Futures; import com.google.inject.Inject; /** @@ -97,16 +95,14 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr for (int i = 0; i < maxErrors; ) { // fetch partial directory listing try { - listing = connection.list(containerName, options).get(); - } catch (ExecutionException ee) { + listing = Futures.getUnchecked(connection.list(containerName, options)); + } catch (RuntimeException ee) { ++i; if (i == maxErrors) { - throw new BlobRuntimeException("list error", ee.getCause()); + throw Throwables.propagate(ee.getCause()); } retryHandler.imposeBackoffExponentialDelay(i, message); continue; - } catch (InterruptedException ie) { - throw Throwables.propagate(ie); } // recurse on subdirectories diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java index 6b90d4d4fe..a5de06bed7 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java @@ -48,6 +48,7 @@ import java.util.zip.GZIPInputStream; import javax.ws.rs.core.MediaType; +import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobBuilder.PayloadBlobBuilder; @@ -628,4 +629,4 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { assertEquals(metadata.getContentMetadata().getContentMD5(), CryptoStreams.md5(InputSuppliers.of(TEST_STRING))); } -} \ No newline at end of file +} diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobStoreIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobStoreIntegrationTest.java index 22111211fb..85b84c1ec5 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobStoreIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobStoreIntegrationTest.java @@ -24,6 +24,8 @@ import static org.jclouds.blobstore.util.BlobStoreUtils.getContentAsStringOrNull import static org.testng.Assert.assertEquals; import java.io.IOException; +import java.security.SecureRandom; +import java.util.Date; import java.util.Map; import java.util.Set; import java.util.Map.Entry; @@ -35,6 +37,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import javax.annotation.Nullable; import javax.ws.rs.core.MediaType; import org.jclouds.blobstore.BlobStoreContext; @@ -43,6 +46,7 @@ import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.domain.StorageType; +import org.jclouds.domain.Location; import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule; import org.testng.ITestContext; import org.testng.annotations.AfterClass; @@ -78,7 +82,7 @@ public class BaseBlobStoreIntegrationTest { protected volatile BlobStoreContext context; protected static volatile int containerCount = Integer.parseInt(System.getProperty("test.blobstore.container-count", "10")); - public static final String CONTAINER_PREFIX = System.getProperty("user.name") + "-blobstore"; + public static final String CONTAINER_PREFIX = (System.getProperty("user.name") + "-blobstore").toLowerCase(); /** * two test groups integration and live. */ @@ -140,7 +144,12 @@ public class BaseBlobStoreIntegrationTest { } else { try { createContainerAndEnsureEmpty(context, containerName); - containerNames.put(containerName); + if (context.getBlobStore().containerExists(containerName)) + containerNames.put(containerName); + else { + deleteContainerOrWarnIfUnable(context, containerName); + containerCount++; + } } catch (Throwable e) { e.printStackTrace(); // throw away the container and try again with the next @@ -182,7 +191,7 @@ public class BaseBlobStoreIntegrationTest { new Predicate() { public boolean apply(StorageMetadata input) { return (input.getType() == StorageType.CONTAINER || input.getType() == StorageType.FOLDER) - && input.getName().startsWith(CONTAINER_PREFIX.toLowerCase()); + && input.getName().startsWith(CONTAINER_PREFIX); } }); for (StorageMetadata container : testContainers) { @@ -340,6 +349,58 @@ public class BaseBlobStoreIntegrationTest { }); } + protected void assertConsistencyAwareContainerExists(final String containerName) throws InterruptedException { + assertConsistencyAware(new Runnable() { + public void run() { + try { + assert context.getBlobStore().containerExists(containerName) : String.format("container %s doesn't exist", containerName); + } catch (Exception e) { + Throwables.propagate(e); + } + } + }); + } + + protected void assertConsistencyAwareContainerInLocation(final String containerName, final Location loc) + throws InterruptedException { + assertConsistencyAware(new Runnable() { + public void run() { + try { + StorageMetadata container = Iterables.find(context.getBlobStore().list(), new Predicate() { + + @Override + public boolean apply(@Nullable StorageMetadata input) { + return input.getName().equals(containerName); + } + + }); + Location actualLoc = container.getLocation(); + + assert loc.equals(actualLoc) : String.format("blob %s, in location %s instead of %s", containerName, + actualLoc, loc); + } catch (Exception e) { + Throwables.propagate(e); + } + } + }); + } + + protected void assertConsistencyAwareBlobInLocation(final String containerName, final String blobName, final Location loc) + throws InterruptedException { + assertConsistencyAware(new Runnable() { + public void run() { + try { + Location actualLoc = context.getBlobStore().getBlob(containerName, blobName).getMetadata().getLocation(); + + assert loc.equals(actualLoc) : String.format( + "blob %s in %s, in location %s instead of %s", blobName, containerName, actualLoc, loc); + } catch (Exception e) { + Throwables.propagate(e); + } + } + }); + } + public String getContainerName() throws InterruptedException { String containerName = containerNames.poll(30, TimeUnit.SECONDS); assert containerName != null : "unable to get a container for the test"; @@ -422,7 +483,7 @@ public class BaseBlobStoreIntegrationTest { deleteContainerOrWarnIfUnable(context, container); } }); - String newScratchContainer = container + containerIndex.incrementAndGet(); + String newScratchContainer = container + new SecureRandom().nextLong(); System.err.printf("*** allocated new container %s...%n", container); return newScratchContainer; } @@ -431,4 +492,4 @@ public class BaseBlobStoreIntegrationTest { return new JavaUrlHttpCommandExecutorServiceModule(); } -} \ No newline at end of file +} diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerLiveTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerLiveTest.java index 40e094528b..cf08d91465 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerLiveTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerLiveTest.java @@ -23,23 +23,50 @@ import static org.testng.Assert.assertEquals; import java.io.IOException; import java.net.MalformedURLException; +import java.util.Collections; +import java.util.List; +import java.util.logging.Logger; +import javax.annotation.Nullable; + +import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.domain.Location; import org.jclouds.util.Strings2; +import org.testng.SkipException; import org.testng.annotations.Test; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.base.Strings; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + /** * * @author Adrian Cole */ public class BaseContainerLiveTest extends BaseBlobStoreIntegrationTest { - @Test(groups = { "live" }) + private Location defaultLocation; + + @Test(groups = "live") public void testPublicAccess() throws InterruptedException, MalformedURLException, IOException { final String containerName = getScratchContainerName(); try { context.getBlobStore().createContainerInLocation(null, containerName, publicRead()); - assertConsistencyAwareContainerSize(containerName, 0); + context.getBlobStore().createContainerInLocation(null, containerName, publicRead()); + assertConsistencyAwareContainerExists(containerName); + + defaultLocation = Iterables.find(context.getBlobStore().list(), new Predicate() { + + @Override + public boolean apply(@Nullable StorageMetadata input) { + return input.getName().equals(containerName); + } + + }).getLocation(); context.getBlobStore().putBlob(containerName, context.getBlobStore().blobBuilder("hello").payload(TEST_STRING).build()); @@ -56,4 +83,58 @@ public class BaseContainerLiveTest extends BaseBlobStoreIntegrationTest { recycleContainer(containerName); } } -} \ No newline at end of file + + static Location findNonDefaultLocationOrSkip(BlobStore blobStore, Location defaultLocation) { + List locs = Lists.newArrayList(Iterables.filter(blobStore.listAssignableLocations(), + Predicates.not(Predicates.equalTo(defaultLocation)))); + if (locs.size() == 0) + throw new SkipException("No non-default location found in " + locs); + // try to use a diverse location + Collections.shuffle(locs); + return locs.get(0); + } + + @Test(groups = "live", dependsOnMethods = "testPublicAccess") + public void testPublicAccessInNonDefaultLocation() throws InterruptedException, MalformedURLException, IOException { + Location nonDefault = findNonDefaultLocationOrSkip(context.getBlobStore(), defaultLocation); + + String payload = "my data"; + runCreateContainerInLocation(payload, nonDefault); + } + + @Test(groups = "live", dependsOnMethods = "testPublicAccess") + public void testPublicAccessInNonDefaultLocationWithBigBlob() throws InterruptedException, MalformedURLException, + IOException { + Location nonDefault = findNonDefaultLocationOrSkip(context.getBlobStore(), defaultLocation); + String payload = Strings.repeat("a", 1024 * 1024); // 1MB + runCreateContainerInLocation(payload, nonDefault); + } + + private void runCreateContainerInLocation(String payload, Location nonDefault) throws InterruptedException, + IOException { + String blobName = "hello"; + BlobStore blobStore = context.getBlobStore(); + final String containerName = getScratchContainerName(); + try { + Logger.getAnonymousLogger().info( + String.format("creating public container %s in location %s", containerName, nonDefault.getId())); + blobStore.createContainerInLocation(nonDefault, containerName, publicRead()); + assertConsistencyAwareContainerExists(containerName); + assertConsistencyAwareContainerInLocation(containerName, nonDefault); + + blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload(payload).build()); + + assertConsistencyAwareContainerSize(containerName, 1); + + BlobMetadata metadata = context.getBlobStore().blobMetadata(containerName, blobName); + assertEquals(Strings2.toStringAndClose(context.utils().http().get(metadata.getPublicUri())), payload); + + assertConsistencyAwareBlobInLocation(containerName, blobName, nonDefault); + + } finally { + // this container is now public, so we can't reuse it directly + recycleContainer(containerName); + } + } + +} diff --git a/common/aws/src/main/java/org/jclouds/aws/domain/Region.java b/common/aws/src/main/java/org/jclouds/aws/domain/Region.java index e7ca0fd228..3c8c50745f 100644 --- a/common/aws/src/main/java/org/jclouds/aws/domain/Region.java +++ b/common/aws/src/main/java/org/jclouds/aws/domain/Region.java @@ -48,8 +48,6 @@ public class Region { * In Amazon S3, the EU (Ireland) Region provides read-after-write consistency for PUTS of new * objects in your Amazon S3 bucket and eventual consistency for overwrite PUTS and DELETES. */ - public static final String EU = "EU"; - public static final String EU_WEST_1 = "eu-west-1"; /** @@ -106,7 +104,7 @@ public class Region { */ public static final String AP_NORTHEAST_1 = "ap-northeast-1"; - public static Set DEFAULT_S3 = ImmutableSet.of(EU, US_STANDARD, US_WEST_1, US_WEST_2, SA_EAST_1, AP_SOUTHEAST_1, + public static Set DEFAULT_S3 = ImmutableSet.of(US_STANDARD, US_WEST_1, US_WEST_2, EU_WEST_1, SA_EAST_1, AP_SOUTHEAST_1, AP_NORTHEAST_1); public static Set DEFAULT_REGIONS = ImmutableSet.of(US_EAST_1, US_WEST_1, US_WEST_2, SA_EAST_1, EU_WEST_1, @@ -119,7 +117,7 @@ public class Region { // note that due to US_STANDARD the codes include US instead of US-VA properties.setProperty(PROPERTY_ISO3166_CODES, "US,US-CA,US-OR,BR-SP,IE,SG,JP-13"); properties.setProperty(PROPERTY_REGION + "." + US_STANDARD + "." + ISO3166_CODES, "US"); - properties.setProperty(PROPERTY_REGION + "." + EU + "." + ISO3166_CODES, "IE"); + properties.setProperty(PROPERTY_REGION + "." + EU_WEST_1 + "." + ISO3166_CODES, "IE"); return properties; } diff --git a/common/aws/src/main/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandler.java b/common/aws/src/main/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandler.java index e17c77f438..f074f99a0f 100644 --- a/common/aws/src/main/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandler.java +++ b/common/aws/src/main/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandler.java @@ -20,17 +20,14 @@ package org.jclouds.aws.handlers; import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream; -import javax.annotation.Resource; -import javax.inject.Named; - -import org.jclouds.Constants; import org.jclouds.aws.domain.AWSError; import org.jclouds.aws.util.AWSUtils; import org.jclouds.http.HttpCommand; import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpRetryHandler; -import org.jclouds.logging.Logger; +import org.jclouds.http.handlers.BackoffLimitedRetryHandler; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; /** @@ -40,35 +37,25 @@ import com.google.inject.Inject; */ public class AWSClientErrorRetryHandler implements HttpRetryHandler { - @Inject(optional = true) - @Named(Constants.PROPERTY_MAX_RETRIES) - private int retryCountLimit = 5; - private final AWSUtils utils; - - @Resource - protected Logger logger = Logger.NULL; + private final BackoffLimitedRetryHandler backoffLimitedRetryHandler; @Inject - public AWSClientErrorRetryHandler(AWSUtils utils) { + public AWSClientErrorRetryHandler(AWSUtils utils, BackoffLimitedRetryHandler backoffLimitedRetryHandler) { this.utils = utils; + this.backoffLimitedRetryHandler = backoffLimitedRetryHandler; } public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) { - if (command.getFailureCount() > retryCountLimit) - return false; - if (response.getStatusCode() == 400 || response.getStatusCode() == 403 - || response.getStatusCode() == 409) { - command.incrementFailureCount(); + if (response.getStatusCode() == 400 || response.getStatusCode() == 403 || response.getStatusCode() == 409) { // Content can be null in the case of HEAD requests if (response.getPayload() != null) { closeClientButKeepContentStream(response); AWSError error = utils.parseAWSErrorFromContent(command.getCurrentRequest(), response); if (error != null - && ("RequestTimeout".equals(error.getCode()) - || "OperationAborted".equals(error.getCode()) || "SignatureDoesNotMatch" - .equals(error.getCode()))) { - return true; + && ImmutableSet.of("RequestTimeout", "OperationAborted", "SignatureDoesNotMatch").contains( + error.getCode())) { + return backoffLimitedRetryHandler.shouldRetryRequest(command, response); } } } diff --git a/common/aws/src/test/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandlerTest.java b/common/aws/src/test/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandlerTest.java index e8e1e7d5f2..0f2f3cf2e2 100644 --- a/common/aws/src/test/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandlerTest.java +++ b/common/aws/src/test/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandlerTest.java @@ -23,9 +23,16 @@ import static org.easymock.classextension.EasyMock.createMock; import static org.easymock.classextension.EasyMock.replay; import static org.easymock.classextension.EasyMock.verify; +import java.net.URI; + +import org.jclouds.aws.domain.AWSError; import org.jclouds.aws.util.AWSUtils; import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpRequest; import org.jclouds.http.HttpResponse; +import org.jclouds.http.handlers.BackoffLimitedRetryHandler; +import org.jclouds.io.Payloads; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; /** @@ -33,29 +40,60 @@ import org.testng.annotations.Test; * * @author Adrian Cole */ -@Test(groups = "unit") +@Test(groups = "unit", testName = "AWSClientErrorRetryHandlerTest") public class AWSClientErrorRetryHandlerTest { @Test public void test401DoesNotRetry() { AWSUtils utils = createMock(AWSUtils.class); + BackoffLimitedRetryHandler backoffLimitedRetryHandler = createMock(BackoffLimitedRetryHandler.class); HttpCommand command = createMock(HttpCommand.class); - HttpResponse response = createMock(HttpResponse.class); - expect(command.getFailureCount()).andReturn(0); - expect(response.getStatusCode()).andReturn(401).atLeastOnce(); + replay(utils, backoffLimitedRetryHandler, command); - replay(utils); - replay(command); - replay(response); + AWSClientErrorRetryHandler retry = new AWSClientErrorRetryHandler(utils, backoffLimitedRetryHandler); - AWSClientErrorRetryHandler retry = new AWSClientErrorRetryHandler(utils); + assert !retry.shouldRetryRequest(command, HttpResponse.builder().statusCode(401).build()); - assert !retry.shouldRetryRequest(command, response); - - verify(utils); - verify(command); - verify(response); + verify(utils, backoffLimitedRetryHandler, command); } + + @DataProvider(name = "codes") + public Object[][] createData() { + return new Object[][] { { "RequestTimeout" }, { "OperationAborted" }, { "SignatureDoesNotMatch" } }; + } + + @Test(dataProvider = "codes") + public void test409DoesBackoffAndRetryForCode(String code) { + + AWSUtils utils = createMock(AWSUtils.class); + BackoffLimitedRetryHandler backoffLimitedRetryHandler = createMock(BackoffLimitedRetryHandler.class); + HttpCommand command = createMock(HttpCommand.class); + + HttpRequest putBucket = HttpRequest.builder().method("PUT") + .endpoint(URI.create("https://adriancole-blobstore113.s3.amazonaws.com/")).build(); + + HttpResponse operationAborted = HttpResponse.builder().statusCode(409) + .payload(Payloads.newStringPayload(String.format("%s", code))).build(); + + expect(command.getCurrentRequest()).andReturn(putBucket); + + AWSError error = new AWSError(); + error.setCode(code); + + expect(utils.parseAWSErrorFromContent(putBucket, operationAborted)).andReturn(error); + + expect(backoffLimitedRetryHandler.shouldRetryRequest(command, operationAborted)).andReturn(Boolean.TRUE); + + replay(utils, backoffLimitedRetryHandler, command); + + AWSClientErrorRetryHandler retry = new AWSClientErrorRetryHandler(utils, backoffLimitedRetryHandler); + + assert retry.shouldRetryRequest(command, operationAborted); + + verify(utils, backoffLimitedRetryHandler, command); + + } + } diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java index fc193341b7..179796769b 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java @@ -22,10 +22,9 @@ import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER; import java.util.Map; -import org.jclouds.aws.s3.blobstore.options.AWSS3PutObjectOptions; -import org.jclouds.javax.annotation.Nullable; import javax.ws.rs.DELETE; import javax.ws.rs.GET; +import javax.ws.rs.HEAD; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; @@ -38,10 +37,13 @@ import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex; import org.jclouds.aws.s3.functions.ObjectMetadataKey; import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex; import org.jclouds.blobstore.attr.BlobScope; +import org.jclouds.blobstore.functions.ReturnFalseOnContainerNotFound; import org.jclouds.http.functions.ParseETagHeader; import org.jclouds.io.Payload; +import org.jclouds.javax.annotation.Nullable; import org.jclouds.rest.annotations.BinderParam; import org.jclouds.rest.annotations.Endpoint; +import org.jclouds.rest.annotations.EndpointParam; import org.jclouds.rest.annotations.ExceptionParser; import org.jclouds.rest.annotations.ParamParser; import org.jclouds.rest.annotations.ParamValidators; @@ -55,13 +57,13 @@ import org.jclouds.s3.Bucket; import org.jclouds.s3.S3AsyncClient; import org.jclouds.s3.S3Client; import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured; -import org.jclouds.s3.binders.BindS3ObjectMetadataToRequest; import org.jclouds.s3.domain.ObjectMetadata; -import org.jclouds.s3.domain.S3Object; import org.jclouds.s3.filters.RequestAuthorizeSignature; +import org.jclouds.s3.functions.AssignCorrectHostnameForBucket; import org.jclouds.s3.functions.BindRegionToXmlPayload; -import org.jclouds.s3.functions.ObjectKey; -import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState; +import org.jclouds.s3.functions.DefaultEndpointThenInvalidateRegion; +import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists; +import org.jclouds.s3.functions.ReturnTrueOn404OrNotFoundFalseOnIllegalState; import org.jclouds.s3.options.PutBucketOptions; import org.jclouds.s3.options.PutObjectOptions; import org.jclouds.s3.predicates.validators.BucketNameValidator; @@ -72,38 +74,13 @@ import com.google.common.util.concurrent.ListenableFuture; /** * Provides access to amazon-specific S3 features * - * @author Adrian Cole + * @author Adrian Cole, Jeremy Whitlock */ @SkipEncoding('/') @RequestFilters(RequestAuthorizeSignature.class) @BlobScope(CONTAINER) public interface AWSS3AsyncClient extends S3AsyncClient { - - /** - * @see S3Client#putBucketInRegion - */ - @Override - @PUT - @Path("/") - @Endpoint(Bucket.class) - @ExceptionParser(ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class) - ListenableFuture putBucketInRegion( - @BinderParam(BindRegionToXmlPayload.class) @Nullable String region, - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - PutBucketOptions... options); - - /** - * @see S3Client#getBucketLocation - */ - @Override - @GET - @QueryParams(keys = "location") - @Path("/") - @Endpoint(Bucket.class) - @XMLResponseParser(LocationConstraintHandler.class) - ListenableFuture getBucketLocation( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); - + /** * @see AWSS3Client#initiateMultipartUpload */ @@ -112,9 +89,9 @@ public interface AWSS3AsyncClient extends S3AsyncClient { @Path("/{key}") @ResponseParser(UploadIdFromHttpResponseViaRegex.class) ListenableFuture initiateMultipartUpload( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class) ObjectMetadata objectMetadata, - PutObjectOptions... options); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class) ObjectMetadata objectMetadata, + PutObjectOptions... options); /** * @see AWSS3Client#abortMultipartUpload @@ -123,8 +100,8 @@ public interface AWSS3AsyncClient extends S3AsyncClient { @Path("/{key}") @ExceptionParser(ReturnVoidOnNotFoundOr404.class) ListenableFuture abortMultipartUpload( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key, @QueryParam("uploadId") String uploadId); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, @QueryParam("uploadId") String uploadId); /** * @see AWSS3Client#uploadPart @@ -133,9 +110,9 @@ public interface AWSS3AsyncClient extends S3AsyncClient { @Path("/{key}") @ResponseParser(ParseETagHeader.class) ListenableFuture uploadPart( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key, @QueryParam("partNumber") int partNumber, - @QueryParam("uploadId") String uploadId, Payload part); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, @QueryParam("partNumber") int partNumber, + @QueryParam("uploadId") String uploadId, Payload part); /** * @see AWSS3Client#completeMultipartUpload @@ -144,8 +121,8 @@ public interface AWSS3AsyncClient extends S3AsyncClient { @Path("/{key}") @ResponseParser(ETagFromHttpResponseViaRegex.class) ListenableFuture completeMultipartUpload( - @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, - @PathParam("key") String key, @QueryParam("uploadId") String uploadId, - @BinderParam(BindPartIdsAndETagsToRequest.class) Map parts); + @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, @QueryParam("uploadId") String uploadId, + @BinderParam(BindPartIdsAndETagsToRequest.class) Map parts); } diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3PropertiesBuilder.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3PropertiesBuilder.java index 37f07d74d6..31f7e29432 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3PropertiesBuilder.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3PropertiesBuilder.java @@ -22,6 +22,7 @@ import static org.jclouds.Constants.PROPERTY_ENDPOINT; import static org.jclouds.aws.domain.Region.AP_NORTHEAST_1; import static org.jclouds.aws.domain.Region.AP_SOUTHEAST_1; import static org.jclouds.aws.domain.Region.SA_EAST_1; +import static org.jclouds.aws.domain.Region.EU_WEST_1; import static org.jclouds.aws.domain.Region.US_STANDARD; import static org.jclouds.aws.domain.Region.US_WEST_1; import static org.jclouds.aws.domain.Region.US_WEST_2; @@ -48,7 +49,7 @@ public class AWSS3PropertiesBuilder extends org.jclouds.s3.S3PropertiesBuilder { properties.setProperty(PROPERTY_REGION + "." + US_WEST_1 + "." + ENDPOINT, "https://s3-us-west-1.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + US_WEST_2 + "." + ENDPOINT, "https://s3-us-west-2.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + SA_EAST_1 + "." + ENDPOINT, "https://s3-sa-east-1.amazonaws.com"); - properties.setProperty(PROPERTY_REGION + "." + "EU" + "." + ENDPOINT, "https://s3-eu-west-1.amazonaws.com"); + properties.setProperty(PROPERTY_REGION + "." + EU_WEST_1 + "." + ENDPOINT, "https://s3-eu-west-1.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + AP_SOUTHEAST_1 + "." + ENDPOINT, "https://s3-ap-southeast-1.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + AP_NORTHEAST_1 + "." + ENDPOINT, diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfigured.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfigured.java deleted file mode 100644 index aa8795472c..0000000000 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfigured.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to jclouds, Inc. (jclouds) under one or more - * contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. jclouds licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.jclouds.aws.s3.binders; - -import static org.jclouds.http.utils.ModifyRequest.endpoint; -import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH; -import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS; - -import java.net.URI; -import java.util.Map; - -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Provider; -import javax.inject.Singleton; -import javax.ws.rs.core.UriBuilder; - -import org.jclouds.http.HttpRequest; -import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; -import org.jclouds.rest.binders.BindAsHostPrefix; -import org.jclouds.s3.Bucket; -import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured; - -/** - * - * @author Adrian Cole - */ -@Singleton -public class AssignCorrectHostnameAndBindAsHostPrefixIfConfigured extends BindAsHostPrefixIfConfigured { - private final Map bucketToRegion; - private final RegionToEndpointOrProviderIfNull r2; - - @Inject - public AssignCorrectHostnameAndBindAsHostPrefixIfConfigured(BindAsHostPrefix bindAsHostPrefix, - @Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle, - @Named(PROPERTY_S3_SERVICE_PATH) String servicePath, - RegionToEndpointOrProviderIfNull r2, Provider uriBuilderProvider, - @Bucket Map bucketToRegion) { - super(bindAsHostPrefix, isVhostStyle, servicePath, uriBuilderProvider); - this.bucketToRegion = bucketToRegion; - this.r2 = r2; - } - - @Override - public R bindToRequest(R request, Object payload) { - String bucket = payload.toString(); - String region = bucketToRegion.get(bucket); - if (region != null) { - URI endpoint = r2.apply(region); - request = endpoint(request, uriBuilderProvider.get().uri(endpoint).path(request.getEndpoint().getPath()) - .replaceQuery(request.getEndpoint().getQuery()).build()); - } - return super.bindToRequest(request, payload); - } -} diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3AsyncBlobStore.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3AsyncBlobStore.java index 02d72315cb..a3b09b6624 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3AsyncBlobStore.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3AsyncBlobStore.java @@ -34,6 +34,8 @@ import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions; import org.jclouds.aws.s3.blobstore.strategy.AsyncMultipartUploadStrategy; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.options.PutOptions; import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata; @@ -45,12 +47,15 @@ import org.jclouds.s3.S3Client; import org.jclouds.s3.blobstore.S3AsyncBlobStore; import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BucketToResourceList; -import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; import org.jclouds.s3.domain.AccessControlList; +import org.jclouds.s3.domain.BucketMetadata; +import org.jclouds.s3.domain.CannedAccessPolicy; +import org.jclouds.s3.domain.ObjectMetadata; +import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.cache.LoadingCache; import com.google.common.util.concurrent.ListenableFuture; @@ -73,12 +78,13 @@ public class AWSS3AsyncBlobStore extends S3AsyncBlobStore { public AWSS3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier defaultLocation, @Memoized Supplier> locations, AWSS3AsyncClient async, AWSS3Client sync, - BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, - BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, - BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, - Provider fetchBlobMetadataProvider, LoadingCache bucketAcls, + Function, PageSet> convertBucketsToStorageMetadata, + ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList, + ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, + ObjectToBlobMetadata object2BlobMd, Provider fetchBlobMetadataProvider, + LoadingCache bucketAcls, Provider multipartUploadStrategy) { - super(context, blobUtils, service, defaultLocation, locations, async, sync, bucket2ResourceMd, + super(context, blobUtils, service, defaultLocation, locations, async, sync, convertBucketsToStorageMetadata, container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object, object2BlobMd, fetchBlobMetadataProvider, bucketAcls); this.multipartUploadStrategy = multipartUploadStrategy; diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java index f981c3c85a..f70ea9d462 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java @@ -30,6 +30,8 @@ import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions; import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.options.PutOptions; import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata; @@ -40,12 +42,15 @@ import org.jclouds.s3.S3Client; import org.jclouds.s3.blobstore.S3BlobStore; import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BucketToResourceList; -import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; import org.jclouds.s3.domain.AccessControlList; +import org.jclouds.s3.domain.BucketMetadata; +import org.jclouds.s3.domain.CannedAccessPolicy; +import org.jclouds.s3.domain.ObjectMetadata; +import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.cache.LoadingCache; import org.jclouds.s3.domain.CannedAccessPolicy; @@ -68,14 +73,15 @@ public class AWSS3BlobStore extends S3BlobStore { @Inject AWSS3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier defaultLocation, @Memoized Supplier> locations, AWSS3Client sync, - BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, - BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, - BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, - Provider fetchBlobMetadataProvider, LoadingCache bucketAcls, + Function, PageSet> convertBucketsToStorageMetadata, + ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList, + ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, + ObjectToBlobMetadata object2BlobMd, Provider fetchBlobMetadataProvider, + LoadingCache bucketAcls, Provider multipartUploadStrategy) { - super(context, blobUtils, defaultLocation, locations, sync, bucket2ResourceMd, container2BucketListOptions, - bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object, object2BlobMd, - fetchBlobMetadataProvider, bucketAcls); + super(context, blobUtils, defaultLocation, locations, sync, convertBucketsToStorageMetadata, + container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object, + object2BlobMd, fetchBlobMetadataProvider, bucketAcls); this.multipartUploadStrategy = multipartUploadStrategy; this.bucketAcls = bucketAcls; this.blob2Object = blob2Object; diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java index a56a0f2c6f..c8087333c8 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java @@ -29,7 +29,6 @@ import javax.inject.Singleton; import org.jclouds.aws.s3.AWSS3AsyncClient; import org.jclouds.aws.s3.AWSS3Client; -import org.jclouds.aws.s3.binders.AssignCorrectHostnameAndBindAsHostPrefixIfConfigured; import org.jclouds.http.RequiresHttp; import org.jclouds.location.Region; import org.jclouds.rest.ConfiguresRestClient; @@ -53,24 +52,6 @@ import com.google.inject.Provides; @ConfiguresRestClient public class AWSS3RestClientModule extends S3RestClientModule { - @Override - protected Supplier defaultRegionForBucket(@Region Supplier defaultRegion) { - return Suppliers.ofInstance(US_STANDARD); - } - - @Override - protected void configure() { - bind(BindAsHostPrefixIfConfigured.class).to(AssignCorrectHostnameAndBindAsHostPrefixIfConfigured.class); - super.configure(); - } - - @Provides - @Singleton - @Bucket - protected Supplier provideBucketURI(@Named(PROPERTY_REGION + "." + US_STANDARD + "." + ENDPOINT) String endpoint){ - return Suppliers.ofInstance(URI.create(endpoint)); - } - public AWSS3RestClientModule() { super(AWSS3Client.class, AWSS3AsyncClient.class); } diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java index da375f3532..ef04209c9c 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java @@ -23,6 +23,9 @@ import java.lang.reflect.Method; import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentMap; +import java.util.Set; + +import javax.inject.Named; import org.jclouds.aws.s3.config.AWSS3RestClientModule; import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex; @@ -37,6 +40,8 @@ import org.jclouds.http.functions.ReleasePayloadAndReturn; import org.jclouds.http.functions.ReturnTrueIf2xx; import org.jclouds.io.Payload; import org.jclouds.io.Payloads; +import org.jclouds.location.Region; +import org.jclouds.location.reference.LocationConstants; import org.jclouds.rest.ConfiguresRestClient; import org.jclouds.rest.RestContextFactory; import org.jclouds.rest.RestContextSpec; @@ -46,7 +51,14 @@ import org.jclouds.rest.internal.RestAnnotationProcessor; import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.domain.ObjectMetadataBuilder; import org.jclouds.s3.domain.S3Object; -import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState; +import org.jclouds.s3.S3AsyncClient; +import org.jclouds.s3.S3AsyncClientTest; +import org.jclouds.s3.S3Client; +import org.jclouds.s3.domain.ObjectMetadata; +import org.jclouds.s3.domain.ObjectMetadataBuilder; +import org.jclouds.s3.domain.S3Object; +import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists; +import org.jclouds.s3.options.CopyObjectOptions; import org.jclouds.s3.options.PutBucketOptions; import org.jclouds.s3.options.PutObjectOptions; import org.jclouds.s3.xml.LocationConstraintHandler; @@ -55,6 +67,12 @@ import org.testng.annotations.Test; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import com.google.common.base.Functions; +import com.google.common.base.Optional; +import com.google.common.base.Supplier; +import com.google.common.cache.CacheLoader; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.TypeLiteral; @@ -70,12 +88,12 @@ public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest bucketToRegion() { - ConcurrentMap returnVal = Maps.newConcurrentMap(); - returnVal.put("eubucket", "EU"); - return returnVal; + protected CacheLoader> bucketToRegion(@Region Supplier> regionSupplier, + final S3Client client) { + return CacheLoader.> from(Functions.forMap(ImmutableMap + .> builder() + .put("bucket", Optional. absent()) + .put("destinationbucket", Optional. absent()) + .put("bucket-us-standard", Optional.of("us-standard")) + .put("bucket-us-west-1", Optional.of("us-west-1")) + .put("bucket-us-west-2", Optional.of("us-west-2")) + .put("bucket-eu-west-1", Optional.of("eu-west-1")) + .put("bucket-sa-east-1", Optional.of("sa-east-1")) + .put("bucket-ap-southeast-1", Optional.of("ap-southeast-1")) + .put("bucket-ap-northeast-1", Optional.of("ap-northeast-1")) + .build())); } @Override diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientExpectTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientExpectTest.java index 63055cb45f..5e64501bc4 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientExpectTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientExpectTest.java @@ -18,35 +18,65 @@ */ package org.jclouds.aws.s3; -import com.google.common.collect.ImmutableMultimap; +import static org.jclouds.aws.s3.blobstore.options.AWSS3PutObjectOptions.Builder.storageClass; + +import java.net.URI; + import org.jclouds.aws.s3.internal.BaseAWSS3ClientExpectTest; import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobBuilder; import org.jclouds.http.HttpRequest; import org.jclouds.http.HttpResponse; import org.jclouds.io.payloads.StringPayload; import org.jclouds.s3.blobstore.functions.BlobToObject; +import org.jclouds.s3.domain.ObjectMetadata.StorageClass; import org.testng.annotations.Test; -import java.net.URI; - -import static org.jclouds.aws.s3.blobstore.options.AWSS3PutObjectOptions.Builder.storageClass; -import static org.jclouds.s3.domain.ObjectMetadata.StorageClass; +import com.google.common.base.Functions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.inject.Injector; /** * @author Andrei Savu */ @Test public class AWSS3ClientExpectTest extends BaseAWSS3ClientExpectTest { - + HttpRequest bucketLocationRequest = HttpRequest.builder() + .method("GET") + .endpoint(URI.create("https://test.s3.amazonaws.com/?location")) + .headers(ImmutableMultimap.of( + "Host", "test.s3.amazonaws.com", + "Date", CONSTANT_DATE, + "Authorization", "AWS identity:D1rymKrEdvzvhmZXeg+Z0R+tiug=" + )) + .build(); + + HttpResponse bucketLocationResponse = HttpResponse.builder() + .statusCode(200) + .payload( + payloadFromStringWithContentType("eu-west-1", "application/xml")) + .headers(ImmutableMultimap.of( + "x-amz-id-2", "BtioT9wIK04YkE2DPgWUrQFiAbjwJVP8cLyfOkJ1FHMbn2hVjBZvkMMuXPDHfGVw", + "x-amz-request-id", "51BF4F45D49B1B34", + "Date", CONSTANT_DATE, + "Server", "AmazonS3" + )) + .build(); @Test public void testPutWithReducedRedundancy() { - AWSS3Client client = requestSendsResponse( + Injector injector = createInjector(Functions.forMap(ImmutableMap.of()), createModule(), setupProperties()); + + Blob blob = injector.getInstance(BlobBuilder.class).name("test").payload("content").build(); + BlobToObject blobToObject = injector.getInstance(BlobToObject.class); + + AWSS3Client client = requestsSendResponses(bucketLocationRequest, bucketLocationResponse, HttpRequest.builder() .method("PUT") - .endpoint(URI.create("https://test.s3.amazonaws.com/test")) + .endpoint(URI.create("https://test.s3-eu-west-1.amazonaws.com/test")) .headers(ImmutableMultimap.of( "x-amz-storage-class", "REDUCED_REDUNDANCY", - "Host", "test.s3.amazonaws.com", + "Host", "test.s3-eu-west-1.amazonaws.com", "Date", CONSTANT_DATE, "Authorization", "AWS identity:1mJrW85/mqZpYTFIK5Ebtt2MM6E=" )) @@ -64,14 +94,7 @@ public class AWSS3ClientExpectTest extends BaseAWSS3ClientExpectTest { .build() ); - Blob blob = blobStore.blobBuilder("test").payload("content").build(); - BlobToObject blobToObject = getInstance(BlobToObject.class); - client.putObject("test", blobToObject.apply(blob), storageClass(StorageClass.REDUCED_REDUNDANCY)); } - - public T getInstance(Class klass) { - return blobStoreContext.utils().injector().getInstance(klass); - } } diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest.java deleted file mode 100644 index aae81e2686..0000000000 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to jclouds, Inc. (jclouds) under one or more - * contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. jclouds licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.jclouds.aws.s3.binders; - -import static org.testng.Assert.assertEquals; - -import java.net.URI; -import java.util.Map; - -import javax.inject.Provider; -import javax.ws.rs.core.UriBuilder; - -import org.jclouds.http.HttpRequest; -import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull; -import org.jclouds.rest.binders.BindAsHostPrefix; -import org.testng.annotations.Test; - -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import com.google.common.collect.ImmutableMap; -import com.sun.jersey.api.uri.UriBuilderImpl; - -/** - * Tests behavior of {@code AssignCorrectHostnameAndBindAsHostPrefixIfConfigured} - * - * @author Adrian Cole - */ -// NOTE:without testName, this will not call @Before* and fail w/NPE during -// surefire -@Test(groups = "unit", testName = "AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest") -public class AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest { - Provider uriBuilderProvider = new Provider() { - - @Override - public UriBuilder get() { - return new UriBuilderImpl(); - } - - }; - - public void testWhenNoBucketRegionMappingInCache() { - HttpRequest request = new HttpRequest("GET", URI.create("https://s3.amazonaws.com")); - - AssignCorrectHostnameAndBindAsHostPrefixIfConfigured binder = new AssignCorrectHostnameAndBindAsHostPrefixIfConfigured( - new BindAsHostPrefix(uriBuilderProvider), true, "/", new RegionToEndpointOrProviderIfNull("aws-s3", Suppliers - .ofInstance(URI.create("https://s3.amazonaws.com")), - - Suppliers.>> ofInstance(ImmutableMap.of("us-standard", Suppliers - .ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1", Suppliers.ofInstance(URI - .create("https://s3-us-west-1.amazonaws.com"))))), - - uriBuilderProvider, ImmutableMap. of()); - - request = binder.bindToRequest(request, "bucket"); - assertEquals(request.getRequestLine(), "GET https://bucket.s3.amazonaws.com HTTP/1.1"); - - } - - public void testWhenBucketRegionMappingInCache() { - HttpRequest request = new HttpRequest("GET", URI.create("https://s3.amazonaws.com")); - - AssignCorrectHostnameAndBindAsHostPrefixIfConfigured binder = new AssignCorrectHostnameAndBindAsHostPrefixIfConfigured( - new BindAsHostPrefix(uriBuilderProvider), true, "/", new RegionToEndpointOrProviderIfNull("aws-s3", Suppliers - .ofInstance(URI.create("https://s3.amazonaws.com")), - - Suppliers.>> ofInstance(ImmutableMap.of("us-standard", Suppliers - .ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1", Suppliers.ofInstance(URI - .create("https://s3-us-west-1.amazonaws.com"))))), - - uriBuilderProvider, ImmutableMap. of("bucket", "us-west-1")); - - request = binder.bindToRequest(request, "bucket"); - assertEquals(request.getRequestLine(), "GET https://bucket.s3-us-west-1.amazonaws.com HTTP/1.1"); - - } -} diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/services/AWSBucketsLiveTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/services/AWSBucketsLiveTest.java new file mode 100644 index 0000000000..5033472a7f --- /dev/null +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/services/AWSBucketsLiveTest.java @@ -0,0 +1,78 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.aws.s3.services; + +import static org.jclouds.s3.options.PutBucketOptions.Builder.withBucketAcl; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import org.jclouds.aws.domain.Region; +import org.jclouds.s3.domain.AccessControlList; +import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI; +import org.jclouds.s3.domain.AccessControlList.Permission; +import org.jclouds.s3.domain.CannedAccessPolicy; +import org.jclouds.s3.services.BucketsLiveTest; +import org.testng.annotations.Test; + +import com.google.common.base.Throwables; + +/** + * @author Adrian Cole + */ +@Test(groups = "live", testName = "AWSBucketsLiveTest") +public class AWSBucketsLiveTest extends BucketsLiveTest { + + public void testDefaultBucketLocation() throws Exception { + + String bucketName = getContainerName(); + try { + String location = getApi().getBucketLocation(bucketName); + assert location.equals(Region.US_STANDARD) : "bucket: " + bucketName + " location: " + location; + } finally { + returnContainer(bucketName); + } + } + + /** + * using scratch bucketName as we are changing location + */ + public void testEu() throws Exception { + final String bucketName = getScratchContainerName(); + try { + getApi().putBucketInRegion(Region.EU_WEST_1, bucketName + "eu", withBucketAcl(CannedAccessPolicy.PUBLIC_READ)); + assertConsistencyAware(new Runnable() { + public void run() { + try { + AccessControlList acl = getApi().getBucketACL(bucketName + "eu"); + assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ), acl.toString()); + } catch (Exception e) { + Throwables.propagateIfPossible(e); + } + } + }); + assertEquals(Region.EU_WEST_1, getApi().getBucketLocation(bucketName + "eu")); + // TODO: I believe that the following should work based on the above acl assertion passing. + // However, it fails on 403 + // URL url = new URL(String.format("http://%s.s3.amazonaws.com", bucketName)); + // Utils.toStringAndClose(url.openStream()); + } finally { + destroyContainer(bucketName + "eu"); + } + } +} diff --git a/providers/hpcloud-objectstorage-lvs/src/test/java/org/jclouds/hpcloud/objectstorage/lvs/blobstore/integration/HPCloudObjectStorageLasVegasContainerLiveTest.java b/providers/hpcloud-objectstorage-lvs/src/test/java/org/jclouds/hpcloud/objectstorage/lvs/blobstore/integration/HPCloudObjectStorageLasVegasContainerLiveTest.java index 1e765bca0f..083f324b99 100644 --- a/providers/hpcloud-objectstorage-lvs/src/test/java/org/jclouds/hpcloud/objectstorage/lvs/blobstore/integration/HPCloudObjectStorageLasVegasContainerLiveTest.java +++ b/providers/hpcloud-objectstorage-lvs/src/test/java/org/jclouds/hpcloud/objectstorage/lvs/blobstore/integration/HPCloudObjectStorageLasVegasContainerLiveTest.java @@ -18,9 +18,6 @@ */ package org.jclouds.hpcloud.objectstorage.lvs.blobstore.integration; -import java.io.IOException; -import java.net.MalformedURLException; - import org.jclouds.blobstore.integration.internal.BaseContainerLiveTest; import org.testng.annotations.Test; @@ -29,10 +26,4 @@ import org.testng.annotations.Test; */ @Test(groups = { "live" }) public class HPCloudObjectStorageLasVegasContainerLiveTest extends BaseContainerLiveTest { - - @Test(enabled = false) - //@Test(expectedExceptions=UnsupportedOperationException.class) - public void testPublicAccess() throws MalformedURLException, InterruptedException, IOException { - super.testPublicAccess(); - } }