Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 1008:aws-s3 could not get location for region eu-west-1 error

This commit is contained in:
Adrian Cole 2012-07-06 22:54:15 -07:00
parent e9bfb9d1a2
commit 74b404eab8
49 changed files with 1525 additions and 737 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
apis/s3/src/main/java/org/jclouds/s3/S3AsyncClient.java
View File

@ -41,8 +41,8 @@ import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404;
import org.jclouds.http.functions.ParseETagHeader; import org.jclouds.http.functions.ParseETagHeader;
import org.jclouds.http.options.GetOptions; import org.jclouds.http.options.GetOptions;
import org.jclouds.javax.annotation.Nullable; import org.jclouds.javax.annotation.Nullable;
import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull;
import org.jclouds.rest.annotations.BinderParam; import org.jclouds.rest.annotations.BinderParam;
import org.jclouds.rest.annotations.Endpoint;
import org.jclouds.rest.annotations.EndpointParam; import org.jclouds.rest.annotations.EndpointParam;
import org.jclouds.rest.annotations.ExceptionParser; import org.jclouds.rest.annotations.ExceptionParser;
import org.jclouds.rest.annotations.Headers; import org.jclouds.rest.annotations.Headers;
@ -69,11 +69,13 @@ import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.Payer; import org.jclouds.s3.domain.Payer;
import org.jclouds.s3.domain.S3Object; import org.jclouds.s3.domain.S3Object;
import org.jclouds.s3.filters.RequestAuthorizeSignature; import org.jclouds.s3.filters.RequestAuthorizeSignature;
import org.jclouds.s3.functions.AssignCorrectHostnameForBucket;
import org.jclouds.s3.functions.BindRegionToXmlPayload; import org.jclouds.s3.functions.BindRegionToXmlPayload;
import org.jclouds.s3.functions.DefaultEndpointThenInvalidateRegion;
import org.jclouds.s3.functions.ObjectKey; import org.jclouds.s3.functions.ObjectKey;
import org.jclouds.s3.functions.ParseObjectFromHeadersAndHttpContent; import org.jclouds.s3.functions.ParseObjectFromHeadersAndHttpContent;
import org.jclouds.s3.functions.ParseObjectMetadataFromHeaders; import org.jclouds.s3.functions.ParseObjectMetadataFromHeaders;
import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState; import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists;
import org.jclouds.s3.functions.ReturnTrueOn404OrNotFoundFalseOnIllegalState; import org.jclouds.s3.functions.ReturnTrueOn404OrNotFoundFalseOnIllegalState;
import org.jclouds.s3.options.CopyObjectOptions; import org.jclouds.s3.options.CopyObjectOptions;
import org.jclouds.s3.options.ListBucketOptions; import org.jclouds.s3.options.ListBucketOptions;
@ -123,8 +125,8 @@ public interface S3AsyncClient {
@ExceptionParser(ReturnNullOnKeyNotFound.class) @ExceptionParser(ReturnNullOnKeyNotFound.class)
@ResponseParser(ParseObjectFromHeadersAndHttpContent.class) @ResponseParser(ParseObjectFromHeadersAndHttpContent.class)
ListenableFuture<S3Object> getObject( ListenableFuture<S3Object> getObject(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") String key, GetOptions... options); @PathParam("key") String key, GetOptions... options);
/** /**
* @see S3Client#headObject * @see S3Client#headObject
@ -134,8 +136,8 @@ public interface S3AsyncClient {
@ExceptionParser(ReturnNullOnKeyNotFound.class) @ExceptionParser(ReturnNullOnKeyNotFound.class)
@ResponseParser(ParseObjectMetadataFromHeaders.class) @ResponseParser(ParseObjectMetadataFromHeaders.class)
ListenableFuture<ObjectMetadata> headObject( ListenableFuture<ObjectMetadata> headObject(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") String key); @PathParam("key") String key);
/** /**
* @see S3Client#objectExists * @see S3Client#objectExists
@ -144,8 +146,8 @@ public interface S3AsyncClient {
@Path("/{key}") @Path("/{key}")
@ExceptionParser(ReturnFalseOnKeyNotFound.class) @ExceptionParser(ReturnFalseOnKeyNotFound.class)
ListenableFuture<Boolean> objectExists( ListenableFuture<Boolean> objectExists(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") String key); @PathParam("key") String key);
/** /**
* @see S3Client#deleteObject * @see S3Client#deleteObject
@ -154,8 +156,8 @@ public interface S3AsyncClient {
@Path("/{key}") @Path("/{key}")
@ExceptionParser(ReturnVoidOnNotFoundOr404.class) @ExceptionParser(ReturnVoidOnNotFoundOr404.class)
ListenableFuture<Void> deleteObject( ListenableFuture<Void> deleteObject(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") String key); @PathParam("key") String key);
/** /**
* @see S3Client#putObject * @see S3Client#putObject
@ -164,39 +166,43 @@ public interface S3AsyncClient {
@Path("/{key}") @Path("/{key}")
@ResponseParser(ParseETagHeader.class) @ResponseParser(ParseETagHeader.class)
ListenableFuture<String> putObject( ListenableFuture<String> putObject(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") @ParamParser(ObjectKey.class) @BinderParam(BindS3ObjectMetadataToRequest.class) S3Object object, @PathParam("key") @ParamParser(ObjectKey.class) @BinderParam(BindS3ObjectMetadataToRequest.class) S3Object object,
PutObjectOptions... options); PutObjectOptions... options);
/** /**
* @see S3Client#putBucketInRegion * @see S3Client#putBucketInRegion
*/ */
@PUT @PUT
@Path("/") @Path("/")
@ExceptionParser(ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class) @Endpoint(Bucket.class)
@ExceptionParser(ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.class)
ListenableFuture<Boolean> putBucketInRegion( ListenableFuture<Boolean> putBucketInRegion(
@EndpointParam(parser = RegionToEndpointOrProviderIfNull.class) @BinderParam(BindRegionToXmlPayload.class) @Nullable String region, @BinderParam(BindRegionToXmlPayload.class) @Nullable String region,
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
PutBucketOptions... options); PutBucketOptions... options);
/** /**
* @see S3Client#deleteBucketIfEmpty * @see S3Client#deleteBucketIfEmpty
*/ */
@DELETE @DELETE
@Path("/") @Path("/")
@Endpoint(Bucket.class)
@ExceptionParser(ReturnTrueOn404OrNotFoundFalseOnIllegalState.class) @ExceptionParser(ReturnTrueOn404OrNotFoundFalseOnIllegalState.class)
ListenableFuture<Boolean> deleteBucketIfEmpty( ListenableFuture<Boolean> deleteBucketIfEmpty(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); @Bucket @EndpointParam(parser = DefaultEndpointThenInvalidateRegion.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName);
/** /**
* @see S3Client#bucketExists * @see S3Client#bucketExists
*/ */
@HEAD @HEAD
@Path("/") @Path("/")
@Endpoint(Bucket.class)
@QueryParams(keys = "max-keys", values = "0") @QueryParams(keys = "max-keys", values = "0")
@ExceptionParser(ReturnFalseOnContainerNotFound.class) @ExceptionParser(ReturnFalseOnContainerNotFound.class)
ListenableFuture<Boolean> bucketExists( ListenableFuture<Boolean> bucketExists(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName);
/** /**
* @see S3Client#getBucketLocation * @see S3Client#getBucketLocation
@ -204,9 +210,10 @@ public interface S3AsyncClient {
@GET @GET
@QueryParams(keys = "location") @QueryParams(keys = "location")
@Path("/") @Path("/")
@Endpoint(Bucket.class)
@XMLResponseParser(LocationConstraintHandler.class) @XMLResponseParser(LocationConstraintHandler.class)
ListenableFuture<String> getBucketLocation( ListenableFuture<String> getBucketLocation(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName);
/** /**
* @see S3Client#getBucketPayer * @see S3Client#getBucketPayer
@ -216,7 +223,7 @@ public interface S3AsyncClient {
@Path("/") @Path("/")
@XMLResponseParser(PayerHandler.class) @XMLResponseParser(PayerHandler.class)
ListenableFuture<Payer> getBucketPayer( ListenableFuture<Payer> getBucketPayer(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName);
/** /**
* @see S3Client#setBucketPayer * @see S3Client#setBucketPayer
@ -225,8 +232,8 @@ public interface S3AsyncClient {
@QueryParams(keys = "requestPayment") @QueryParams(keys = "requestPayment")
@Path("/") @Path("/")
ListenableFuture<Void> setBucketPayer( ListenableFuture<Void> setBucketPayer(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@BinderParam(BindPayerToXmlPayload.class) Payer payer); @BinderParam(BindPayerToXmlPayload.class) Payer payer);
/** /**
* @see S3Client#listBucket * @see S3Client#listBucket
@ -235,8 +242,8 @@ public interface S3AsyncClient {
@Path("/") @Path("/")
@XMLResponseParser(ListBucketHandler.class) @XMLResponseParser(ListBucketHandler.class)
ListenableFuture<ListBucketResponse> listBucket( ListenableFuture<ListBucketResponse> listBucket(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
ListBucketOptions... options); ListBucketOptions... options);
/** /**
* @see S3Client#listOwnedBuckets * @see S3Client#listOwnedBuckets
@ -255,10 +262,10 @@ public interface S3AsyncClient {
@Headers(keys = "x-amz-copy-source", values = "/{sourceBucket}/{sourceObject}") @Headers(keys = "x-amz-copy-source", values = "/{sourceBucket}/{sourceObject}")
@XMLResponseParser(CopyObjectHandler.class) @XMLResponseParser(CopyObjectHandler.class)
ListenableFuture<ObjectMetadata> copyObject( ListenableFuture<ObjectMetadata> copyObject(
@PathParam("sourceBucket") String sourceBucket, @PathParam("sourceBucket") String sourceBucket,
@PathParam("sourceObject") String sourceObject, @PathParam("sourceObject") String sourceObject,
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String destinationBucket, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String destinationBucket,
@PathParam("destinationObject") String destinationObject, CopyObjectOptions... options); @PathParam("destinationObject") String destinationObject, CopyObjectOptions... options);
/** /**
* @see S3Client#getBucketACL * @see S3Client#getBucketACL
@ -269,7 +276,7 @@ public interface S3AsyncClient {
@ExceptionParser(ThrowContainerNotFoundOn404.class) @ExceptionParser(ThrowContainerNotFoundOn404.class)
@Path("/") @Path("/")
ListenableFuture<AccessControlList> getBucketACL( ListenableFuture<AccessControlList> getBucketACL(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName);
/** /**
* @see S3Client#putBucketACL * @see S3Client#putBucketACL
@ -278,8 +285,8 @@ public interface S3AsyncClient {
@Path("/") @Path("/")
@QueryParams(keys = "acl") @QueryParams(keys = "acl")
ListenableFuture<Boolean> putBucketACL( ListenableFuture<Boolean> putBucketACL(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@BinderParam(BindACLToXMLPayload.class) AccessControlList acl); @BinderParam(BindACLToXMLPayload.class) AccessControlList acl);
/** /**
* @see S3Client#getObjectACL * @see S3Client#getObjectACL
@ -290,8 +297,8 @@ public interface S3AsyncClient {
@XMLResponseParser(AccessControlListHandler.class) @XMLResponseParser(AccessControlListHandler.class)
@ExceptionParser(ThrowKeyNotFoundOn404.class) @ExceptionParser(ThrowKeyNotFoundOn404.class)
ListenableFuture<AccessControlList> getObjectACL( ListenableFuture<AccessControlList> getObjectACL(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") String key); @PathParam("key") String key);
/** /**
* @see S3Client#putObjectACL * @see S3Client#putObjectACL
@ -300,8 +307,8 @@ public interface S3AsyncClient {
@QueryParams(keys = "acl") @QueryParams(keys = "acl")
@Path("/{key}") @Path("/{key}")
ListenableFuture<Boolean> putObjectACL( ListenableFuture<Boolean> putObjectACL(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") String key, @BinderParam(BindACLToXMLPayload.class) AccessControlList acl); @PathParam("key") String key, @BinderParam(BindACLToXMLPayload.class) AccessControlList acl);
/** /**
* @see S3Client#getBucketLogging * @see S3Client#getBucketLogging
@ -312,7 +319,7 @@ public interface S3AsyncClient {
@ExceptionParser(ThrowContainerNotFoundOn404.class) @ExceptionParser(ThrowContainerNotFoundOn404.class)
@Path("/") @Path("/")
ListenableFuture<BucketLogging> getBucketLogging( ListenableFuture<BucketLogging> getBucketLogging(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName);
/** /**
* @see S3Client#enableBucketLogging * @see S3Client#enableBucketLogging
@ -321,8 +328,8 @@ public interface S3AsyncClient {
@Path("/") @Path("/")
@QueryParams(keys = "logging") @QueryParams(keys = "logging")
ListenableFuture<Void> enableBucketLogging( ListenableFuture<Void> enableBucketLogging(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@BinderParam(BindBucketLoggingToXmlPayload.class) BucketLogging logging); @BinderParam(BindBucketLoggingToXmlPayload.class) BucketLogging logging);
/** /**
* @see S3Client#putBucketLogging * @see S3Client#putBucketLogging
@ -332,6 +339,6 @@ public interface S3AsyncClient {
@QueryParams(keys = "logging") @QueryParams(keys = "logging")
@Produces(MediaType.TEXT_XML) @Produces(MediaType.TEXT_XML)
ListenableFuture<Void> disableBucketLogging( ListenableFuture<Void> disableBucketLogging(
@Bucket @BinderParam(BindNoBucketLoggingToXmlPayload.class) @ParamValidators({ BucketNameValidator.class }) String bucketName); @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindNoBucketLoggingToXmlPayload.class) @ParamValidators(BucketNameValidator.class) String bucketName);
} }

2
apis/s3/src/main/java/org/jclouds/s3/S3Client.java
View File

@ -196,7 +196,7 @@ public interface S3Client {
* *
*/ */
@Timeout(duration = 90, timeUnit = TimeUnit.SECONDS) @Timeout(duration = 90, timeUnit = TimeUnit.SECONDS)
boolean putBucketInRegion(@Nullable String region, String bucketName, PutBucketOptions... options); boolean putBucketInRegion(@Nullable String region, @Bucket String bucketName, PutBucketOptions... options);
/** /**
* Deletes the bucket, if it is empty. * Deletes the bucket, if it is empty.

22
apis/s3/src/main/java/org/jclouds/s3/blobstore/S3AsyncBlobStore.java
View File

@ -34,7 +34,6 @@ import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseAsyncBlobStore; import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
import org.jclouds.blobstore.options.CreateContainerOptions; import org.jclouds.blobstore.options.CreateContainerOptions;
@ -50,7 +49,6 @@ import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.S3Client; import org.jclouds.s3.S3Client;
import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BlobToObject;
import org.jclouds.s3.blobstore.functions.BucketToResourceList; import org.jclouds.s3.blobstore.functions.BucketToResourceList;
import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata;
import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions;
import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlob;
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
@ -70,7 +68,6 @@ import com.google.common.base.Function;
import com.google.common.base.Supplier; import com.google.common.base.Supplier;
import com.google.common.cache.CacheLoader; import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache; import com.google.common.cache.LoadingCache;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
/** /**
@ -82,7 +79,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
private final S3AsyncClient async; private final S3AsyncClient async;
private final S3Client sync; private final S3Client sync;
private final BucketToResourceMetadata bucket2ResourceMd; private final Function<Set<BucketMetadata>, PageSet<? extends StorageMetadata>> convertBucketsToStorageMetadata;
private final ContainerToBucketListOptions container2BucketListOptions; private final ContainerToBucketListOptions container2BucketListOptions;
private final BlobToHttpGetOptions blob2ObjectGetOptions; private final BlobToHttpGetOptions blob2ObjectGetOptions;
private final BucketToResourceList bucket2ResourceList; private final BucketToResourceList bucket2ResourceList;
@ -94,17 +91,18 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
@Inject @Inject
protected S3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, protected S3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils,
@Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier<Location> defaultLocation, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations, S3AsyncClient async, S3Client sync, @Memoized Supplier<Set<? extends Location>> locations, S3AsyncClient async, S3Client sync,
BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, Function<Set<BucketMetadata>, PageSet<? extends StorageMetadata>> convertBucketsToStorageMetadata,
BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList,
BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object,
Provider<FetchBlobMetadata> fetchBlobMetadataProvider, LoadingCache<String, AccessControlList> bucketAcls) { ObjectToBlobMetadata object2BlobMd, Provider<FetchBlobMetadata> fetchBlobMetadataProvider,
LoadingCache<String, AccessControlList> bucketAcls) {
super(context, blobUtils, service, defaultLocation, locations); super(context, blobUtils, service, defaultLocation, locations);
this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions"); this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
this.async = checkNotNull(async, "async"); this.async = checkNotNull(async, "async");
this.sync = checkNotNull(sync, "sync"); this.sync = checkNotNull(sync, "sync");
this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd"); this.convertBucketsToStorageMetadata = checkNotNull(convertBucketsToStorageMetadata, "convertBucketsToStorageMetadata");
this.container2BucketListOptions = checkNotNull(container2BucketListOptions, "container2BucketListOptions"); this.container2BucketListOptions = checkNotNull(container2BucketListOptions, "container2BucketListOptions");
this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList"); this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList");
this.object2Blob = checkNotNull(object2Blob, "object2Blob"); this.object2Blob = checkNotNull(object2Blob, "object2Blob");
@ -122,7 +120,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
return Futures.compose(async.listOwnedBuckets(), return Futures.compose(async.listOwnedBuckets(),
new Function<Set<BucketMetadata>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() { new Function<Set<BucketMetadata>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() {
public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(Set<BucketMetadata> from) { public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(Set<BucketMetadata> from) {
return new PageSetImpl<StorageMetadata>(Iterables.transform(from, bucket2ResourceMd), null); return convertBucketsToStorageMetadata.apply(from);
} }
}, service); }, service);
} }

34
apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java
View File

@ -27,11 +27,11 @@ import javax.inject.Provider;
import javax.inject.Singleton; import javax.inject.Singleton;
import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.ContainerNotFoundException;
import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseBlobStore; import org.jclouds.blobstore.internal.BaseBlobStore;
import org.jclouds.blobstore.options.CreateContainerOptions; import org.jclouds.blobstore.options.CreateContainerOptions;
@ -45,7 +45,6 @@ import org.jclouds.http.options.GetOptions;
import org.jclouds.s3.S3Client; import org.jclouds.s3.S3Client;
import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BlobToObject;
import org.jclouds.s3.blobstore.functions.BucketToResourceList; import org.jclouds.s3.blobstore.functions.BucketToResourceList;
import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata;
import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions;
import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlob;
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
@ -64,7 +63,6 @@ import com.google.common.base.Function;
import com.google.common.base.Supplier; import com.google.common.base.Supplier;
import com.google.common.cache.CacheLoader; import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache; import com.google.common.cache.LoadingCache;
import com.google.common.collect.Iterables;
/** /**
* *
@ -73,7 +71,7 @@ import com.google.common.collect.Iterables;
@Singleton @Singleton
public class S3BlobStore extends BaseBlobStore { public class S3BlobStore extends BaseBlobStore {
private final S3Client sync; private final S3Client sync;
private final BucketToResourceMetadata bucket2ResourceMd; private final Function<Set<BucketMetadata>, PageSet<? extends StorageMetadata>> convertBucketsToStorageMetadata;
private final ContainerToBucketListOptions container2BucketListOptions; private final ContainerToBucketListOptions container2BucketListOptions;
private final BucketToResourceList bucket2ResourceList; private final BucketToResourceList bucket2ResourceList;
private final ObjectToBlob object2Blob; private final ObjectToBlob object2Blob;
@ -85,15 +83,16 @@ public class S3BlobStore extends BaseBlobStore {
@Inject @Inject
protected S3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation, protected S3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations, S3Client sync, @Memoized Supplier<Set<? extends Location>> locations, S3Client sync,
BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, Function<Set<BucketMetadata>, PageSet<? extends StorageMetadata>> convertBucketsToStorageMetadata,
BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList,
BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object,
Provider<FetchBlobMetadata> fetchBlobMetadataProvider, LoadingCache<String, AccessControlList> bucketAcls) { ObjectToBlobMetadata object2BlobMd, Provider<FetchBlobMetadata> fetchBlobMetadataProvider,
LoadingCache<String, AccessControlList> bucketAcls) {
super(context, blobUtils, defaultLocation, locations); super(context, blobUtils, defaultLocation, locations);
this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions"); this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
this.sync = checkNotNull(sync, "sync"); this.sync = checkNotNull(sync, "sync");
this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd"); this.convertBucketsToStorageMetadata = checkNotNull(convertBucketsToStorageMetadata, "convertBucketsToStorageMetadata");
this.container2BucketListOptions = checkNotNull(container2BucketListOptions, "container2BucketListOptions"); this.container2BucketListOptions = checkNotNull(container2BucketListOptions, "container2BucketListOptions");
this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList"); this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList");
this.object2Blob = checkNotNull(object2Blob, "object2Blob"); this.object2Blob = checkNotNull(object2Blob, "object2Blob");
@ -108,11 +107,7 @@ public class S3BlobStore extends BaseBlobStore {
*/ */
@Override @Override
public PageSet<? extends StorageMetadata> list() { public PageSet<? extends StorageMetadata> list() {
return new Function<Set<BucketMetadata>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() { return convertBucketsToStorageMetadata.apply(sync.listOwnedBuckets());
public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(Set<BucketMetadata> from) {
return new PageSetImpl<StorageMetadata>(Iterables.transform(from, bucket2ResourceMd), null);
}
}.apply(sync.listOwnedBuckets());
} }
/** /**
@ -169,10 +164,15 @@ public class S3BlobStore extends BaseBlobStore {
*/ */
public void clearAndDeleteContainer(final String container) { public void clearAndDeleteContainer(final String container) {
try { try {
//TODO: probably it is better to use a retryable predicate
if (!Assertions.eventuallyTrue(new Supplier<Boolean>() { if (!Assertions.eventuallyTrue(new Supplier<Boolean>() {
public Boolean get() { public Boolean get() {
clearContainer(container); try {
return sync.deleteBucketIfEmpty(container); clearContainer(container);
return sync.deleteBucketIfEmpty(container);
} catch (ContainerNotFoundException e) {
return true;
}
} }
}, 30000)) { }, 30000)) {
throw new IllegalStateException(container + " still exists after deleting!"); throw new IllegalStateException(container + " still exists after deleting!");

11
apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java
View File

@ -33,9 +33,8 @@ import org.jclouds.s3.blobstore.S3AsyncBlobStore;
import org.jclouds.s3.blobstore.S3BlobRequestSigner; import org.jclouds.s3.blobstore.S3BlobRequestSigner;
import org.jclouds.s3.blobstore.S3BlobStore; import org.jclouds.s3.blobstore.S3BlobStore;
import org.jclouds.s3.blobstore.S3BlobStoreContext; import org.jclouds.s3.blobstore.S3BlobStoreContext;
import org.jclouds.s3.blobstore.functions.LocationFromBucketLocation; import org.jclouds.s3.blobstore.functions.LocationFromBucketName;
import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.AccessControlList;
import org.jclouds.s3.domain.BucketMetadata;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheBuilder;
@ -60,12 +59,8 @@ public class S3BlobStoreContextModule extends AbstractModule {
bind(AsyncBlobStore.class).to(S3AsyncBlobStore.class).in(Scopes.SINGLETON); bind(AsyncBlobStore.class).to(S3AsyncBlobStore.class).in(Scopes.SINGLETON);
bind(BlobStore.class).to(S3BlobStore.class).in(Scopes.SINGLETON); bind(BlobStore.class).to(S3BlobStore.class).in(Scopes.SINGLETON);
bind(BlobRequestSigner.class).to(S3BlobRequestSigner.class); bind(BlobRequestSigner.class).to(S3BlobRequestSigner.class);
bindBucketLocationStrategy(); bind(new TypeLiteral<Function<String, Location>>() {
} }).to(LocationFromBucketName.class);
protected void bindBucketLocationStrategy() {
bind(new TypeLiteral<Function<BucketMetadata, Location>>() {
}).to(LocationFromBucketLocation.class);
} }
@Provides @Provides

6
apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BucketToResourceMetadata.java
View File

@ -36,10 +36,10 @@ import com.google.common.base.Function;
*/ */
@Singleton @Singleton
public class BucketToResourceMetadata implements Function<BucketMetadata, StorageMetadata> { public class BucketToResourceMetadata implements Function<BucketMetadata, StorageMetadata> {
private final Function<BucketMetadata, Location> locationOfBucket; private final Function<String, Location> locationOfBucket;
@Inject @Inject
BucketToResourceMetadata(Function<BucketMetadata, Location> locationOfBucket) { BucketToResourceMetadata(Function<String, Location> locationOfBucket) {
this.locationOfBucket = locationOfBucket; this.locationOfBucket = locationOfBucket;
} }
@ -47,7 +47,7 @@ public class BucketToResourceMetadata implements Function<BucketMetadata, Storag
MutableStorageMetadata to = new MutableStorageMetadataImpl(); MutableStorageMetadata to = new MutableStorageMetadataImpl();
to.setName(from.getName()); to.setName(from.getName());
to.setType(StorageType.CONTAINER); to.setType(StorageType.CONTAINER);
to.setLocation(locationOfBucket.apply(from)); to.setLocation(locationOfBucket.apply(from.getName()));
return to; return to;
} }
} }

86
apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BucketsToStorageMetadata.java Normal file
View File

@ -0,0 +1,86 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.blobstore.functions;
import static com.google.common.base.Preconditions.checkNotNull;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import org.jclouds.Constants;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.concurrent.FutureIterables;
import org.jclouds.logging.Logger;
import org.jclouds.s3.domain.BucketMetadata;
import com.google.common.base.Function;
@Singleton
public class BucketsToStorageMetadata implements
Function<Set<BucketMetadata>, PageSet<? extends StorageMetadata>> {
@Resource
protected Logger logger = Logger.NULL;
private final ExecutorService userExecutor;
private final BucketToResourceMetadata bucket2ResourceMd;
@Inject
public BucketsToStorageMetadata(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, BucketToResourceMetadata bucket2ResourceMd) {
this.userExecutor = checkNotNull(userExecutor, "userExecutor");
this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd");
}
@Override
public PageSet<? extends StorageMetadata> apply(Set<BucketMetadata> input) {
// parallel as listing buckets is slow when looking up regions
Iterable<? extends StorageMetadata> buckets = FutureIterables
.<BucketMetadata, StorageMetadata> transformParallel(input,
new Function<BucketMetadata, Future<? extends StorageMetadata>>() {
@Override
public Future<? extends StorageMetadata> apply(final BucketMetadata from) {
return userExecutor.submit(new Callable<StorageMetadata>() {
@Override
public StorageMetadata call() throws Exception {
return bucket2ResourceMd.apply(from);
}
@Override
public String toString() {
return "bucket2ResourceMd.apply(" + from + ")";
}
});
}
}, userExecutor, null, logger, "my buckets");
return new PageSetImpl<StorageMetadata>(buckets, null);
}
}

87
apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketLocation.java
View File

@ -1,87 +0,0 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.blobstore.functions;
import java.util.NoSuchElementException;
import java.util.Set;
import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.blobstore.ContainerNotFoundException;
import org.jclouds.collect.Memoized;
import org.jclouds.domain.Location;
import org.jclouds.logging.Logger;
import org.jclouds.s3.S3Client;
import org.jclouds.s3.domain.BucketMetadata;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Supplier;
import com.google.common.collect.Iterables;
/**
* @author Adrian Cole
*/
@Singleton
public class LocationFromBucketLocation implements Function<BucketMetadata, Location> {
private final Location onlyLocation;
private final Supplier<Set<? extends Location>> locations;
private final S3Client client;
@Resource
protected Logger logger = Logger.NULL;
@Inject
LocationFromBucketLocation(S3Client client, @Memoized Supplier<Set<? extends Location>> locations) {
this.client = client;
this.onlyLocation = locations.get().size() == 1 ? Iterables.get(locations.get(), 0) : null;
this.locations = locations;
}
public Location apply(BucketMetadata from) {
if (onlyLocation != null)
return onlyLocation;
try {
Set<? extends Location> locations = this.locations.get();
final String region = client.getBucketLocation(from.getName());
assert region != null : String.format("could not get region for %s", from.getName());
if (region != null) {
try {
return Iterables.find(locations, new Predicate<Location>() {
@Override
public boolean apply(Location input) {
return input.getId().equalsIgnoreCase(region.toString());
}
});
} catch (NoSuchElementException e) {
logger.error("could not get location for region %s in %s", region, locations);
}
} else {
logger.error("could not get region for %s", from.getName());
}
} catch (ContainerNotFoundException e) {
logger.error(e, "could not get region for %s, as service suggests the bucket doesn't exist", from.getName());
}
return null;
}
}

75
apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/LocationFromBucketName.java Normal file
View File

@ -0,0 +1,75 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.blobstore.functions;
import static com.google.common.collect.Iterables.find;
import static com.google.common.collect.Iterables.get;
import static org.jclouds.location.predicates.LocationPredicates.idEquals;
import java.util.NoSuchElementException;
import java.util.Set;
import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.collect.Memoized;
import org.jclouds.domain.Location;
import org.jclouds.logging.Logger;
import org.jclouds.s3.Bucket;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Supplier;
/**
* @author Adrian Cole
*/
@Singleton
public class LocationFromBucketName implements Function<String, Location> {
private final Supplier<Set<? extends Location>> locations;
private final Function<String, Optional<String>> bucketToRegion;
@Resource
protected Logger logger = Logger.NULL;
@Inject
LocationFromBucketName(@Bucket Function<String, Optional<String>> bucketToRegion,
@Memoized Supplier<Set<? extends Location>> locations) {
this.bucketToRegion = bucketToRegion;
this.locations = locations;
}
public Location apply(String bucket) {
Set<? extends Location> locations = this.locations.get();
if (locations.size() == 1)
return get(locations, 0);
final Optional<String> region = bucketToRegion.apply(bucket);
if (region.isPresent()) {
try {
return find(locations, idEquals(region.get()));
} catch (NoSuchElementException e) {
logger.debug("could not get location for region %s in %s", region, locations);
}
} else {
logger.debug("could not get region for %s", bucket);
}
return null;
}
}

6
apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java
View File

@ -25,6 +25,7 @@ import org.jclouds.blobstore.domain.MutableBlobMetadata;
import org.jclouds.blobstore.domain.StorageType; import org.jclouds.blobstore.domain.StorageType;
import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl; import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl;
import org.jclouds.blobstore.strategy.IfDirectoryReturnNameStrategy; import org.jclouds.blobstore.strategy.IfDirectoryReturnNameStrategy;
import org.jclouds.domain.Location;
import org.jclouds.http.HttpUtils; import org.jclouds.http.HttpUtils;
import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.AccessControlList;
import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI; import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI;
@ -42,12 +43,14 @@ import com.google.common.cache.LoadingCache;
public class ObjectToBlobMetadata implements Function<ObjectMetadata, MutableBlobMetadata> { public class ObjectToBlobMetadata implements Function<ObjectMetadata, MutableBlobMetadata> {
private final IfDirectoryReturnNameStrategy ifDirectoryReturnName; private final IfDirectoryReturnNameStrategy ifDirectoryReturnName;
private final LoadingCache<String, AccessControlList> bucketAcls; private final LoadingCache<String, AccessControlList> bucketAcls;
private final Function<String, Location> locationOfBucket;
@Inject @Inject
public ObjectToBlobMetadata(IfDirectoryReturnNameStrategy ifDirectoryReturnName, public ObjectToBlobMetadata(IfDirectoryReturnNameStrategy ifDirectoryReturnName,
LoadingCache<String, AccessControlList> bucketAcls) { LoadingCache<String, AccessControlList> bucketAcls,Function<String, Location> locationOfBucket) {
this.ifDirectoryReturnName = ifDirectoryReturnName; this.ifDirectoryReturnName = ifDirectoryReturnName;
this.bucketAcls = bucketAcls; this.bucketAcls = bucketAcls;
this.locationOfBucket = locationOfBucket;
} }
public MutableBlobMetadata apply(ObjectMetadata from) { public MutableBlobMetadata apply(ObjectMetadata from) {
@ -68,6 +71,7 @@ public class ObjectToBlobMetadata implements Function<ObjectMetadata, MutableBlo
to.setName(from.getKey()); to.setName(from.getKey());
to.setLastModified(from.getLastModified()); to.setLastModified(from.getLastModified());
to.setUserMetadata(from.getUserMetadata()); to.setUserMetadata(from.getUserMetadata());
to.setLocation(locationOfBucket.apply(from.getBucket()));
String directoryName = ifDirectoryReturnName.execute(to); String directoryName = ifDirectoryReturnName.execute(to);
if (directoryName != null) { if (directoryName != null) {
to.setName(directoryName); to.setName(directoryName);

91
apis/s3/src/main/java/org/jclouds/s3/config/S3RestClientModule.java
View File

@ -18,7 +18,8 @@
*/ */
package org.jclouds.s3.config; package org.jclouds.s3.config;
import java.util.Map; import java.net.URI;
import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import javax.inject.Named; import javax.inject.Named;
@ -26,6 +27,10 @@ import javax.inject.Singleton;
import org.jclouds.Constants; import org.jclouds.Constants;
import org.jclouds.aws.config.AWSRestClientModule; import org.jclouds.aws.config.AWSRestClientModule;
import org.jclouds.aws.handlers.AWSClientErrorRetryHandler;
import org.jclouds.blobstore.ContainerNotFoundException;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.date.DateService; import org.jclouds.date.DateService;
import org.jclouds.date.TimeStamp; import org.jclouds.date.TimeStamp;
import org.jclouds.http.HttpErrorHandler; import org.jclouds.http.HttpErrorHandler;
@ -34,21 +39,31 @@ import org.jclouds.http.annotation.ClientError;
import org.jclouds.http.annotation.Redirection; import org.jclouds.http.annotation.Redirection;
import org.jclouds.http.annotation.ServerError; import org.jclouds.http.annotation.ServerError;
import org.jclouds.location.Region; import org.jclouds.location.Region;
import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull;
import org.jclouds.rest.ConfiguresRestClient; import org.jclouds.rest.ConfiguresRestClient;
import org.jclouds.rest.RequestSigner; import org.jclouds.rest.RequestSigner;
import org.jclouds.s3.Bucket; import org.jclouds.s3.Bucket;
import org.jclouds.s3.S3AsyncClient; import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.S3Client; import org.jclouds.s3.S3Client;
import org.jclouds.s3.blobstore.functions.BucketsToStorageMetadata;
import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.filters.RequestAuthorizeSignature; import org.jclouds.s3.filters.RequestAuthorizeSignature;
import org.jclouds.s3.functions.GetRegionForBucket;
import org.jclouds.s3.handlers.ParseS3ErrorFromXmlContent; import org.jclouds.s3.handlers.ParseS3ErrorFromXmlContent;
import org.jclouds.s3.handlers.S3RedirectionRetryHandler; import org.jclouds.s3.handlers.S3RedirectionRetryHandler;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Supplier; import com.google.common.base.Supplier;
import com.google.common.base.Suppliers; import com.google.common.base.Suppliers;
import com.google.common.collect.Maps; import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.Iterables;
import com.google.common.reflect.TypeToken; import com.google.common.reflect.TypeToken;
import com.google.inject.Provides; import com.google.inject.Provides;
import com.google.inject.Scopes; import com.google.inject.Scopes;
import com.google.inject.TypeLiteral;
/** /**
* Configures the S3 connection, including logging and http transport. * Configures the S3 connection, including logging and http transport.
@ -60,7 +75,7 @@ public class S3RestClientModule<S extends S3Client, A extends S3AsyncClient> ext
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public S3RestClientModule() { public S3RestClientModule() {
this((TypeToken) TypeToken.of(S3Client.class), (TypeToken) TypeToken.of(S3AsyncClient.class)); this(TypeToken.class.cast(TypeToken.of(S3Client.class)), TypeToken.class.cast(TypeToken.of(S3AsyncClient.class)));
} }
protected S3RestClientModule(TypeToken<S> syncClientType, TypeToken<A> asyncClientType) { protected S3RestClientModule(TypeToken<S> syncClientType, TypeToken<A> asyncClientType) {
@ -70,8 +85,61 @@ public class S3RestClientModule<S extends S3Client, A extends S3AsyncClient> ext
@Provides @Provides
@Bucket @Bucket
@Singleton @Singleton
protected Map<String, String> bucketToRegion() { protected CacheLoader<String, Optional<String>> bucketToRegion(@Region Supplier<Set<String>> regionSupplier,
return Maps.newConcurrentMap(); final S3Client client) {
Set<String> regions = regionSupplier.get();
if (regions.size() == 0) {
return new CacheLoader<String, Optional<String>>() {
@Override
public Optional<String> load(String bucket) {
return Optional.absent();
}
@Override
public String toString() {
return "noRegions()";
}
};
} else if (regions.size() == 1) {
final String onlyRegion = Iterables.getOnlyElement(regions);
return new CacheLoader<String, Optional<String>>() {
Optional<String> onlyRegionOption = Optional.of(onlyRegion);
@Override
public Optional<String> load(String bucket) {
return onlyRegionOption;
}
@Override
public String toString() {
return "onlyRegion(" + onlyRegion + ")";
}
};
} else {
return new CacheLoader<String, Optional<String>>() {
@Override
public Optional<String> load(String bucket) {
try {
return Optional.fromNullable(client.getBucketLocation(bucket));
} catch (ContainerNotFoundException e) {
return null;
}
}
@Override
public String toString() {
return "bucketToRegion()";
}
};
}
}
@Provides
@Bucket
@Singleton
protected LoadingCache<String, Optional<String>> bucketToRegion(@Bucket CacheLoader<String, Optional<String>> loader) {
return CacheBuilder.newBuilder().build(loader);
} }
@Provides @Provides
@ -81,12 +149,24 @@ public class S3RestClientModule<S extends S3Client, A extends S3AsyncClient> ext
return defaultRegion; return defaultRegion;
} }
@Provides
@Singleton
@Bucket
protected Supplier<URI> provideBucketURI(@Bucket Supplier<String> defaultRegion,
RegionToEndpointOrProviderIfNull regionToEndpoint) {
return Suppliers.compose(regionToEndpoint, defaultRegion);
}
@Override @Override
protected void configure() { protected void configure() {
super.configure(); super.configure();
install(new S3ObjectModule()); install(new S3ObjectModule());
install(new S3ParserModule()); install(new S3ParserModule());
bind(RequestAuthorizeSignature.class).in(Scopes.SINGLETON); bind(RequestAuthorizeSignature.class).in(Scopes.SINGLETON);
bind(new TypeLiteral<Function<String, Optional<String>>>() {
}).annotatedWith(Bucket.class).to(GetRegionForBucket.class);
bind(new TypeLiteral<Function<Set<BucketMetadata>, PageSet<? extends StorageMetadata>>>() {
}).to(BucketsToStorageMetadata.class);
} }
@Override @Override
@ -105,6 +185,7 @@ public class S3RestClientModule<S extends S3Client, A extends S3AsyncClient> ext
@Override @Override
protected void bindRetryHandlers() { protected void bindRetryHandlers() {
bind(HttpRetryHandler.class).annotatedWith(Redirection.class).to(S3RedirectionRetryHandler.class); bind(HttpRetryHandler.class).annotatedWith(Redirection.class).to(S3RedirectionRetryHandler.class);
bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(AWSClientErrorRetryHandler.class);
} }
@Provides @Provides

2
apis/s3/src/main/java/org/jclouds/s3/domain/BucketMetadata.java
View File

@ -27,8 +27,8 @@ import java.util.Date;
*/ */
public class BucketMetadata implements Comparable<BucketMetadata> { public class BucketMetadata implements Comparable<BucketMetadata> {
private final Date creationDate; private final Date creationDate;
private final CanonicalUser owner;
private final String name; private final String name;
private final CanonicalUser owner;
public BucketMetadata(String name, Date creationDate, CanonicalUser owner) { public BucketMetadata(String name, Date creationDate, CanonicalUser owner) {
this.name = name; this.name = name;

31
apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java
View File

@ -18,8 +18,6 @@
*/ */
package org.jclouds.s3.filters; package org.jclouds.s3.filters;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.Iterables.any;
import static com.google.common.collect.Iterables.get; import static com.google.common.collect.Iterables.get;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AUTH_TAG; import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AUTH_TAG;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG; import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
@ -29,8 +27,6 @@ import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS; import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
import static org.jclouds.util.Strings2.toInputStream; import static org.jclouds.util.Strings2.toInputStream;
import java.lang.annotation.Annotation;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Locale; import java.util.Locale;
import java.util.Map.Entry; import java.util.Map.Entry;
@ -57,11 +53,9 @@ import org.jclouds.logging.Logger;
import org.jclouds.rest.RequestSigner; import org.jclouds.rest.RequestSigner;
import org.jclouds.rest.annotations.Credential; import org.jclouds.rest.annotations.Credential;
import org.jclouds.rest.annotations.Identity; import org.jclouds.rest.annotations.Identity;
import org.jclouds.rest.internal.GeneratedHttpRequest; import org.jclouds.s3.util.S3Utils;
import org.jclouds.s3.Bucket;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.base.Strings; import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
@ -81,11 +75,6 @@ import com.google.common.collect.TreeMultimap;
*/ */
@Singleton @Singleton
public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSigner { public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSigner {
private static final Predicate<Annotation> ANNOTATIONTYPE_BUCKET = new Predicate<Annotation>() {
public boolean apply(Annotation input) {
return input.annotationType().equals(Bucket.class);
}
};
private static final Collection<String> FIRST_HEADERS_TO_SIGN = ImmutableList.of(HttpHeaders.DATE); private static final Collection<String> FIRST_HEADERS_TO_SIGN = ImmutableList.of(HttpHeaders.DATE);
@ -231,7 +220,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
@VisibleForTesting @VisibleForTesting
void appendBucketName(HttpRequest req, StringBuilder toSign) { void appendBucketName(HttpRequest req, StringBuilder toSign) {
String bucketName = getBucketName(req); String bucketName = S3Utils.getBucketName(req);
// If we have a payload/bucket/container that is not all lowercase, vhost-style URLs are not an option and must be // If we have a payload/bucket/container that is not all lowercase, vhost-style URLs are not an option and must be
// automatically converted to their path-based equivalent. This should only be possible for AWS-S3 since it is // automatically converted to their path-based equivalent. This should only be possible for AWS-S3 since it is
@ -266,20 +255,4 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
} }
} }
private String getBucketName(HttpRequest req) {
checkArgument(req instanceof GeneratedHttpRequest<?>, "this should be a generated http request");
GeneratedHttpRequest<?> request = GeneratedHttpRequest.class.cast(req);
String bucketName = null;
for (int i = 0; i < request.getJavaMethod().getParameterAnnotations().length; i++) {
if (any(Arrays.asList(request.getJavaMethod().getParameterAnnotations()[i]), ANNOTATIONTYPE_BUCKET)) {
bucketName = (String) request.getArgs().get(i);
break;
}
}
return bucketName;
}
} }

64
apis/s3/src/main/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucket.java Normal file
View File

@ -0,0 +1,64 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.functions;
import java.net.URI;
import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.javax.annotation.Nullable;
import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull;
import org.jclouds.logging.Logger;
import org.jclouds.s3.Bucket;
import com.google.common.base.Function;
import com.google.common.base.Optional;
/**
*
* @author Adrian Cole
*/
@Singleton
public class AssignCorrectHostnameForBucket implements Function<Object, URI> {
@Resource
protected Logger logger = Logger.NULL;
protected final RegionToEndpointOrProviderIfNull r2;
protected final Function<String, Optional<String>> bucketToRegion;
@Inject
public AssignCorrectHostnameForBucket(RegionToEndpointOrProviderIfNull r2,
@Bucket Function<String, Optional<String>> bucketToRegion) {
this.bucketToRegion = bucketToRegion;
this.r2 = r2;
}
@Override
public URI apply(@Nullable Object from) {
String bucket = from.toString();
Optional<String> region = bucketToRegion.apply(bucket);
if (region.isPresent()) {
return r2.apply(region.get());
}
return r2.apply(null);
}
}

37
apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.java → apis/s3/src/main/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegion.java
View File

@ -18,29 +18,42 @@
*/ */
package org.jclouds.s3.functions; package org.jclouds.s3.functions;
import java.net.URI;
import javax.inject.Inject;
import javax.inject.Singleton; import javax.inject.Singleton;
import org.jclouds.aws.AWSResponseException; import org.jclouds.javax.annotation.Nullable;
import org.jclouds.util.Throwables2; import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull;
import org.jclouds.s3.Bucket;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.base.Throwables; import com.google.common.base.Optional;
import com.google.common.cache.LoadingCache;
/** /**
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
@Singleton @Singleton
public class ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState implements Function<Exception, Boolean> { public class DefaultEndpointThenInvalidateRegion implements Function<Object, URI> {
public Boolean apply(Exception from) { private final LoadingCache<String, Optional<String>> bucketToRegionCache;
AWSResponseException exception = Throwables2.getFirstThrowableOfType(from, AWSResponseException.class); private final RegionToEndpointOrProviderIfNull r2;
if (exception != null && exception.getError() != null
&& exception.getError().getCode().equals("BucketAlreadyOwnedByYou")) { @Inject
return false; public DefaultEndpointThenInvalidateRegion(RegionToEndpointOrProviderIfNull r2,
} else if (Throwables2.getFirstThrowableOfType(from, IllegalStateException.class) != null) { @Bucket LoadingCache<String, Optional<String>> bucketToRegionCache) {
return false; this.r2 = r2;
this.bucketToRegionCache = bucketToRegionCache;
}
@Override
public URI apply(@Nullable Object from) {
try {
return r2.apply(null);
} finally {
bucketToRegionCache.invalidate(from.toString());
} }
throw Throwables.propagate(from);
} }
} }

65
apis/s3/src/main/java/org/jclouds/s3/functions/GetRegionForBucket.java Normal file
View File

@ -0,0 +1,65 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.functions;
import java.util.concurrent.ExecutionException;
import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.logging.Logger;
import org.jclouds.s3.Bucket;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.cache.CacheLoader.InvalidCacheLoadException;
import com.google.common.cache.LoadingCache;
import com.google.common.util.concurrent.UncheckedExecutionException;
/**
*
* @author Adrian Cole
*/
@Singleton
public class GetRegionForBucket implements Function<String, Optional<String>> {
@Resource
protected Logger logger = Logger.NULL;
protected final LoadingCache<String, Optional<String>> bucketToRegion;
@Inject
public GetRegionForBucket(@Bucket LoadingCache<String, Optional<String>> bucketToRegion) {
this.bucketToRegion = bucketToRegion;
}
@Override
public Optional<String> apply(String bucket) {
try {
return bucketToRegion.get(bucket);
} catch (ExecutionException e) {
logger.debug("error looking up region for bucket %s: %s", bucket, e);
} catch (UncheckedExecutionException e) {
logger.debug("error looking up region for bucket %s: %s", bucket, e);
} catch (InvalidCacheLoadException e) {
logger.trace("bucket %s not found: %s", bucket, e);
}
return Optional.absent();
}
}

70
apis/s3/src/main/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.java Normal file
View File

@ -0,0 +1,70 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.functions;
import javax.inject.Inject;
import org.jclouds.aws.AWSResponseException;
import org.jclouds.http.HttpRequest;
import org.jclouds.javax.annotation.Nullable;
import org.jclouds.rest.InvocationContext;
import org.jclouds.s3.S3Client;
import org.jclouds.s3.util.S3Utils;
import org.jclouds.util.Throwables2;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
/**
*
* @author Adrian Cole
*/
public class ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists implements
Function<Exception, Boolean>,
InvocationContext<ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists> {
private final S3Client client;
private String bucket;
@Inject
ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(S3Client client) {
this.client = client;
}
public Boolean apply(Exception from) {
AWSResponseException exception = Throwables2.getFirstThrowableOfType(from, AWSResponseException.class);
if (exception != null && exception.getError() != null && exception.getError().getCode() != null) {
String code = exception.getError().getCode();
if (code.equals("BucketAlreadyOwnedByYou"))
return false;
else if (code.equals("OperationAborted") && bucket != null && client.bucketExists(bucket))
return false;
}
throw Throwables.propagate(from);
}
@Override
public ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists setContext(
@Nullable HttpRequest request) {
if (request != null)
this.bucket = S3Utils.getBucketName(request);
return this;
}
}

1
apis/s3/src/main/java/org/jclouds/s3/functions/ReturnTrueOn404OrNotFoundFalseOnIllegalState.java
View File

@ -36,6 +36,7 @@ import com.google.common.base.Throwables;
@Singleton @Singleton
public class ReturnTrueOn404OrNotFoundFalseOnIllegalState implements Function<Exception, Boolean> { public class ReturnTrueOn404OrNotFoundFalseOnIllegalState implements Function<Exception, Boolean> {
//TODO: invalidate
public Boolean apply(Exception from) { public Boolean apply(Exception from) {
if (getFirstThrowableOfType(from, IllegalStateException.class) != null) { if (getFirstThrowableOfType(from, IllegalStateException.class) != null) {
return false; return false;

31
apis/s3/src/main/java/org/jclouds/s3/util/S3Utils.java
View File

@ -20,12 +20,20 @@ package org.jclouds.s3.util;
import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.any;
import java.lang.annotation.Annotation;
import java.util.Arrays;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.internal.GeneratedHttpRequest;
import org.jclouds.s3.Bucket;
import org.jclouds.s3.S3Client; import org.jclouds.s3.S3Client;
import org.jclouds.util.Patterns; import org.jclouds.util.Patterns;
import com.google.common.base.Predicate;
/** /**
* Encryption, Hashing, and IO Utilities needed to sign and verify S3 requests and responses. * Encryption, Hashing, and IO Utilities needed to sign and verify S3 requests and responses.
* *
@ -56,4 +64,27 @@ public class S3Utils {
sync.deleteBucketIfEmpty(container); sync.deleteBucketIfEmpty(container);
return sync.bucketExists(container); return sync.bucketExists(container);
} }
private static final Predicate<Annotation> ANNOTATIONTYPE_BUCKET = new Predicate<Annotation>() {
public boolean apply(Annotation input) {
return input.annotationType().equals(Bucket.class);
}
};
public static String getBucketName(HttpRequest req) {
checkArgument(req instanceof GeneratedHttpRequest<?>, "this should be a generated http request");
GeneratedHttpRequest<?> request = GeneratedHttpRequest.class.cast(req);
String bucketName = null;
for (int i = 0; i < request.getJavaMethod().getParameterAnnotations().length; i++) {
if (any(Arrays.asList(request.getJavaMethod().getParameterAnnotations()[i]), ANNOTATIONTYPE_BUCKET)) {
bucketName = (String) request.getArgs().get(i);
break;
}
}
return bucketName;
}
} }

13
apis/s3/src/main/java/org/jclouds/s3/xml/LocationConstraintHandler.java
View File

@ -20,8 +20,6 @@ package org.jclouds.s3.xml;
import static org.jclouds.util.SaxUtils.currentOrNull; import static org.jclouds.util.SaxUtils.currentOrNull;
import java.util.Map;
import javax.inject.Inject; import javax.inject.Inject;
import org.jclouds.aws.domain.Region; import org.jclouds.aws.domain.Region;
@ -30,6 +28,9 @@ import org.jclouds.http.functions.ParseSax;
import org.jclouds.rest.internal.GeneratedHttpRequest; import org.jclouds.rest.internal.GeneratedHttpRequest;
import org.jclouds.s3.Bucket; import org.jclouds.s3.Bucket;
import com.google.common.base.Optional;
import com.google.common.cache.LoadingCache;
/** /**
* Parses the response from Amazon S3 GET Bucket Location * Parses the response from Amazon S3 GET Bucket Location
* <p/> * <p/>
@ -41,13 +42,13 @@ import org.jclouds.s3.Bucket;
* @author Adrian Cole * @author Adrian Cole
*/ */
public class LocationConstraintHandler extends ParseSax.HandlerWithResult<String> { public class LocationConstraintHandler extends ParseSax.HandlerWithResult<String> {
private final Map<String, String> bucketToRegion; private final LoadingCache<String, Optional<String>> bucketToRegion;
private StringBuilder currentText = new StringBuilder(); private StringBuilder currentText = new StringBuilder();
private String region; private String region;
private String bucket; private String bucket;
@Inject @Inject
public LocationConstraintHandler(@Bucket Map<String, String> bucketToRegion) { public LocationConstraintHandler(@Bucket LoadingCache<String, Optional<String>> bucketToRegion) {
this.bucketToRegion = bucketToRegion; this.bucketToRegion = bucketToRegion;
} }
@ -57,7 +58,7 @@ public class LocationConstraintHandler extends ParseSax.HandlerWithResult<String
public void endElement(String uri, String name, String qName) { public void endElement(String uri, String name, String qName) {
region = fromValue(currentOrNull(currentText)); region = fromValue(currentOrNull(currentText));
bucketToRegion.put(bucket, region); bucketToRegion.put(bucket, Optional.fromNullable(region));
} }
@Override @Override
@ -79,6 +80,8 @@ public class LocationConstraintHandler extends ParseSax.HandlerWithResult<String
public static String fromValue(String v) { public static String fromValue(String v) {
if (v == null || "".equals(v)) if (v == null || "".equals(v))
return Region.US_STANDARD; return Region.US_STANDARD;
if ("EU".equals(v))
return "eu-west-1";
return v; return v;
} }

6
apis/s3/src/test/java/org/jclouds/s3/S3AsyncClientTest.java
View File

@ -51,7 +51,7 @@ import org.jclouds.s3.domain.AccessControlList.Grant;
import org.jclouds.s3.domain.AccessControlList.Permission; import org.jclouds.s3.domain.AccessControlList.Permission;
import org.jclouds.s3.functions.ParseObjectFromHeadersAndHttpContent; import org.jclouds.s3.functions.ParseObjectFromHeadersAndHttpContent;
import org.jclouds.s3.functions.ParseObjectMetadataFromHeaders; import org.jclouds.s3.functions.ParseObjectMetadataFromHeaders;
import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState; import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists;
import org.jclouds.s3.functions.ReturnTrueOn404OrNotFoundFalseOnIllegalState; import org.jclouds.s3.functions.ReturnTrueOn404OrNotFoundFalseOnIllegalState;
import org.jclouds.s3.internal.BaseS3AsyncClientTest; import org.jclouds.s3.internal.BaseS3AsyncClientTest;
import org.jclouds.s3.options.CopyObjectOptions; import org.jclouds.s3.options.CopyObjectOptions;
@ -87,7 +87,7 @@ public abstract class S3AsyncClientTest<T extends S3AsyncClient> extends BaseS3A
Method method = S3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class, Array.newInstance( Method method = S3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class, Array.newInstance(
PutBucketOptions.class, 0).getClass()); PutBucketOptions.class, 0).getClass());
for (String region : Region.DEFAULT_S3) { for (String region : Region.DEFAULT_S3) {
processor.createRequest(method, region, "bucket-name"); processor.createRequest(method, region, "bucket-" + region);
} }
} }
@ -383,7 +383,7 @@ public abstract class S3AsyncClientTest<T extends S3AsyncClient> extends BaseS3A
assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class); assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class);
assertSaxResponseParserClassEquals(method, null); assertSaxResponseParserClassEquals(method, null);
assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class); assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.class);
checkFilters(request); checkFilters(request);
} }

70
apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/BucketsToStorageMetadataTest.java Normal file
View File

@ -0,0 +1,70 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.blobstore.functions;
import static org.testng.Assert.assertEquals;
import org.jclouds.blobstore.domain.MutableStorageMetadata;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.StorageType;
import org.jclouds.blobstore.domain.internal.MutableStorageMetadataImpl;
import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.concurrent.MoreExecutors;
import org.jclouds.domain.Location;
import org.jclouds.domain.LocationBuilder;
import org.jclouds.domain.LocationScope;
import org.jclouds.s3.domain.BucketMetadata;
import org.testng.annotations.Test;
import com.google.common.base.Functions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
/**
* Tests behavior of {@code BucketsToStorageMetadata}
*
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during
// surefire
@Test(groups = "unit", testName = "BucketsToStorageMetadataTest")
public class BucketsToStorageMetadataTest {
protected Location provider = new LocationBuilder().scope(LocationScope.PROVIDER).id("aws-ec2")
.description("aws-ec2").build();
protected Location region = new LocationBuilder().scope(LocationScope.REGION).id("us-east-1")
.description("us-east-1").parent(provider).build();
public void test() {
BucketsToStorageMetadata fn = new BucketsToStorageMetadata(
MoreExecutors.sameThreadExecutor(),
new BucketToResourceMetadata(Functions.forMap(ImmutableMap.<String, Location> of("mycontainer", region))));
MutableStorageMetadata expected = new MutableStorageMetadataImpl();
expected.setName("mycontainer");
expected.setType(StorageType.CONTAINER);
expected.setLocation(region);
assertEquals(
fn.apply(ImmutableSet.of(new BucketMetadata("mycontainer", null, null))).toString(),
new PageSetImpl<StorageMetadata>(ImmutableSet.<StorageMetadata> of(expected), null).toString());
}
}

89
apis/s3/src/test/java/org/jclouds/s3/blobstore/functions/LocationFromBucketNameTest.java Normal file
View File

@ -0,0 +1,89 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.blobstore.functions;
import static org.testng.Assert.assertEquals;
import java.util.Set;
import org.jclouds.domain.Location;
import org.jclouds.domain.LocationBuilder;
import org.jclouds.domain.LocationScope;
import org.testng.annotations.Test;
import com.google.common.base.Functions;
import com.google.common.base.Optional;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
/**
* Tests behavior of {@code LocationFromBucketName}
*
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during
// surefire
@Test(groups = "unit", testName = "LocationFromBucketNameTest")
public class LocationFromBucketNameTest {
protected Location provider = new LocationBuilder().scope(LocationScope.PROVIDER).id("aws-ec2")
.description("aws-ec2").build();
protected Location region = new LocationBuilder().scope(LocationScope.REGION).id("us-east-1")
.description("us-east-1").parent(provider).build();
protected Location region2 = new LocationBuilder().scope(LocationScope.REGION).id("eu-west-1")
.description("eu-west-1").parent(provider).build();
public void testOnlyLocationDoesntNeedMapping() {
LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap
.<String, Optional<String>> of()), Suppliers.<Set<? extends Location>> ofInstance(ImmutableSet
.of(provider)));
assertEquals(fn.apply("mybucket"), provider);
}
public void testMapsToCorrectRegion() {
LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap
.<String, Optional<String>> of("mybucket", Optional.of("eu-west-1"))),
Suppliers.<Set<? extends Location>> ofInstance(ImmutableSet.of(region, region2)));
assertEquals(fn.apply("mybucket"), region2);
}
public void testNullOnUnmatchedRegion() {
LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap
.<String, Optional<String>> of("mybucket", Optional.of("eu-west-2"))),
Suppliers.<Set<? extends Location>> ofInstance(ImmutableSet.of(region, region2)));
assertEquals(fn.apply("mybucket"), null);
}
public void testNullOnAbsentData() {
LocationFromBucketName fn = new LocationFromBucketName(Functions.forMap(ImmutableMap
.<String, Optional<String>> of("mybucket", Optional.<String> absent())),
Suppliers.<Set<? extends Location>> ofInstance(ImmutableSet.of(region, region2)));
assertEquals(fn.apply("mybucket"), null);
}
}

86
apis/s3/src/test/java/org/jclouds/s3/functions/AssignCorrectHostnameForBucketTest.java Normal file
View File

@ -0,0 +1,86 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.functions;
import static org.testng.Assert.assertEquals;
import java.net.URI;
import java.util.Map;
import javax.inject.Provider;
import javax.ws.rs.core.UriBuilder;
import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull;
import org.testng.annotations.Test;
import com.google.common.base.Functions;
import com.google.common.base.Optional;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableMap;
import com.sun.jersey.api.uri.UriBuilderImpl;
/**
* Tests behavior of {@code AssignCorrectHostnameForBucket}
*
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during
// surefire
@Test(groups = "unit", testName = "AssignCorrectHostnameForBucketTest")
public class AssignCorrectHostnameForBucketTest {
Provider<UriBuilder> uriBuilderProvider = new Provider<UriBuilder>() {
@Override
public UriBuilder get() {
return new UriBuilderImpl();
}
};
public void testWhenNoBucketRegionMappingInCache() {
AssignCorrectHostnameForBucket fn = new AssignCorrectHostnameForBucket(new RegionToEndpointOrProviderIfNull(
"aws-s3", Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")),
Suppliers.<Map<String, Supplier<URI>>> ofInstance(ImmutableMap.of("us-standard",
Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1",
Suppliers.ofInstance(URI.create("https://s3-us-west-1.amazonaws.com"))))),
Functions.forMap(ImmutableMap.<String, Optional<String>> of("bucket", Optional.<String> absent())));
assertEquals(fn.apply("bucket"), URI.create("https://s3.amazonaws.com"));
}
public void testWhenBucketRegionMappingInCache() {
AssignCorrectHostnameForBucket fn = new AssignCorrectHostnameForBucket(new RegionToEndpointOrProviderIfNull(
"aws-s3", Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")),
Suppliers.<Map<String, Supplier<URI>>> ofInstance(ImmutableMap.of("us-standard",
Suppliers.ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1",
Suppliers.ofInstance(URI.create("https://s3-us-west-1.amazonaws.com"))))),
Functions.forMap(ImmutableMap.<String, Optional<String>> of("bucket", Optional.of("us-west-1"))));
assertEquals(fn.apply("bucket"), URI.create("https://s3-us-west-1.amazonaws.com"));
}
}

56
apis/s3/src/test/java/org/jclouds/s3/functions/DefaultEndpointThenInvalidateRegionTest.java Normal file
View File

@ -0,0 +1,56 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.functions;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import java.net.URI;
import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull;
import org.testng.annotations.Test;
import com.google.common.base.Optional;
import com.google.common.cache.LoadingCache;
/**
* @author Adrian Cole
*/
@Test(testName = "DefaultEndpointThenInvalidateRegionTest")
public class DefaultEndpointThenInvalidateRegionTest {
@SuppressWarnings("unchecked")
@Test
void testInvalidate() throws Exception {
RegionToEndpointOrProviderIfNull r2 = createMock(RegionToEndpointOrProviderIfNull.class);
LoadingCache<String, Optional<String>> bucketToRegionCache = createMock(LoadingCache.class);
expect(r2.apply(null)).andReturn(URI.create("http://east-url"));
bucketToRegionCache.invalidate("mybucket");
replay(r2, bucketToRegionCache);
new DefaultEndpointThenInvalidateRegion(r2, bucketToRegionCache).apply("mybucket");
verify(r2, bucketToRegionCache);
}
}

81
apis/s3/src/test/java/org/jclouds/s3/functions/GetRegionForBucketTest.java Normal file
View File

@ -0,0 +1,81 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.functions;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import static org.testng.Assert.assertEquals;
import java.util.concurrent.ExecutionException;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import com.google.common.base.Optional;
import com.google.common.cache.CacheLoader.InvalidCacheLoadException;
import com.google.common.cache.LoadingCache;
import com.google.common.util.concurrent.UncheckedExecutionException;
/**
* @author Adrian Cole
*/
@Test(testName = "GetRegionForBucketTest")
public class GetRegionForBucketTest {
@SuppressWarnings("unchecked")
@Test
void test() throws Exception {
LoadingCache<String, Optional<String>> bucketToRegionCache = createMock(LoadingCache.class);
expect(bucketToRegionCache.get("bucket")).andReturn(Optional.of("us-east-1"));
replay(bucketToRegionCache);
GetRegionForBucket fn = new GetRegionForBucket(bucketToRegionCache);
assertEquals(fn.apply("bucket"), Optional.of("us-east-1"));
verify(bucketToRegionCache);
}
@SuppressWarnings("serial")
@DataProvider(name = "exceptions")
public Object[][] createExceptions() {
return new Object[][] { { new ExecutionException() {
} }, { new UncheckedExecutionException() {
} }, { new InvalidCacheLoadException("foo") } };
}
@SuppressWarnings("unchecked")
@Test(dataProvider = "exceptions")
void testGracefulOnException(Exception exception) throws Exception {
LoadingCache<String, Optional<String>> bucketToRegionCache = createMock(LoadingCache.class);
expect(bucketToRegionCache.get("bucket")).andThrow(exception);
replay(bucketToRegionCache);
GetRegionForBucket fn = new GetRegionForBucket(bucketToRegionCache);
assertEquals(fn.apply("bucket"), Optional.absent());
verify(bucketToRegionCache);
}
}

54
apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest.java
View File

@ -1,54 +0,0 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.functions;
import org.jclouds.aws.AWSResponseException;
import org.jclouds.aws.domain.AWSError;
import org.testng.annotations.Test;
/**
* @author Adrian Cole
*/
@Test(testName = "ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest")
public class ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalStateTest {
@Test
void testBucketAlreadyOwnedByYouIsOk() throws Exception {
Exception e = getErrorWithCode("BucketAlreadyOwnedByYou");
assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState().apply(e);
}
@Test
void testIllegalStateIsOk() throws Exception {
Exception e = new IllegalStateException();
assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState().apply(e);
}
@Test(expectedExceptions = AWSResponseException.class)
void testBlahIsNotOk() throws Exception {
Exception e = getErrorWithCode("blah");
new ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState().apply(e);
}
private Exception getErrorWithCode(String code) {
AWSError error = new AWSError();
error.setCode(code);
return new AWSResponseException(null, null, null, error);
}
}

114
apis/s3/src/test/java/org/jclouds/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest.java Normal file
View File

@ -0,0 +1,114 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3.functions;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import java.net.URI;
import org.jclouds.aws.AWSResponseException;
import org.jclouds.aws.domain.AWSError;
import org.jclouds.rest.internal.GeneratedHttpRequest;
import org.jclouds.s3.S3Client;
import org.jclouds.s3.options.PutBucketOptions;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
/**
* @author Adrian Cole
*/
@Test(testName = "ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest")
public class ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExistsTest {
GeneratedHttpRequest<S3Client> putBucket;
@BeforeClass
void setUp() throws SecurityException, NoSuchMethodException {
putBucket = GeneratedHttpRequest
.<S3Client> requestBuilder()
.method("PUT")
.endpoint(URI.create("https://adriancole-blobstore113.s3.amazonaws.com/"))
.declaring(S3Client.class)
.javaMethod(
S3Client.class.getMethod("putBucketInRegion", String.class, String.class,
PutBucketOptions[].class)).args(new Object[] { null, "bucket" }).build();
}
@Test
void testBucketAlreadyOwnedByYouIsOk() throws Exception {
S3Client client = createMock(S3Client.class);
replay(client);
Exception e = getErrorWithCode("BucketAlreadyOwnedByYou");
assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).setContext(putBucket)
.apply(e);
verify(client);
}
@Test
void testOperationAbortedIsOkWhenBucketExists() throws Exception {
S3Client client = createMock(S3Client.class);
expect(client.bucketExists("bucket")).andReturn(true);
replay(client);
Exception e = getErrorWithCode("OperationAborted");
assert !new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).setContext(putBucket)
.apply(e);
verify(client);
}
@Test(expectedExceptions = Exception.class)
void testOperationAbortedNotOkWhenBucketDoesntExist() throws Exception {
S3Client client = createMock(S3Client.class);
expect(client.bucketExists("bucket")).andReturn(false);
replay(client);
Exception e = getErrorWithCode("OperationAborted");
new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).setContext(putBucket).apply(e);
Assert.fail();
}
@Test(expectedExceptions = IllegalStateException.class)
void testIllegalStateIsNotOk() throws Exception {
S3Client client = createMock(S3Client.class);
replay(client);
Exception e = new IllegalStateException();
new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).apply(e);
Assert.fail();
}
@Test(expectedExceptions = AWSResponseException.class)
void testBlahIsNotOk() throws Exception {
S3Client client = createMock(S3Client.class);
replay(client);
Exception e = getErrorWithCode("blah");
new ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists(client).apply(e);
Assert.fail();
}
private Exception getErrorWithCode(String code) {
AWSError error = new AWSError();
error.setCode(code);
return new AWSResponseException(null, null, null, error);
}
}

51
apis/s3/src/test/java/org/jclouds/s3/services/BucketsLiveTest.java
View File

@ -34,22 +34,21 @@ import java.util.Set;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import org.jclouds.aws.domain.Region;
import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest; import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
import org.jclouds.s3.S3ApiMetadata; import org.jclouds.s3.S3ApiMetadata;
import org.jclouds.s3.S3Client; import org.jclouds.s3.S3Client;
import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.AccessControlList;
import org.jclouds.s3.domain.AccessControlList.CanonicalUserGrantee;
import org.jclouds.s3.domain.AccessControlList.EmailAddressGrantee;
import org.jclouds.s3.domain.AccessControlList.Grant;
import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI;
import org.jclouds.s3.domain.AccessControlList.Permission;
import org.jclouds.s3.domain.BucketLogging; import org.jclouds.s3.domain.BucketLogging;
import org.jclouds.s3.domain.BucketMetadata; import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.domain.CannedAccessPolicy; import org.jclouds.s3.domain.CannedAccessPolicy;
import org.jclouds.s3.domain.ListBucketResponse; import org.jclouds.s3.domain.ListBucketResponse;
import org.jclouds.s3.domain.Payer; import org.jclouds.s3.domain.Payer;
import org.jclouds.s3.domain.S3Object; import org.jclouds.s3.domain.S3Object;
import org.jclouds.s3.domain.AccessControlList.CanonicalUserGrantee;
import org.jclouds.s3.domain.AccessControlList.EmailAddressGrantee;
import org.jclouds.s3.domain.AccessControlList.Grant;
import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI;
import org.jclouds.s3.domain.AccessControlList.Permission;
import org.jclouds.s3.internal.StubS3AsyncClient; import org.jclouds.s3.internal.StubS3AsyncClient;
import org.jclouds.util.Strings2; import org.jclouds.util.Strings2;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@ -71,7 +70,7 @@ public class BucketsLiveTest extends BaseBlobStoreIntegrationTest {
} }
public S3Client getApi() { public S3Client getApi() {
return (S3Client) view.unwrap(S3ApiMetadata.CONTEXT_TOKEN).getApi(); return view.unwrap(S3ApiMetadata.CONTEXT_TOKEN).getApi();
} }
/** /**
@ -177,17 +176,6 @@ public class BucketsLiveTest extends BaseBlobStoreIntegrationTest {
} }
public void testDefaultBucketLocation() throws Exception {
String bucketName = getContainerName();
try {
String location = getApi().getBucketLocation(bucketName);
assert location.equals(Region.US_STANDARD) : "bucket: " + bucketName + " location: " + location;
} finally {
returnContainer(bucketName);
}
}
public void testBucketPayer() throws Exception { public void testBucketPayer() throws Exception {
final String bucketName = getContainerName(); final String bucketName = getContainerName();
try { try {
@ -276,33 +264,6 @@ public class BucketsLiveTest extends BaseBlobStoreIntegrationTest {
assertTrue(getApi().putBucketACL(targetBucket, acl)); assertTrue(getApi().putBucketACL(targetBucket, acl));
} }
/**
* using scratch bucketName as we are changing location
*/
public void testEu() throws Exception {
final String bucketName = getScratchContainerName();
try {
getApi().putBucketInRegion(Region.EU, bucketName + "eu", withBucketAcl(CannedAccessPolicy.PUBLIC_READ));
assertConsistencyAware(new Runnable() {
public void run() {
try {
AccessControlList acl = getApi().getBucketACL(bucketName + "eu");
assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ), acl.toString());
} catch (Exception e) {
Throwables.propagateIfPossible(e);
}
}
});
assertEquals(Region.EU, getApi().getBucketLocation(bucketName + "eu"));
// TODO: I believe that the following should work based on the above acl assertion passing.
// However, it fails on 403
// URL url = new URL(String.format("http://%s.s3.amazonaws.com", bucketName));
// Utils.toStringAndClose(url.openStream());
} finally {
destroyContainer(bucketName + "eu");
}
}
void bucketExists() throws Exception { void bucketExists() throws Exception {
String bucketName = getContainerName(); String bucketName = getContainerName();
try { try {

4
apis/walrus/src/main/java/org/jclouds/walrus/WalrusAsyncClient.java
View File

@ -26,6 +26,7 @@ import javax.ws.rs.Path;
import org.jclouds.blobstore.attr.BlobScope; import org.jclouds.blobstore.attr.BlobScope;
import org.jclouds.blobstore.functions.ReturnFalseOnContainerNotFound; import org.jclouds.blobstore.functions.ReturnFalseOnContainerNotFound;
import org.jclouds.rest.annotations.BinderParam; import org.jclouds.rest.annotations.BinderParam;
import org.jclouds.rest.annotations.Endpoint;
import org.jclouds.rest.annotations.ExceptionParser; import org.jclouds.rest.annotations.ExceptionParser;
import org.jclouds.rest.annotations.ParamValidators; import org.jclouds.rest.annotations.ParamValidators;
import org.jclouds.rest.annotations.QueryParams; import org.jclouds.rest.annotations.QueryParams;
@ -55,9 +56,10 @@ public interface WalrusAsyncClient extends S3AsyncClient {
@Override @Override
@GET @GET
@Path("/") @Path("/")
@Endpoint(Bucket.class)
@QueryParams(keys = "max-keys", values = "0") @QueryParams(keys = "max-keys", values = "0")
@ExceptionParser(ReturnFalseOnContainerNotFound.class) @ExceptionParser(ReturnFalseOnContainerNotFound.class)
ListenableFuture<Boolean> bucketExists( ListenableFuture<Boolean> bucketExists(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName); @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName);
} }

12
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java
View File

@ -22,7 +22,6 @@ import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursi
import static org.jclouds.concurrent.FutureIterables.awaitCompletion; import static org.jclouds.concurrent.FutureIterables.awaitCompletion;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future; import java.util.concurrent.Future;
@ -42,10 +41,9 @@ import org.jclouds.blobstore.strategy.ClearListStrategy;
import org.jclouds.http.handlers.BackoffLimitedRetryHandler; import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
import org.jclouds.logging.Logger; import org.jclouds.logging.Logger;
import com.google.common.base.Predicate;
import com.google.common.base.Throwables; import com.google.common.base.Throwables;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import com.google.common.util.concurrent.Futures;
import com.google.inject.Inject; import com.google.inject.Inject;
/** /**
@ -97,16 +95,14 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
for (int i = 0; i < maxErrors; ) { for (int i = 0; i < maxErrors; ) {
// fetch partial directory listing // fetch partial directory listing
try { try {
listing = connection.list(containerName, options).get(); listing = Futures.getUnchecked(connection.list(containerName, options));
} catch (ExecutionException ee) { } catch (RuntimeException ee) {
++i; ++i;
if (i == maxErrors) { if (i == maxErrors) {
throw new BlobRuntimeException("list error", ee.getCause()); throw Throwables.propagate(ee.getCause());
} }
retryHandler.imposeBackoffExponentialDelay(i, message); retryHandler.imposeBackoffExponentialDelay(i, message);
continue; continue;
} catch (InterruptedException ie) {
throw Throwables.propagate(ie);
} }
// recurse on subdirectories // recurse on subdirectories

18
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
View File

@ -48,6 +48,7 @@ import java.util.zip.GZIPInputStream;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.ContainerNotFoundException;
import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobBuilder.PayloadBlobBuilder; import org.jclouds.blobstore.domain.BlobBuilder.PayloadBlobBuilder;
@ -239,6 +240,23 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
} }
@Test(groups = { "integration", "live" })
public void testCreateBlobWithExpiry() throws InterruptedException {
String container = getContainerName();
BlobStore blobStore = view.getBlobStore();
try {
final String blobName = "hello";
final Date expires = new Date((System.currentTimeMillis() / 1000) * 1000 + 60 * 1000);
blobStore.putBlob(container, blobStore.blobBuilder(blobName).payload(TEST_STRING).expires(expires).build());
assertConsistencyAwareBlobExpiryMetadata(container, blobName, expires);
} finally {
returnContainer(container);
}
}
@Test(groups = { "integration", "live" }) @Test(groups = { "integration", "live" })
public void testGetIfUnmodifiedSince() throws InterruptedException { public void testGetIfUnmodifiedSince() throws InterruptedException {
String container = getContainerName(); String container = getContainerName();

80
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobStoreIntegrationTest.java
View File

@ -23,6 +23,7 @@ import static org.jclouds.blobstore.util.BlobStoreUtils.getContentAsStringOrNull
import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertEquals;
import java.io.IOException; import java.io.IOException;
import java.security.SecureRandom;
import java.util.Date; import java.util.Date;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
@ -35,6 +36,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nullable;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import org.jclouds.apis.BaseViewLiveTest; import org.jclouds.apis.BaseViewLiveTest;
@ -80,7 +82,7 @@ public class BaseBlobStoreIntegrationTest extends BaseViewLiveTest<BlobStoreCont
protected static volatile int containerCount = Integer.parseInt(System.getProperty("test.blobstore.container-count", protected static volatile int containerCount = Integer.parseInt(System.getProperty("test.blobstore.container-count",
"10")); "10"));
public static final String CONTAINER_PREFIX = System.getProperty("user.name") + "-blobstore"; public static final String CONTAINER_PREFIX = (System.getProperty("user.name") + "-blobstore").toLowerCase();
/** /**
* two test groups integration and live. * two test groups integration and live.
*/ */
@ -139,7 +141,12 @@ public class BaseBlobStoreIntegrationTest extends BaseViewLiveTest<BlobStoreCont
} else { } else {
try { try {
createContainerAndEnsureEmpty(context, containerName); createContainerAndEnsureEmpty(context, containerName);
containerNames.put(containerName); if (context.getBlobStore().containerExists(containerName))
containerNames.put(containerName);
else {
deleteContainerOrWarnIfUnable(context, containerName);
containerCount++;
}
} catch (Throwable e) { } catch (Throwable e) {
e.printStackTrace(); e.printStackTrace();
// throw away the container and try again with the next // throw away the container and try again with the next
@ -181,7 +188,7 @@ public class BaseBlobStoreIntegrationTest extends BaseViewLiveTest<BlobStoreCont
new Predicate<StorageMetadata>() { new Predicate<StorageMetadata>() {
public boolean apply(StorageMetadata input) { public boolean apply(StorageMetadata input) {
return (input.getType() == StorageType.CONTAINER || input.getType() == StorageType.FOLDER) return (input.getType() == StorageType.CONTAINER || input.getType() == StorageType.FOLDER)
&& input.getName().startsWith(CONTAINER_PREFIX.toLowerCase()); && input.getName().startsWith(CONTAINER_PREFIX);
} }
}); });
for (StorageMetadata container : testContainers) { for (StorageMetadata container : testContainers) {
@ -339,20 +346,57 @@ public class BaseBlobStoreIntegrationTest extends BaseViewLiveTest<BlobStoreCont
}); });
} }
protected void assertConsistencyAwareBlobExpiryMetadata(final String containerName, final String blobName, final Date expectedExpires) protected void assertConsistencyAwareContainerExists(final String containerName) throws InterruptedException {
throws InterruptedException { assertConsistencyAware(new Runnable() {
assertConsistencyAware(new Runnable() { public void run() {
public void run() { try {
try { assert view.getBlobStore().containerExists(containerName) : String.format("container %s doesn't exist", containerName);
Blob blob = view.getBlobStore().getBlob(containerName, blobName); } catch (Exception e) {
Date actualExpires = blob.getPayload().getContentMetadata().getExpires(); Throwables.propagate(e);
assert expectedExpires.equals(actualExpires) : "expires="+actualExpires+"; expected="+expectedExpires; }
} catch (Exception e) { }
Throwables.propagateIfPossible(e); });
} }
}
}); protected void assertConsistencyAwareContainerInLocation(final String containerName, final Location loc)
} throws InterruptedException {
assertConsistencyAware(new Runnable() {
public void run() {
try {
StorageMetadata container = Iterables.find(view.getBlobStore().list(), new Predicate<StorageMetadata>() {
@Override
public boolean apply(@Nullable StorageMetadata input) {
return input.getName().equals(containerName);
}
});
Location actualLoc = container.getLocation();
assert loc.equals(actualLoc) : String.format("blob %s, in location %s instead of %s", containerName,
actualLoc, loc);
} catch (Exception e) {
Throwables.propagate(e);
}
}
});
}
protected void assertConsistencyAwareBlobExpiryMetadata(final String containerName, final String blobName,
final Date expectedExpires) throws InterruptedException {
assertConsistencyAware(new Runnable() {
public void run() {
try {
Blob blob = view.getBlobStore().getBlob(containerName, blobName);
Date actualExpires = blob.getPayload().getContentMetadata().getExpires();
assert expectedExpires.equals(actualExpires) : "expires=" + actualExpires + "; expected="
+ expectedExpires;
} catch (Exception e) {
Throwables.propagateIfPossible(e);
}
}
});
}
protected void assertConsistencyAwareBlobInLocation(final String containerName, final String blobName, final Location loc) protected void assertConsistencyAwareBlobInLocation(final String containerName, final String blobName, final Location loc)
throws InterruptedException { throws InterruptedException {
@ -452,7 +496,7 @@ public class BaseBlobStoreIntegrationTest extends BaseViewLiveTest<BlobStoreCont
deleteContainerOrWarnIfUnable(view, container); deleteContainerOrWarnIfUnable(view, container);
} }
}); });
String newScratchContainer = container + containerIndex.incrementAndGet(); String newScratchContainer = container + new SecureRandom().nextLong();
System.err.printf("*** allocated new container %s...%n", container); System.err.printf("*** allocated new container %s...%n", container);
return newScratchContainer; return newScratchContainer;
} }

84
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerLiveTest.java
View File

@ -23,23 +23,49 @@ import static org.testng.Assert.assertEquals;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.util.Collections;
import java.util.List;
import java.util.logging.Logger;
import javax.annotation.Nullable;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.domain.Location;
import org.jclouds.util.Strings2; import org.jclouds.util.Strings2;
import org.testng.SkipException;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Strings;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
/** /**
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
public class BaseContainerLiveTest extends BaseBlobStoreIntegrationTest { public class BaseContainerLiveTest extends BaseBlobStoreIntegrationTest {
@Test(groups = { "live" }) private Location defaultLocation;
@Test(groups = "live")
public void testPublicAccess() throws InterruptedException, MalformedURLException, IOException { public void testPublicAccess() throws InterruptedException, MalformedURLException, IOException {
final String containerName = getScratchContainerName(); final String containerName = getScratchContainerName();
try { try {
view.getBlobStore().createContainerInLocation(null, containerName, publicRead()); view.getBlobStore().createContainerInLocation(null, containerName, publicRead());
assertConsistencyAwareContainerSize(containerName, 0); assertConsistencyAwareContainerExists(containerName);
defaultLocation = Iterables.find(view.getBlobStore().list(), new Predicate<StorageMetadata>() {
@Override
public boolean apply(@Nullable StorageMetadata input) {
return input.getName().equals(containerName);
}
}).getLocation();
view.getBlobStore().putBlob(containerName, view.getBlobStore().putBlob(containerName,
view.getBlobStore().blobBuilder("hello").payload(TEST_STRING).build()); view.getBlobStore().blobBuilder("hello").payload(TEST_STRING).build());
@ -56,4 +82,58 @@ public class BaseContainerLiveTest extends BaseBlobStoreIntegrationTest {
recycleContainer(containerName); recycleContainer(containerName);
} }
} }
static Location findNonDefaultLocationOrSkip(BlobStore blobStore, Location defaultLocation) {
List<? extends Location> locs = Lists.newArrayList(Iterables.filter(blobStore.listAssignableLocations(),
Predicates.not(Predicates.equalTo(defaultLocation))));
if (locs.size() == 0)
throw new SkipException("No non-default location found in " + locs);
// try to use a diverse location
Collections.shuffle(locs);
return locs.get(0);
}
@Test(groups = "live", dependsOnMethods = "testPublicAccess")
public void testPublicAccessInNonDefaultLocation() throws InterruptedException, MalformedURLException, IOException {
Location nonDefault = findNonDefaultLocationOrSkip(view.getBlobStore(), defaultLocation);
String payload = "my data";
runCreateContainerInLocation(payload, nonDefault);
}
@Test(groups = "live", dependsOnMethods = "testPublicAccess")
public void testPublicAccessInNonDefaultLocationWithBigBlob() throws InterruptedException, MalformedURLException,
IOException {
Location nonDefault = findNonDefaultLocationOrSkip(view.getBlobStore(), defaultLocation);
String payload = Strings.repeat("a", 1024 * 1024); // 1MB
runCreateContainerInLocation(payload, nonDefault);
}
private void runCreateContainerInLocation(String payload, Location nonDefault) throws InterruptedException,
IOException {
String blobName = "hello";
BlobStore blobStore = view.getBlobStore();
final String containerName = getScratchContainerName();
try {
Logger.getAnonymousLogger().info(
String.format("creating public container %s in location %s", containerName, nonDefault.getId()));
blobStore.createContainerInLocation(nonDefault, containerName, publicRead());
assertConsistencyAwareContainerExists(containerName);
assertConsistencyAwareContainerInLocation(containerName, nonDefault);
blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload(payload).build());
assertConsistencyAwareContainerSize(containerName, 1);
BlobMetadata metadata = view.getBlobStore().blobMetadata(containerName, blobName);
assertEquals(Strings2.toStringAndClose(view.utils().http().get(metadata.getPublicUri())), payload);
assertConsistencyAwareBlobInLocation(containerName, blobName, nonDefault);
} finally {
// this container is now public, so we can't reuse it directly
recycleContainer(containerName);
}
}
} }

6
common/aws/src/main/java/org/jclouds/aws/domain/Region.java
View File

@ -48,8 +48,6 @@ public class Region {
* In Amazon S3, the EU (Ireland) Region provides read-after-write consistency for PUTS of new * In Amazon S3, the EU (Ireland) Region provides read-after-write consistency for PUTS of new
* objects in your Amazon S3 bucket and eventual consistency for overwrite PUTS and DELETES. * objects in your Amazon S3 bucket and eventual consistency for overwrite PUTS and DELETES.
*/ */
public static final String EU = "EU";
public static final String EU_WEST_1 = "eu-west-1"; public static final String EU_WEST_1 = "eu-west-1";
/** /**
@ -106,7 +104,7 @@ public class Region {
*/ */
public static final String AP_NORTHEAST_1 = "ap-northeast-1"; public static final String AP_NORTHEAST_1 = "ap-northeast-1";
public static Set<String> DEFAULT_S3 = ImmutableSet.of(EU, US_STANDARD, US_WEST_1, US_WEST_2, SA_EAST_1, AP_SOUTHEAST_1, public static Set<String> DEFAULT_S3 = ImmutableSet.of(US_STANDARD, US_WEST_1, US_WEST_2, EU_WEST_1, SA_EAST_1, AP_SOUTHEAST_1,
AP_NORTHEAST_1); AP_NORTHEAST_1);
public static Set<String> DEFAULT_REGIONS = ImmutableSet.of(US_EAST_1, US_WEST_1, US_WEST_2, SA_EAST_1, EU_WEST_1, public static Set<String> DEFAULT_REGIONS = ImmutableSet.of(US_EAST_1, US_WEST_1, US_WEST_2, SA_EAST_1, EU_WEST_1,
@ -119,7 +117,7 @@ public class Region {
// note that due to US_STANDARD the codes include US instead of US-VA // note that due to US_STANDARD the codes include US instead of US-VA
properties.setProperty(PROPERTY_ISO3166_CODES, "US,US-CA,US-OR,BR-SP,IE,SG,JP-13"); properties.setProperty(PROPERTY_ISO3166_CODES, "US,US-CA,US-OR,BR-SP,IE,SG,JP-13");
properties.setProperty(PROPERTY_REGION + "." + US_STANDARD + "." + ISO3166_CODES, "US"); properties.setProperty(PROPERTY_REGION + "." + US_STANDARD + "." + ISO3166_CODES, "US");
properties.setProperty(PROPERTY_REGION + "." + EU + "." + ISO3166_CODES, "IE"); properties.setProperty(PROPERTY_REGION + "." + EU_WEST_1 + "." + ISO3166_CODES, "IE");
return properties; return properties;
} }

31
common/aws/src/main/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandler.java
View File

@ -20,17 +20,14 @@ package org.jclouds.aws.handlers;
import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream; import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
import javax.annotation.Resource;
import javax.inject.Named;
import org.jclouds.Constants;
import org.jclouds.aws.domain.AWSError; import org.jclouds.aws.domain.AWSError;
import org.jclouds.aws.util.AWSUtils; import org.jclouds.aws.util.AWSUtils;
import org.jclouds.http.HttpCommand; import org.jclouds.http.HttpCommand;
import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpResponse;
import org.jclouds.http.HttpRetryHandler; import org.jclouds.http.HttpRetryHandler;
import org.jclouds.logging.Logger; import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Inject; import com.google.inject.Inject;
/** /**
@ -40,35 +37,25 @@ import com.google.inject.Inject;
*/ */
public class AWSClientErrorRetryHandler implements HttpRetryHandler { public class AWSClientErrorRetryHandler implements HttpRetryHandler {
@Inject(optional = true)
@Named(Constants.PROPERTY_MAX_RETRIES)
private int retryCountLimit = 5;
private final AWSUtils utils; private final AWSUtils utils;
private final BackoffLimitedRetryHandler backoffLimitedRetryHandler;
@Resource
protected Logger logger = Logger.NULL;
@Inject @Inject
public AWSClientErrorRetryHandler(AWSUtils utils) { public AWSClientErrorRetryHandler(AWSUtils utils, BackoffLimitedRetryHandler backoffLimitedRetryHandler) {
this.utils = utils; this.utils = utils;
this.backoffLimitedRetryHandler = backoffLimitedRetryHandler;
} }
public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) { public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) {
if (command.getFailureCount() > retryCountLimit) if (response.getStatusCode() == 400 || response.getStatusCode() == 403 || response.getStatusCode() == 409) {
return false;
if (response.getStatusCode() == 400 || response.getStatusCode() == 403
|| response.getStatusCode() == 409) {
command.incrementFailureCount();
// Content can be null in the case of HEAD requests // Content can be null in the case of HEAD requests
if (response.getPayload() != null) { if (response.getPayload() != null) {
closeClientButKeepContentStream(response); closeClientButKeepContentStream(response);
AWSError error = utils.parseAWSErrorFromContent(command.getCurrentRequest(), response); AWSError error = utils.parseAWSErrorFromContent(command.getCurrentRequest(), response);
if (error != null if (error != null
&& ("RequestTimeout".equals(error.getCode()) && ImmutableSet.of("RequestTimeout", "OperationAborted", "SignatureDoesNotMatch").contains(
|| "OperationAborted".equals(error.getCode()) || "SignatureDoesNotMatch" error.getCode())) {
.equals(error.getCode()))) { return backoffLimitedRetryHandler.shouldRetryRequest(command, response);
return true;
} }
} }
} }

64
common/aws/src/test/java/org/jclouds/aws/handlers/AWSClientErrorRetryHandlerTest.java
View File

@ -23,9 +23,16 @@ import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify; import static org.easymock.EasyMock.verify;
import java.net.URI;
import org.jclouds.aws.domain.AWSError;
import org.jclouds.aws.util.AWSUtils; import org.jclouds.aws.util.AWSUtils;
import org.jclouds.http.HttpCommand; import org.jclouds.http.HttpCommand;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpResponse;
import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
import org.jclouds.io.Payloads;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test; import org.testng.annotations.Test;
/** /**
@ -33,29 +40,60 @@ import org.testng.annotations.Test;
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(groups = "unit") @Test(groups = "unit", testName = "AWSClientErrorRetryHandlerTest")
public class AWSClientErrorRetryHandlerTest { public class AWSClientErrorRetryHandlerTest {
@Test @Test
public void test401DoesNotRetry() { public void test401DoesNotRetry() {
AWSUtils utils = createMock(AWSUtils.class); AWSUtils utils = createMock(AWSUtils.class);
BackoffLimitedRetryHandler backoffLimitedRetryHandler = createMock(BackoffLimitedRetryHandler.class);
HttpCommand command = createMock(HttpCommand.class); HttpCommand command = createMock(HttpCommand.class);
HttpResponse response = createMock(HttpResponse.class);
expect(command.getFailureCount()).andReturn(0); replay(utils, backoffLimitedRetryHandler, command);
expect(response.getStatusCode()).andReturn(401).atLeastOnce();
replay(utils); AWSClientErrorRetryHandler retry = new AWSClientErrorRetryHandler(utils, backoffLimitedRetryHandler);
replay(command);
replay(response);
AWSClientErrorRetryHandler retry = new AWSClientErrorRetryHandler(utils); assert !retry.shouldRetryRequest(command, HttpResponse.builder().statusCode(401).build());
assert !retry.shouldRetryRequest(command, response); verify(utils, backoffLimitedRetryHandler, command);
verify(utils);
verify(command);
verify(response);
} }
@DataProvider(name = "codes")
public Object[][] createData() {
return new Object[][] { { "RequestTimeout" }, { "OperationAborted" }, { "SignatureDoesNotMatch" } };
}
@Test(dataProvider = "codes")
public void test409DoesBackoffAndRetryForCode(String code) {
AWSUtils utils = createMock(AWSUtils.class);
BackoffLimitedRetryHandler backoffLimitedRetryHandler = createMock(BackoffLimitedRetryHandler.class);
HttpCommand command = createMock(HttpCommand.class);
HttpRequest putBucket = HttpRequest.builder().method("PUT")
.endpoint(URI.create("https://adriancole-blobstore113.s3.amazonaws.com/")).build();
HttpResponse operationAborted = HttpResponse.builder().statusCode(409)
.payload(Payloads.newStringPayload(String.format("<Error><Code>%s</Code></Error>", code))).build();
expect(command.getCurrentRequest()).andReturn(putBucket);
AWSError error = new AWSError();
error.setCode(code);
expect(utils.parseAWSErrorFromContent(putBucket, operationAborted)).andReturn(error);
expect(backoffLimitedRetryHandler.shouldRetryRequest(command, operationAborted)).andReturn(Boolean.TRUE);
replay(utils, backoffLimitedRetryHandler, command);
AWSClientErrorRetryHandler retry = new AWSClientErrorRetryHandler(utils, backoffLimitedRetryHandler);
assert retry.shouldRetryRequest(command, operationAborted);
verify(utils, backoffLimitedRetryHandler, command);
}
} }

58
providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java
View File

@ -21,8 +21,10 @@ package org.jclouds.aws.s3;
import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER; import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER;
import java.util.Map; import java.util.Map;
import javax.ws.rs.DELETE; import javax.ws.rs.DELETE;
import javax.ws.rs.GET; import javax.ws.rs.GET;
import javax.ws.rs.HEAD;
import javax.ws.rs.POST; import javax.ws.rs.POST;
import javax.ws.rs.PUT; import javax.ws.rs.PUT;
import javax.ws.rs.Path; import javax.ws.rs.Path;
@ -35,11 +37,13 @@ import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex;
import org.jclouds.aws.s3.functions.ObjectMetadataKey; import org.jclouds.aws.s3.functions.ObjectMetadataKey;
import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex; import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex;
import org.jclouds.blobstore.attr.BlobScope; import org.jclouds.blobstore.attr.BlobScope;
import org.jclouds.blobstore.functions.ReturnFalseOnContainerNotFound;
import org.jclouds.http.functions.ParseETagHeader; import org.jclouds.http.functions.ParseETagHeader;
import org.jclouds.io.Payload; import org.jclouds.io.Payload;
import org.jclouds.javax.annotation.Nullable; import org.jclouds.javax.annotation.Nullable;
import org.jclouds.rest.annotations.BinderParam; import org.jclouds.rest.annotations.BinderParam;
import org.jclouds.rest.annotations.Endpoint; import org.jclouds.rest.annotations.Endpoint;
import org.jclouds.rest.annotations.EndpointParam;
import org.jclouds.rest.annotations.ExceptionParser; import org.jclouds.rest.annotations.ExceptionParser;
import org.jclouds.rest.annotations.ParamParser; import org.jclouds.rest.annotations.ParamParser;
import org.jclouds.rest.annotations.ParamValidators; import org.jclouds.rest.annotations.ParamValidators;
@ -55,8 +59,11 @@ import org.jclouds.s3.S3Client;
import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured; import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured;
import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.filters.RequestAuthorizeSignature; import org.jclouds.s3.filters.RequestAuthorizeSignature;
import org.jclouds.s3.functions.AssignCorrectHostnameForBucket;
import org.jclouds.s3.functions.BindRegionToXmlPayload; import org.jclouds.s3.functions.BindRegionToXmlPayload;
import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState; import org.jclouds.s3.functions.DefaultEndpointThenInvalidateRegion;
import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists;
import org.jclouds.s3.functions.ReturnTrueOn404OrNotFoundFalseOnIllegalState;
import org.jclouds.s3.options.PutBucketOptions; import org.jclouds.s3.options.PutBucketOptions;
import org.jclouds.s3.options.PutObjectOptions; import org.jclouds.s3.options.PutObjectOptions;
import org.jclouds.s3.predicates.validators.BucketNameValidator; import org.jclouds.s3.predicates.validators.BucketNameValidator;
@ -73,32 +80,7 @@ import com.google.common.util.concurrent.ListenableFuture;
@RequestFilters(RequestAuthorizeSignature.class) @RequestFilters(RequestAuthorizeSignature.class)
@BlobScope(CONTAINER) @BlobScope(CONTAINER)
public interface AWSS3AsyncClient extends S3AsyncClient { public interface AWSS3AsyncClient extends S3AsyncClient {
/**
* @see S3Client#putBucketInRegion
*/
@Override
@PUT
@Path("/")
@Endpoint(Bucket.class)
@ExceptionParser(ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class)
ListenableFuture<Boolean> putBucketInRegion(
@BinderParam(BindRegionToXmlPayload.class) @Nullable String region,
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName,
PutBucketOptions... options);
/**
* @see S3Client#getBucketLocation
*/
@Override
@GET
@QueryParams(keys = "location")
@Path("/")
@Endpoint(Bucket.class)
@XMLResponseParser(LocationConstraintHandler.class)
ListenableFuture<String> getBucketLocation(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName);
/** /**
* @see AWSS3Client#initiateMultipartUpload * @see AWSS3Client#initiateMultipartUpload
*/ */
@ -107,9 +89,9 @@ public interface AWSS3AsyncClient extends S3AsyncClient {
@Path("/{key}") @Path("/{key}")
@ResponseParser(UploadIdFromHttpResponseViaRegex.class) @ResponseParser(UploadIdFromHttpResponseViaRegex.class)
ListenableFuture<String> initiateMultipartUpload( ListenableFuture<String> initiateMultipartUpload(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class) ObjectMetadata objectMetadata, @PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class) ObjectMetadata objectMetadata,
PutObjectOptions... options); PutObjectOptions... options);
/** /**
* @see AWSS3Client#abortMultipartUpload * @see AWSS3Client#abortMultipartUpload
@ -118,8 +100,8 @@ public interface AWSS3AsyncClient extends S3AsyncClient {
@Path("/{key}") @Path("/{key}")
@ExceptionParser(ReturnVoidOnNotFoundOr404.class) @ExceptionParser(ReturnVoidOnNotFoundOr404.class)
ListenableFuture<Void> abortMultipartUpload( ListenableFuture<Void> abortMultipartUpload(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") String key, @QueryParam("uploadId") String uploadId); @PathParam("key") String key, @QueryParam("uploadId") String uploadId);
/** /**
* @see AWSS3Client#uploadPart * @see AWSS3Client#uploadPart
@ -128,9 +110,9 @@ public interface AWSS3AsyncClient extends S3AsyncClient {
@Path("/{key}") @Path("/{key}")
@ResponseParser(ParseETagHeader.class) @ResponseParser(ParseETagHeader.class)
ListenableFuture<String> uploadPart( ListenableFuture<String> uploadPart(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") String key, @QueryParam("partNumber") int partNumber, @PathParam("key") String key, @QueryParam("partNumber") int partNumber,
@QueryParam("uploadId") String uploadId, Payload part); @QueryParam("uploadId") String uploadId, Payload part);
/** /**
* @see AWSS3Client#completeMultipartUpload * @see AWSS3Client#completeMultipartUpload
@ -139,8 +121,8 @@ public interface AWSS3AsyncClient extends S3AsyncClient {
@Path("/{key}") @Path("/{key}")
@ResponseParser(ETagFromHttpResponseViaRegex.class) @ResponseParser(ETagFromHttpResponseViaRegex.class)
ListenableFuture<String> completeMultipartUpload( ListenableFuture<String> completeMultipartUpload(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators({ BucketNameValidator.class }) String bucketName, @Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") String key, @QueryParam("uploadId") String uploadId, @PathParam("key") String key, @QueryParam("uploadId") String uploadId,
@BinderParam(BindPartIdsAndETagsToRequest.class) Map<Integer, String> parts); @BinderParam(BindPartIdsAndETagsToRequest.class) Map<Integer, String> parts);
} }

3
providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ProviderMetadata.java
View File

@ -21,6 +21,7 @@ package org.jclouds.aws.s3;
import static org.jclouds.Constants.PROPERTY_ENDPOINT; import static org.jclouds.Constants.PROPERTY_ENDPOINT;
import static org.jclouds.aws.domain.Region.AP_NORTHEAST_1; import static org.jclouds.aws.domain.Region.AP_NORTHEAST_1;
import static org.jclouds.aws.domain.Region.AP_SOUTHEAST_1; import static org.jclouds.aws.domain.Region.AP_SOUTHEAST_1;
import static org.jclouds.aws.domain.Region.EU_WEST_1;
import static org.jclouds.aws.domain.Region.SA_EAST_1; import static org.jclouds.aws.domain.Region.SA_EAST_1;
import static org.jclouds.aws.domain.Region.US_STANDARD; import static org.jclouds.aws.domain.Region.US_STANDARD;
import static org.jclouds.aws.domain.Region.US_WEST_1; import static org.jclouds.aws.domain.Region.US_WEST_1;
@ -71,7 +72,7 @@ public class AWSS3ProviderMetadata extends BaseProviderMetadata {
properties.setProperty(PROPERTY_REGION + "." + US_WEST_1 + "." + ENDPOINT, "https://s3-us-west-1.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + US_WEST_1 + "." + ENDPOINT, "https://s3-us-west-1.amazonaws.com");
properties.setProperty(PROPERTY_REGION + "." + US_WEST_2 + "." + ENDPOINT, "https://s3-us-west-2.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + US_WEST_2 + "." + ENDPOINT, "https://s3-us-west-2.amazonaws.com");
properties.setProperty(PROPERTY_REGION + "." + SA_EAST_1 + "." + ENDPOINT, "https://s3-sa-east-1.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + SA_EAST_1 + "." + ENDPOINT, "https://s3-sa-east-1.amazonaws.com");
properties.setProperty(PROPERTY_REGION + "." + "EU" + "." + ENDPOINT, "https://s3-eu-west-1.amazonaws.com"); properties.setProperty(PROPERTY_REGION + "." + EU_WEST_1 + "." + ENDPOINT, "https://s3-eu-west-1.amazonaws.com");
properties.setProperty(PROPERTY_REGION + "." + AP_SOUTHEAST_1 + "." + ENDPOINT, properties.setProperty(PROPERTY_REGION + "." + AP_SOUTHEAST_1 + "." + ENDPOINT,
"https://s3-ap-southeast-1.amazonaws.com"); "https://s3-ap-southeast-1.amazonaws.com");
properties.setProperty(PROPERTY_REGION + "." + AP_NORTHEAST_1 + "." + ENDPOINT, properties.setProperty(PROPERTY_REGION + "." + AP_NORTHEAST_1 + "." + ENDPOINT,

71
providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfigured.java
View File

@ -1,71 +0,0 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.aws.s3.binders;
import static org.jclouds.http.utils.ModifyRequest.endpoint;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
import java.net.URI;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Provider;
import javax.inject.Singleton;
import javax.ws.rs.core.UriBuilder;
import org.jclouds.http.HttpRequest;
import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull;
import org.jclouds.rest.binders.BindAsHostPrefix;
import org.jclouds.s3.Bucket;
import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured;
/**
*
* @author Adrian Cole
*/
@Singleton
public class AssignCorrectHostnameAndBindAsHostPrefixIfConfigured extends BindAsHostPrefixIfConfigured {
private final Map<String, String> bucketToRegion;
private final RegionToEndpointOrProviderIfNull r2;
@Inject
public AssignCorrectHostnameAndBindAsHostPrefixIfConfigured(BindAsHostPrefix bindAsHostPrefix,
@Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
@Named(PROPERTY_S3_SERVICE_PATH) String servicePath,
RegionToEndpointOrProviderIfNull r2, Provider<UriBuilder> uriBuilderProvider,
@Bucket Map<String, String> bucketToRegion) {
super(bindAsHostPrefix, isVhostStyle, servicePath, uriBuilderProvider);
this.bucketToRegion = bucketToRegion;
this.r2 = r2;
}
@Override
public <R extends HttpRequest> R bindToRequest(R request, Object payload) {
String bucket = payload.toString();
String region = bucketToRegion.get(bucket);
if (region != null) {
URI endpoint = r2.apply(region);
request = endpoint(request, uriBuilderProvider.get().uri(endpoint).path(request.getEndpoint().getPath())
.replaceQuery(request.getEndpoint().getQuery()).build());
}
return super.bindToRequest(request, payload);
}
}

16
providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3AsyncBlobStore.java
View File

@ -36,6 +36,8 @@ import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions;
import org.jclouds.aws.s3.blobstore.strategy.AsyncMultipartUploadStrategy; import org.jclouds.aws.s3.blobstore.strategy.AsyncMultipartUploadStrategy;
import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.options.PutOptions; import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata; import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
@ -45,14 +47,15 @@ import org.jclouds.domain.Location;
import org.jclouds.s3.blobstore.S3AsyncBlobStore; import org.jclouds.s3.blobstore.S3AsyncBlobStore;
import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BlobToObject;
import org.jclouds.s3.blobstore.functions.BucketToResourceList; import org.jclouds.s3.blobstore.functions.BucketToResourceList;
import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata;
import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions;
import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlob;
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.AccessControlList;
import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.domain.CannedAccessPolicy; import org.jclouds.s3.domain.CannedAccessPolicy;
import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.domain.ObjectMetadata;
import com.google.common.base.Function;
import com.google.common.base.Supplier; import com.google.common.base.Supplier;
import com.google.common.cache.CacheLoader; import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache; import com.google.common.cache.LoadingCache;
@ -72,12 +75,13 @@ public class AWSS3AsyncBlobStore extends S3AsyncBlobStore {
public AWSS3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, public AWSS3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils,
@Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier<Location> defaultLocation, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations, AWSS3AsyncClient async, AWSS3Client sync, @Memoized Supplier<Set<? extends Location>> locations, AWSS3AsyncClient async, AWSS3Client sync,
BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, Function<Set<BucketMetadata>, PageSet<? extends StorageMetadata>> convertBucketsToStorageMetadata,
BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList,
BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object,
Provider<FetchBlobMetadata> fetchBlobMetadataProvider, LoadingCache<String, AccessControlList> bucketAcls, ObjectToBlobMetadata object2BlobMd, Provider<FetchBlobMetadata> fetchBlobMetadataProvider,
LoadingCache<String, AccessControlList> bucketAcls,
Provider<AsyncMultipartUploadStrategy> multipartUploadStrategy) { Provider<AsyncMultipartUploadStrategy> multipartUploadStrategy) {
super(context, blobUtils, service, defaultLocation, locations, async, sync, bucket2ResourceMd, super(context, blobUtils, service, defaultLocation, locations, async, sync, convertBucketsToStorageMetadata,
container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object, container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object,
object2BlobMd, fetchBlobMetadataProvider, bucketAcls); object2BlobMd, fetchBlobMetadataProvider, bucketAcls);
this.multipartUploadStrategy = multipartUploadStrategy; this.multipartUploadStrategy = multipartUploadStrategy;

20
providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java
View File

@ -32,6 +32,8 @@ import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions;
import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy; import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy;
import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.options.PutOptions; import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata; import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
@ -41,14 +43,15 @@ import org.jclouds.domain.Location;
import org.jclouds.s3.blobstore.S3BlobStore; import org.jclouds.s3.blobstore.S3BlobStore;
import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.blobstore.functions.BlobToObject;
import org.jclouds.s3.blobstore.functions.BucketToResourceList; import org.jclouds.s3.blobstore.functions.BucketToResourceList;
import org.jclouds.s3.blobstore.functions.BucketToResourceMetadata;
import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions;
import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlob;
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.AccessControlList;
import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.domain.CannedAccessPolicy; import org.jclouds.s3.domain.CannedAccessPolicy;
import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.domain.ObjectMetadata;
import com.google.common.base.Function;
import com.google.common.base.Supplier; import com.google.common.base.Supplier;
import com.google.common.cache.CacheLoader; import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache; import com.google.common.cache.LoadingCache;
@ -67,14 +70,15 @@ public class AWSS3BlobStore extends S3BlobStore {
@Inject @Inject
AWSS3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation, AWSS3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations, AWSS3Client sync, @Memoized Supplier<Set<? extends Location>> locations, AWSS3Client sync,
BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, Function<Set<BucketMetadata>, PageSet<? extends StorageMetadata>> convertBucketsToStorageMetadata,
BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList,
BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object,
Provider<FetchBlobMetadata> fetchBlobMetadataProvider, LoadingCache<String, AccessControlList> bucketAcls, ObjectToBlobMetadata object2BlobMd, Provider<FetchBlobMetadata> fetchBlobMetadataProvider,
LoadingCache<String, AccessControlList> bucketAcls,
Provider<MultipartUploadStrategy> multipartUploadStrategy) { Provider<MultipartUploadStrategy> multipartUploadStrategy) {
super(context, blobUtils, defaultLocation, locations, sync, bucket2ResourceMd, container2BucketListOptions, super(context, blobUtils, defaultLocation, locations, sync, convertBucketsToStorageMetadata,
bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object, object2BlobMd, container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object,
fetchBlobMetadataProvider, bucketAcls); object2BlobMd, fetchBlobMetadataProvider, bucketAcls);
this.multipartUploadStrategy = multipartUploadStrategy; this.multipartUploadStrategy = multipartUploadStrategy;
this.bucketAcls = bucketAcls; this.bucketAcls = bucketAcls;
this.blob2Object = blob2Object; this.blob2Object = blob2Object;

17
providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java
View File

@ -19,24 +19,17 @@
package org.jclouds.aws.s3.config; package org.jclouds.aws.s3.config;
import static org.jclouds.aws.domain.Region.US_STANDARD; import static org.jclouds.aws.domain.Region.US_STANDARD;
import static org.jclouds.location.reference.LocationConstants.ENDPOINT;
import static org.jclouds.location.reference.LocationConstants.PROPERTY_REGION;
import java.net.URI;
import javax.inject.Named;
import javax.inject.Singleton; import javax.inject.Singleton;
import org.jclouds.aws.s3.AWSS3AsyncClient; import org.jclouds.aws.s3.AWSS3AsyncClient;
import org.jclouds.aws.s3.AWSS3Client; import org.jclouds.aws.s3.AWSS3Client;
import org.jclouds.aws.s3.binders.AssignCorrectHostnameAndBindAsHostPrefixIfConfigured;
import org.jclouds.aws.s3.predicates.validators.AWSS3BucketNameValidator; import org.jclouds.aws.s3.predicates.validators.AWSS3BucketNameValidator;
import org.jclouds.location.Region; import org.jclouds.location.Region;
import org.jclouds.rest.ConfiguresRestClient; import org.jclouds.rest.ConfiguresRestClient;
import org.jclouds.rest.RestContext; import org.jclouds.rest.RestContext;
import org.jclouds.s3.Bucket;
import org.jclouds.s3.S3AsyncClient; import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.S3Client; import org.jclouds.s3.S3Client;
import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured;
import org.jclouds.s3.config.S3RestClientModule; import org.jclouds.s3.config.S3RestClientModule;
import org.jclouds.s3.predicates.validators.BucketNameValidator; import org.jclouds.s3.predicates.validators.BucketNameValidator;
@ -61,20 +54,12 @@ public class AWSS3RestClientModule extends S3RestClientModule<AWSS3Client, AWSS3
protected Supplier<String> defaultRegionForBucket(@Region Supplier<String> defaultRegion) { protected Supplier<String> defaultRegionForBucket(@Region Supplier<String> defaultRegion) {
return Suppliers.ofInstance(US_STANDARD); return Suppliers.ofInstance(US_STANDARD);
} }
@Override @Override
protected void configure() { protected void configure() {
bind(BindAsHostPrefixIfConfigured.class).to(AssignCorrectHostnameAndBindAsHostPrefixIfConfigured.class);
bind(BucketNameValidator.class).to(AWSS3BucketNameValidator.class); bind(BucketNameValidator.class).to(AWSS3BucketNameValidator.class);
super.configure(); super.configure();
} }
@Provides
@Singleton
@Bucket
protected Supplier<URI> provideBucketURI(@Named(PROPERTY_REGION + "." + US_STANDARD + "." + ENDPOINT) String endpoint){
return Suppliers.ofInstance(URI.create(endpoint));
}
@Singleton @Singleton
@Provides @Provides

61
providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java
View File

@ -22,13 +22,10 @@ import java.io.IOException;
import java.lang.reflect.Array; import java.lang.reflect.Array;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentMap; import java.util.Set;
import javax.inject.Named;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.inject.Module;
import com.google.inject.TypeLiteral;
import org.jclouds.aws.s3.config.AWSS3RestClientModule; import org.jclouds.aws.s3.config.AWSS3RestClientModule;
import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex; import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex;
import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex; import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex;
@ -41,22 +38,34 @@ import org.jclouds.http.functions.ReleasePayloadAndReturn;
import org.jclouds.http.functions.ReturnTrueIf2xx; import org.jclouds.http.functions.ReturnTrueIf2xx;
import org.jclouds.io.Payload; import org.jclouds.io.Payload;
import org.jclouds.io.Payloads; import org.jclouds.io.Payloads;
import org.jclouds.location.Region;
import org.jclouds.location.reference.LocationConstants;
import org.jclouds.rest.ConfiguresRestClient; import org.jclouds.rest.ConfiguresRestClient;
import org.jclouds.rest.functions.MapHttp4xxCodesToExceptions; import org.jclouds.rest.functions.MapHttp4xxCodesToExceptions;
import org.jclouds.rest.functions.ReturnVoidOnNotFoundOr404; import org.jclouds.rest.functions.ReturnVoidOnNotFoundOr404;
import org.jclouds.rest.internal.RestAnnotationProcessor; import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.S3AsyncClient; import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.S3AsyncClientTest; import org.jclouds.s3.S3AsyncClientTest;
import org.jclouds.s3.S3Client;
import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.ObjectMetadataBuilder; import org.jclouds.s3.domain.ObjectMetadataBuilder;
import org.jclouds.s3.domain.S3Object; import org.jclouds.s3.domain.S3Object;
import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState; import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists;
import org.jclouds.s3.options.CopyObjectOptions; import org.jclouds.s3.options.CopyObjectOptions;
import org.jclouds.s3.options.PutBucketOptions; import org.jclouds.s3.options.PutBucketOptions;
import org.jclouds.s3.options.PutObjectOptions; import org.jclouds.s3.options.PutObjectOptions;
import org.jclouds.s3.xml.LocationConstraintHandler; import org.jclouds.s3.xml.LocationConstraintHandler;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.common.base.Functions;
import com.google.common.base.Optional;
import com.google.common.base.Supplier;
import com.google.common.cache.CacheLoader;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.TypeLiteral;
/** /**
* @author Adrian Cole * @author Adrian Cole
*/ */
@ -72,15 +81,15 @@ public class AWSS3AsyncClientTest extends S3AsyncClientTest<AWSS3AsyncClient> {
Method method = S3AsyncClient.class.getMethod("copyObject", String.class, String.class, String.class, Method method = S3AsyncClient.class.getMethod("copyObject", String.class, String.class, String.class,
String.class, String.class,
Array.newInstance(CopyObjectOptions.class, 0).getClass()); Array.newInstance(CopyObjectOptions.class, 0).getClass());
processor.createRequest(method, "sourceBucket", "sourceObject", "destinationBucket", "destinationObject"); processor.createRequest(method, "sourceBucket", "sourceObject", "destinationbucket", "destinationObject");
} }
public void testGetBucketLocationEU() throws SecurityException, NoSuchMethodException, IOException { public void testGetBucketLocationEUIsStillDefault() throws SecurityException, NoSuchMethodException, IOException {
Method method = AWSS3AsyncClient.class.getMethod("getBucketLocation", String.class); Method method = AWSS3AsyncClient.class.getMethod("getBucketLocation", String.class);
HttpRequest request = processor.createRequest(method, "eubucket"); HttpRequest request = processor.createRequest(method, "bucket-eu-west-1");
assertRequestLineEquals(request, "GET https://eubucket.s3-eu-west-1.amazonaws.com/?location HTTP/1.1"); assertRequestLineEquals(request, "GET https://bucket-eu-west-1.s3.amazonaws.com/?location HTTP/1.1");
assertNonPayloadHeadersEqual(request, "Host: eubucket.s3-eu-west-1.amazonaws.com\n"); assertNonPayloadHeadersEqual(request, "Host: bucket-eu-west-1.s3.amazonaws.com\n");
assertPayloadEquals(request, null, null, false); assertPayloadEquals(request, null, null, false);
assertResponseParserClassEquals(method, request, ParseSax.class); assertResponseParserClassEquals(method, request, ParseSax.class);
@ -147,7 +156,7 @@ public class AWSS3AsyncClientTest extends S3AsyncClientTest<AWSS3AsyncClient> {
assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class); assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class);
assertSaxResponseParserClassEquals(method, null); assertSaxResponseParserClassEquals(method, null);
assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class); assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.class);
checkFilters(request); checkFilters(request);
} }
@ -257,23 +266,29 @@ public class AWSS3AsyncClientTest extends S3AsyncClientTest<AWSS3AsyncClient> {
assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class); assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class);
assertSaxResponseParserClassEquals(method, null); assertSaxResponseParserClassEquals(method, null);
assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState.class); assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists.class);
checkFilters(request); checkFilters(request);
} }
@ConfiguresRestClient @ConfiguresRestClient
private static final class TestAWSS3RestClientModule extends AWSS3RestClientModule { private static final class TestAWSS3RestClientModule extends AWSS3RestClientModule {
public TestAWSS3RestClientModule() {
super();
}
@Override @Override
protected ConcurrentMap<String, String> bucketToRegion() { protected CacheLoader<String, Optional<String>> bucketToRegion(@Region Supplier<Set<String>> regionSupplier,
ConcurrentMap<String, String> returnVal = Maps.newConcurrentMap(); final S3Client client) {
returnVal.put("eubucket", "EU"); return CacheLoader.<String, Optional<String>> from(Functions.forMap(ImmutableMap
return returnVal; .<String, Optional<String>> builder()
.put("bucket", Optional.<String> absent())
.put("destinationbucket", Optional.<String> absent())
.put("bucket-us-standard", Optional.of("us-standard"))
.put("bucket-us-west-1", Optional.of("us-west-1"))
.put("bucket-us-west-2", Optional.of("us-west-2"))
.put("bucket-eu-west-1", Optional.of("eu-west-1"))
.put("bucket-sa-east-1", Optional.of("sa-east-1"))
.put("bucket-ap-southeast-1", Optional.of("ap-southeast-1"))
.put("bucket-ap-northeast-1", Optional.of("ap-northeast-1"))
.build()));
} }
@Override @Override

28
providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientExpectTest.java
View File

@ -42,7 +42,27 @@ import com.google.inject.Injector;
*/ */
@Test @Test
public class AWSS3ClientExpectTest extends BaseAWSS3ClientExpectTest { public class AWSS3ClientExpectTest extends BaseAWSS3ClientExpectTest {
HttpRequest bucketLocationRequest = HttpRequest.builder()
.method("GET")
.endpoint(URI.create("https://test.s3.amazonaws.com/?location"))
.headers(ImmutableMultimap.of(
"Host", "test.s3.amazonaws.com",
"Date", CONSTANT_DATE,
"Authorization", "AWS identity:D1rymKrEdvzvhmZXeg+Z0R+tiug="
))
.build();
HttpResponse bucketLocationResponse = HttpResponse.builder()
.statusCode(200)
.payload(
payloadFromStringWithContentType("<LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">eu-west-1</LocationConstraint>", "application/xml"))
.headers(ImmutableMultimap.of(
"x-amz-id-2", "BtioT9wIK04YkE2DPgWUrQFiAbjwJVP8cLyfOkJ1FHMbn2hVjBZvkMMuXPDHfGVw",
"x-amz-request-id", "51BF4F45D49B1B34",
"Date", CONSTANT_DATE,
"Server", "AmazonS3"
))
.build();
@Test @Test
public void testPutWithReducedRedundancy() { public void testPutWithReducedRedundancy() {
Injector injector = createInjector(Functions.forMap(ImmutableMap.<HttpRequest, HttpResponse>of()), createModule(), setupProperties()); Injector injector = createInjector(Functions.forMap(ImmutableMap.<HttpRequest, HttpResponse>of()), createModule(), setupProperties());
@ -50,13 +70,13 @@ public class AWSS3ClientExpectTest extends BaseAWSS3ClientExpectTest {
Blob blob = injector.getInstance(BlobBuilder.class).name("test").payload("content").build(); Blob blob = injector.getInstance(BlobBuilder.class).name("test").payload("content").build();
BlobToObject blobToObject = injector.getInstance(BlobToObject.class); BlobToObject blobToObject = injector.getInstance(BlobToObject.class);
AWSS3Client client = requestSendsResponse( AWSS3Client client = requestsSendResponses(bucketLocationRequest, bucketLocationResponse,
HttpRequest.builder() HttpRequest.builder()
.method("PUT") .method("PUT")
.endpoint(URI.create("https://test.s3.amazonaws.com/test")) .endpoint(URI.create("https://test.s3-eu-west-1.amazonaws.com/test"))
.headers(ImmutableMultimap.of( .headers(ImmutableMultimap.of(
"x-amz-storage-class", "REDUCED_REDUNDANCY", "x-amz-storage-class", "REDUCED_REDUNDANCY",
"Host", "test.s3.amazonaws.com", "Host", "test.s3-eu-west-1.amazonaws.com",
"Date", CONSTANT_DATE, "Date", CONSTANT_DATE,
"Authorization", "AWS identity:1mJrW85/mqZpYTFIK5Ebtt2MM6E=" "Authorization", "AWS identity:1mJrW85/mqZpYTFIK5Ebtt2MM6E="
)) ))

92
providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest.java
View File

@ -1,92 +0,0 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.aws.s3.binders;
import static org.testng.Assert.assertEquals;
import java.net.URI;
import java.util.Map;
import javax.inject.Provider;
import javax.ws.rs.core.UriBuilder;
import org.jclouds.http.HttpRequest;
import org.jclouds.location.functions.RegionToEndpointOrProviderIfNull;
import org.jclouds.rest.binders.BindAsHostPrefix;
import org.testng.annotations.Test;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableMap;
import com.sun.jersey.api.uri.UriBuilderImpl;
/**
* Tests behavior of {@code AssignCorrectHostnameAndBindAsHostPrefixIfConfigured}
*
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during
// surefire
@Test(groups = "unit", testName = "AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest")
public class AssignCorrectHostnameAndBindAsHostPrefixIfConfiguredTest {
Provider<UriBuilder> uriBuilderProvider = new Provider<UriBuilder>() {
@Override
public UriBuilder get() {
return new UriBuilderImpl();
}
};
public void testWhenNoBucketRegionMappingInCache() {
HttpRequest request = new HttpRequest("GET", URI.create("https://s3.amazonaws.com"));
AssignCorrectHostnameAndBindAsHostPrefixIfConfigured binder = new AssignCorrectHostnameAndBindAsHostPrefixIfConfigured(
new BindAsHostPrefix(uriBuilderProvider), true, "/", new RegionToEndpointOrProviderIfNull("aws-s3", Suppliers
.ofInstance(URI.create("https://s3.amazonaws.com")),
Suppliers.<Map<String, Supplier<URI>>> ofInstance(ImmutableMap.of("us-standard", Suppliers
.ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1", Suppliers.ofInstance(URI
.create("https://s3-us-west-1.amazonaws.com"))))),
uriBuilderProvider, ImmutableMap.<String, String> of());
request = binder.bindToRequest(request, "bucket");
assertEquals(request.getRequestLine(), "GET https://bucket.s3.amazonaws.com HTTP/1.1");
}
public void testWhenBucketRegionMappingInCache() {
HttpRequest request = new HttpRequest("GET", URI.create("https://s3.amazonaws.com"));
AssignCorrectHostnameAndBindAsHostPrefixIfConfigured binder = new AssignCorrectHostnameAndBindAsHostPrefixIfConfigured(
new BindAsHostPrefix(uriBuilderProvider), true, "/", new RegionToEndpointOrProviderIfNull("aws-s3", Suppliers
.ofInstance(URI.create("https://s3.amazonaws.com")),
Suppliers.<Map<String, Supplier<URI>>> ofInstance(ImmutableMap.of("us-standard", Suppliers
.ofInstance(URI.create("https://s3.amazonaws.com")), "us-west-1", Suppliers.ofInstance(URI
.create("https://s3-us-west-1.amazonaws.com"))))),
uriBuilderProvider, ImmutableMap.<String, String> of("bucket", "us-west-1"));
request = binder.bindToRequest(request, "bucket");
assertEquals(request.getRequestLine(), "GET https://bucket.s3-us-west-1.amazonaws.com HTTP/1.1");
}
}

79
providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3ContainerLiveTest.java
View File

@ -18,93 +18,16 @@
*/ */
package org.jclouds.aws.s3.blobstore.integration; package org.jclouds.aws.s3.blobstore.integration;
import static org.jclouds.blobstore.options.CreateContainerOptions.Builder.publicRead;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.NoSuchElementException;
import java.util.Set;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.domain.Location;
import org.jclouds.s3.blobstore.integration.S3ContainerLiveTest; import org.jclouds.s3.blobstore.integration.S3ContainerLiveTest;
import org.jclouds.util.Strings2;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.common.base.Strings;
/** /**
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(groups = "live", testName = "AWSS3ContainerLiveTest") @Test(groups = "live", singleThreaded = true, testName = "AWSS3ContainerLiveTest")
public class AWSS3ContainerLiveTest extends S3ContainerLiveTest { public class AWSS3ContainerLiveTest extends S3ContainerLiveTest {
public AWSS3ContainerLiveTest() { public AWSS3ContainerLiveTest() {
provider = "aws-s3"; provider = "aws-s3";
} }
@Test(groups = { "live" })
public void testCreateBlobWithExpiry() throws InterruptedException, MalformedURLException, IOException {
final String containerName = getScratchContainerName();
BlobStore blobStore = view.getBlobStore();
try {
final String blobName = "hello";
final Date expires = new Date( (System.currentTimeMillis() / 1000) * 1000 + 60*1000);
blobStore.createContainerInLocation(null, containerName, publicRead());
blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload(TEST_STRING).expires(expires).build());
assertConsistencyAwareBlobExpiryMetadata(containerName, blobName, expires);
} finally {
recycleContainer(containerName);
}
}
@Test(groups = { "live" })
public void testCreateBlobInLocation() throws InterruptedException, MalformedURLException, IOException {
String payload = "my data";
runCreateContainerInLocation(payload);
}
@Test(groups = { "live" })
public void testCreateBigBlobInLocation() throws InterruptedException, MalformedURLException, IOException {
String payload = Strings.repeat("a", 1024*1024); // 1MB
runCreateContainerInLocation(payload);
}
private void runCreateContainerInLocation(String payload) throws InterruptedException, MalformedURLException, IOException {
String blobName = "hello";
BlobStore blobStore = view.getBlobStore();
final String containerName = getScratchContainerName();
try {
String locationId = "EU";
Location location = findLocation(blobStore, locationId);
blobStore.createContainerInLocation(location, containerName, publicRead());
blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload(payload).build());
assertConsistencyAwareContainerSize(containerName, 1);
BlobMetadata metadata = view.getBlobStore().blobMetadata(containerName, blobName);
assertEquals(Strings2.toStringAndClose(view.utils().http().get(metadata.getPublicUri())), payload);
assertConsistencyAwareBlobInLocation(containerName, blobName, location);
} finally {
// this container is now public, so we can't reuse it directly
recycleContainer(containerName);
}
}
private Location findLocation(BlobStore blobStore, String id) {
Set<? extends Location> locs = blobStore.listAssignableLocations();
for (Location loc : locs) {
if (loc.getId().equals(id)) {
return loc;
}
}
throw new NoSuchElementException("No location found with id '"+id+"'; contenders were "+locs);
}
} }

81
providers/aws-s3/src/test/java/org/jclouds/aws/s3/services/AWSBucketsLiveTest.java Normal file
View File

@ -0,0 +1,81 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.aws.s3.services;
import static org.jclouds.s3.options.PutBucketOptions.Builder.withBucketAcl;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import org.jclouds.aws.domain.Region;
import org.jclouds.s3.domain.AccessControlList;
import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI;
import org.jclouds.s3.domain.AccessControlList.Permission;
import org.jclouds.s3.domain.CannedAccessPolicy;
import org.jclouds.s3.services.BucketsLiveTest;
import org.testng.annotations.Test;
import com.google.common.base.Throwables;
/**
* @author Adrian Cole
*/
@Test(groups = "live", testName = "AWSBucketsLiveTest")
public class AWSBucketsLiveTest extends BucketsLiveTest {
public AWSBucketsLiveTest() {
provider = "aws-s3";
}
public void testDefaultBucketLocation() throws Exception {
String bucketName = getContainerName();
try {
String location = getApi().getBucketLocation(bucketName);
assert location.equals(Region.US_STANDARD) : "bucket: " + bucketName + " location: " + location;
} finally {
returnContainer(bucketName);
}
}
/**
* using scratch bucketName as we are changing location
*/
public void testEu() throws Exception {
final String bucketName = getScratchContainerName();
try {
getApi().putBucketInRegion(Region.EU_WEST_1, bucketName + "eu", withBucketAcl(CannedAccessPolicy.PUBLIC_READ));
assertConsistencyAware(new Runnable() {
public void run() {
try {
AccessControlList acl = getApi().getBucketACL(bucketName + "eu");
assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ), acl.toString());
} catch (Exception e) {
Throwables.propagateIfPossible(e);
}
}
});
assertEquals(Region.EU_WEST_1, getApi().getBucketLocation(bucketName + "eu"));
// TODO: I believe that the following should work based on the above acl assertion passing.
// However, it fails on 403
// URL url = new URL(String.format("http://%s.s3.amazonaws.com", bucketName));
// Utils.toStringAndClose(url.openStream());
} finally {
destroyContainer(bucketName + "eu");
}
}
}

9
providers/hpcloud-objectstorage/src/test/java/org/jclouds/hpcloud/objectstorage/blobstore/integration/HPCloudObjectStorageContainerLiveTest.java
View File

@ -18,9 +18,6 @@
*/ */
package org.jclouds.hpcloud.objectstorage.blobstore.integration; package org.jclouds.hpcloud.objectstorage.blobstore.integration;
import java.io.IOException;
import java.net.MalformedURLException;
import org.jclouds.blobstore.integration.internal.BaseContainerLiveTest; import org.jclouds.blobstore.integration.internal.BaseContainerLiveTest;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@ -31,11 +28,5 @@ import org.testng.annotations.Test;
public class HPCloudObjectStorageContainerLiveTest extends BaseContainerLiveTest { public class HPCloudObjectStorageContainerLiveTest extends BaseContainerLiveTest {
public HPCloudObjectStorageContainerLiveTest() { public HPCloudObjectStorageContainerLiveTest() {
provider = "hpcloud-objectstorage"; provider = "hpcloud-objectstorage";
}
@Test(enabled = false)
//@Test(expectedExceptions=UnsupportedOperationException.class)
public void testPublicAccess() throws MalformedURLException, InterruptedException, IOException {
super.testPublicAccess();
} }
} }