From ba2f8ac2328ccf7a38e578da7b0c7344de65d6ee Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 9 Feb 2015 17:34:48 -0800 Subject: [PATCH] JCLOUDS-258: Support MPU for generic S3 Tested against AWS-S3 and DreamObjects. This commit only moves and renames code although some classes are duplicated for deprecation purposes. --- .../main/java/org/jclouds/s3/S3Client.java | 141 ++++++++++++++++ .../binders/BindObjectMetadataToRequest.java | 2 +- .../binders/BindPartIdsAndETagsToRequest.java | 2 +- .../org/jclouds/s3/blobstore/S3BlobStore.java | 11 +- .../config/S3BlobStoreContextModule.java | 6 + .../AsyncMultipartUploadStrategy.java | 4 +- .../blobstore/strategy/MultipartUpload.java | 2 +- .../strategy/MultipartUploadStrategy.java | 4 +- .../MultipartUploadSlicingAlgorithm.java | 4 +- .../ParallelMultipartUploadStrategy.java | 16 +- .../SequentialMultipartUploadStrategy.java | 10 +- .../ETagFromHttpResponseViaRegex.java | 2 +- .../s3/functions/ObjectMetadataKey.java | 2 +- .../UploadIdFromHttpResponseViaRegex.java | 2 +- .../java/org/jclouds/s3/S3ClientLiveTest.java | 58 +++++++ .../java/org/jclouds/s3/S3ClientTest.java | 98 ++++++++++++ .../BindObjectMetadataToRequestTest.java | 2 +- .../BindPartIdsAndETagsToRequestTest.java | 2 +- .../S3BlobIntegrationLiveTest.java | 14 ++ .../strategy/internal/MpuGraphData.java | 4 +- .../MpuPartitioningAlgorithmTest.java | 4 +- ...entialMultipartUploadStrategyMockTest.java | 4 +- .../ETagFromHttpResponseViaRegexTest.java | 2 +- .../UploadIdFromHttpResponseViaRegexTest.java | 2 +- .../resources/complete-multipart-upload.xml | 0 .../resources/initiate-multipart-upload.xml | 0 .../java/org/jclouds/aws/s3/AWSS3Client.java | 151 ------------------ .../aws/s3/blobstore/AWSS3BlobStore.java | 7 +- .../config/AWSS3BlobStoreContextModule.java | 6 - .../jclouds/aws/s3/AWSS3ClientLiveTest.java | 61 ------- .../org/jclouds/aws/s3/AWSS3ClientTest.java | 99 ------------ .../AWSS3BlobIntegrationLiveTest.java | 15 -- 32 files changed, 365 insertions(+), 372 deletions(-) rename {providers/aws-s3/src/main/java/org/jclouds/aws => apis/s3/src/main/java/org/jclouds}/s3/binders/BindObjectMetadataToRequest.java (98%) rename {providers/aws-s3/src/main/java/org/jclouds/aws => apis/s3/src/main/java/org/jclouds}/s3/binders/BindPartIdsAndETagsToRequest.java (98%) rename {providers/aws-s3/src/main/java/org/jclouds/aws => apis/s3/src/main/java/org/jclouds}/s3/blobstore/strategy/AsyncMultipartUploadStrategy.java (89%) rename {providers/aws-s3/src/main/java/org/jclouds/aws => apis/s3/src/main/java/org/jclouds}/s3/blobstore/strategy/MultipartUpload.java (96%) rename {providers/aws-s3/src/main/java/org/jclouds/aws => apis/s3/src/main/java/org/jclouds}/s3/blobstore/strategy/MultipartUploadStrategy.java (88%) rename {providers/aws-s3/src/main/java/org/jclouds/aws => apis/s3/src/main/java/org/jclouds}/s3/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java (97%) rename {providers/aws-s3/src/main/java/org/jclouds/aws => apis/s3/src/main/java/org/jclouds}/s3/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java (96%) rename {providers/aws-s3/src/main/java/org/jclouds/aws => apis/s3/src/main/java/org/jclouds}/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java (94%) rename {providers/aws-s3/src/main/java/org/jclouds/aws => apis/s3/src/main/java/org/jclouds}/s3/functions/ETagFromHttpResponseViaRegex.java (98%) rename {providers/aws-s3/src/main/java/org/jclouds/aws => apis/s3/src/main/java/org/jclouds}/s3/functions/ObjectMetadataKey.java (96%) rename {providers/aws-s3/src/main/java/org/jclouds/aws => apis/s3/src/main/java/org/jclouds}/s3/functions/UploadIdFromHttpResponseViaRegex.java (97%) rename {providers/aws-s3/src/test/java/org/jclouds/aws => apis/s3/src/test/java/org/jclouds}/s3/binders/BindObjectMetadataToRequestTest.java (99%) rename {providers/aws-s3/src/test/java/org/jclouds/aws => apis/s3/src/test/java/org/jclouds}/s3/binders/BindPartIdsAndETagsToRequestTest.java (98%) rename {providers/aws-s3/src/test/java/org/jclouds/aws => apis/s3/src/test/java/org/jclouds}/s3/blobstore/strategy/internal/MpuGraphData.java (95%) rename {providers/aws-s3/src/test/java/org/jclouds/aws => apis/s3/src/test/java/org/jclouds}/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java (98%) rename {providers/aws-s3/src/test/java/org/jclouds/aws => apis/s3/src/test/java/org/jclouds}/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategyMockTest.java (98%) rename {providers/aws-s3/src/test/java/org/jclouds/aws => apis/s3/src/test/java/org/jclouds}/s3/functions/ETagFromHttpResponseViaRegexTest.java (97%) rename {providers/aws-s3/src/test/java/org/jclouds/aws => apis/s3/src/test/java/org/jclouds}/s3/functions/UploadIdFromHttpResponseViaRegexTest.java (97%) rename {providers/aws-s3 => apis/s3}/src/test/resources/complete-multipart-upload.xml (100%) rename {providers/aws-s3 => apis/s3}/src/test/resources/initiate-multipart-upload.xml (100%) diff --git a/apis/s3/src/main/java/org/jclouds/s3/S3Client.java b/apis/s3/src/main/java/org/jclouds/s3/S3Client.java index ba1199233f..76d246a385 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/S3Client.java +++ b/apis/s3/src/main/java/org/jclouds/s3/S3Client.java @@ -21,16 +21,19 @@ import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER; import static org.jclouds.s3.S3Fallbacks.TrueOn404OrNotFoundFalseOnIllegalState; import java.io.Closeable; +import java.util.Map; import java.util.Set; import javax.inject.Named; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.HEAD; +import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import org.jclouds.Fallbacks.VoidOnNotFoundOr404; @@ -42,6 +45,7 @@ import org.jclouds.blobstore.BlobStoreFallbacks.ThrowKeyNotFoundOn404; import org.jclouds.blobstore.attr.BlobScope; import org.jclouds.http.functions.ParseETagHeader; import org.jclouds.http.options.GetOptions; +import org.jclouds.io.Payload; import org.jclouds.javax.annotation.Nullable; import org.jclouds.rest.annotations.BinderParam; import org.jclouds.rest.annotations.Endpoint; @@ -59,6 +63,8 @@ import org.jclouds.s3.binders.BindACLToXMLPayload; import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured; import org.jclouds.s3.binders.BindBucketLoggingToXmlPayload; import org.jclouds.s3.binders.BindNoBucketLoggingToXmlPayload; +import org.jclouds.s3.binders.BindObjectMetadataToRequest; +import org.jclouds.s3.binders.BindPartIdsAndETagsToRequest; import org.jclouds.s3.binders.BindPayerToXmlPayload; import org.jclouds.s3.binders.BindS3ObjectMetadataToRequest; import org.jclouds.s3.domain.AccessControlList; @@ -73,9 +79,12 @@ import org.jclouds.s3.filters.RequestAuthorizeSignature; import org.jclouds.s3.functions.AssignCorrectHostnameForBucket; import org.jclouds.s3.functions.BindRegionToXmlPayload; import org.jclouds.s3.functions.DefaultEndpointThenInvalidateRegion; +import org.jclouds.s3.functions.ETagFromHttpResponseViaRegex; import org.jclouds.s3.functions.ObjectKey; +import org.jclouds.s3.functions.ObjectMetadataKey; import org.jclouds.s3.functions.ParseObjectFromHeadersAndHttpContent; import org.jclouds.s3.functions.ParseObjectMetadataFromHeaders; +import org.jclouds.s3.functions.UploadIdFromHttpResponseViaRegex; import org.jclouds.s3.options.CopyObjectOptions; import org.jclouds.s3.options.ListBucketOptions; import org.jclouds.s3.options.PutBucketOptions; @@ -545,4 +554,136 @@ public interface S3Client extends Closeable { @Produces(MediaType.TEXT_XML) void disableBucketLogging(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam( BindNoBucketLoggingToXmlPayload.class) @ParamValidators(BucketNameValidator.class) String bucketName); + + /** + * This operation initiates a multipart upload and returns an upload ID. This upload ID is used + * to associate all the parts in the specific multipart upload. You specify this upload ID in + * each of your subsequent upload part requests (see Upload Part). You also include this upload + * ID in the final request to either complete or abort the multipart upload request. + * + *

Note

If you create an object using the multipart upload APIs, currently you cannot + * copy the object between regions. + * + * + * @param bucketName + * namespace of the object you are to upload + * @param objectMetadata + * metadata around the object you wish to upload + * @param options + * controls optional parameters such as canned ACL + * @return ID for the initiated multipart upload. + */ + @Named("PutObject") + @POST + @QueryParams(keys = "uploads") + @Path("/{key}") + @ResponseParser(UploadIdFromHttpResponseViaRegex.class) + String initiateMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam( + BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class) + ObjectMetadata objectMetadata, PutObjectOptions... options); + + /** + * This operation aborts a multipart upload. After a multipart upload is aborted, no additional + * parts can be uploaded using that upload ID. The storage consumed by any previously uploaded + * parts will be freed. However, if any part uploads are currently in progress, those part + * uploads might or might not succeed. As a result, it might be necessary to abort a given + * multipart upload multiple times in order to completely free all storage consumed by all parts. + * + * + * @param bucketName + * namespace of the object you are deleting + * @param key + * unique key in the s3Bucket identifying the object + * @param uploadId + * id of the multipart upload in progress. + */ + @Named("AbortMultipartUpload") + @DELETE + @Path("/{key}") + @Fallback(VoidOnNotFoundOr404.class) + void abortMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam( + BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, @QueryParam("uploadId") String uploadId); + + /** + * This operation uploads a part in a multipart upload. You must initiate a multipart upload (see + * Initiate Multipart Upload) before you can upload any part. In response to your initiate + * request. Amazon S3 returns an upload ID, a unique identifier, that you must include in your + * upload part request. + * + *

+ * Part numbers can be any number from 1 to 10,000, inclusive. A part number uniquely identifies + * a part and also defines its position within the object being created. If you upload a new part + * using the same part number that was used with a previous part, the previously uploaded part is + * overwritten. Each part must be at least 5 MB in size, except the last part. There is no size + * limit on the last part of your multipart upload. + * + *

+ * To ensure that data is not corrupted when traversing the network, specify the Content-MD5 + * header in the upload part request. Amazon S3 checks the part data against the provided MD5 + * value. If they do not match, Amazon S3 returns an error. + * + * + * @param bucketName + * namespace of the object you are storing + * @param key + * unique key in the s3Bucket identifying the object + * @param partNumber + * which part is this. + * @param uploadId + * id of the multipart upload in progress. + * @param part + * contains the data to create or overwrite + * @return ETag of the content uploaded + */ + @Named("PutObject") + @PUT + @Path("/{key}") + @ResponseParser(ParseETagHeader.class) + String uploadPart(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam( + BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, @QueryParam("partNumber") int partNumber, + @QueryParam("uploadId") String uploadId, Payload part); + + /** + * + This operation completes a multipart upload by assembling previously uploaded parts. + *

+ * You first initiate the multipart upload and then upload all parts using the Upload Parts + * operation (see Upload Part). After successfully uploading all relevant parts of an upload, you + * call this operation to complete the upload. Upon receiving this request, Amazon S3 + * concatenates all the parts in ascending order by part number to create a new object. In the + * Complete Multipart Upload request, you must provide the parts list. For each part in the list, + * you must provide the part number and the ETag header value, returned after that part was + * uploaded. + *

+ * Processing of a Complete Multipart Upload request could take several minutes to complete. + * After Amazon S3 begins processing the request, it sends an HTTP response header that specifies + * a 200 OK response. While processing is in progress, Amazon S3 periodically sends whitespace + * characters to keep the connection from timing out. Because a request could fail after the + * initial 200 OK response has been sent, it is important that you check the response body to + * determine whether the request succeeded. + *

+ * Note that if Complete Multipart Upload fails, applications should be prepared to retry the + * failed requests. + * + * @param bucketName + * namespace of the object you are deleting + * @param key + * unique key in the s3Bucket identifying the object + * @param uploadId + * id of the multipart upload in progress. + * @param parts + * a map of part id to eTag from the {@link #uploadPart} command. + * @return ETag of the content uploaded + */ + @Named("PutObject") + @POST + @Path("/{key}") + @ResponseParser(ETagFromHttpResponseViaRegex.class) + String completeMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam( + BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, + @PathParam("key") String key, @QueryParam("uploadId") String uploadId, + @BinderParam(BindPartIdsAndETagsToRequest.class) Map parts); } diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequest.java b/apis/s3/src/main/java/org/jclouds/s3/binders/BindObjectMetadataToRequest.java similarity index 98% rename from providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequest.java rename to apis/s3/src/main/java/org/jclouds/s3/binders/BindObjectMetadataToRequest.java index f880353cb3..1f96b7a252 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequest.java +++ b/apis/s3/src/main/java/org/jclouds/s3/binders/BindObjectMetadataToRequest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.binders; +package org.jclouds.s3.binders; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequest.java b/apis/s3/src/main/java/org/jclouds/s3/binders/BindPartIdsAndETagsToRequest.java similarity index 98% rename from providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequest.java rename to apis/s3/src/main/java/org/jclouds/s3/binders/BindPartIdsAndETagsToRequest.java index 464c2065f5..16794c79ca 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequest.java +++ b/apis/s3/src/main/java/org/jclouds/s3/binders/BindPartIdsAndETagsToRequest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.binders; +package org.jclouds.s3.binders; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java index 4d28a12e08..3118073a16 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java @@ -49,6 +49,7 @@ import org.jclouds.s3.blobstore.functions.BucketToResourceList; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; +import org.jclouds.s3.blobstore.strategy.MultipartUploadStrategy; import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI; import org.jclouds.s3.domain.AccessControlList.Permission; @@ -77,6 +78,7 @@ public class S3BlobStore extends BaseBlobStore { private final BlobToHttpGetOptions blob2ObjectGetOptions; private final Provider fetchBlobMetadataProvider; private final LoadingCache bucketAcls; + protected final Provider multipartUploadStrategy; @Inject protected S3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier defaultLocation, @@ -85,7 +87,8 @@ public class S3BlobStore extends BaseBlobStore { ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, Provider fetchBlobMetadataProvider, - LoadingCache bucketAcls) { + LoadingCache bucketAcls, + Provider multipartUploadStrategy) { super(context, blobUtils, defaultLocation, locations); this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions"); this.sync = checkNotNull(sync, "sync"); @@ -97,6 +100,7 @@ public class S3BlobStore extends BaseBlobStore { this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd"); this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider"); this.bucketAcls = checkNotNull(bucketAcls, "bucketAcls"); + this.multipartUploadStrategy = checkNotNull(multipartUploadStrategy, "multipartUploadStrategy"); } /** @@ -249,6 +253,11 @@ public class S3BlobStore extends BaseBlobStore { */ @Override public String putBlob(String container, Blob blob, PutOptions overrides) { + if (overrides.isMultipart()) { + // need to use a provider if the strategy object is stateful + return multipartUploadStrategy.get().execute(container, blob); + } + // TODO: Make use of options overrides PutObjectOptions options = new PutObjectOptions(); try { diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java index f82030821e..99a220b0bf 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java @@ -31,6 +31,10 @@ import org.jclouds.s3.blobstore.S3BlobRequestSigner; import org.jclouds.s3.blobstore.S3BlobStore; import org.jclouds.s3.blobstore.functions.LocationFromBucketName; import org.jclouds.s3.blobstore.internal.BackoffOnNotFoundWhenGetBucketACL; +import org.jclouds.s3.blobstore.strategy.AsyncMultipartUploadStrategy; +import org.jclouds.s3.blobstore.strategy.MultipartUploadStrategy; +import org.jclouds.s3.blobstore.strategy.internal.ParallelMultipartUploadStrategy; +import org.jclouds.s3.blobstore.strategy.internal.SequentialMultipartUploadStrategy; import org.jclouds.s3.domain.AccessControlList; import com.google.common.base.Function; @@ -49,6 +53,8 @@ public class S3BlobStoreContextModule extends AbstractModule { bind(new TypeLiteral>() { }).to(LocationFromBucketName.class); bindRequestSigner(); + bind(MultipartUploadStrategy.class).to(SequentialMultipartUploadStrategy.class); + bind(AsyncMultipartUploadStrategy.class).to(ParallelMultipartUploadStrategy.class); } protected void bindRequestSigner() { diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/AsyncMultipartUploadStrategy.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/AsyncMultipartUploadStrategy.java similarity index 89% rename from providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/AsyncMultipartUploadStrategy.java rename to apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/AsyncMultipartUploadStrategy.java index 3f638ae053..adbc842d75 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/AsyncMultipartUploadStrategy.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/AsyncMultipartUploadStrategy.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.blobstore.strategy; +package org.jclouds.s3.blobstore.strategy; -import org.jclouds.aws.s3.blobstore.strategy.internal.ParallelMultipartUploadStrategy; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.s3.blobstore.strategy.internal.ParallelMultipartUploadStrategy; import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.ImplementedBy; diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/MultipartUpload.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/MultipartUpload.java similarity index 96% rename from providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/MultipartUpload.java rename to apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/MultipartUpload.java index 5342e67817..a82543d648 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/MultipartUpload.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/MultipartUpload.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.blobstore.strategy; +package org.jclouds.s3.blobstore.strategy; public final class MultipartUpload { diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/MultipartUploadStrategy.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/MultipartUploadStrategy.java similarity index 88% rename from providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/MultipartUploadStrategy.java rename to apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/MultipartUploadStrategy.java index 12b7ef37e5..a6800ad76a 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/MultipartUploadStrategy.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/MultipartUploadStrategy.java @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.blobstore.strategy; +package org.jclouds.s3.blobstore.strategy; -import org.jclouds.aws.s3.blobstore.strategy.internal.SequentialMultipartUploadStrategy; import org.jclouds.blobstore.domain.Blob; +import org.jclouds.s3.blobstore.strategy.internal.SequentialMultipartUploadStrategy; import com.google.inject.ImplementedBy; diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java similarity index 97% rename from providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java rename to apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java index 86205412f7..c9125ad3a8 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java @@ -23,12 +23,12 @@ * History */ -package org.jclouds.aws.s3.blobstore.strategy.internal; +package org.jclouds.s3.blobstore.strategy.internal; import javax.annotation.Resource; import javax.inject.Named; -import org.jclouds.aws.s3.blobstore.strategy.MultipartUpload; +import org.jclouds.s3.blobstore.strategy.MultipartUpload; import org.jclouds.blobstore.reference.BlobStoreConstants; import org.jclouds.logging.Logger; diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java similarity index 96% rename from providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java rename to apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java index 9694274c2a..789fb192af 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.blobstore.strategy.internal; +package org.jclouds.s3.blobstore.strategy.internal; import static com.google.common.base.Preconditions.checkNotNull; @@ -36,9 +36,7 @@ import javax.annotation.Resource; import javax.inject.Named; import org.jclouds.Constants; -import org.jclouds.aws.s3.AWSS3Client; -import org.jclouds.aws.s3.blobstore.AWSS3BlobStore; -import org.jclouds.aws.s3.blobstore.strategy.AsyncMultipartUploadStrategy; +import org.jclouds.s3.blobstore.strategy.AsyncMultipartUploadStrategy; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.internal.BlobRuntimeException; import org.jclouds.blobstore.options.PutOptions; @@ -46,6 +44,8 @@ import org.jclouds.blobstore.reference.BlobStoreConstants; import org.jclouds.io.Payload; import org.jclouds.io.PayloadSlicer; import org.jclouds.logging.Logger; +import org.jclouds.s3.S3Client; +import org.jclouds.s3.blobstore.S3BlobStore; import org.jclouds.s3.domain.ObjectMetadataBuilder; import org.jclouds.util.Throwables2; @@ -91,11 +91,11 @@ public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStra @Named(Constants.PROPERTY_REQUEST_TIMEOUT) protected Long maxTime; - protected final AWSS3BlobStore blobstore; + protected final S3BlobStore blobstore; protected final PayloadSlicer slicer; @Inject - public ParallelMultipartUploadStrategy(AWSS3BlobStore blobstore, PayloadSlicer slicer, + public ParallelMultipartUploadStrategy(S3BlobStore blobstore, PayloadSlicer slicer, @Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService executor) { this.blobstore = checkNotNull(blobstore, "blobstore"); this.slicer = checkNotNull(slicer, "slicer"); @@ -114,7 +114,7 @@ public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStra latch.countDown(); return; } - final AWSS3Client client = blobstore.getContext().unwrapApi(AWSS3Client.class); + final S3Client client = blobstore.getContext().unwrapApi(S3Client.class); final Payload chunkedPart = slicer.slice(payload, offset, size); logger.debug(String.format("async uploading part %s of %s to container %s with uploadId %s", part, key, container, uploadId)); final long start = System.currentTimeMillis(); @@ -166,7 +166,7 @@ public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStra long chunkSize = algorithm.getChunkSize(); long remaining = algorithm.getRemaining(); if (parts > 0) { - final AWSS3Client client = blobstore.getContext().unwrapApi(AWSS3Client.class); + final S3Client client = blobstore.getContext().unwrapApi(S3Client.class); String uploadId = null; final Map> futureParts = new ConcurrentHashMap>(); diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java similarity index 94% rename from providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java rename to apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java index 9c15b4ef2b..04003417aa 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.blobstore.strategy.internal; +package org.jclouds.s3.blobstore.strategy.internal; import static com.google.common.base.Preconditions.checkNotNull; @@ -23,8 +23,7 @@ import java.util.SortedMap; import javax.annotation.Resource; import javax.inject.Named; -import org.jclouds.aws.s3.AWSS3Client; -import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy; +import org.jclouds.s3.blobstore.strategy.MultipartUploadStrategy; import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.reference.BlobStoreConstants; @@ -32,6 +31,7 @@ import org.jclouds.io.ContentMetadata; import org.jclouds.io.Payload; import org.jclouds.io.PayloadSlicer; import org.jclouds.logging.Logger; +import org.jclouds.s3.S3Client; import org.jclouds.s3.blobstore.functions.BlobToObject; import org.jclouds.s3.domain.ObjectMetadataBuilder; @@ -54,13 +54,13 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg @Named(BlobStoreConstants.BLOBSTORE_LOGGER) private Logger logger = Logger.NULL; - private final AWSS3Client client; + private final S3Client client; private final BlobToObject blobToObject; private final MultipartUploadSlicingAlgorithm algorithm; private final PayloadSlicer slicer; @Inject - public SequentialMultipartUploadStrategy(AWSS3Client client, BlobToObject blobToObject, + public SequentialMultipartUploadStrategy(S3Client client, BlobToObject blobToObject, MultipartUploadSlicingAlgorithm algorithm, PayloadSlicer slicer) { this.client = checkNotNull(client, "client"); this.blobToObject = checkNotNull(blobToObject, "blobToObject"); diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegex.java b/apis/s3/src/main/java/org/jclouds/s3/functions/ETagFromHttpResponseViaRegex.java similarity index 98% rename from providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegex.java rename to apis/s3/src/main/java/org/jclouds/s3/functions/ETagFromHttpResponseViaRegex.java index 46f583775c..35ac9cc2de 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegex.java +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/ETagFromHttpResponseViaRegex.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.functions; +package org.jclouds.s3.functions; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ObjectMetadataKey.java b/apis/s3/src/main/java/org/jclouds/s3/functions/ObjectMetadataKey.java similarity index 96% rename from providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ObjectMetadataKey.java rename to apis/s3/src/main/java/org/jclouds/s3/functions/ObjectMetadataKey.java index 2fdc9ec782..f5c555e27a 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ObjectMetadataKey.java +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/ObjectMetadataKey.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.functions; +package org.jclouds.s3.functions; import javax.inject.Singleton; diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegex.java b/apis/s3/src/main/java/org/jclouds/s3/functions/UploadIdFromHttpResponseViaRegex.java similarity index 97% rename from providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegex.java rename to apis/s3/src/main/java/org/jclouds/s3/functions/UploadIdFromHttpResponseViaRegex.java index 9b4271480d..1d3edb14ac 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegex.java +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/UploadIdFromHttpResponseViaRegex.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.functions; +package org.jclouds.s3.functions; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/apis/s3/src/test/java/org/jclouds/s3/S3ClientLiveTest.java b/apis/s3/src/test/java/org/jclouds/s3/S3ClientLiveTest.java index 378eb11cdd..dba1d2615c 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/S3ClientLiveTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/S3ClientLiveTest.java @@ -16,6 +16,8 @@ */ package org.jclouds.s3; +import static com.google.common.hash.Hashing.md5; +import static org.jclouds.io.Payloads.newByteArrayPayload; import static org.jclouds.s3.options.CopyObjectOptions.Builder.ifSourceETagDoesntMatch; import static org.jclouds.s3.options.CopyObjectOptions.Builder.ifSourceETagMatches; import static org.jclouds.s3.options.CopyObjectOptions.Builder.ifSourceModifiedSince; @@ -37,8 +39,11 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; +import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest; import org.jclouds.http.HttpResponseException; +import org.jclouds.io.ByteStreams2; +import org.jclouds.io.Payload; import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.AccessControlList.CanonicalUserGrantee; import org.jclouds.s3.domain.AccessControlList.EmailAddressGrantee; @@ -46,19 +51,25 @@ import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI; import org.jclouds.s3.domain.AccessControlList.Permission; import org.jclouds.s3.domain.CannedAccessPolicy; import org.jclouds.s3.domain.ObjectMetadata; +import org.jclouds.s3.domain.ObjectMetadataBuilder; import org.jclouds.s3.domain.S3Object; import org.jclouds.s3.options.PutObjectOptions; import org.jclouds.util.Strings2; +import org.jclouds.utils.TestUtils; import org.testng.annotations.Test; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import com.google.common.hash.HashCode; +import com.google.common.io.ByteSource; @Test(groups = { "integration", "live" }) public class S3ClientLiveTest extends BaseBlobStoreIntegrationTest { public static final String TEST_ACL_ID = "1a405254c932b52e5b5caaa88186bc431a1bacb9ece631f835daddaf0c47677c"; public static final String TEST_ACL_EMAIL = "james@misterm.org"; public static final String DEFAULT_OWNER_ID = "abc123"; + private static final ByteSource oneHundredOneConstitutions = TestUtils.randomByteSource().slice(0, 5 * 1024 * 1024 + 1); public S3ClientLiveTest() { this.provider = "s3"; @@ -480,6 +491,53 @@ public class S3ClientLiveTest extends BaseBlobStoreIntegrationTest { } } + public void testMultipartSynchronously() throws InterruptedException, IOException { + HashCode oneHundredOneConstitutionsMD5 = oneHundredOneConstitutions.hash(md5()); + String containerName = getContainerName(); + S3Object object = null; + try { + String key = "constitution.txt"; + String uploadId = getApi().initiateMultipartUpload(containerName, + ObjectMetadataBuilder.create().key(key).contentMD5(oneHundredOneConstitutionsMD5.asBytes()).build()); + byte[] buffer = oneHundredOneConstitutions.read(); + assertEquals(oneHundredOneConstitutions.size(), (long) buffer.length); + + Payload part1 = newByteArrayPayload(buffer); + part1.getContentMetadata().setContentLength((long) buffer.length); + part1.getContentMetadata().setContentMD5(oneHundredOneConstitutionsMD5); + + String eTagOf1 = null; + try { + eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1); + } catch (KeyNotFoundException e) { + // note that because of eventual consistency, the upload id may not be present yet + // we may wish to add this condition to the retry handler + + // we may also choose to implement ListParts and wait for the uploadId to become + // available there. + eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1); + } + + String eTag = getApi().completeMultipartUpload(containerName, key, uploadId, ImmutableMap.of(1, eTagOf1)); + + assert !eTagOf1.equals(eTag); + + object = getApi().getObject(containerName, key); + assertEquals(ByteStreams2.toByteArrayAndClose(object.getPayload().openStream()), buffer); + + // noticing amazon does not return content-md5 header or a parsable ETag after a multi-part + // upload is complete: + // https://forums.aws.amazon.com/thread.jspa?threadID=61344 + assertEquals(object.getPayload().getContentMetadata().getContentMD5(), null); + assertEquals(getApi().headObject(containerName, key).getContentMetadata().getContentMD5(), null); + + } finally { + if (object != null) + object.getPayload().close(); + returnContainer(containerName); + } + } + private void checkGrants(AccessControlList acl) { String ownerId = acl.getOwner().getId(); diff --git a/apis/s3/src/test/java/org/jclouds/s3/S3ClientTest.java b/apis/s3/src/test/java/org/jclouds/s3/S3ClientTest.java index fb5aeb97c4..4432efcd09 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/S3ClientTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/S3ClientTest.java @@ -20,6 +20,7 @@ import static org.jclouds.reflect.Reflection2.method; import static org.testng.Assert.assertEquals; import java.io.IOException; +import java.util.Map; import org.jclouds.Fallbacks.VoidOnNotFoundOr404; import org.jclouds.aws.domain.Region; @@ -30,11 +31,14 @@ import org.jclouds.blobstore.BlobStoreFallbacks.ThrowContainerNotFoundOn404; import org.jclouds.blobstore.BlobStoreFallbacks.ThrowKeyNotFoundOn404; import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest; import org.jclouds.date.TimeStamp; +import org.jclouds.fallbacks.MapHttp4xxCodesToExceptions; import org.jclouds.http.functions.ParseETagHeader; import org.jclouds.http.functions.ParseSax; import org.jclouds.http.functions.ReleasePayloadAndReturn; import org.jclouds.http.functions.ReturnTrueIf2xx; import org.jclouds.http.options.GetOptions; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; import org.jclouds.rest.ConfiguresHttpApi; import org.jclouds.rest.internal.GeneratedHttpRequest; import org.jclouds.s3.S3Fallbacks.TrueOn404OrNotFoundFalseOnIllegalState; @@ -45,11 +49,15 @@ import org.jclouds.s3.domain.AccessControlList.Grant; import org.jclouds.s3.domain.AccessControlList.Permission; import org.jclouds.s3.domain.BucketLogging; import org.jclouds.s3.domain.CannedAccessPolicy; +import org.jclouds.s3.domain.ObjectMetadata; +import org.jclouds.s3.domain.ObjectMetadataBuilder; import org.jclouds.s3.domain.Payer; import org.jclouds.s3.domain.S3Object; import org.jclouds.s3.fallbacks.FalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists; +import org.jclouds.s3.functions.ETagFromHttpResponseViaRegex; import org.jclouds.s3.functions.ParseObjectFromHeadersAndHttpContent; import org.jclouds.s3.functions.ParseObjectMetadataFromHeaders; +import org.jclouds.s3.functions.UploadIdFromHttpResponseViaRegex; import org.jclouds.s3.internal.BaseS3ClientTest; import org.jclouds.s3.options.CopyObjectOptions; import org.jclouds.s3.options.ListBucketOptions; @@ -67,6 +75,7 @@ import org.testng.annotations.Test; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.reflect.Invokable; @@ -473,6 +482,95 @@ public abstract class S3ClientTest extends BaseS3ClientTest< checkFilters(request); } + public void testInitiateMultipartUpload() throws SecurityException, NegativeArraySizeException, + NoSuchMethodException { + Invokable method = method(S3Client.class, "initiateMultipartUpload", String.class, ObjectMetadata.class, + PutObjectOptions[].class); + GeneratedHttpRequest request = processor.createRequest(method, ImmutableList. of("bucket", ObjectMetadataBuilder.create().key("foo") + .contentMD5(new byte[16]).build())); + + assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1"); + assertNonPayloadHeadersEqual(request, + "Content-MD5: AAAAAAAAAAAAAAAAAAAAAA==\n" + + "Content-Type: binary/octet-stream\n" + + "Host: bucket." + url + "\n"); + assertPayloadEquals(request, null, null, false); + + // as this is a payload-related command, but with no payload, be careful + // that we check + // filtering and do not ignore if this fails later. + request = (GeneratedHttpRequest) request.getFilters().get(0).filter(request); + + assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1"); + assertNonPayloadHeadersEqual(request, + "Authorization: AWS identity:972m/Bqn2L5FIaB+wWDeY83mGvU=\n" + + "Content-MD5: AAAAAAAAAAAAAAAAAAAAAA==\n" + + "Content-Type: binary/octet-stream\n" + + "Date: 2009-11-08T15:54:08.897Z\n" + + "Host: bucket." + url + "\n"); + assertPayloadEquals(request, null, null, false); + + assertResponseParserClassEquals(method, request, UploadIdFromHttpResponseViaRegex.class); + assertSaxResponseParserClassEquals(method, null); + assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class); + + checkFilters(request); + } + + public void testAbortMultipartUpload() throws SecurityException, NegativeArraySizeException, NoSuchMethodException { + Invokable method = method(S3Client.class, "abortMultipartUpload", String.class, String.class, String.class); + GeneratedHttpRequest request = processor.createRequest(method, ImmutableList. of("bucket", "foo", "asdsadasdas", 1, + Payloads.newStringPayload(""))); + + assertRequestLineEquals(request, "DELETE https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1"); + assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n"); + assertPayloadEquals(request, "", "application/unknown", false); + + assertResponseParserClassEquals(method, request, ReleasePayloadAndReturn.class); + assertSaxResponseParserClassEquals(method, null); + assertFallbackClassEquals(method, VoidOnNotFoundOr404.class); + + checkFilters(request); + } + + public void testUploadPart() throws SecurityException, NegativeArraySizeException, NoSuchMethodException { + Invokable method = method(S3Client.class, "uploadPart", String.class, String.class, int.class, + String.class, Payload.class); + GeneratedHttpRequest request = processor.createRequest(method, ImmutableList. of("bucket", "foo", 1, "asdsadasdas", + Payloads.newStringPayload(""))); + + assertRequestLineEquals(request, "PUT https://bucket." + url + "/foo?partNumber=1&uploadId=asdsadasdas HTTP/1.1"); + assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n"); + assertPayloadEquals(request, "", "application/unknown", false); + + assertResponseParserClassEquals(method, request, ParseETagHeader.class); + assertSaxResponseParserClassEquals(method, null); + assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class); + + checkFilters(request); + } + + public void testCompleteMultipartUpload() throws SecurityException, NegativeArraySizeException, + NoSuchMethodException { + Invokable method = method(S3Client.class, "completeMultipartUpload", String.class, String.class, + String.class, Map.class); + GeneratedHttpRequest request = processor.createRequest(method, ImmutableList. of("bucket", "foo", "asdsadasdas", + ImmutableMap. of(1, "\"a54357aff0632cce46d942af68356b38\""))); + + assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1"); + assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n"); + assertPayloadEquals( + request, + "1\"a54357aff0632cce46d942af68356b38\"", + "text/xml", false); + + assertResponseParserClassEquals(method, request, ETagFromHttpResponseViaRegex.class); + assertSaxResponseParserClassEquals(method, null); + assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class); + + checkFilters(request); + } + @ConfiguresHttpApi private static final class TestS3HttpApiModule extends S3HttpApiModule { diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequestTest.java b/apis/s3/src/test/java/org/jclouds/s3/binders/BindObjectMetadataToRequestTest.java similarity index 99% rename from providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequestTest.java rename to apis/s3/src/test/java/org/jclouds/s3/binders/BindObjectMetadataToRequestTest.java index ff60120f6f..fb43d25d9c 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequestTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/binders/BindObjectMetadataToRequestTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.binders; +package org.jclouds.s3.binders; import static org.testng.Assert.assertEquals; diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequestTest.java b/apis/s3/src/test/java/org/jclouds/s3/binders/BindPartIdsAndETagsToRequestTest.java similarity index 98% rename from providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequestTest.java rename to apis/s3/src/test/java/org/jclouds/s3/binders/BindPartIdsAndETagsToRequestTest.java index 109b9e4719..fde3e29f3f 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequestTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/binders/BindPartIdsAndETagsToRequestTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.binders; +package org.jclouds.s3.binders; import static org.testng.Assert.assertEquals; diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3BlobIntegrationLiveTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3BlobIntegrationLiveTest.java index 650ee1bca7..54890e9a8b 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3BlobIntegrationLiveTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3BlobIntegrationLiveTest.java @@ -17,10 +17,12 @@ package org.jclouds.s3.blobstore.integration; import java.io.IOException; +import java.util.Properties; import java.util.concurrent.ExecutionException; import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest; import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest; +import org.jclouds.s3.blobstore.strategy.MultipartUpload; import org.testng.annotations.Test; @Test(groups = "live", testName = "S3BlobIntegrationLiveTest") @@ -30,6 +32,18 @@ public class S3BlobIntegrationLiveTest extends BaseBlobIntegrationTest { provider = "s3"; BaseBlobStoreIntegrationTest.SANITY_CHECK_RETURNED_BUCKET_NAME = true; } + + @Override + protected Properties setupProperties() { + Properties props = super.setupProperties(); + props.setProperty("jclouds.mpu.parts.size", String.valueOf(MultipartUpload.MIN_PART_SIZE)); + return props; + } + + @Override + protected long getMinimumMultipartBlobSize() { + return MultipartUpload.MIN_PART_SIZE + 1; + } @Override @Test(expectedExceptions = IllegalArgumentException.class) diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/strategy/internal/MpuGraphData.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuGraphData.java similarity index 95% rename from providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/strategy/internal/MpuGraphData.java rename to apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuGraphData.java index 0f1d9ba826..1bf0fe7404 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/strategy/internal/MpuGraphData.java +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuGraphData.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.blobstore.strategy.internal; +package org.jclouds.s3.blobstore.strategy.internal; -import org.jclouds.aws.s3.blobstore.strategy.MultipartUpload; +import org.jclouds.s3.blobstore.strategy.MultipartUpload; /** * Print out on the console some graph data regarding the partitioning algorithm. diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java similarity index 98% rename from providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java rename to apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java index 1bc0df51d7..ac6dc7c6d4 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.blobstore.strategy.internal; +package org.jclouds.s3.blobstore.strategy.internal; import static org.testng.Assert.assertEquals; -import org.jclouds.aws.s3.blobstore.strategy.MultipartUpload; +import org.jclouds.s3.blobstore.strategy.MultipartUpload; import org.testng.annotations.Test; /** diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategyMockTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategyMockTest.java similarity index 98% rename from providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategyMockTest.java rename to apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategyMockTest.java index 6b9269b704..cdff2ccf18 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategyMockTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategyMockTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.blobstore.strategy.internal; +package org.jclouds.s3.blobstore.strategy.internal; import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; import static org.jclouds.Constants.PROPERTY_MAX_RETRIES; @@ -137,7 +137,7 @@ public class SequentialMultipartUploadStrategyMockTest { overrides.setProperty(PROPERTY_SO_TIMEOUT, "0"); overrides.setProperty(PROPERTY_MAX_RETRIES, "1"); overrides.setProperty("jclouds.mpu.parts.size", String.valueOf(partSize)); - return ContextBuilder.newBuilder("aws-s3") + return ContextBuilder.newBuilder("s3") .credentials("accessKey", "secretKey") .endpoint(uri) .overrides(overrides) diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegexTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/ETagFromHttpResponseViaRegexTest.java similarity index 97% rename from providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegexTest.java rename to apis/s3/src/test/java/org/jclouds/s3/functions/ETagFromHttpResponseViaRegexTest.java index f363baca61..8b85f1a861 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegexTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/functions/ETagFromHttpResponseViaRegexTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.functions; +package org.jclouds.s3.functions; import static org.testng.Assert.assertEquals; diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegexTest.java b/apis/s3/src/test/java/org/jclouds/s3/functions/UploadIdFromHttpResponseViaRegexTest.java similarity index 97% rename from providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegexTest.java rename to apis/s3/src/test/java/org/jclouds/s3/functions/UploadIdFromHttpResponseViaRegexTest.java index 2e2f66db7d..9aee98554e 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegexTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/functions/UploadIdFromHttpResponseViaRegexTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.aws.s3.functions; +package org.jclouds.s3.functions; import static org.testng.Assert.assertEquals; diff --git a/providers/aws-s3/src/test/resources/complete-multipart-upload.xml b/apis/s3/src/test/resources/complete-multipart-upload.xml similarity index 100% rename from providers/aws-s3/src/test/resources/complete-multipart-upload.xml rename to apis/s3/src/test/resources/complete-multipart-upload.xml diff --git a/providers/aws-s3/src/test/resources/initiate-multipart-upload.xml b/apis/s3/src/test/resources/initiate-multipart-upload.xml similarity index 100% rename from providers/aws-s3/src/test/resources/initiate-multipart-upload.xml rename to apis/s3/src/test/resources/initiate-multipart-upload.xml diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java index 7dccd707da..0f9176cf64 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java @@ -16,46 +16,27 @@ */ package org.jclouds.aws.s3; -import static org.jclouds.Fallbacks.VoidOnNotFoundOr404; import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER; -import java.util.Map; - import javax.inject.Named; -import javax.ws.rs.DELETE; import javax.ws.rs.POST; -import javax.ws.rs.PUT; import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.QueryParam; import org.jclouds.aws.s3.binders.BindIterableAsPayloadToDeleteRequest; -import org.jclouds.aws.s3.binders.BindObjectMetadataToRequest; -import org.jclouds.aws.s3.binders.BindPartIdsAndETagsToRequest; import org.jclouds.aws.s3.domain.DeleteResult; -import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex; -import org.jclouds.aws.s3.functions.ObjectMetadataKey; -import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex; import org.jclouds.aws.s3.xml.DeleteResultHandler; import org.jclouds.blobstore.attr.BlobScope; -import org.jclouds.http.functions.ParseETagHeader; -import org.jclouds.io.Payload; import org.jclouds.rest.annotations.BinderParam; import org.jclouds.rest.annotations.EndpointParam; -import org.jclouds.rest.annotations.Fallback; -import org.jclouds.rest.annotations.ParamParser; import org.jclouds.rest.annotations.ParamValidators; import org.jclouds.rest.annotations.QueryParams; import org.jclouds.rest.annotations.RequestFilters; -import org.jclouds.rest.annotations.ResponseParser; import org.jclouds.rest.annotations.XMLResponseParser; import org.jclouds.s3.Bucket; import org.jclouds.s3.S3Client; import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured; -import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.filters.RequestAuthorizeSignature; import org.jclouds.s3.functions.AssignCorrectHostnameForBucket; -import org.jclouds.s3.options.PutObjectOptions; import org.jclouds.s3.predicates.validators.BucketNameValidator; /** @@ -65,138 +46,6 @@ import org.jclouds.s3.predicates.validators.BucketNameValidator; @BlobScope(CONTAINER) public interface AWSS3Client extends S3Client { - /** - * This operation initiates a multipart upload and returns an upload ID. This upload ID is used - * to associate all the parts in the specific multipart upload. You specify this upload ID in - * each of your subsequent upload part requests (see Upload Part). You also include this upload - * ID in the final request to either complete or abort the multipart upload request. - * - *

Note

If you create an object using the multipart upload APIs, currently you cannot - * copy the object between regions. - * - * - * @param bucketName - * namespace of the object you are to upload - * @param objectMetadata - * metadata around the object you wish to upload - * @param options - * controls optional parameters such as canned ACL - * @return ID for the initiated multipart upload. - */ - @Named("PutObject") - @POST - @QueryParams(keys = "uploads") - @Path("/{key}") - @ResponseParser(UploadIdFromHttpResponseViaRegex.class) - String initiateMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam( - BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, - @PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class) - ObjectMetadata objectMetadata, PutObjectOptions... options); - - /** - * This operation aborts a multipart upload. After a multipart upload is aborted, no additional - * parts can be uploaded using that upload ID. The storage consumed by any previously uploaded - * parts will be freed. However, if any part uploads are currently in progress, those part - * uploads might or might not succeed. As a result, it might be necessary to abort a given - * multipart upload multiple times in order to completely free all storage consumed by all parts. - * - * - * @param bucketName - * namespace of the object you are deleting - * @param key - * unique key in the s3Bucket identifying the object - * @param uploadId - * id of the multipart upload in progress. - */ - @Named("AbortMultipartUpload") - @DELETE - @Path("/{key}") - @Fallback(VoidOnNotFoundOr404.class) - void abortMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam( - BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, - @PathParam("key") String key, @QueryParam("uploadId") String uploadId); - - /** - * This operation uploads a part in a multipart upload. You must initiate a multipart upload (see - * Initiate Multipart Upload) before you can upload any part. In response to your initiate - * request. Amazon S3 returns an upload ID, a unique identifier, that you must include in your - * upload part request. - * - *

- * Part numbers can be any number from 1 to 10,000, inclusive. A part number uniquely identifies - * a part and also defines its position within the object being created. If you upload a new part - * using the same part number that was used with a previous part, the previously uploaded part is - * overwritten. Each part must be at least 5 MB in size, except the last part. There is no size - * limit on the last part of your multipart upload. - * - *

- * To ensure that data is not corrupted when traversing the network, specify the Content-MD5 - * header in the upload part request. Amazon S3 checks the part data against the provided MD5 - * value. If they do not match, Amazon S3 returns an error. - * - * - * @param bucketName - * namespace of the object you are storing - * @param key - * unique key in the s3Bucket identifying the object - * @param partNumber - * which part is this. - * @param uploadId - * id of the multipart upload in progress. - * @param part - * contains the data to create or overwrite - * @return ETag of the content uploaded - */ - @Named("PutObject") - @PUT - @Path("/{key}") - @ResponseParser(ParseETagHeader.class) - String uploadPart(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam( - BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, - @PathParam("key") String key, @QueryParam("partNumber") int partNumber, - @QueryParam("uploadId") String uploadId, Payload part); - - /** - * - This operation completes a multipart upload by assembling previously uploaded parts. - *

- * You first initiate the multipart upload and then upload all parts using the Upload Parts - * operation (see Upload Part). After successfully uploading all relevant parts of an upload, you - * call this operation to complete the upload. Upon receiving this request, Amazon S3 - * concatenates all the parts in ascending order by part number to create a new object. In the - * Complete Multipart Upload request, you must provide the parts list. For each part in the list, - * you must provide the part number and the ETag header value, returned after that part was - * uploaded. - *

- * Processing of a Complete Multipart Upload request could take several minutes to complete. - * After Amazon S3 begins processing the request, it sends an HTTP response header that specifies - * a 200 OK response. While processing is in progress, Amazon S3 periodically sends whitespace - * characters to keep the connection from timing out. Because a request could fail after the - * initial 200 OK response has been sent, it is important that you check the response body to - * determine whether the request succeeded. - *

- * Note that if Complete Multipart Upload fails, applications should be prepared to retry the - * failed requests. - * - * @param bucketName - * namespace of the object you are deleting - * @param key - * unique key in the s3Bucket identifying the object - * @param uploadId - * id of the multipart upload in progress. - * @param parts - * a map of part id to eTag from the {@link #uploadPart} command. - * @return ETag of the content uploaded - */ - @Named("PutObject") - @POST - @Path("/{key}") - @ResponseParser(ETagFromHttpResponseViaRegex.class) - String completeMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam( - BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName, - @PathParam("key") String key, @QueryParam("uploadId") String uploadId, - @BinderParam(BindPartIdsAndETagsToRequest.class) Map parts); - /** * The Multi-Object Delete operation enables you to delete multiple objects from a bucket using a * single HTTP request. If you know the object keys that you want to delete, then this operation diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java index a9b8b1e210..44507374b9 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java @@ -27,7 +27,6 @@ import org.jclouds.aws.domain.Region; import org.jclouds.aws.s3.AWSS3Client; import org.jclouds.aws.s3.blobstore.options.AWSS3PutObjectOptions; import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions; -import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.PageSet; @@ -45,6 +44,7 @@ import org.jclouds.s3.blobstore.functions.BucketToResourceList; import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; +import org.jclouds.s3.blobstore.strategy.MultipartUploadStrategy; import org.jclouds.s3.domain.AccessControlList; import org.jclouds.s3.domain.BucketMetadata; import org.jclouds.s3.domain.CannedAccessPolicy; @@ -60,7 +60,6 @@ import com.google.common.cache.LoadingCache; */ public class AWSS3BlobStore extends S3BlobStore { - private final Provider multipartUploadStrategy; private final LoadingCache bucketAcls; private final BlobToObject blob2Object; @@ -75,8 +74,8 @@ public class AWSS3BlobStore extends S3BlobStore { Provider multipartUploadStrategy) { super(context, blobUtils, defaultLocation, locations, sync, convertBucketsToStorageMetadata, container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object, - object2BlobMd, fetchBlobMetadataProvider, bucketAcls); - this.multipartUploadStrategy = multipartUploadStrategy; + object2BlobMd, fetchBlobMetadataProvider, bucketAcls, + multipartUploadStrategy); this.bucketAcls = bucketAcls; this.blob2Object = blob2Object; } diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/config/AWSS3BlobStoreContextModule.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/config/AWSS3BlobStoreContextModule.java index bfd671630d..6c551d5d71 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/config/AWSS3BlobStoreContextModule.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/config/AWSS3BlobStoreContextModule.java @@ -18,10 +18,6 @@ package org.jclouds.aws.s3.blobstore.config; import org.jclouds.aws.s3.blobstore.AWSS3BlobRequestSigner; import org.jclouds.aws.s3.blobstore.AWSS3BlobStore; -import org.jclouds.aws.s3.blobstore.strategy.AsyncMultipartUploadStrategy; -import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy; -import org.jclouds.aws.s3.blobstore.strategy.internal.ParallelMultipartUploadStrategy; -import org.jclouds.aws.s3.blobstore.strategy.internal.SequentialMultipartUploadStrategy; import org.jclouds.blobstore.BlobRequestSigner; import org.jclouds.s3.blobstore.S3BlobStore; import org.jclouds.s3.blobstore.config.S3BlobStoreContextModule; @@ -34,8 +30,6 @@ public class AWSS3BlobStoreContextModule extends S3BlobStoreContextModule { protected void configure() { super.configure(); bind(S3BlobStore.class).to(AWSS3BlobStore.class).in(Scopes.SINGLETON); - bind(MultipartUploadStrategy.class).to(SequentialMultipartUploadStrategy.class); - bind(AsyncMultipartUploadStrategy.class).to(ParallelMultipartUploadStrategy.class); } @Override diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java index d6c71b706d..f660b8d924 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java @@ -16,16 +16,13 @@ */ package org.jclouds.aws.s3; -import static com.google.common.hash.Hashing.md5; import static org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions.Builder.storageClass; -import static org.jclouds.io.Payloads.newByteArrayPayload; import static org.jclouds.s3.options.ListBucketOptions.Builder.withPrefix; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import java.io.IOException; import java.util.Set; import java.util.UUID; @@ -33,28 +30,19 @@ import org.jclouds.aws.AWSResponseException; import org.jclouds.aws.domain.Region; import org.jclouds.aws.s3.domain.DeleteResult; import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.domain.Location; -import org.jclouds.io.ByteStreams2; -import org.jclouds.io.Payload; import org.jclouds.s3.S3Client; import org.jclouds.s3.S3ClientLiveTest; import org.jclouds.s3.domain.ListBucketResponse; import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.domain.ObjectMetadata.StorageClass; -import org.jclouds.s3.domain.ObjectMetadataBuilder; -import org.jclouds.s3.domain.S3Object; -import org.jclouds.utils.TestUtils; import org.testng.ITestContext; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.hash.HashCode; -import com.google.common.io.ByteSource; /** * Tests behavior of {@code S3Client} @@ -65,8 +53,6 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest { provider = "aws-s3"; } - private static final ByteSource oneHundredOneConstitutions = TestUtils.randomByteSource().slice(0, 5 * 1024 * 1024 + 1); - @Override public AWSS3Client getApi() { return view.unwrapApi(AWSS3Client.class); @@ -78,53 +64,6 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest { super.setUpResourcesOnThisThread(testContext); } - public void testMultipartSynchronously() throws InterruptedException, IOException { - HashCode oneHundredOneConstitutionsMD5 = oneHundredOneConstitutions.hash(md5()); - String containerName = getContainerName(); - S3Object object = null; - try { - String key = "constitution.txt"; - String uploadId = getApi().initiateMultipartUpload(containerName, - ObjectMetadataBuilder.create().key(key).contentMD5(oneHundredOneConstitutionsMD5.asBytes()).build()); - byte[] buffer = oneHundredOneConstitutions.read(); - assertEquals(oneHundredOneConstitutions.size(), (long) buffer.length); - - Payload part1 = newByteArrayPayload(buffer); - part1.getContentMetadata().setContentLength((long) buffer.length); - part1.getContentMetadata().setContentMD5(oneHundredOneConstitutionsMD5); - - String eTagOf1 = null; - try { - eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1); - } catch (KeyNotFoundException e) { - // note that because of eventual consistency, the upload id may not be present yet - // we may wish to add this condition to the retry handler - - // we may also choose to implement ListParts and wait for the uploadId to become - // available there. - eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1); - } - - String eTag = getApi().completeMultipartUpload(containerName, key, uploadId, ImmutableMap.of(1, eTagOf1)); - - assert !eTagOf1.equals(eTag); - - object = getApi().getObject(containerName, key); - assertEquals(ByteStreams2.toByteArrayAndClose(object.getPayload().openStream()), buffer); - - // noticing amazon does not return content-md5 header or a parsable ETag after a multi-part - // upload is complete: - // https://forums.aws.amazon.com/thread.jspa?threadID=61344 - assertEquals(object.getPayload().getContentMetadata().getContentMD5(), null); - assertEquals(getApi().headObject(containerName, key).getContentMetadata().getContentMD5(), null); - - } finally { - if (object != null) - object.getPayload().close(); - returnContainer(containerName); - } - } - public void testPutWithReducedRedundancyStorage() throws InterruptedException { String containerName = getContainerName(); try { diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientTest.java index c68b7634f0..f260c591cb 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientTest.java @@ -20,31 +20,21 @@ import static org.jclouds.reflect.Reflection2.method; import static org.testng.Assert.assertEquals; import java.io.IOException; -import java.util.Map; import java.util.Set; -import org.jclouds.Fallbacks.VoidOnNotFoundOr404; import org.jclouds.aws.s3.config.AWSS3HttpApiModule; import org.jclouds.aws.s3.filters.AWSRequestAuthorizeSignature; -import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex; -import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex; import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest; import org.jclouds.date.TimeStamp; -import org.jclouds.fallbacks.MapHttp4xxCodesToExceptions; import org.jclouds.http.HttpRequest; import org.jclouds.http.functions.ParseETagHeader; import org.jclouds.http.functions.ParseSax; -import org.jclouds.http.functions.ReleasePayloadAndReturn; import org.jclouds.http.functions.ReturnTrueIf2xx; -import org.jclouds.io.Payload; -import org.jclouds.io.Payloads; import org.jclouds.location.Region; import org.jclouds.rest.ConfiguresHttpApi; import org.jclouds.rest.internal.GeneratedHttpRequest; import org.jclouds.s3.S3Client; import org.jclouds.s3.S3ClientTest; -import org.jclouds.s3.domain.ObjectMetadata; -import org.jclouds.s3.domain.ObjectMetadataBuilder; import org.jclouds.s3.domain.S3Object; import org.jclouds.s3.fallbacks.FalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists; import org.jclouds.s3.options.CopyObjectOptions; @@ -161,95 +151,6 @@ public class AWSS3ClientTest extends S3ClientTest { checkFilters(request); } - public void testInitiateMultipartUpload() throws SecurityException, NegativeArraySizeException, - NoSuchMethodException { - Invokable method = method(AWSS3Client.class, "initiateMultipartUpload", String.class, ObjectMetadata.class, - PutObjectOptions[].class); - GeneratedHttpRequest request = processor.createRequest(method, ImmutableList. of("bucket", ObjectMetadataBuilder.create().key("foo") - .contentMD5(new byte[16]).build())); - - assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1"); - assertNonPayloadHeadersEqual(request, - "Content-MD5: AAAAAAAAAAAAAAAAAAAAAA==\n" + - "Content-Type: binary/octet-stream\n" + - "Host: bucket." + url + "\n"); - assertPayloadEquals(request, null, null, false); - - // as this is a payload-related command, but with no payload, be careful - // that we check - // filtering and do not ignore if this fails later. - request = (GeneratedHttpRequest) request.getFilters().get(0).filter(request); - - assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1"); - assertNonPayloadHeadersEqual(request, - "Authorization: AWS identity:972m/Bqn2L5FIaB+wWDeY83mGvU=\n" + - "Content-MD5: AAAAAAAAAAAAAAAAAAAAAA==\n" + - "Content-Type: binary/octet-stream\n" + - "Date: 2009-11-08T15:54:08.897Z\n" + - "Host: bucket." + url + "\n"); - assertPayloadEquals(request, null, null, false); - - assertResponseParserClassEquals(method, request, UploadIdFromHttpResponseViaRegex.class); - assertSaxResponseParserClassEquals(method, null); - assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class); - - checkFilters(request); - } - - public void testAbortMultipartUpload() throws SecurityException, NegativeArraySizeException, NoSuchMethodException { - Invokable method = method(AWSS3Client.class, "abortMultipartUpload", String.class, String.class, String.class); - GeneratedHttpRequest request = processor.createRequest(method, ImmutableList. of("bucket", "foo", "asdsadasdas", 1, - Payloads.newStringPayload(""))); - - assertRequestLineEquals(request, "DELETE https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1"); - assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n"); - assertPayloadEquals(request, "", "application/unknown", false); - - assertResponseParserClassEquals(method, request, ReleasePayloadAndReturn.class); - assertSaxResponseParserClassEquals(method, null); - assertFallbackClassEquals(method, VoidOnNotFoundOr404.class); - - checkFilters(request); - } - - public void testUploadPart() throws SecurityException, NegativeArraySizeException, NoSuchMethodException { - Invokable method = method(AWSS3Client.class, "uploadPart", String.class, String.class, int.class, - String.class, Payload.class); - GeneratedHttpRequest request = processor.createRequest(method, ImmutableList. of("bucket", "foo", 1, "asdsadasdas", - Payloads.newStringPayload(""))); - - assertRequestLineEquals(request, "PUT https://bucket." + url + "/foo?partNumber=1&uploadId=asdsadasdas HTTP/1.1"); - assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n"); - assertPayloadEquals(request, "", "application/unknown", false); - - assertResponseParserClassEquals(method, request, ParseETagHeader.class); - assertSaxResponseParserClassEquals(method, null); - assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class); - - checkFilters(request); - } - - public void testCompleteMultipartUpload() throws SecurityException, NegativeArraySizeException, - NoSuchMethodException { - Invokable method = method(AWSS3Client.class, "completeMultipartUpload", String.class, String.class, - String.class, Map.class); - GeneratedHttpRequest request = processor.createRequest(method, ImmutableList. of("bucket", "foo", "asdsadasdas", - ImmutableMap. of(1, "\"a54357aff0632cce46d942af68356b38\""))); - - assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1"); - assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n"); - assertPayloadEquals( - request, - "1\"a54357aff0632cce46d942af68356b38\"", - "text/xml", false); - - assertResponseParserClassEquals(method, request, ETagFromHttpResponseViaRegex.class); - assertSaxResponseParserClassEquals(method, null); - assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class); - - checkFilters(request); - } - public void testPutBucketEu() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException, NoSuchMethodException, IOException { Invokable method = method(AWSS3Client.class, "putBucketInRegion", String.class, String.class, diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3BlobIntegrationLiveTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3BlobIntegrationLiveTest.java index 775926897b..cba1f72536 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3BlobIntegrationLiveTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3BlobIntegrationLiveTest.java @@ -16,9 +16,6 @@ */ package org.jclouds.aws.s3.blobstore.integration; -import java.util.Properties; - -import org.jclouds.aws.s3.blobstore.strategy.MultipartUpload; import org.jclouds.s3.blobstore.integration.S3BlobIntegrationLiveTest; import org.testng.annotations.Test; @@ -27,16 +24,4 @@ public class AWSS3BlobIntegrationLiveTest extends S3BlobIntegrationLiveTest { public AWSS3BlobIntegrationLiveTest() { provider = "aws-s3"; } - - @Override - protected Properties setupProperties() { - Properties props = super.setupProperties(); - props.setProperty("jclouds.mpu.parts.size", String.valueOf(MultipartUpload.MIN_PART_SIZE)); - return props; - } - - @Override - protected long getMinimumMultipartBlobSize() { - return MultipartUpload.MIN_PART_SIZE + 1; - } }