mirror of https://github.com/apache/jclouds.git
JCLOUDS-258: Support MPU for generic S3
Tested against AWS-S3 and DreamObjects. This commit only moves and renames code although some classes are duplicated for deprecation purposes.
This commit is contained in:
Andrew Gaul
parent
fcc991fa22
commit
ba2f8ac232
|
|
@ -21,16 +21,19 @@ import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER;
|
|||
import static org.jclouds.s3.S3Fallbacks.TrueOn404OrNotFoundFalseOnIllegalState;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.inject.Named;
|
||||
import javax.ws.rs.DELETE;
|
||||
import javax.ws.rs.GET;
|
||||
import javax.ws.rs.HEAD;
|
||||
import javax.ws.rs.POST;
|
||||
import javax.ws.rs.PUT;
|
||||
import javax.ws.rs.Path;
|
||||
import javax.ws.rs.PathParam;
|
||||
import javax.ws.rs.Produces;
|
||||
import javax.ws.rs.QueryParam;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
|
||||
import org.jclouds.Fallbacks.VoidOnNotFoundOr404;
|
||||
|
|
@ -42,6 +45,7 @@ import org.jclouds.blobstore.BlobStoreFallbacks.ThrowKeyNotFoundOn404;
|
|||
import org.jclouds.blobstore.attr.BlobScope;
|
||||
import org.jclouds.http.functions.ParseETagHeader;
|
||||
import org.jclouds.http.options.GetOptions;
|
||||
import org.jclouds.io.Payload;
|
||||
import org.jclouds.javax.annotation.Nullable;
|
||||
import org.jclouds.rest.annotations.BinderParam;
|
||||
import org.jclouds.rest.annotations.Endpoint;
|
||||
|
|
@ -59,6 +63,8 @@ import org.jclouds.s3.binders.BindACLToXMLPayload;
|
|||
import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured;
|
||||
import org.jclouds.s3.binders.BindBucketLoggingToXmlPayload;
|
||||
import org.jclouds.s3.binders.BindNoBucketLoggingToXmlPayload;
|
||||
import org.jclouds.s3.binders.BindObjectMetadataToRequest;
|
||||
import org.jclouds.s3.binders.BindPartIdsAndETagsToRequest;
|
||||
import org.jclouds.s3.binders.BindPayerToXmlPayload;
|
||||
import org.jclouds.s3.binders.BindS3ObjectMetadataToRequest;
|
||||
import org.jclouds.s3.domain.AccessControlList;
|
||||
|
|
@ -73,9 +79,12 @@ import org.jclouds.s3.filters.RequestAuthorizeSignature;
|
|||
import org.jclouds.s3.functions.AssignCorrectHostnameForBucket;
|
||||
import org.jclouds.s3.functions.BindRegionToXmlPayload;
|
||||
import org.jclouds.s3.functions.DefaultEndpointThenInvalidateRegion;
|
||||
import org.jclouds.s3.functions.ETagFromHttpResponseViaRegex;
|
||||
import org.jclouds.s3.functions.ObjectKey;
|
||||
import org.jclouds.s3.functions.ObjectMetadataKey;
|
||||
import org.jclouds.s3.functions.ParseObjectFromHeadersAndHttpContent;
|
||||
import org.jclouds.s3.functions.ParseObjectMetadataFromHeaders;
|
||||
import org.jclouds.s3.functions.UploadIdFromHttpResponseViaRegex;
|
||||
import org.jclouds.s3.options.CopyObjectOptions;
|
||||
import org.jclouds.s3.options.ListBucketOptions;
|
||||
import org.jclouds.s3.options.PutBucketOptions;
|
||||
|
|
@ -545,4 +554,136 @@ public interface S3Client extends Closeable {
|
|||
@Produces(MediaType.TEXT_XML)
|
||||
void disableBucketLogging(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(
|
||||
BindNoBucketLoggingToXmlPayload.class) @ParamValidators(BucketNameValidator.class) String bucketName);
|
||||
|
||||
/**
|
||||
* This operation initiates a multipart upload and returns an upload ID. This upload ID is used
|
||||
* to associate all the parts in the specific multipart upload. You specify this upload ID in
|
||||
* each of your subsequent upload part requests (see Upload Part). You also include this upload
|
||||
* ID in the final request to either complete or abort the multipart upload request.
|
||||
*
|
||||
* <h4>Note</h4> If you create an object using the multipart upload APIs, currently you cannot
|
||||
* copy the object between regions.
|
||||
*
|
||||
*
|
||||
* @param bucketName
|
||||
* namespace of the object you are to upload
|
||||
* @param objectMetadata
|
||||
* metadata around the object you wish to upload
|
||||
* @param options
|
||||
* controls optional parameters such as canned ACL
|
||||
* @return ID for the initiated multipart upload.
|
||||
*/
|
||||
@Named("PutObject")
|
||||
@POST
|
||||
@QueryParams(keys = "uploads")
|
||||
@Path("/{key}")
|
||||
@ResponseParser(UploadIdFromHttpResponseViaRegex.class)
|
||||
String initiateMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(
|
||||
BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
|
||||
@PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class)
|
||||
ObjectMetadata objectMetadata, PutObjectOptions... options);
|
||||
|
||||
/**
|
||||
* This operation aborts a multipart upload. After a multipart upload is aborted, no additional
|
||||
* parts can be uploaded using that upload ID. The storage consumed by any previously uploaded
|
||||
* parts will be freed. However, if any part uploads are currently in progress, those part
|
||||
* uploads might or might not succeed. As a result, it might be necessary to abort a given
|
||||
* multipart upload multiple times in order to completely free all storage consumed by all parts.
|
||||
*
|
||||
*
|
||||
* @param bucketName
|
||||
* namespace of the object you are deleting
|
||||
* @param key
|
||||
* unique key in the s3Bucket identifying the object
|
||||
* @param uploadId
|
||||
* id of the multipart upload in progress.
|
||||
*/
|
||||
@Named("AbortMultipartUpload")
|
||||
@DELETE
|
||||
@Path("/{key}")
|
||||
@Fallback(VoidOnNotFoundOr404.class)
|
||||
void abortMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(
|
||||
BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
|
||||
@PathParam("key") String key, @QueryParam("uploadId") String uploadId);
|
||||
|
||||
/**
|
||||
* This operation uploads a part in a multipart upload. You must initiate a multipart upload (see
|
||||
* Initiate Multipart Upload) before you can upload any part. In response to your initiate
|
||||
* request. Amazon S3 returns an upload ID, a unique identifier, that you must include in your
|
||||
* upload part request.
|
||||
*
|
||||
* <p/>
|
||||
* Part numbers can be any number from 1 to 10,000, inclusive. A part number uniquely identifies
|
||||
* a part and also defines its position within the object being created. If you upload a new part
|
||||
* using the same part number that was used with a previous part, the previously uploaded part is
|
||||
* overwritten. Each part must be at least 5 MB in size, except the last part. There is no size
|
||||
* limit on the last part of your multipart upload.
|
||||
*
|
||||
* <p/>
|
||||
* To ensure that data is not corrupted when traversing the network, specify the Content-MD5
|
||||
* header in the upload part request. Amazon S3 checks the part data against the provided MD5
|
||||
* value. If they do not match, Amazon S3 returns an error.
|
||||
*
|
||||
*
|
||||
* @param bucketName
|
||||
* namespace of the object you are storing
|
||||
* @param key
|
||||
* unique key in the s3Bucket identifying the object
|
||||
* @param partNumber
|
||||
* which part is this.
|
||||
* @param uploadId
|
||||
* id of the multipart upload in progress.
|
||||
* @param part
|
||||
* contains the data to create or overwrite
|
||||
* @return ETag of the content uploaded
|
||||
*/
|
||||
@Named("PutObject")
|
||||
@PUT
|
||||
@Path("/{key}")
|
||||
@ResponseParser(ParseETagHeader.class)
|
||||
String uploadPart(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(
|
||||
BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
|
||||
@PathParam("key") String key, @QueryParam("partNumber") int partNumber,
|
||||
@QueryParam("uploadId") String uploadId, Payload part);
|
||||
|
||||
/**
|
||||
*
|
||||
This operation completes a multipart upload by assembling previously uploaded parts.
|
||||
* <p/>
|
||||
* You first initiate the multipart upload and then upload all parts using the Upload Parts
|
||||
* operation (see Upload Part). After successfully uploading all relevant parts of an upload, you
|
||||
* call this operation to complete the upload. Upon receiving this request, Amazon S3
|
||||
* concatenates all the parts in ascending order by part number to create a new object. In the
|
||||
* Complete Multipart Upload request, you must provide the parts list. For each part in the list,
|
||||
* you must provide the part number and the ETag header value, returned after that part was
|
||||
* uploaded.
|
||||
* <p/>
|
||||
* Processing of a Complete Multipart Upload request could take several minutes to complete.
|
||||
* After Amazon S3 begins processing the request, it sends an HTTP response header that specifies
|
||||
* a 200 OK response. While processing is in progress, Amazon S3 periodically sends whitespace
|
||||
* characters to keep the connection from timing out. Because a request could fail after the
|
||||
* initial 200 OK response has been sent, it is important that you check the response body to
|
||||
* determine whether the request succeeded.
|
||||
* <p/>
|
||||
* Note that if Complete Multipart Upload fails, applications should be prepared to retry the
|
||||
* failed requests.
|
||||
*
|
||||
* @param bucketName
|
||||
* namespace of the object you are deleting
|
||||
* @param key
|
||||
* unique key in the s3Bucket identifying the object
|
||||
* @param uploadId
|
||||
* id of the multipart upload in progress.
|
||||
* @param parts
|
||||
* a map of part id to eTag from the {@link #uploadPart} command.
|
||||
* @return ETag of the content uploaded
|
||||
*/
|
||||
@Named("PutObject")
|
||||
@POST
|
||||
@Path("/{key}")
|
||||
@ResponseParser(ETagFromHttpResponseViaRegex.class)
|
||||
String completeMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(
|
||||
BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
|
||||
@PathParam("key") String key, @QueryParam("uploadId") String uploadId,
|
||||
@BinderParam(BindPartIdsAndETagsToRequest.class) Map<Integer, String> parts);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.binders;
|
||||
package org.jclouds.s3.binders;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.binders;
|
||||
package org.jclouds.s3.binders;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
|
@ -49,6 +49,7 @@ import org.jclouds.s3.blobstore.functions.BucketToResourceList;
|
|||
import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions;
|
||||
import org.jclouds.s3.blobstore.functions.ObjectToBlob;
|
||||
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
|
||||
import org.jclouds.s3.blobstore.strategy.MultipartUploadStrategy;
|
||||
import org.jclouds.s3.domain.AccessControlList;
|
||||
import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI;
|
||||
import org.jclouds.s3.domain.AccessControlList.Permission;
|
||||
|
|
@ -77,6 +78,7 @@ public class S3BlobStore extends BaseBlobStore {
|
|||
private final BlobToHttpGetOptions blob2ObjectGetOptions;
|
||||
private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
|
||||
private final LoadingCache<String, AccessControlList> bucketAcls;
|
||||
protected final Provider<MultipartUploadStrategy> multipartUploadStrategy;
|
||||
|
||||
@Inject
|
||||
protected S3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
|
||||
|
|
@ -85,7 +87,8 @@ public class S3BlobStore extends BaseBlobStore {
|
|||
ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList,
|
||||
ObjectToBlob object2Blob, BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object,
|
||||
ObjectToBlobMetadata object2BlobMd, Provider<FetchBlobMetadata> fetchBlobMetadataProvider,
|
||||
LoadingCache<String, AccessControlList> bucketAcls) {
|
||||
LoadingCache<String, AccessControlList> bucketAcls,
|
||||
Provider<MultipartUploadStrategy> multipartUploadStrategy) {
|
||||
super(context, blobUtils, defaultLocation, locations);
|
||||
this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
|
||||
this.sync = checkNotNull(sync, "sync");
|
||||
|
|
@ -97,6 +100,7 @@ public class S3BlobStore extends BaseBlobStore {
|
|||
this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
|
||||
this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider");
|
||||
this.bucketAcls = checkNotNull(bucketAcls, "bucketAcls");
|
||||
this.multipartUploadStrategy = checkNotNull(multipartUploadStrategy, "multipartUploadStrategy");
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -249,6 +253,11 @@ public class S3BlobStore extends BaseBlobStore {
|
|||
*/
|
||||
@Override
|
||||
public String putBlob(String container, Blob blob, PutOptions overrides) {
|
||||
if (overrides.isMultipart()) {
|
||||
// need to use a provider if the strategy object is stateful
|
||||
return multipartUploadStrategy.get().execute(container, blob);
|
||||
}
|
||||
|
||||
// TODO: Make use of options overrides
|
||||
PutObjectOptions options = new PutObjectOptions();
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,10 @@ import org.jclouds.s3.blobstore.S3BlobRequestSigner;
|
|||
import org.jclouds.s3.blobstore.S3BlobStore;
|
||||
import org.jclouds.s3.blobstore.functions.LocationFromBucketName;
|
||||
import org.jclouds.s3.blobstore.internal.BackoffOnNotFoundWhenGetBucketACL;
|
||||
import org.jclouds.s3.blobstore.strategy.AsyncMultipartUploadStrategy;
|
||||
import org.jclouds.s3.blobstore.strategy.MultipartUploadStrategy;
|
||||
import org.jclouds.s3.blobstore.strategy.internal.ParallelMultipartUploadStrategy;
|
||||
import org.jclouds.s3.blobstore.strategy.internal.SequentialMultipartUploadStrategy;
|
||||
import org.jclouds.s3.domain.AccessControlList;
|
||||
|
||||
import com.google.common.base.Function;
|
||||
|
|
@ -49,6 +53,8 @@ public class S3BlobStoreContextModule extends AbstractModule {
|
|||
bind(new TypeLiteral<Function<String, Location>>() {
|
||||
}).to(LocationFromBucketName.class);
|
||||
bindRequestSigner();
|
||||
bind(MultipartUploadStrategy.class).to(SequentialMultipartUploadStrategy.class);
|
||||
bind(AsyncMultipartUploadStrategy.class).to(ParallelMultipartUploadStrategy.class);
|
||||
}
|
||||
|
||||
protected void bindRequestSigner() {
|
||||
|
|
|
|||
|
|
@ -14,11 +14,11 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.blobstore.strategy;
|
||||
package org.jclouds.s3.blobstore.strategy;
|
||||
|
||||
import org.jclouds.aws.s3.blobstore.strategy.internal.ParallelMultipartUploadStrategy;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
import org.jclouds.blobstore.options.PutOptions;
|
||||
import org.jclouds.s3.blobstore.strategy.internal.ParallelMultipartUploadStrategy;
|
||||
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
import com.google.inject.ImplementedBy;
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.blobstore.strategy;
|
||||
package org.jclouds.s3.blobstore.strategy;
|
||||
|
||||
public final class MultipartUpload {
|
||||
|
||||
|
|
@ -14,10 +14,10 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.blobstore.strategy;
|
||||
package org.jclouds.s3.blobstore.strategy;
|
||||
|
||||
import org.jclouds.aws.s3.blobstore.strategy.internal.SequentialMultipartUploadStrategy;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
import org.jclouds.s3.blobstore.strategy.internal.SequentialMultipartUploadStrategy;
|
||||
|
||||
import com.google.inject.ImplementedBy;
|
||||
|
||||
|
|
@ -23,12 +23,12 @@
|
|||
* History
|
||||
*/
|
||||
|
||||
package org.jclouds.aws.s3.blobstore.strategy.internal;
|
||||
package org.jclouds.s3.blobstore.strategy.internal;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import javax.inject.Named;
|
||||
|
||||
import org.jclouds.aws.s3.blobstore.strategy.MultipartUpload;
|
||||
import org.jclouds.s3.blobstore.strategy.MultipartUpload;
|
||||
import org.jclouds.blobstore.reference.BlobStoreConstants;
|
||||
import org.jclouds.logging.Logger;
|
||||
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.blobstore.strategy.internal;
|
||||
package org.jclouds.s3.blobstore.strategy.internal;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
|
|
@ -36,9 +36,7 @@ import javax.annotation.Resource;
|
|||
import javax.inject.Named;
|
||||
|
||||
import org.jclouds.Constants;
|
||||
import org.jclouds.aws.s3.AWSS3Client;
|
||||
import org.jclouds.aws.s3.blobstore.AWSS3BlobStore;
|
||||
import org.jclouds.aws.s3.blobstore.strategy.AsyncMultipartUploadStrategy;
|
||||
import org.jclouds.s3.blobstore.strategy.AsyncMultipartUploadStrategy;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
import org.jclouds.blobstore.internal.BlobRuntimeException;
|
||||
import org.jclouds.blobstore.options.PutOptions;
|
||||
|
|
@ -46,6 +44,8 @@ import org.jclouds.blobstore.reference.BlobStoreConstants;
|
|||
import org.jclouds.io.Payload;
|
||||
import org.jclouds.io.PayloadSlicer;
|
||||
import org.jclouds.logging.Logger;
|
||||
import org.jclouds.s3.S3Client;
|
||||
import org.jclouds.s3.blobstore.S3BlobStore;
|
||||
import org.jclouds.s3.domain.ObjectMetadataBuilder;
|
||||
import org.jclouds.util.Throwables2;
|
||||
|
||||
|
|
@ -91,11 +91,11 @@ public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStra
|
|||
@Named(Constants.PROPERTY_REQUEST_TIMEOUT)
|
||||
protected Long maxTime;
|
||||
|
||||
protected final AWSS3BlobStore blobstore;
|
||||
protected final S3BlobStore blobstore;
|
||||
protected final PayloadSlicer slicer;
|
||||
|
||||
@Inject
|
||||
public ParallelMultipartUploadStrategy(AWSS3BlobStore blobstore, PayloadSlicer slicer,
|
||||
public ParallelMultipartUploadStrategy(S3BlobStore blobstore, PayloadSlicer slicer,
|
||||
@Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService executor) {
|
||||
this.blobstore = checkNotNull(blobstore, "blobstore");
|
||||
this.slicer = checkNotNull(slicer, "slicer");
|
||||
|
|
@ -114,7 +114,7 @@ public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStra
|
|||
latch.countDown();
|
||||
return;
|
||||
}
|
||||
final AWSS3Client client = blobstore.getContext().unwrapApi(AWSS3Client.class);
|
||||
final S3Client client = blobstore.getContext().unwrapApi(S3Client.class);
|
||||
final Payload chunkedPart = slicer.slice(payload, offset, size);
|
||||
logger.debug(String.format("async uploading part %s of %s to container %s with uploadId %s", part, key, container, uploadId));
|
||||
final long start = System.currentTimeMillis();
|
||||
|
|
@ -166,7 +166,7 @@ public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStra
|
|||
long chunkSize = algorithm.getChunkSize();
|
||||
long remaining = algorithm.getRemaining();
|
||||
if (parts > 0) {
|
||||
final AWSS3Client client = blobstore.getContext().unwrapApi(AWSS3Client.class);
|
||||
final S3Client client = blobstore.getContext().unwrapApi(S3Client.class);
|
||||
String uploadId = null;
|
||||
final Map<Integer, ListenableFuture<String>> futureParts =
|
||||
new ConcurrentHashMap<Integer, ListenableFuture<String>>();
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.blobstore.strategy.internal;
|
||||
package org.jclouds.s3.blobstore.strategy.internal;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
|
|
@ -23,8 +23,7 @@ import java.util.SortedMap;
|
|||
import javax.annotation.Resource;
|
||||
import javax.inject.Named;
|
||||
|
||||
import org.jclouds.aws.s3.AWSS3Client;
|
||||
import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy;
|
||||
import org.jclouds.s3.blobstore.strategy.MultipartUploadStrategy;
|
||||
import org.jclouds.blobstore.KeyNotFoundException;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
import org.jclouds.blobstore.reference.BlobStoreConstants;
|
||||
|
|
@ -32,6 +31,7 @@ import org.jclouds.io.ContentMetadata;
|
|||
import org.jclouds.io.Payload;
|
||||
import org.jclouds.io.PayloadSlicer;
|
||||
import org.jclouds.logging.Logger;
|
||||
import org.jclouds.s3.S3Client;
|
||||
import org.jclouds.s3.blobstore.functions.BlobToObject;
|
||||
import org.jclouds.s3.domain.ObjectMetadataBuilder;
|
||||
|
||||
|
|
@ -54,13 +54,13 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
|
|||
@Named(BlobStoreConstants.BLOBSTORE_LOGGER)
|
||||
private Logger logger = Logger.NULL;
|
||||
|
||||
private final AWSS3Client client;
|
||||
private final S3Client client;
|
||||
private final BlobToObject blobToObject;
|
||||
private final MultipartUploadSlicingAlgorithm algorithm;
|
||||
private final PayloadSlicer slicer;
|
||||
|
||||
@Inject
|
||||
public SequentialMultipartUploadStrategy(AWSS3Client client, BlobToObject blobToObject,
|
||||
public SequentialMultipartUploadStrategy(S3Client client, BlobToObject blobToObject,
|
||||
MultipartUploadSlicingAlgorithm algorithm, PayloadSlicer slicer) {
|
||||
this.client = checkNotNull(client, "client");
|
||||
this.blobToObject = checkNotNull(blobToObject, "blobToObject");
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.functions;
|
||||
package org.jclouds.s3.functions;
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.functions;
|
||||
package org.jclouds.s3.functions;
|
||||
|
||||
import javax.inject.Singleton;
|
||||
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.functions;
|
||||
package org.jclouds.s3.functions;
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
|
@ -16,6 +16,8 @@
|
|||
*/
|
||||
package org.jclouds.s3;
|
||||
|
||||
import static com.google.common.hash.Hashing.md5;
|
||||
import static org.jclouds.io.Payloads.newByteArrayPayload;
|
||||
import static org.jclouds.s3.options.CopyObjectOptions.Builder.ifSourceETagDoesntMatch;
|
||||
import static org.jclouds.s3.options.CopyObjectOptions.Builder.ifSourceETagMatches;
|
||||
import static org.jclouds.s3.options.CopyObjectOptions.Builder.ifSourceModifiedSince;
|
||||
|
|
@ -37,8 +39,11 @@ import java.util.Map;
|
|||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import org.jclouds.blobstore.KeyNotFoundException;
|
||||
import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
|
||||
import org.jclouds.http.HttpResponseException;
|
||||
import org.jclouds.io.ByteStreams2;
|
||||
import org.jclouds.io.Payload;
|
||||
import org.jclouds.s3.domain.AccessControlList;
|
||||
import org.jclouds.s3.domain.AccessControlList.CanonicalUserGrantee;
|
||||
import org.jclouds.s3.domain.AccessControlList.EmailAddressGrantee;
|
||||
|
|
@ -46,19 +51,25 @@ import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI;
|
|||
import org.jclouds.s3.domain.AccessControlList.Permission;
|
||||
import org.jclouds.s3.domain.CannedAccessPolicy;
|
||||
import org.jclouds.s3.domain.ObjectMetadata;
|
||||
import org.jclouds.s3.domain.ObjectMetadataBuilder;
|
||||
import org.jclouds.s3.domain.S3Object;
|
||||
import org.jclouds.s3.options.PutObjectOptions;
|
||||
import org.jclouds.util.Strings2;
|
||||
import org.jclouds.utils.TestUtils;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.hash.HashCode;
|
||||
import com.google.common.io.ByteSource;
|
||||
|
||||
@Test(groups = { "integration", "live" })
|
||||
public class S3ClientLiveTest extends BaseBlobStoreIntegrationTest {
|
||||
public static final String TEST_ACL_ID = "1a405254c932b52e5b5caaa88186bc431a1bacb9ece631f835daddaf0c47677c";
|
||||
public static final String TEST_ACL_EMAIL = "james@misterm.org";
|
||||
public static final String DEFAULT_OWNER_ID = "abc123";
|
||||
private static final ByteSource oneHundredOneConstitutions = TestUtils.randomByteSource().slice(0, 5 * 1024 * 1024 + 1);
|
||||
|
||||
public S3ClientLiveTest() {
|
||||
this.provider = "s3";
|
||||
|
|
@ -480,6 +491,53 @@ public class S3ClientLiveTest extends BaseBlobStoreIntegrationTest {
|
|||
}
|
||||
}
|
||||
|
||||
public void testMultipartSynchronously() throws InterruptedException, IOException {
|
||||
HashCode oneHundredOneConstitutionsMD5 = oneHundredOneConstitutions.hash(md5());
|
||||
String containerName = getContainerName();
|
||||
S3Object object = null;
|
||||
try {
|
||||
String key = "constitution.txt";
|
||||
String uploadId = getApi().initiateMultipartUpload(containerName,
|
||||
ObjectMetadataBuilder.create().key(key).contentMD5(oneHundredOneConstitutionsMD5.asBytes()).build());
|
||||
byte[] buffer = oneHundredOneConstitutions.read();
|
||||
assertEquals(oneHundredOneConstitutions.size(), (long) buffer.length);
|
||||
|
||||
Payload part1 = newByteArrayPayload(buffer);
|
||||
part1.getContentMetadata().setContentLength((long) buffer.length);
|
||||
part1.getContentMetadata().setContentMD5(oneHundredOneConstitutionsMD5);
|
||||
|
||||
String eTagOf1 = null;
|
||||
try {
|
||||
eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
|
||||
} catch (KeyNotFoundException e) {
|
||||
// note that because of eventual consistency, the upload id may not be present yet
|
||||
// we may wish to add this condition to the retry handler
|
||||
|
||||
// we may also choose to implement ListParts and wait for the uploadId to become
|
||||
// available there.
|
||||
eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
|
||||
}
|
||||
|
||||
String eTag = getApi().completeMultipartUpload(containerName, key, uploadId, ImmutableMap.of(1, eTagOf1));
|
||||
|
||||
assert !eTagOf1.equals(eTag);
|
||||
|
||||
object = getApi().getObject(containerName, key);
|
||||
assertEquals(ByteStreams2.toByteArrayAndClose(object.getPayload().openStream()), buffer);
|
||||
|
||||
// noticing amazon does not return content-md5 header or a parsable ETag after a multi-part
|
||||
// upload is complete:
|
||||
// https://forums.aws.amazon.com/thread.jspa?threadID=61344
|
||||
assertEquals(object.getPayload().getContentMetadata().getContentMD5(), null);
|
||||
assertEquals(getApi().headObject(containerName, key).getContentMetadata().getContentMD5(), null);
|
||||
|
||||
} finally {
|
||||
if (object != null)
|
||||
object.getPayload().close();
|
||||
returnContainer(containerName);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkGrants(AccessControlList acl) {
|
||||
String ownerId = acl.getOwner().getId();
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import static org.jclouds.reflect.Reflection2.method;
|
|||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
import org.jclouds.Fallbacks.VoidOnNotFoundOr404;
|
||||
import org.jclouds.aws.domain.Region;
|
||||
|
|
@ -30,11 +31,14 @@ import org.jclouds.blobstore.BlobStoreFallbacks.ThrowContainerNotFoundOn404;
|
|||
import org.jclouds.blobstore.BlobStoreFallbacks.ThrowKeyNotFoundOn404;
|
||||
import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
|
||||
import org.jclouds.date.TimeStamp;
|
||||
import org.jclouds.fallbacks.MapHttp4xxCodesToExceptions;
|
||||
import org.jclouds.http.functions.ParseETagHeader;
|
||||
import org.jclouds.http.functions.ParseSax;
|
||||
import org.jclouds.http.functions.ReleasePayloadAndReturn;
|
||||
import org.jclouds.http.functions.ReturnTrueIf2xx;
|
||||
import org.jclouds.http.options.GetOptions;
|
||||
import org.jclouds.io.Payload;
|
||||
import org.jclouds.io.Payloads;
|
||||
import org.jclouds.rest.ConfiguresHttpApi;
|
||||
import org.jclouds.rest.internal.GeneratedHttpRequest;
|
||||
import org.jclouds.s3.S3Fallbacks.TrueOn404OrNotFoundFalseOnIllegalState;
|
||||
|
|
@ -45,11 +49,15 @@ import org.jclouds.s3.domain.AccessControlList.Grant;
|
|||
import org.jclouds.s3.domain.AccessControlList.Permission;
|
||||
import org.jclouds.s3.domain.BucketLogging;
|
||||
import org.jclouds.s3.domain.CannedAccessPolicy;
|
||||
import org.jclouds.s3.domain.ObjectMetadata;
|
||||
import org.jclouds.s3.domain.ObjectMetadataBuilder;
|
||||
import org.jclouds.s3.domain.Payer;
|
||||
import org.jclouds.s3.domain.S3Object;
|
||||
import org.jclouds.s3.fallbacks.FalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists;
|
||||
import org.jclouds.s3.functions.ETagFromHttpResponseViaRegex;
|
||||
import org.jclouds.s3.functions.ParseObjectFromHeadersAndHttpContent;
|
||||
import org.jclouds.s3.functions.ParseObjectMetadataFromHeaders;
|
||||
import org.jclouds.s3.functions.UploadIdFromHttpResponseViaRegex;
|
||||
import org.jclouds.s3.internal.BaseS3ClientTest;
|
||||
import org.jclouds.s3.options.CopyObjectOptions;
|
||||
import org.jclouds.s3.options.ListBucketOptions;
|
||||
|
|
@ -67,6 +75,7 @@ import org.testng.annotations.Test;
|
|||
|
||||
import com.google.common.base.Supplier;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.reflect.Invokable;
|
||||
|
|
@ -473,6 +482,95 @@ public abstract class S3ClientTest<T extends S3Client> extends BaseS3ClientTest<
|
|||
checkFilters(request);
|
||||
}
|
||||
|
||||
public void testInitiateMultipartUpload() throws SecurityException, NegativeArraySizeException,
|
||||
NoSuchMethodException {
|
||||
Invokable<?, ?> method = method(S3Client.class, "initiateMultipartUpload", String.class, ObjectMetadata.class,
|
||||
PutObjectOptions[].class);
|
||||
GeneratedHttpRequest request = processor.createRequest(method, ImmutableList.<Object> of("bucket", ObjectMetadataBuilder.create().key("foo")
|
||||
.contentMD5(new byte[16]).build()));
|
||||
|
||||
assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1");
|
||||
assertNonPayloadHeadersEqual(request,
|
||||
"Content-MD5: AAAAAAAAAAAAAAAAAAAAAA==\n" +
|
||||
"Content-Type: binary/octet-stream\n" +
|
||||
"Host: bucket." + url + "\n");
|
||||
assertPayloadEquals(request, null, null, false);
|
||||
|
||||
// as this is a payload-related command, but with no payload, be careful
|
||||
// that we check
|
||||
// filtering and do not ignore if this fails later.
|
||||
request = (GeneratedHttpRequest) request.getFilters().get(0).filter(request);
|
||||
|
||||
assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1");
|
||||
assertNonPayloadHeadersEqual(request,
|
||||
"Authorization: AWS identity:972m/Bqn2L5FIaB+wWDeY83mGvU=\n" +
|
||||
"Content-MD5: AAAAAAAAAAAAAAAAAAAAAA==\n" +
|
||||
"Content-Type: binary/octet-stream\n" +
|
||||
"Date: 2009-11-08T15:54:08.897Z\n" +
|
||||
"Host: bucket." + url + "\n");
|
||||
assertPayloadEquals(request, null, null, false);
|
||||
|
||||
assertResponseParserClassEquals(method, request, UploadIdFromHttpResponseViaRegex.class);
|
||||
assertSaxResponseParserClassEquals(method, null);
|
||||
assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class);
|
||||
|
||||
checkFilters(request);
|
||||
}
|
||||
|
||||
public void testAbortMultipartUpload() throws SecurityException, NegativeArraySizeException, NoSuchMethodException {
|
||||
Invokable<?, ?> method = method(S3Client.class, "abortMultipartUpload", String.class, String.class, String.class);
|
||||
GeneratedHttpRequest request = processor.createRequest(method, ImmutableList.<Object> of("bucket", "foo", "asdsadasdas", 1,
|
||||
Payloads.newStringPayload("")));
|
||||
|
||||
assertRequestLineEquals(request, "DELETE https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1");
|
||||
assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
|
||||
assertPayloadEquals(request, "", "application/unknown", false);
|
||||
|
||||
assertResponseParserClassEquals(method, request, ReleasePayloadAndReturn.class);
|
||||
assertSaxResponseParserClassEquals(method, null);
|
||||
assertFallbackClassEquals(method, VoidOnNotFoundOr404.class);
|
||||
|
||||
checkFilters(request);
|
||||
}
|
||||
|
||||
public void testUploadPart() throws SecurityException, NegativeArraySizeException, NoSuchMethodException {
|
||||
Invokable<?, ?> method = method(S3Client.class, "uploadPart", String.class, String.class, int.class,
|
||||
String.class, Payload.class);
|
||||
GeneratedHttpRequest request = processor.createRequest(method, ImmutableList.<Object> of("bucket", "foo", 1, "asdsadasdas",
|
||||
Payloads.newStringPayload("")));
|
||||
|
||||
assertRequestLineEquals(request, "PUT https://bucket." + url + "/foo?partNumber=1&uploadId=asdsadasdas HTTP/1.1");
|
||||
assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
|
||||
assertPayloadEquals(request, "", "application/unknown", false);
|
||||
|
||||
assertResponseParserClassEquals(method, request, ParseETagHeader.class);
|
||||
assertSaxResponseParserClassEquals(method, null);
|
||||
assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class);
|
||||
|
||||
checkFilters(request);
|
||||
}
|
||||
|
||||
public void testCompleteMultipartUpload() throws SecurityException, NegativeArraySizeException,
|
||||
NoSuchMethodException {
|
||||
Invokable<?, ?> method = method(S3Client.class, "completeMultipartUpload", String.class, String.class,
|
||||
String.class, Map.class);
|
||||
GeneratedHttpRequest request = processor.createRequest(method, ImmutableList.<Object> of("bucket", "foo", "asdsadasdas",
|
||||
ImmutableMap.<Integer, String> of(1, "\"a54357aff0632cce46d942af68356b38\"")));
|
||||
|
||||
assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1");
|
||||
assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
|
||||
assertPayloadEquals(
|
||||
request,
|
||||
"<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>\"a54357aff0632cce46d942af68356b38\"</ETag></Part></CompleteMultipartUpload>",
|
||||
"text/xml", false);
|
||||
|
||||
assertResponseParserClassEquals(method, request, ETagFromHttpResponseViaRegex.class);
|
||||
assertSaxResponseParserClassEquals(method, null);
|
||||
assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class);
|
||||
|
||||
checkFilters(request);
|
||||
}
|
||||
|
||||
@ConfiguresHttpApi
|
||||
private static final class TestS3HttpApiModule extends S3HttpApiModule<S3Client> {
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.binders;
|
||||
package org.jclouds.s3.binders;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.binders;
|
||||
package org.jclouds.s3.binders;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
|
|
@ -17,10 +17,12 @@
|
|||
package org.jclouds.s3.blobstore.integration;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
||||
import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
|
||||
import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
|
||||
import org.jclouds.s3.blobstore.strategy.MultipartUpload;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
@Test(groups = "live", testName = "S3BlobIntegrationLiveTest")
|
||||
|
|
@ -30,6 +32,18 @@ public class S3BlobIntegrationLiveTest extends BaseBlobIntegrationTest {
|
|||
provider = "s3";
|
||||
BaseBlobStoreIntegrationTest.SANITY_CHECK_RETURNED_BUCKET_NAME = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Properties setupProperties() {
|
||||
Properties props = super.setupProperties();
|
||||
props.setProperty("jclouds.mpu.parts.size", String.valueOf(MultipartUpload.MIN_PART_SIZE));
|
||||
return props;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected long getMinimumMultipartBlobSize() {
|
||||
return MultipartUpload.MIN_PART_SIZE + 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test(expectedExceptions = IllegalArgumentException.class)
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.blobstore.strategy.internal;
|
||||
package org.jclouds.s3.blobstore.strategy.internal;
|
||||
|
||||
import org.jclouds.aws.s3.blobstore.strategy.MultipartUpload;
|
||||
import org.jclouds.s3.blobstore.strategy.MultipartUpload;
|
||||
|
||||
/**
|
||||
* Print out on the console some graph data regarding the partitioning algorithm.
|
||||
|
|
@ -14,11 +14,11 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.blobstore.strategy.internal;
|
||||
package org.jclouds.s3.blobstore.strategy.internal;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
import org.jclouds.aws.s3.blobstore.strategy.MultipartUpload;
|
||||
import org.jclouds.s3.blobstore.strategy.MultipartUpload;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
/**
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.blobstore.strategy.internal;
|
||||
package org.jclouds.s3.blobstore.strategy.internal;
|
||||
|
||||
import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor;
|
||||
import static org.jclouds.Constants.PROPERTY_MAX_RETRIES;
|
||||
|
|
@ -137,7 +137,7 @@ public class SequentialMultipartUploadStrategyMockTest {
|
|||
overrides.setProperty(PROPERTY_SO_TIMEOUT, "0");
|
||||
overrides.setProperty(PROPERTY_MAX_RETRIES, "1");
|
||||
overrides.setProperty("jclouds.mpu.parts.size", String.valueOf(partSize));
|
||||
return ContextBuilder.newBuilder("aws-s3")
|
||||
return ContextBuilder.newBuilder("s3")
|
||||
.credentials("accessKey", "secretKey")
|
||||
.endpoint(uri)
|
||||
.overrides(overrides)
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.functions;
|
||||
package org.jclouds.s3.functions;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.jclouds.aws.s3.functions;
|
||||
package org.jclouds.s3.functions;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
|
|
@ -16,46 +16,27 @@
|
|||
*/
|
||||
package org.jclouds.aws.s3;
|
||||
|
||||
import static org.jclouds.Fallbacks.VoidOnNotFoundOr404;
|
||||
import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import javax.inject.Named;
|
||||
import javax.ws.rs.DELETE;
|
||||
import javax.ws.rs.POST;
|
||||
import javax.ws.rs.PUT;
|
||||
import javax.ws.rs.Path;
|
||||
import javax.ws.rs.PathParam;
|
||||
import javax.ws.rs.QueryParam;
|
||||
|
||||
import org.jclouds.aws.s3.binders.BindIterableAsPayloadToDeleteRequest;
|
||||
import org.jclouds.aws.s3.binders.BindObjectMetadataToRequest;
|
||||
import org.jclouds.aws.s3.binders.BindPartIdsAndETagsToRequest;
|
||||
import org.jclouds.aws.s3.domain.DeleteResult;
|
||||
import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex;
|
||||
import org.jclouds.aws.s3.functions.ObjectMetadataKey;
|
||||
import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex;
|
||||
import org.jclouds.aws.s3.xml.DeleteResultHandler;
|
||||
import org.jclouds.blobstore.attr.BlobScope;
|
||||
import org.jclouds.http.functions.ParseETagHeader;
|
||||
import org.jclouds.io.Payload;
|
||||
import org.jclouds.rest.annotations.BinderParam;
|
||||
import org.jclouds.rest.annotations.EndpointParam;
|
||||
import org.jclouds.rest.annotations.Fallback;
|
||||
import org.jclouds.rest.annotations.ParamParser;
|
||||
import org.jclouds.rest.annotations.ParamValidators;
|
||||
import org.jclouds.rest.annotations.QueryParams;
|
||||
import org.jclouds.rest.annotations.RequestFilters;
|
||||
import org.jclouds.rest.annotations.ResponseParser;
|
||||
import org.jclouds.rest.annotations.XMLResponseParser;
|
||||
import org.jclouds.s3.Bucket;
|
||||
import org.jclouds.s3.S3Client;
|
||||
import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured;
|
||||
import org.jclouds.s3.domain.ObjectMetadata;
|
||||
import org.jclouds.s3.filters.RequestAuthorizeSignature;
|
||||
import org.jclouds.s3.functions.AssignCorrectHostnameForBucket;
|
||||
import org.jclouds.s3.options.PutObjectOptions;
|
||||
import org.jclouds.s3.predicates.validators.BucketNameValidator;
|
||||
|
||||
/**
|
||||
|
|
@ -65,138 +46,6 @@ import org.jclouds.s3.predicates.validators.BucketNameValidator;
|
|||
@BlobScope(CONTAINER)
|
||||
public interface AWSS3Client extends S3Client {
|
||||
|
||||
/**
|
||||
* This operation initiates a multipart upload and returns an upload ID. This upload ID is used
|
||||
* to associate all the parts in the specific multipart upload. You specify this upload ID in
|
||||
* each of your subsequent upload part requests (see Upload Part). You also include this upload
|
||||
* ID in the final request to either complete or abort the multipart upload request.
|
||||
*
|
||||
* <h4>Note</h4> If you create an object using the multipart upload APIs, currently you cannot
|
||||
* copy the object between regions.
|
||||
*
|
||||
*
|
||||
* @param bucketName
|
||||
* namespace of the object you are to upload
|
||||
* @param objectMetadata
|
||||
* metadata around the object you wish to upload
|
||||
* @param options
|
||||
* controls optional parameters such as canned ACL
|
||||
* @return ID for the initiated multipart upload.
|
||||
*/
|
||||
@Named("PutObject")
|
||||
@POST
|
||||
@QueryParams(keys = "uploads")
|
||||
@Path("/{key}")
|
||||
@ResponseParser(UploadIdFromHttpResponseViaRegex.class)
|
||||
String initiateMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(
|
||||
BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
|
||||
@PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class)
|
||||
ObjectMetadata objectMetadata, PutObjectOptions... options);
|
||||
|
||||
/**
|
||||
* This operation aborts a multipart upload. After a multipart upload is aborted, no additional
|
||||
* parts can be uploaded using that upload ID. The storage consumed by any previously uploaded
|
||||
* parts will be freed. However, if any part uploads are currently in progress, those part
|
||||
* uploads might or might not succeed. As a result, it might be necessary to abort a given
|
||||
* multipart upload multiple times in order to completely free all storage consumed by all parts.
|
||||
*
|
||||
*
|
||||
* @param bucketName
|
||||
* namespace of the object you are deleting
|
||||
* @param key
|
||||
* unique key in the s3Bucket identifying the object
|
||||
* @param uploadId
|
||||
* id of the multipart upload in progress.
|
||||
*/
|
||||
@Named("AbortMultipartUpload")
|
||||
@DELETE
|
||||
@Path("/{key}")
|
||||
@Fallback(VoidOnNotFoundOr404.class)
|
||||
void abortMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(
|
||||
BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
|
||||
@PathParam("key") String key, @QueryParam("uploadId") String uploadId);
|
||||
|
||||
/**
|
||||
* This operation uploads a part in a multipart upload. You must initiate a multipart upload (see
|
||||
* Initiate Multipart Upload) before you can upload any part. In response to your initiate
|
||||
* request. Amazon S3 returns an upload ID, a unique identifier, that you must include in your
|
||||
* upload part request.
|
||||
*
|
||||
* <p/>
|
||||
* Part numbers can be any number from 1 to 10,000, inclusive. A part number uniquely identifies
|
||||
* a part and also defines its position within the object being created. If you upload a new part
|
||||
* using the same part number that was used with a previous part, the previously uploaded part is
|
||||
* overwritten. Each part must be at least 5 MB in size, except the last part. There is no size
|
||||
* limit on the last part of your multipart upload.
|
||||
*
|
||||
* <p/>
|
||||
* To ensure that data is not corrupted when traversing the network, specify the Content-MD5
|
||||
* header in the upload part request. Amazon S3 checks the part data against the provided MD5
|
||||
* value. If they do not match, Amazon S3 returns an error.
|
||||
*
|
||||
*
|
||||
* @param bucketName
|
||||
* namespace of the object you are storing
|
||||
* @param key
|
||||
* unique key in the s3Bucket identifying the object
|
||||
* @param partNumber
|
||||
* which part is this.
|
||||
* @param uploadId
|
||||
* id of the multipart upload in progress.
|
||||
* @param part
|
||||
* contains the data to create or overwrite
|
||||
* @return ETag of the content uploaded
|
||||
*/
|
||||
@Named("PutObject")
|
||||
@PUT
|
||||
@Path("/{key}")
|
||||
@ResponseParser(ParseETagHeader.class)
|
||||
String uploadPart(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(
|
||||
BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
|
||||
@PathParam("key") String key, @QueryParam("partNumber") int partNumber,
|
||||
@QueryParam("uploadId") String uploadId, Payload part);
|
||||
|
||||
/**
|
||||
*
|
||||
This operation completes a multipart upload by assembling previously uploaded parts.
|
||||
* <p/>
|
||||
* You first initiate the multipart upload and then upload all parts using the Upload Parts
|
||||
* operation (see Upload Part). After successfully uploading all relevant parts of an upload, you
|
||||
* call this operation to complete the upload. Upon receiving this request, Amazon S3
|
||||
* concatenates all the parts in ascending order by part number to create a new object. In the
|
||||
* Complete Multipart Upload request, you must provide the parts list. For each part in the list,
|
||||
* you must provide the part number and the ETag header value, returned after that part was
|
||||
* uploaded.
|
||||
* <p/>
|
||||
* Processing of a Complete Multipart Upload request could take several minutes to complete.
|
||||
* After Amazon S3 begins processing the request, it sends an HTTP response header that specifies
|
||||
* a 200 OK response. While processing is in progress, Amazon S3 periodically sends whitespace
|
||||
* characters to keep the connection from timing out. Because a request could fail after the
|
||||
* initial 200 OK response has been sent, it is important that you check the response body to
|
||||
* determine whether the request succeeded.
|
||||
* <p/>
|
||||
* Note that if Complete Multipart Upload fails, applications should be prepared to retry the
|
||||
* failed requests.
|
||||
*
|
||||
* @param bucketName
|
||||
* namespace of the object you are deleting
|
||||
* @param key
|
||||
* unique key in the s3Bucket identifying the object
|
||||
* @param uploadId
|
||||
* id of the multipart upload in progress.
|
||||
* @param parts
|
||||
* a map of part id to eTag from the {@link #uploadPart} command.
|
||||
* @return ETag of the content uploaded
|
||||
*/
|
||||
@Named("PutObject")
|
||||
@POST
|
||||
@Path("/{key}")
|
||||
@ResponseParser(ETagFromHttpResponseViaRegex.class)
|
||||
String completeMultipartUpload(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(
|
||||
BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
|
||||
@PathParam("key") String key, @QueryParam("uploadId") String uploadId,
|
||||
@BinderParam(BindPartIdsAndETagsToRequest.class) Map<Integer, String> parts);
|
||||
|
||||
/**
|
||||
* The Multi-Object Delete operation enables you to delete multiple objects from a bucket using a
|
||||
* single HTTP request. If you know the object keys that you want to delete, then this operation
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ import org.jclouds.aws.domain.Region;
|
|||
import org.jclouds.aws.s3.AWSS3Client;
|
||||
import org.jclouds.aws.s3.blobstore.options.AWSS3PutObjectOptions;
|
||||
import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions;
|
||||
import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy;
|
||||
import org.jclouds.blobstore.BlobStoreContext;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
import org.jclouds.blobstore.domain.PageSet;
|
||||
|
|
@ -45,6 +44,7 @@ import org.jclouds.s3.blobstore.functions.BucketToResourceList;
|
|||
import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions;
|
||||
import org.jclouds.s3.blobstore.functions.ObjectToBlob;
|
||||
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
|
||||
import org.jclouds.s3.blobstore.strategy.MultipartUploadStrategy;
|
||||
import org.jclouds.s3.domain.AccessControlList;
|
||||
import org.jclouds.s3.domain.BucketMetadata;
|
||||
import org.jclouds.s3.domain.CannedAccessPolicy;
|
||||
|
|
@ -60,7 +60,6 @@ import com.google.common.cache.LoadingCache;
|
|||
*/
|
||||
public class AWSS3BlobStore extends S3BlobStore {
|
||||
|
||||
private final Provider<MultipartUploadStrategy> multipartUploadStrategy;
|
||||
private final LoadingCache<String, AccessControlList> bucketAcls;
|
||||
private final BlobToObject blob2Object;
|
||||
|
||||
|
|
@ -75,8 +74,8 @@ public class AWSS3BlobStore extends S3BlobStore {
|
|||
Provider<MultipartUploadStrategy> multipartUploadStrategy) {
|
||||
super(context, blobUtils, defaultLocation, locations, sync, convertBucketsToStorageMetadata,
|
||||
container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object,
|
||||
object2BlobMd, fetchBlobMetadataProvider, bucketAcls);
|
||||
this.multipartUploadStrategy = multipartUploadStrategy;
|
||||
object2BlobMd, fetchBlobMetadataProvider, bucketAcls,
|
||||
multipartUploadStrategy);
|
||||
this.bucketAcls = bucketAcls;
|
||||
this.blob2Object = blob2Object;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,10 +18,6 @@ package org.jclouds.aws.s3.blobstore.config;
|
|||
|
||||
import org.jclouds.aws.s3.blobstore.AWSS3BlobRequestSigner;
|
||||
import org.jclouds.aws.s3.blobstore.AWSS3BlobStore;
|
||||
import org.jclouds.aws.s3.blobstore.strategy.AsyncMultipartUploadStrategy;
|
||||
import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy;
|
||||
import org.jclouds.aws.s3.blobstore.strategy.internal.ParallelMultipartUploadStrategy;
|
||||
import org.jclouds.aws.s3.blobstore.strategy.internal.SequentialMultipartUploadStrategy;
|
||||
import org.jclouds.blobstore.BlobRequestSigner;
|
||||
import org.jclouds.s3.blobstore.S3BlobStore;
|
||||
import org.jclouds.s3.blobstore.config.S3BlobStoreContextModule;
|
||||
|
|
@ -34,8 +30,6 @@ public class AWSS3BlobStoreContextModule extends S3BlobStoreContextModule {
|
|||
protected void configure() {
|
||||
super.configure();
|
||||
bind(S3BlobStore.class).to(AWSS3BlobStore.class).in(Scopes.SINGLETON);
|
||||
bind(MultipartUploadStrategy.class).to(SequentialMultipartUploadStrategy.class);
|
||||
bind(AsyncMultipartUploadStrategy.class).to(ParallelMultipartUploadStrategy.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -16,16 +16,13 @@
|
|||
*/
|
||||
package org.jclouds.aws.s3;
|
||||
|
||||
import static com.google.common.hash.Hashing.md5;
|
||||
import static org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions.Builder.storageClass;
|
||||
import static org.jclouds.io.Payloads.newByteArrayPayload;
|
||||
import static org.jclouds.s3.options.ListBucketOptions.Builder.withPrefix;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertNotNull;
|
||||
import static org.testng.Assert.assertTrue;
|
||||
import static org.testng.Assert.fail;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
|
|
@ -33,28 +30,19 @@ import org.jclouds.aws.AWSResponseException;
|
|||
import org.jclouds.aws.domain.Region;
|
||||
import org.jclouds.aws.s3.domain.DeleteResult;
|
||||
import org.jclouds.blobstore.BlobStore;
|
||||
import org.jclouds.blobstore.KeyNotFoundException;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
import org.jclouds.blobstore.domain.StorageMetadata;
|
||||
import org.jclouds.domain.Location;
|
||||
import org.jclouds.io.ByteStreams2;
|
||||
import org.jclouds.io.Payload;
|
||||
import org.jclouds.s3.S3Client;
|
||||
import org.jclouds.s3.S3ClientLiveTest;
|
||||
import org.jclouds.s3.domain.ListBucketResponse;
|
||||
import org.jclouds.s3.domain.ObjectMetadata;
|
||||
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
|
||||
import org.jclouds.s3.domain.ObjectMetadataBuilder;
|
||||
import org.jclouds.s3.domain.S3Object;
|
||||
import org.jclouds.utils.TestUtils;
|
||||
import org.testng.ITestContext;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.hash.HashCode;
|
||||
import com.google.common.io.ByteSource;
|
||||
|
||||
/**
|
||||
* Tests behavior of {@code S3Client}
|
||||
|
|
@ -65,8 +53,6 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
|
|||
provider = "aws-s3";
|
||||
}
|
||||
|
||||
private static final ByteSource oneHundredOneConstitutions = TestUtils.randomByteSource().slice(0, 5 * 1024 * 1024 + 1);
|
||||
|
||||
@Override
|
||||
public AWSS3Client getApi() {
|
||||
return view.unwrapApi(AWSS3Client.class);
|
||||
|
|
@ -78,53 +64,6 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
|
|||
super.setUpResourcesOnThisThread(testContext);
|
||||
}
|
||||
|
||||
public void testMultipartSynchronously() throws InterruptedException, IOException {
|
||||
HashCode oneHundredOneConstitutionsMD5 = oneHundredOneConstitutions.hash(md5());
|
||||
String containerName = getContainerName();
|
||||
S3Object object = null;
|
||||
try {
|
||||
String key = "constitution.txt";
|
||||
String uploadId = getApi().initiateMultipartUpload(containerName,
|
||||
ObjectMetadataBuilder.create().key(key).contentMD5(oneHundredOneConstitutionsMD5.asBytes()).build());
|
||||
byte[] buffer = oneHundredOneConstitutions.read();
|
||||
assertEquals(oneHundredOneConstitutions.size(), (long) buffer.length);
|
||||
|
||||
Payload part1 = newByteArrayPayload(buffer);
|
||||
part1.getContentMetadata().setContentLength((long) buffer.length);
|
||||
part1.getContentMetadata().setContentMD5(oneHundredOneConstitutionsMD5);
|
||||
|
||||
String eTagOf1 = null;
|
||||
try {
|
||||
eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
|
||||
} catch (KeyNotFoundException e) {
|
||||
// note that because of eventual consistency, the upload id may not be present yet
|
||||
// we may wish to add this condition to the retry handler
|
||||
|
||||
// we may also choose to implement ListParts and wait for the uploadId to become
|
||||
// available there.
|
||||
eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
|
||||
}
|
||||
|
||||
String eTag = getApi().completeMultipartUpload(containerName, key, uploadId, ImmutableMap.of(1, eTagOf1));
|
||||
|
||||
assert !eTagOf1.equals(eTag);
|
||||
|
||||
object = getApi().getObject(containerName, key);
|
||||
assertEquals(ByteStreams2.toByteArrayAndClose(object.getPayload().openStream()), buffer);
|
||||
|
||||
// noticing amazon does not return content-md5 header or a parsable ETag after a multi-part
|
||||
// upload is complete:
|
||||
// https://forums.aws.amazon.com/thread.jspa?threadID=61344
|
||||
assertEquals(object.getPayload().getContentMetadata().getContentMD5(), null);
|
||||
assertEquals(getApi().headObject(containerName, key).getContentMetadata().getContentMD5(), null);
|
||||
|
||||
} finally {
|
||||
if (object != null)
|
||||
object.getPayload().close();
|
||||
returnContainer(containerName);
|
||||
}
|
||||
}
|
||||
|
||||
public void testPutWithReducedRedundancyStorage() throws InterruptedException {
|
||||
String containerName = getContainerName();
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -20,31 +20,21 @@ import static org.jclouds.reflect.Reflection2.method;
|
|||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.jclouds.Fallbacks.VoidOnNotFoundOr404;
|
||||
import org.jclouds.aws.s3.config.AWSS3HttpApiModule;
|
||||
import org.jclouds.aws.s3.filters.AWSRequestAuthorizeSignature;
|
||||
import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex;
|
||||
import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex;
|
||||
import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
|
||||
import org.jclouds.date.TimeStamp;
|
||||
import org.jclouds.fallbacks.MapHttp4xxCodesToExceptions;
|
||||
import org.jclouds.http.HttpRequest;
|
||||
import org.jclouds.http.functions.ParseETagHeader;
|
||||
import org.jclouds.http.functions.ParseSax;
|
||||
import org.jclouds.http.functions.ReleasePayloadAndReturn;
|
||||
import org.jclouds.http.functions.ReturnTrueIf2xx;
|
||||
import org.jclouds.io.Payload;
|
||||
import org.jclouds.io.Payloads;
|
||||
import org.jclouds.location.Region;
|
||||
import org.jclouds.rest.ConfiguresHttpApi;
|
||||
import org.jclouds.rest.internal.GeneratedHttpRequest;
|
||||
import org.jclouds.s3.S3Client;
|
||||
import org.jclouds.s3.S3ClientTest;
|
||||
import org.jclouds.s3.domain.ObjectMetadata;
|
||||
import org.jclouds.s3.domain.ObjectMetadataBuilder;
|
||||
import org.jclouds.s3.domain.S3Object;
|
||||
import org.jclouds.s3.fallbacks.FalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists;
|
||||
import org.jclouds.s3.options.CopyObjectOptions;
|
||||
|
|
@ -161,95 +151,6 @@ public class AWSS3ClientTest extends S3ClientTest<AWSS3Client> {
|
|||
checkFilters(request);
|
||||
}
|
||||
|
||||
public void testInitiateMultipartUpload() throws SecurityException, NegativeArraySizeException,
|
||||
NoSuchMethodException {
|
||||
Invokable<?, ?> method = method(AWSS3Client.class, "initiateMultipartUpload", String.class, ObjectMetadata.class,
|
||||
PutObjectOptions[].class);
|
||||
GeneratedHttpRequest request = processor.createRequest(method, ImmutableList.<Object> of("bucket", ObjectMetadataBuilder.create().key("foo")
|
||||
.contentMD5(new byte[16]).build()));
|
||||
|
||||
assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1");
|
||||
assertNonPayloadHeadersEqual(request,
|
||||
"Content-MD5: AAAAAAAAAAAAAAAAAAAAAA==\n" +
|
||||
"Content-Type: binary/octet-stream\n" +
|
||||
"Host: bucket." + url + "\n");
|
||||
assertPayloadEquals(request, null, null, false);
|
||||
|
||||
// as this is a payload-related command, but with no payload, be careful
|
||||
// that we check
|
||||
// filtering and do not ignore if this fails later.
|
||||
request = (GeneratedHttpRequest) request.getFilters().get(0).filter(request);
|
||||
|
||||
assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1");
|
||||
assertNonPayloadHeadersEqual(request,
|
||||
"Authorization: AWS identity:972m/Bqn2L5FIaB+wWDeY83mGvU=\n" +
|
||||
"Content-MD5: AAAAAAAAAAAAAAAAAAAAAA==\n" +
|
||||
"Content-Type: binary/octet-stream\n" +
|
||||
"Date: 2009-11-08T15:54:08.897Z\n" +
|
||||
"Host: bucket." + url + "\n");
|
||||
assertPayloadEquals(request, null, null, false);
|
||||
|
||||
assertResponseParserClassEquals(method, request, UploadIdFromHttpResponseViaRegex.class);
|
||||
assertSaxResponseParserClassEquals(method, null);
|
||||
assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class);
|
||||
|
||||
checkFilters(request);
|
||||
}
|
||||
|
||||
public void testAbortMultipartUpload() throws SecurityException, NegativeArraySizeException, NoSuchMethodException {
|
||||
Invokable<?, ?> method = method(AWSS3Client.class, "abortMultipartUpload", String.class, String.class, String.class);
|
||||
GeneratedHttpRequest request = processor.createRequest(method, ImmutableList.<Object> of("bucket", "foo", "asdsadasdas", 1,
|
||||
Payloads.newStringPayload("")));
|
||||
|
||||
assertRequestLineEquals(request, "DELETE https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1");
|
||||
assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
|
||||
assertPayloadEquals(request, "", "application/unknown", false);
|
||||
|
||||
assertResponseParserClassEquals(method, request, ReleasePayloadAndReturn.class);
|
||||
assertSaxResponseParserClassEquals(method, null);
|
||||
assertFallbackClassEquals(method, VoidOnNotFoundOr404.class);
|
||||
|
||||
checkFilters(request);
|
||||
}
|
||||
|
||||
public void testUploadPart() throws SecurityException, NegativeArraySizeException, NoSuchMethodException {
|
||||
Invokable<?, ?> method = method(AWSS3Client.class, "uploadPart", String.class, String.class, int.class,
|
||||
String.class, Payload.class);
|
||||
GeneratedHttpRequest request = processor.createRequest(method, ImmutableList.<Object> of("bucket", "foo", 1, "asdsadasdas",
|
||||
Payloads.newStringPayload("")));
|
||||
|
||||
assertRequestLineEquals(request, "PUT https://bucket." + url + "/foo?partNumber=1&uploadId=asdsadasdas HTTP/1.1");
|
||||
assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
|
||||
assertPayloadEquals(request, "", "application/unknown", false);
|
||||
|
||||
assertResponseParserClassEquals(method, request, ParseETagHeader.class);
|
||||
assertSaxResponseParserClassEquals(method, null);
|
||||
assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class);
|
||||
|
||||
checkFilters(request);
|
||||
}
|
||||
|
||||
public void testCompleteMultipartUpload() throws SecurityException, NegativeArraySizeException,
|
||||
NoSuchMethodException {
|
||||
Invokable<?, ?> method = method(AWSS3Client.class, "completeMultipartUpload", String.class, String.class,
|
||||
String.class, Map.class);
|
||||
GeneratedHttpRequest request = processor.createRequest(method, ImmutableList.<Object> of("bucket", "foo", "asdsadasdas",
|
||||
ImmutableMap.<Integer, String> of(1, "\"a54357aff0632cce46d942af68356b38\"")));
|
||||
|
||||
assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1");
|
||||
assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
|
||||
assertPayloadEquals(
|
||||
request,
|
||||
"<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>\"a54357aff0632cce46d942af68356b38\"</ETag></Part></CompleteMultipartUpload>",
|
||||
"text/xml", false);
|
||||
|
||||
assertResponseParserClassEquals(method, request, ETagFromHttpResponseViaRegex.class);
|
||||
assertSaxResponseParserClassEquals(method, null);
|
||||
assertFallbackClassEquals(method, MapHttp4xxCodesToExceptions.class);
|
||||
|
||||
checkFilters(request);
|
||||
}
|
||||
|
||||
public void testPutBucketEu() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
|
||||
NoSuchMethodException, IOException {
|
||||
Invokable<?, ?> method = method(AWSS3Client.class, "putBucketInRegion", String.class, String.class,
|
||||
|
|
|
|||
|
|
@ -16,9 +16,6 @@
|
|||
*/
|
||||
package org.jclouds.aws.s3.blobstore.integration;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.jclouds.aws.s3.blobstore.strategy.MultipartUpload;
|
||||
import org.jclouds.s3.blobstore.integration.S3BlobIntegrationLiveTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
|
|
@ -27,16 +24,4 @@ public class AWSS3BlobIntegrationLiveTest extends S3BlobIntegrationLiveTest {
|
|||
public AWSS3BlobIntegrationLiveTest() {
|
||||
provider = "aws-s3";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Properties setupProperties() {
|
||||
Properties props = super.setupProperties();
|
||||
props.setProperty("jclouds.mpu.parts.size", String.valueOf(MultipartUpload.MIN_PART_SIZE));
|
||||
return props;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected long getMinimumMultipartBlobSize() {
|
||||
return MultipartUpload.MIN_PART_SIZE + 1;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue