From e11dd37237616ba33bb8f8062caf82597371b549 Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Thu, 29 Mar 2012 10:22:46 +0400 Subject: [PATCH] Stubs for multipart upload support in swift. --- .../blobstore/CloudFilesBlobStore.java | 3 +- .../swift/blobstore/SwiftBlobStore.java | 12 ++++- .../blobstore/strategy/MultipartUpload.java | 21 +++++++++ .../internal/MultipartUploadStrategy.java | 12 +++++ .../SequentialMultipartUploadStrategy.java | 46 +++++++++++++++++++ .../HPCloudObjectStorageBlobStore.java | 2 +- 6 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java diff --git a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java index 42be6d884c..40ba250abd 100644 --- a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java +++ b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java @@ -42,6 +42,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata; import com.google.common.base.Supplier; +import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy; /** * @@ -62,7 +63,7 @@ public class CloudFilesBlobStore extends SwiftBlobStore { Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) { super(context, blobUtils, defaultLocation, locations, sync, container2ResourceMd, container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd, blob2ObjectGetOptions, - fetchBlobMetadataProvider); + fetchBlobMetadataProvider, null); this.enableCDNAndCache = enableCDNAndCache; } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java index e96909ba18..6cc3329861 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java @@ -50,6 +50,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceList; import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceMetadata; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata; +import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy; import org.jclouds.openstack.swift.domain.ContainerMetadata; import com.google.common.base.Function; @@ -71,6 +72,7 @@ public class SwiftBlobStore extends BaseBlobStore { private final ObjectToBlobMetadata object2BlobMd; private final BlobToHttpGetOptions blob2ObjectGetOptions; private final Provider fetchBlobMetadataProvider; + private final Provider multipartUploadStrategy; @Inject protected SwiftBlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier defaultLocation, @@ -79,7 +81,8 @@ public class SwiftBlobStore extends BaseBlobStore { BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions, ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions, - Provider fetchBlobMetadataProvider) { + Provider fetchBlobMetadataProvider, + Provider multipartUploadStrategy) { super(context, blobUtils, defaultLocation, locations); this.sync = sync; this.container2ResourceMd = container2ResourceMd; @@ -90,6 +93,7 @@ public class SwiftBlobStore extends BaseBlobStore { this.object2BlobMd = object2BlobMd; this.blob2ObjectGetOptions = blob2ObjectGetOptions; this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider"); + this.multipartUploadStrategy = multipartUploadStrategy; } /** @@ -207,7 +211,11 @@ public class SwiftBlobStore extends BaseBlobStore { @Override public String putBlob(String container, Blob blob, PutOptions options) { // TODO implement options - return putBlob(container, blob); + if (options.isMultipart()) { + return multipartUploadStrategy.get().execute(container, blob, options); + } else { + return putBlob(container, blob); + } } /** diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java new file mode 100644 index 0000000000..839f4774e2 --- /dev/null +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java @@ -0,0 +1,21 @@ +package org.jclouds.openstack.swift.blobstore.strategy; + +/* +@author Roman Bogorodskiy + */ + +public interface MultipartUpload { + + /* Maximum number of parts per upload */ + public static final int MAX_NUMBER_OF_PARTS = 10000; + /* Maximum number of parts returned for a list parts request */ + public static final int MAX_LIST_PARTS_RETURNED = 1000; + /* Maximum number of multipart uploads returned in a list multipart uploads request */ + public static final int MAX_LIST_MPU_RETURNED = 1000; + + /* + * part size 5 MB to 5 GB, last part can be < 5 MB + */ + public static final long MIN_PART_SIZE = 5242880L; + public static final long MAX_PART_SIZE = 5368709120L; +} diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java new file mode 100644 index 0000000000..40a79ad45a --- /dev/null +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java @@ -0,0 +1,12 @@ +package org.jclouds.openstack.swift.blobstore.strategy.internal; + +import com.google.inject.ImplementedBy; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload; + +@ImplementedBy(SequentialMultipartUploadStrategy.class) +public interface MultipartUploadStrategy extends MultipartUpload { + + String execute(String container, Blob blob, PutOptions options); +} diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java new file mode 100644 index 0000000000..693cee51be --- /dev/null +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java @@ -0,0 +1,46 @@ +package org.jclouds.openstack.swift.blobstore.strategy.internal; + +import javax.annotation.Resource; +import javax.inject.Named; + +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.blobstore.reference.BlobStoreConstants; +import org.jclouds.io.Payload; +import org.jclouds.io.PayloadSlicer; +import org.jclouds.logging.Logger; +import org.jclouds.openstack.swift.blobstore.SwiftBlobStore; + +import static com.google.common.base.Preconditions.checkNotNull; + + +public class SequentialMultipartUploadStrategy implements MultipartUploadStrategy { + @Resource + @Named(BlobStoreConstants.BLOBSTORE_LOGGER) + protected Logger logger = Logger.NULL; + + protected final SwiftBlobStore ablobstore; + protected final PayloadSlicer slicer; + + public SequentialMultipartUploadStrategy(SwiftBlobStore ablobstore, PayloadSlicer slicer) { + this.ablobstore = checkNotNull(ablobstore, "ablobstore"); + this.slicer = checkNotNull(slicer, "slicer"); + } + + @Override + public String execute(String container, Blob blob, PutOptions options) { + String key = blob.getMetadata().getName(); + Payload payload = blob.getPayload(); + /*MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(); + algorithm + .calculateChunkSize(checkNotNull( + payload.getContentMetadata().getContentLength(), + "contentLength required on all uploads to amazon s3; please invoke payload.getContentMetadata().setContentLength(length) first")); + int parts = algorithm.getParts(); + long chunkSize = algorithm.getChunkSize(); + if (parts > 0) { + + } */ + return "NOT IMPLEMENTED"; + } +} diff --git a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java index e05119ed8c..cd11feb720 100644 --- a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java +++ b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java @@ -62,7 +62,7 @@ public class HPCloudObjectStorageBlobStore extends SwiftBlobStore { Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) { super(context, blobUtils, defaultLocation, locations, sync, container2ResourceMd, container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd, blob2ObjectGetOptions, - fetchBlobMetadataProvider); + fetchBlobMetadataProvider, null); this.enableCDNAndCache = enableCDNAndCache; }