AWS S3 sequential Multipart Upload strategy

apis/atmos/src/main/java/org/jclouds/atmos/blobstore/AtmosBlobStore.java

@@ -207,6 +207,16 @@ public class AtmosBlobStore extends BaseBlobStore {
       return AtmosUtils.putBlob(sync, crypto, blob2Object, container, blob);
    }
 
+   /**
+    * This implementation invokes {@link AtmosClient#createFile}
+    * <p/>
+    * Since there is no etag support in atmos, we just return the path.
+    */
+   @Override
+   public String putBlobMultipart(String container, Blob blob) {
+      return putBlob(container, blob);
+   }
+
    /**
     * This implementation invokes {@link AtmosClient#deletePath}
     */

apis/s3/src/main/java/org/jclouds/s3/blobstore/S3AsyncBlobStore.java

@@ -83,7 +83,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
    private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
 
    @Inject
-   S3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils,
+   protected S3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils,
             @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier<Location> defaultLocation,
             @Memoized Supplier<Set<? extends Location>> locations, S3AsyncClient async, S3Client sync,
             BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions,

apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java

@@ -74,7 +74,7 @@ public class S3BlobStore extends BaseBlobStore {
    private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
 
    @Inject
-   S3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
+   protected S3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
             @Memoized Supplier<Set<? extends Location>> locations, S3Client sync,
             BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions,
             BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob,
@@ -225,6 +225,19 @@ public class S3BlobStore extends BaseBlobStore {
       return sync.putObject(container, blob2Object.apply(blob));
    }
 
+   /**
+    * This implementation invokes {@link S3Client#putObject}
+    * 
+    * @param container
+    *           bucket name
+    * @param blob
+    *           object
+    */
+   @Override
+   public String putBlobMultipart(String container, Blob blob) {
+      return sync.putObject(container, blob2Object.apply(blob));
+   }
+
    /**
     * This implementation invokes {@link S3Client#deleteObject}
     * 

apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java

@@ -195,6 +195,19 @@ public class SwiftBlobStore extends BaseBlobStore {
       return sync.putObject(container, blob2Object.apply(blob));
    }
 
+   /**
+    * This implementation invokes {@link CommonSwiftClient#putObject}
+    * 
+    * @param container
+    *           container name
+    * @param blob
+    *           object
+    */
+   @Override
+   public String putBlobMultipart(String container, Blob blob) {
+      return putBlob(container, blob);
+   }
+
    /**
     * This implementation invokes {@link CommonSwiftClient#removeObject}
     * 

blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java

@@ -199,6 +199,22 @@ public interface BlobStore {
     *            if the container doesn't exist
     */
    String putBlob(String container, Blob blob);
+   
+   /**
+    * Adds a {@code Blob} representing the data at location {@code container/blob.metadata.name}
+    * using multipart strategies.
+    * 
+    * @param container
+    *           container to place the blob.
+    * @param blob
+    *           fully qualified name relative to the container.
+    * @param options
+    *           byte range or condition options
+    * @return etag of the blob you uploaded, possibly null where etags are unsupported
+    * @throws ContainerNotFoundException
+    *            if the container doesn't exist
+    */
+   String putBlobMultipart(String container, Blob blob);
 
    /**
     * Retrieves the metadata of a {@code Blob} at location {@code container/name}

core/pom.xml

@@ -119,6 +119,11 @@
             <artifactId>jsr305</artifactId>
             <version>1.3.9</version>
         </dependency>
+        <dependency>
+        	<groupId>org.jboss.netty</groupId>
+        	<artifactId>netty</artifactId>
+        	<version>3.2.4.Final</version>
+        </dependency>
     </dependencies>
 
     <profiles>

core/src/main/java/org/jclouds/io/Payloads.java

@@ -35,6 +35,7 @@ import javax.annotation.Nullable;
 
 import org.jclouds.crypto.CryptoStreams;
 import org.jclouds.io.payloads.ByteArrayPayload;
+import org.jclouds.io.payloads.ChunkedFilePayload;
 import org.jclouds.io.payloads.FilePayload;
 import org.jclouds.io.payloads.InputStreamPayload;
 import org.jclouds.io.payloads.StringPayload;
@@ -83,6 +84,10 @@ public class Payloads {
    public static FilePayload newFilePayload(File data) {
       return new FilePayload(checkNotNull(data, "data"));
    }
+   
+   public static ChunkedFilePayload newChunkedFilePayload(File data, int part, long chunkOffset, long chunkSize) {
+      return new ChunkedFilePayload(checkNotNull(data, "data"), part, chunkOffset, chunkSize);
+   }
 
    public static UrlEncodedFormPayload newUrlEncodedFormPayload(Multimap<String, String> formParams, char... skips) {
       return new UrlEncodedFormPayload(formParams, skips);

providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ContextBuilder.java

@@ -22,8 +22,10 @@ package org.jclouds.aws.s3;
 import java.util.List;
 import java.util.Properties;
 
+import org.jclouds.aws.s3.blobstore.config.AWSS3BlobStoreContextModule;
 import org.jclouds.aws.s3.config.AWSS3RestClientModule;
 import org.jclouds.s3.S3ContextBuilder;
+import org.jclouds.s3.blobstore.config.S3BlobStoreContextModule;
 
 import com.google.inject.Module;
 
@@ -37,6 +39,11 @@ public class AWSS3ContextBuilder extends S3ContextBuilder {
       super(props);
    }
 
+   @Override
+   protected void addContextModule(List<Module> modules) {
+      modules.add(new AWSS3BlobStoreContextModule());
+   }
+
    @Override
    protected void addClientModule(List<Module> modules) {
       modules.add(new AWSS3RestClientModule());

providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java

@@ -27,11 +27,15 @@ import static org.jclouds.io.Payloads.newByteArrayPayload;
 import static org.testng.Assert.assertEquals;
 
 import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.zip.GZIPInputStream;
 
+import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.KeyNotFoundException;
+import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.http.BaseJettyTest;
 import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule;
 import org.jclouds.io.Payload;
@@ -43,6 +47,7 @@ import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.common.io.ByteStreams;
 import com.google.common.io.InputSupplier;
 import com.google.inject.Module;
 
@@ -134,4 +139,23 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
          returnContainer(containerName);
       }
    }
+   
+   public void testMultipartChunkedFileStream() throws IOException, InterruptedException {
+      
+      FileOutputStream fous = new FileOutputStream(new File("target/const.txt"));
+      ByteStreams.copy(oneHundredOneConstitutions.getInput(), fous);
+      fous.flush();
+      fous.close();
+      String containerName = getContainerName();
+      
+      try {
+         BlobStore blobStore = context.getBlobStore();
+         blobStore.createContainerInLocation(null, containerName);
+         Blob blob = blobStore.blobBuilder("const.txt")
+            .payload(new File("target/const.txt")).build();
+         blobStore.putBlobMultipart(containerName, blob);
+      } finally {
+         returnContainer(containerName);
+      }
+   }
 }

providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/AzureBlobStore.java

@@ -192,6 +192,19 @@ public class AzureBlobStore extends BaseBlobStore {
       return sync.putBlob(container, blob2AzureBlob.apply(blob));
    }
 
+   /**
+    * This implementation invokes {@link AzureBlobClient#putObject}
+    * 
+    * @param container
+    *           container name
+    * @param blob
+    *           object
+    */
+   @Override
+   public String putBlobMultipart(String container, Blob blob) {
+      return putBlob(container, blob);
+   }
+
    /**
     * This implementation invokes {@link AzureBlobClient#deleteObject}
     *