Merge branch 'large-blob' of https://github.com/jclouds/jclouds into jclouds-large-blob

apis/atmos/src/main/java/org/jclouds/atmos/blobstore/AtmosAsyncBlobStore.java

@@ -250,4 +250,9 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
       return async.deletePath(container + "/" + key);
    }
 
+   @Override
+   public ListenableFuture<String> putBlobMultipart(String container, Blob blob) {
+      return putBlob(container, blob);
+   }
+
 }

apis/filesystem/src/main/java/org/jclouds/filesystem/FilesystemAsyncBlobStore.java

@@ -660,4 +660,10 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
       String eTag = CryptoStreams.hex(object.getPayload().getContentMetadata().getContentMD5());
       return eTag;
    }
+   
+   @Override
+   public ListenableFuture<String> putBlobMultipart(String container, Blob blob) {
+      return putBlob(container, blob);
+   }
+
 }

apis/s3/src/main/java/org/jclouds/s3/blobstore/S3AsyncBlobStore.java

@@ -238,4 +238,9 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
       return async.deleteObject(container, key);
    }
 
+   @Override
+   public ListenableFuture<String> putBlobMultipart(String container, Blob blob) {
+      return putBlob(container, blob);
+   }
+
 }

apis/s3/src/main/java/org/jclouds/s3/blobstore/config/S3BlobStoreContextModule.java

@@ -55,12 +55,16 @@ public class S3BlobStoreContextModule extends AbstractModule {
       bind(ConsistencyModel.class).toInstance(ConsistencyModel.EVENTUAL);
       bind(AsyncBlobStore.class).to(S3AsyncBlobStore.class).in(Scopes.SINGLETON);
       bind(BlobStore.class).to(S3BlobStore.class).in(Scopes.SINGLETON);
-      bind(BlobStoreContext.class).to(new TypeLiteral<BlobStoreContextImpl<S3Client, S3AsyncClient>>() {
+      bindContext();
-      }).in(Scopes.SINGLETON);
       bind(BlobRequestSigner.class).to(S3BlobRequestSigner.class);
       bindBucketLocationStrategy();
    }
 
+   protected void bindContext() {
+      bind(BlobStoreContext.class).to(new TypeLiteral<BlobStoreContextImpl<S3Client, S3AsyncClient>>() {
+      }).in(Scopes.SINGLETON);
+   }
+
    protected void bindBucketLocationStrategy() {
       bind(new TypeLiteral<Function<BucketMetadata, Location>>() {
       }).to(LocationFromBucketLocation.class);

apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java

@@ -110,8 +110,7 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore {
    public ListenableFuture<PageSet<? extends StorageMetadata>> list() {
       return Futures.compose(async.listContainers(),
             new Function<Set<ContainerMetadata>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() {
-                  public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(
+               public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(Set<ContainerMetadata> from) {
-                           Set<ContainerMetadata> from) {
                   return new PageSetImpl<StorageMetadata>(Iterables.transform(from, container2ResourceMd), null);
                }
             }, service);
@@ -235,4 +234,9 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore {
       return !sync.containerExists(container);
    }
 
+   @Override
+   public ListenableFuture<String> putBlobMultipart(String container, Blob blob) {
+      return putBlob(container, blob);
+   }
+
 }

blobstore/src/main/java/org/jclouds/blobstore/AsyncBlobStore.java

@@ -32,6 +32,7 @@ import org.jclouds.blobstore.options.GetOptions;
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.domain.Location;
 
+import com.google.common.annotations.Beta;
 import com.google.common.util.concurrent.ListenableFuture;
 
 /**
@@ -126,6 +127,12 @@ public interface AsyncBlobStore {
     */
    ListenableFuture<String> putBlob(String container, Blob blob);
 
+   /**
+    * @see BlobStore#putBlobMultipart
+    */
+   @Beta
+   ListenableFuture<String> putBlobMultipart(String container, Blob blob);
+
    /**
     * @see BlobStore#blobMetadata
     */

blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java

@@ -32,6 +32,8 @@ import org.jclouds.blobstore.options.GetOptions;
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.domain.Location;
 
+import com.google.common.annotations.Beta;
+
 /**
  * Synchronous access to a BlobStore such as Amazon S3
  * 
@@ -48,6 +50,7 @@ public interface BlobStore {
 
    /**
     * creates a new blob with the specified name.
+    * 
     * @see #blobBuilder
     */
    @Deprecated
@@ -214,6 +217,7 @@ public interface BlobStore {
     * @throws ContainerNotFoundException
     *            if the container doesn't exist
     */
+   @Beta
    String putBlobMultipart(String container, Blob blob);
 
    /**

blobstore/src/main/java/org/jclouds/blobstore/TransientAsyncBlobStore.java

@@ -675,4 +675,10 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore {
       return containerToLocation;
    }
 
+   @Override
+   public ListenableFuture<String> putBlobMultipart(String container, Blob blob) {
+      // TODO implement
+      return putBlob(container, blob);
+   }
+
 }

core/pom.xml

@@ -119,11 +119,6 @@
             <artifactId>jsr305</artifactId>
             <version>1.3.9</version>
         </dependency>
-        <dependency>
-        	<groupId>org.jboss.netty</groupId>
-        	<artifactId>netty</artifactId>
-        	<version>3.2.4.Final</version>
-        </dependency>
     </dependencies>
 
     <profiles>

core/src/main/java/org/jclouds/io/PayloadSlicer.java

@@ -0,0 +1,47 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.io;
+
+import org.jclouds.io.internal.BasePayloadSlicer;
+
+import com.google.inject.ImplementedBy;
+
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@ImplementedBy(BasePayloadSlicer.class)
+public interface PayloadSlicer {
+   /**
+    * Returns a {@link Payload} that returns input streams from the an underlying payload, where
+    * each stream starts at the given offset and is limited to the specified number of bytes.
+    * 
+    * @param input
+    *           the payload from which to get the raw streams
+    * @param offset
+    *           the offset in bytes into the underlying stream where the returned streams will start
+    * @param length
+    *           the maximum length of the returned streams
+    * @throws IllegalArgumentException
+    *            if offset or length are negative
+    */
+   Payload slice(Payload input, long offset, long length);
+}

core/src/main/java/org/jclouds/io/Payloads.java

@@ -35,7 +35,6 @@ import javax.annotation.Nullable;
 
 import org.jclouds.crypto.CryptoStreams;
 import org.jclouds.io.payloads.ByteArrayPayload;
-import org.jclouds.io.payloads.ChunkedFilePayload;
 import org.jclouds.io.payloads.FilePayload;
 import org.jclouds.io.payloads.InputStreamPayload;
 import org.jclouds.io.payloads.StringPayload;
@@ -85,10 +84,6 @@ public class Payloads {
       return new FilePayload(checkNotNull(data, "data"));
    }
 
-   public static ChunkedFilePayload newChunkedFilePayload(File data, int part, long chunkOffset, long chunkSize) {
-      return new ChunkedFilePayload(checkNotNull(data, "data"), part, chunkOffset, chunkSize);
-   }
-
    public static UrlEncodedFormPayload newUrlEncodedFormPayload(Multimap<String, String> formParams, char... skips) {
       return new UrlEncodedFormPayload(formParams, skips);
    }

core/src/main/java/org/jclouds/io/internal/BasePayloadSlicer.java

@@ -0,0 +1,104 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.io.internal;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+
+import javax.inject.Singleton;
+
+import org.jclouds.io.InputSuppliers;
+import org.jclouds.io.Payload;
+import org.jclouds.io.PayloadSlicer;
+import org.jclouds.io.payloads.BaseMutableContentMetadata;
+import org.jclouds.io.payloads.InputStreamSupplierPayload;
+
+import com.google.common.base.Throwables;
+import com.google.common.io.ByteStreams;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class BasePayloadSlicer implements PayloadSlicer {
+   /**
+    * {@inheritDoc}
+    */
+   @Override
+   public Payload slice(Payload input, long offset, long length) {
+      checkNotNull(input);
+      checkArgument(offset >= 0, "offset is negative");
+      checkArgument(length >= 0, "length is negative");
+      Payload returnVal;
+      if (input.getRawContent() instanceof File) {
+         returnVal = doSlice((File) input.getRawContent(), offset, length);
+      } else if (input.getRawContent() instanceof String) {
+         returnVal = doSlice((byte[]) input.getRawContent(), offset, length);
+      } else if (input.getRawContent() instanceof byte[]) {
+         returnVal = doSlice((byte[]) input.getRawContent(), offset, length);
+      } else {
+         returnVal = doSlice(input.getInput(), offset, length);
+      }
+      return copyMetadataAndSetLength(input, returnVal, length);
+   }
+
+   protected Payload doSlice(Payload content, long offset, long length) {
+      return new InputStreamSupplierPayload(ByteStreams.slice(content, offset, length));
+   }
+
+   protected Payload doSlice(String content, long offset, long length) {
+      return doSlice(content.getBytes(), offset, length);
+   }
+
+   protected Payload doSlice(File content, long offset, long length) {
+      try {
+         return doSlice(new FileInputStream(content), offset, length);
+      } catch (FileNotFoundException e) {
+         Throwables.propagate(e);
+         return null;
+      }
+   }
+
+   protected Payload doSlice(InputStream content, long offset, long length) {
+      return new InputStreamSupplierPayload(ByteStreams.slice(InputSuppliers.of(content), offset, length));
+   }
+
+   protected Payload doSlice(byte[] content, long offset, long length) {
+      Payload returnVal;
+      checkArgument(offset <= Integer.MAX_VALUE, "offset is too big for an array");
+      checkArgument(length <= Integer.MAX_VALUE, "length is too big for an array");
+      returnVal = new InputStreamSupplierPayload(
+            ByteStreams.newInputStreamSupplier(content, (int) offset, (int) length));
+      return returnVal;
+   }
+
+   protected Payload copyMetadataAndSetLength(Payload input, Payload returnVal, long length) {
+      returnVal.setContentMetadata(BaseMutableContentMetadata.fromContentMetadata(input.getContentMetadata()
+            .toBuilder().contentLength(length).contentMD5(null).build()));
+      return returnVal;
+   }
+
+}

core/src/main/java/org/jclouds/io/payloads/InputStreamSupplierPayload.java

@@ -0,0 +1,75 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.io.payloads;
+
+import static com.google.common.io.Closeables.closeQuietly;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import com.google.common.io.InputSupplier;
+
+/**
+ * @author Adrian Cole
+ */
+public class InputStreamSupplierPayload extends BasePayload<InputSupplier<? extends InputStream>> {
+   private List<InputStream> toClose = Lists.newArrayList();
+
+   public InputStreamSupplierPayload(InputSupplier<? extends InputStream> content) {
+      super(content);
+   }
+
+   /**
+    * {@inheritDoc}
+    */
+   @Override
+   public InputStream getInput() {
+      try {
+         InputStream returnVal = content.getInput();
+         toClose.add(returnVal);
+         return returnVal;
+      } catch (IOException e) {
+         Throwables.propagate(e);
+         return null;
+      }
+   }
+
+   /**
+    * {@inheritDoc}
+    */
+   @Override
+   public boolean isRepeatable() {
+      return true;
+   }
+
+   /**
+    * if we created the stream, then it is already consumed on close.
+    */
+   @Override
+   public void release() {
+      if (toClose.size() > 0)
+         for (InputStream content = toClose.remove(0); toClose.size() > 0; content = toClose.remove(0))
+            closeQuietly(content);
+   }
+
+}

drivers/netty/pom.xml

@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+
+    Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+
+    ====================================================================
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+    ====================================================================
+
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.jclouds</groupId>
+        <artifactId>jclouds-project</artifactId>
+        <version>1.0-SNAPSHOT</version>
+        <relativePath>../../project/pom.xml</relativePath>
+    </parent>
+    <groupId>org.jclouds.driver</groupId>
+    <artifactId>jclouds-netty</artifactId>
+    <name>jclouds netty payload module</name>
+    <description>jclouds netty payload module</description>
+
+    <!-- bootstrapping: need to fetch the project POM -->
+    <repositories>
+        <repository>
+            <id>jclouds-sona-snapshots-nexus</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>jboss-public-releases</id>
+            <url>https://repository.jboss.org/nexus/content/groups/public-jboss</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.jclouds</groupId>
+            <artifactId>jclouds-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.jclouds</groupId>
+            <artifactId>jclouds-core</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.jboss.netty</groupId>
+            <artifactId>netty</artifactId>
+            <version>3.2.4.Final</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+</project>
+

drivers/netty/src/main/java/org/jclouds/netty/config/NettyPayloadModule.java

@@ -0,0 +1,20 @@
+package org.jclouds.netty.config;
+
+import org.jclouds.io.PayloadSlicer;
+import org.jclouds.netty.io.NettyPayloadSlicer;
+
+import com.google.inject.AbstractModule;
+
+/**
+ * 
+ * @author Adrian Cole
+ * 
+ */
+public class NettyPayloadModule extends AbstractModule {
+
+   @Override
+   protected void configure() {
+      bind(PayloadSlicer.class).to(NettyPayloadSlicer.class);
+   }
+
+}

drivers/netty/src/main/java/org/jclouds/netty/io/ChunkedFileInputStream.java

@@ -17,10 +17,9 @@
  * ====================================================================
  */
 
-package org.jclouds.io.payloads;
+package org.jclouds.netty.io;
 
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.RandomAccessFile;
@@ -87,7 +86,7 @@ public class ChunkedFileInputStream extends InputStream {
          int readIndex = chunk.readerIndex();
          byte abyte = chunk.getByte(readIndex);
          chunk.readerIndex(readIndex + 1);
-         return (int)abyte;
+         return (int) abyte;
       } catch (Exception e) {
          throw new IOException(e);
       }

drivers/netty/src/main/java/org/jclouds/netty/io/NettyPayloadSlicer.java

@@ -17,34 +17,26 @@
  * ====================================================================
  */
 
-package org.jclouds.io.payloads;
+package org.jclouds.netty.io;
 
 import java.io.File;
-import java.io.InputStream;
 
-public class ChunkedFilePayload extends FilePayload {
+import javax.inject.Singleton;
 
-   private int part;
+import org.jclouds.io.Payload;
-   private long chunkOffset;
+import org.jclouds.io.Payloads;
-   private long chunkSize;
+import org.jclouds.io.internal.BasePayloadSlicer;
 
-   public ChunkedFilePayload(File content) {
+/**
-      this(content, 1, 0, content.length());
+ * 
-   }
+ * @author Adrian Cole
-   
+ */
-   public ChunkedFilePayload(File content, int part, long chunkOffset, long chunkSize) {
+@Singleton
-      super(content);
+public class NettyPayloadSlicer extends BasePayloadSlicer {
-      this.part = part;
-      this.chunkOffset = chunkOffset;
-      this.chunkSize = chunkSize;
-   }
-   
-   public int getPart() {
-      return part;
-   }
 
    @Override
-   public InputStream getInput() {
+   protected Payload doSlice(File content, long offset, long length) {
-      return new ChunkedFileInputStream(getRawContent(), chunkOffset, chunkSize);
+      return Payloads.newInputStreamPayload(new ChunkedFileInputStream(content, offset, length));
    }
+
 }

drivers/pom.xml

@@ -38,6 +38,7 @@
         <module>bouncycastle</module>
         <module>log4j</module>
         <module>jsch</module>
+        <module>netty</module>
         <module>enterprise</module>
     </modules>
 </project>

providers/aws-s3/pom.xml

@@ -42,6 +42,21 @@
         <test.aws-s3.credential>${test.aws.credential}</test.aws-s3.credential>
     </properties>
 
+    <!-- temporary -->
+    <repositories>
+        <repository>
+            <id>jboss-public-releases</id>
+            <url>https://repository.jboss.org/nexus/content/groups/public-jboss</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
+
     <dependencies>
         <dependency>
             <groupId>org.jclouds.api</groupId>
@@ -94,6 +109,18 @@
             <version>1.2.16</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.jclouds.driver</groupId>
+            <artifactId>jclouds-netty</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.jboss.netty</groupId>
+            <artifactId>netty</artifactId>
+            <version>3.2.4.Final</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <profiles>

providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ContextBuilder.java

@@ -25,7 +25,6 @@ import java.util.Properties;
 import org.jclouds.aws.s3.blobstore.config.AWSS3BlobStoreContextModule;
 import org.jclouds.aws.s3.config.AWSS3RestClientModule;
 import org.jclouds.s3.S3ContextBuilder;
-import org.jclouds.s3.blobstore.config.S3BlobStoreContextModule;
 
 import com.google.inject.Module;
 

providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3AsyncBlobStore.java

@@ -29,7 +29,9 @@ import javax.inject.Provider;
 import org.jclouds.Constants;
 import org.jclouds.aws.s3.AWSS3AsyncClient;
 import org.jclouds.aws.s3.AWSS3Client;
+import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy;
 import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
 import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
 import org.jclouds.blobstore.util.BlobUtils;
@@ -44,6 +46,8 @@ import org.jclouds.s3.blobstore.functions.ObjectToBlob;
 import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
 
 import com.google.common.base.Supplier;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
 
 /**
  * 
@@ -51,6 +55,8 @@ import com.google.common.base.Supplier;
  */
 public class AWSS3AsyncBlobStore extends S3AsyncBlobStore {
 
+   private final Provider<MultipartUploadStrategy> multipartUploadStrategy;
+
    @Inject
    public AWSS3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils,
          @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier<Location> defaultLocation,
@@ -58,10 +64,19 @@ public class AWSS3AsyncBlobStore extends S3AsyncBlobStore {
          BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions,
          BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob,
          BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd,
-         Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
+         Provider<FetchBlobMetadata> fetchBlobMetadataProvider,
+         Provider<MultipartUploadStrategy> multipartUploadStrategy) {
       super(context, blobUtils, service, defaultLocation, locations, async, sync, bucket2ResourceMd,
-         container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, 
+            container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object,
-         blob2Object, object2BlobMd, fetchBlobMetadataProvider);
+            object2BlobMd, fetchBlobMetadataProvider);
+      this.multipartUploadStrategy = multipartUploadStrategy;
+   }
+
+   @Override
+   public ListenableFuture<String> putBlobMultipart(String container, Blob blob) {
+      // TODO: make this better
+      // need to use a provider if the strategy object is stateful
+      return Futures.immediateFuture(multipartUploadStrategy.get().execute(container, blob));
    }
 
 }

providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java

@@ -26,7 +26,6 @@ import javax.inject.Provider;
 
 import org.jclouds.aws.s3.AWSS3Client;
 import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy;
-import org.jclouds.aws.s3.blobstore.strategy.internal.SequentialMultipartUploadStrategy;
 import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
@@ -51,7 +50,7 @@ import com.google.common.base.Supplier;
  */
 public class AWSS3BlobStore extends S3BlobStore {
 
-   private MultipartUploadStrategy multipartUploadStrategy;
+   private final Provider<MultipartUploadStrategy> multipartUploadStrategy;
 
    @Inject
    AWSS3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
@@ -59,18 +58,16 @@ public class AWSS3BlobStore extends S3BlobStore {
          BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions,
          BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob,
          BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd,
-            Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
+         Provider<FetchBlobMetadata> fetchBlobMetadataProvider, Provider<MultipartUploadStrategy> multipartUploadStrategy) {
       super(context, blobUtils, defaultLocation, locations, sync, bucket2ResourceMd, container2BucketListOptions,
             bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object, object2BlobMd,
             fetchBlobMetadataProvider);
-      multipartUploadStrategy = new SequentialMultipartUploadStrategy(this);
+      this.multipartUploadStrategy = multipartUploadStrategy;
    }
 
-   /* (non-Javadoc)
-    * @see org.jclouds.blobstore.internal.BaseBlobStore#putBlobMultipart(java.lang.String, org.jclouds.blobstore.domain.Blob)
-    */
    @Override
    public String putBlobMultipart(String container, Blob blob) {
-      return multipartUploadStrategy.execute(container, blob);
+      // need to use a provider if the strategy object is stateful
+      return multipartUploadStrategy.get().execute(container, blob);
    }
 }

providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/config/AWSS3BlobStoreContextModule.java

@@ -23,15 +23,12 @@ import org.jclouds.aws.s3.AWSS3AsyncClient;
 import org.jclouds.aws.s3.AWSS3Client;
 import org.jclouds.aws.s3.blobstore.AWSS3AsyncBlobStore;
 import org.jclouds.aws.s3.blobstore.AWSS3BlobStore;
-import org.jclouds.blobstore.AsyncBlobStore;
+import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy;
-import org.jclouds.blobstore.BlobRequestSigner;
+import org.jclouds.aws.s3.blobstore.strategy.internal.SequentialMultipartUploadStrategy;
-import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.BlobStoreContext;
-import org.jclouds.blobstore.attr.ConsistencyModel;
-import org.jclouds.blobstore.config.BlobStoreMapModule;
 import org.jclouds.blobstore.internal.BlobStoreContextImpl;
-import org.jclouds.location.config.RegionsLocationModule;
+import org.jclouds.s3.blobstore.S3AsyncBlobStore;
-import org.jclouds.s3.blobstore.S3BlobRequestSigner;
+import org.jclouds.s3.blobstore.S3BlobStore;
 import org.jclouds.s3.blobstore.config.S3BlobStoreContextModule;
 
 import com.google.inject.Scopes;
@@ -46,14 +43,16 @@ public class AWSS3BlobStoreContextModule extends S3BlobStoreContextModule {
 
    @Override
    protected void configure() {
-      install(new BlobStoreMapModule());
+      super.configure();
-      install(new RegionsLocationModule());
+      bind(S3AsyncBlobStore.class).to(AWSS3AsyncBlobStore.class).in(Scopes.SINGLETON);
-      bind(ConsistencyModel.class).toInstance(ConsistencyModel.EVENTUAL);
+      bind(S3BlobStore.class).to(AWSS3BlobStore.class).in(Scopes.SINGLETON);
-      bind(AsyncBlobStore.class).to(AWSS3AsyncBlobStore.class).in(Scopes.SINGLETON);
+      bind(MultipartUploadStrategy.class).to(SequentialMultipartUploadStrategy.class);
-      bind(BlobStore.class).to(AWSS3BlobStore.class).in(Scopes.SINGLETON);
+   }
+
+   @Override
+   protected void bindContext() {
       bind(BlobStoreContext.class).to(new TypeLiteral<BlobStoreContextImpl<AWSS3Client, AWSS3AsyncClient>>() {
       }).in(Scopes.SINGLETON);
-      bind(BlobRequestSigner.class).to(S3BlobRequestSigner.class);
-      bindBucketLocationStrategy();
    }
+
 }

providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java

@@ -19,19 +19,23 @@
 
 package org.jclouds.aws.s3.blobstore.strategy.internal;
 
-import static org.jclouds.io.Payloads.newChunkedFilePayload;
+import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.io.File;
 
 import java.util.Map;
 
+import javax.annotation.Resource;
+import javax.inject.Inject;
+import javax.inject.Named;
+
 import org.jclouds.aws.s3.AWSS3Client;
 import org.jclouds.aws.s3.blobstore.AWSS3BlobStore;
 import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy;
 import org.jclouds.blobstore.KeyNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.reference.BlobStoreConstants;
 import org.jclouds.io.Payload;
-import org.jclouds.io.payloads.FilePayload;
+import org.jclouds.io.PayloadSlicer;
+import org.jclouds.logging.Logger;
 import org.jclouds.s3.domain.ObjectMetadataBuilder;
 
 import com.google.common.collect.Maps;
@@ -41,21 +45,23 @@ import com.google.common.collect.Maps;
  * 
  * The file partitioning algorithm:
  * 
- * The default partition size we choose is 32mb. 
+ * The default partition size we choose is 32mb. A multiple of this default partition size is used.
- * A multiple of this default partition size is used. 
+ * The number of parts first grows to a chosen magnitude (for example 100 parts), then it grows the
- * The number of parts first grows to a chosen magnitude (for example 100 parts), 
+ * partition size instead of number of partitions. When we reached the maximum part size, then again
- * then it grows the partition size instead of number of partitions.
+ * it starts to grow the number of partitions.
- * When we reached the maximum part size, then again it starts to grow the number of partitions.
  * 
  * @author Tibor Kiss
  */
-public class SequentialMultipartUploadStrategy implements
+public class SequentialMultipartUploadStrategy implements MultipartUploadStrategy {
-      MultipartUploadStrategy {
+   @Resource
+   @Named(BlobStoreConstants.BLOBSTORE_LOGGER)
+   protected Logger logger = Logger.NULL;
 
    private final long DEFAULT_PART_SIZE = 33554432; // 32mb
    private final int MAGNITUDE_BASE = 100;
 
    private final AWSS3BlobStore ablobstore;
+   private final PayloadSlicer slicer;
 
    // calculated only once, but not from the constructor
    private volatile long parts; // required number of parts with chunkSize
@@ -67,8 +73,10 @@ public class SequentialMultipartUploadStrategy implements
    private volatile long chunkOffset;
    private volatile long copied;
 
-   public SequentialMultipartUploadStrategy(AWSS3BlobStore ablobstore) {
+   @Inject
-      this.ablobstore = ablobstore;
+   public SequentialMultipartUploadStrategy(AWSS3BlobStore ablobstore, PayloadSlicer slicer) {
+      this.ablobstore = checkNotNull(ablobstore, "ablobstore");
+      this.slicer = checkNotNull(slicer, "slicer");
    }
 
    protected long calculateChunkSize(long length) {
@@ -100,9 +108,8 @@ public class SequentialMultipartUploadStrategy implements
       this.chunkSize = partSize;
       this.parts = parts;
       this.remaining = length - partSize * parts;
-      System.out.println(" " + length + " bytes partitioned in " + parts
+      logger.debug(" %d bytes partitioned in %d parts of part size: %d, remaining: %d%s", length, parts, chunkSize,
-            + " parts of part size: " + chunkSize + ", remaining: "
+            remaining, (remaining > MAX_PART_SIZE ? " overflow!" : ""));
-            + remaining + (remaining > MAX_PART_SIZE ? " overflow!" : ""));
       return this.chunkSize;
    }
 
@@ -132,11 +139,8 @@ public class SequentialMultipartUploadStrategy implements
       return remaining;
    }
 
-   private String prepareUploadPart(AWSS3Client client, String container, String key, String uploadId,
+   private String prepareUploadPart(AWSS3Client client, String container, String key, String uploadId, int part,
-         int part, File file, long chunkOffset, long chunkSize) {
+         Payload chunkedPart) {
-      Payload chunkedPart = newChunkedFilePayload(file, part, chunkOffset, chunkSize);
-      chunkedPart.getContentMetadata().setContentLength(chunkSize);
-      //chukedPayload.getContentMetadata().setContentMD5(???);
       String eTag = null;
       try {
          eTag = client.uploadPart(container, key, part, uploadId, chunkedPart);
@@ -153,33 +157,29 @@ public class SequentialMultipartUploadStrategy implements
 
    @Override
    public String execute(String container, Blob blob) {
-      Payload payload = blob.getPayload();
-      if (payload instanceof FilePayload) {
       String key = blob.getMetadata().getName();
-         File file = FilePayload.class.cast(payload).getRawContent();
+      calculateChunkSize(blob.getPayload().getContentMetadata().getContentLength());
-         calculateChunkSize(file.length());
       long parts = getParts();
       if (parts > 0) {
          AWSS3Client client = (AWSS3Client) ablobstore.getContext().getProviderSpecificContext().getApi();
-            String uploadId = client.initiateMultipartUpload(container, 
+         String uploadId = client.initiateMultipartUpload(container, ObjectMetadataBuilder.create().key(key).build()); // TODO
-                  ObjectMetadataBuilder.create().key(key).build()); // TODO md5            
+                                                                                                                       // md5
          Map<Integer, String> etags = Maps.newHashMap();
          int part;
          while ((part = getNextPart()) <= getParts()) {
-               String eTag = prepareUploadPart(client, container, key, uploadId, part, file, getNextChunkOffset(), chunkSize);
+            String eTag = prepareUploadPart(client, container, key, uploadId, part,
+                  slicer.slice(blob.getPayload(), getNextChunkOffset(), chunkSize));
             etags.put(new Integer(part), eTag);
          }
          long remaining = getRemaining();
          if (remaining > 0) {
-               String eTag = prepareUploadPart(client, container, key, uploadId, part, file, getNextChunkOffset(), remaining);
+            String eTag = prepareUploadPart(client, container, key, uploadId, part,
+                  slicer.slice(blob.getPayload(), getNextChunkOffset(), remaining));
             etags.put(new Integer(part), eTag);
          }
          return client.completeMultipartUpload(container, key, uploadId, etags);
       } else {
          return ablobstore.putBlob(container, blob);
       }
-      } else {
-         return ablobstore.putBlob(container, blob);
-      }
    }
 }

providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3TestInitializer.java

@@ -26,6 +26,7 @@ import org.jclouds.blobstore.BlobStoreContextFactory;
 import org.jclouds.blobstore.integration.TransientBlobStoreTestInitializer;
 import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
 import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.netty.config.NettyPayloadModule;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Module;
@@ -45,7 +46,7 @@ public class AWSS3TestInitializer extends TransientBlobStoreTestInitializer {
    protected BlobStoreContext createLiveContext(Module configurationModule, String endpoint, String apiversion,
             String app, String identity, String credential) throws IOException {
       return new BlobStoreContextFactory().createContext(provider, ImmutableSet.of(configurationModule,
-               new Log4JLoggingModule()), setupProperties(endpoint, apiversion, identity, credential));
+               new Log4JLoggingModule(), new NettyPayloadModule()), setupProperties(endpoint, apiversion, identity, credential));
    }
 
 }

providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/AzureAsyncBlobStore.java

@@ -30,6 +30,7 @@ import javax.inject.Named;
 import javax.inject.Singleton;
 
 import org.jclouds.Constants;
+import org.jclouds.azure.storage.domain.BoundedSet;
 import org.jclouds.azureblob.AzureBlobAsyncClient;
 import org.jclouds.azureblob.blobstore.functions.AzureBlobToBlob;
 import org.jclouds.azureblob.blobstore.functions.BlobPropertiesToBlobMetadata;
@@ -42,7 +43,6 @@ import org.jclouds.azureblob.domain.BlobProperties;
 import org.jclouds.azureblob.domain.ContainerProperties;
 import org.jclouds.azureblob.domain.ListBlobsResponse;
 import org.jclouds.azureblob.options.ListBlobsOptions;
-import org.jclouds.azure.storage.domain.BoundedSet;
 import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.BlobMetadata;
@@ -110,8 +110,8 @@ public class AzureAsyncBlobStore extends BaseAsyncBlobStore {
                   new Function<BoundedSet<ContainerProperties>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() {
                      public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(
                            BoundedSet<ContainerProperties> from) {
-                              return new PageSetImpl<StorageMetadata>(Iterables.transform(from, container2ResourceMd),
+                        return new PageSetImpl<StorageMetadata>(Iterables.transform(from, container2ResourceMd), from
-                                       from.getNextMarker());
+                              .getNextMarker());
                      }
                   }, service);
    }
@@ -243,4 +243,9 @@ public class AzureAsyncBlobStore extends BaseAsyncBlobStore {
       throw new UnsupportedOperationException("please use deleteContainer");
    }
 
+   @Override
+   public ListenableFuture<String> putBlobMultipart(String container, Blob blob) {
+      return putBlob(container, blob);
+   }
+
 }