Merge branch 'master' of github.com:jclouds/jclouds

* 'master' of github.com:jclouds/jclouds:
  Issue 430:large blob support on aws-s3
  added debug to http executor
  carved out a place to add aws-s3 specific functionality

apis/atmos/src/main/java/org/jclouds/atmos/domain/internal/DelegatingMutableContentMetadata.java

@@ -20,6 +20,7 @@
 package org.jclouds.atmos.domain.internal;
 
 import org.jclouds.atmos.domain.MutableContentMetadata;
+import org.jclouds.io.ContentMetadataBuilder;
 import org.jclouds.io.payloads.BaseMutableContentMetadata;
 
 import com.google.common.collect.Multimap;
@@ -157,4 +158,9 @@ public class DelegatingMutableContentMetadata implements MutableContentMetadata
       delegate.setPropertiesFromHttpHeaders(headers);
    }
 
+   @Override
+   public ContentMetadataBuilder toBuilder() {
+      return delegate.toBuilder();
+   }
+
 }

apis/filesystem/src/main/java/org/jclouds/filesystem/FilesystemAsyncBlobStore.java

@@ -49,10 +49,10 @@ import java.util.Collection;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.Map.Entry;
 import java.util.concurrent.ExecutorService;
 
 import javax.annotation.Nullable;
@@ -67,13 +67,13 @@ import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.ContainerNotFoundException;
 import org.jclouds.blobstore.KeyNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.blobstore.domain.Blob.Factory;
 import org.jclouds.blobstore.domain.BlobMetadata;
 import org.jclouds.blobstore.domain.MutableBlobMetadata;
 import org.jclouds.blobstore.domain.MutableStorageMetadata;
 import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.domain.StorageType;
+import org.jclouds.blobstore.domain.Blob.Factory;
 import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl;
 import org.jclouds.blobstore.domain.internal.MutableStorageMetadataImpl;
 import org.jclouds.blobstore.domain.internal.PageSetImpl;
@@ -97,6 +97,7 @@ import org.jclouds.http.HttpResponseException;
 import org.jclouds.http.options.HttpRequestOptions;
 import org.jclouds.io.Payload;
 import org.jclouds.io.Payloads;
+import org.jclouds.io.payloads.BaseMutableContentMetadata;
 import org.jclouds.io.payloads.FilePayload;
 import org.jclouds.logging.Logger;
 import org.jclouds.rest.annotations.ParamValidators;
@@ -129,10 +130,10 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
 
    @Inject
    protected FilesystemAsyncBlobStore(BlobStoreContext context, DateService dateService, Crypto crypto,
-         HttpGetOptionsListToGetOptions httpGetOptionsConverter, IfDirectoryReturnNameStrategy ifDirectoryReturnName,
+            HttpGetOptionsListToGetOptions httpGetOptionsConverter,
-         Factory blobFactory, BlobUtils blobUtils, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service,
+            IfDirectoryReturnNameStrategy ifDirectoryReturnName, Factory blobFactory, BlobUtils blobUtils,
-         Supplier<Location> defaultLocation, @Memoized Supplier<Set<? extends Location>> locations,
+            @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier<Location> defaultLocation,
-         FilesystemStorageStrategy storageStrategy) {
+            @Memoized Supplier<Set<? extends Location>> locations, FilesystemStorageStrategy storageStrategy) {
       super(context, blobUtils, service, defaultLocation, locations);
       // super(context, blobUtils, service, null, null);
       this.blobFactory = blobFactory;
@@ -164,22 +165,22 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
       }
 
       SortedSet<StorageMetadata> contents = newTreeSet(transform(blobBelongingToContainer,
-            new Function<String, StorageMetadata>() {
+               new Function<String, StorageMetadata>() {
-               public StorageMetadata apply(String key) {
+                  public StorageMetadata apply(String key) {
-                  Blob oldBlob = loadFileBlob(container, key);
+                     Blob oldBlob = loadFileBlob(container, key);
 
-                  checkState(oldBlob != null, "blob " + key + " is not present although it was in the list of "
+                     checkState(oldBlob != null, "blob " + key + " is not present although it was in the list of "
-                        + container);
+                              + container);
-                  checkState(oldBlob.getMetadata() != null, "blob " + container + "/" + key + " has no metadata");
+                     checkState(oldBlob.getMetadata() != null, "blob " + container + "/" + key + " has no metadata");
-                  MutableBlobMetadata md = copy(oldBlob.getMetadata());
+                     MutableBlobMetadata md = copy(oldBlob.getMetadata());
-                  String directoryName = ifDirectoryReturnName.execute(md);
+                     String directoryName = ifDirectoryReturnName.execute(md);
-                  if (directoryName != null) {
+                     if (directoryName != null) {
-                     md.setName(directoryName);
+                        md.setName(directoryName);
-                     md.setType(StorageType.RELATIVE_PATH);
+                        md.setType(StorageType.RELATIVE_PATH);
+                     }
+                     return md;
                   }
-                  return md;
+               }));
-               }
-            }));
 
       String marker = null;
       if (options != null) {
@@ -219,21 +220,21 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
          if (delimiter != null) {
             SortedSet<String> commonPrefixes = null;
             Iterable<String> iterable = transform(contents, new CommonPrefixes(prefix != null ? prefix : null,
-                  delimiter));
+                     delimiter));
             commonPrefixes = iterable != null ? newTreeSet(iterable) : new TreeSet<String>();
             commonPrefixes.remove(CommonPrefixes.NO_PREFIX);
 
             contents = newTreeSet(filter(contents, new DelimiterFilter(prefix != null ? prefix : null, delimiter)));
 
-            Iterables.<StorageMetadata> addAll(contents,
+            Iterables.<StorageMetadata> addAll(contents, transform(commonPrefixes,
-                  transform(commonPrefixes, new Function<String, StorageMetadata>() {
+                     new Function<String, StorageMetadata>() {
-                     public StorageMetadata apply(String o) {
+                        public StorageMetadata apply(String o) {
-                        MutableStorageMetadata md = new MutableStorageMetadataImpl();
+                           MutableStorageMetadata md = new MutableStorageMetadataImpl();
-                        md.setType(StorageType.RELATIVE_PATH);
+                           md.setType(StorageType.RELATIVE_PATH);
-                        md.setName(o);
+                           md.setName(o);
-                        return md;
+                           return md;
-                     }
+                        }
-                  }));
+                     }));
          }
 
          // trim metadata, if the response isn't supposed to be detailed.
@@ -245,7 +246,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
       }
 
       return Futures.<PageSet<? extends StorageMetadata>> immediateFuture(new PageSetImpl<StorageMetadata>(contents,
-            marker));
+               marker));
 
    }
 
@@ -262,6 +263,8 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
          ObjectInput is = new ObjectInputStream(new ByteArrayInputStream(bout.toByteArray()));
          MutableBlobMetadata metadata = (MutableBlobMetadata) is.readObject();
          convertUserMetadataKeysToLowercase(metadata);
+         metadata.setContentMetadata(BaseMutableContentMetadata.fromContentMetadata(in.getContentMetadata().toBuilder()
+                  .build()));
          return metadata;
       } catch (Exception e) {
          propagate(e);
@@ -310,14 +313,14 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
       Iterable<String> containers = storageStrategy.getAllContainerNames();
 
       return Futures.<PageSet<? extends StorageMetadata>> immediateFuture(new PageSetImpl<StorageMetadata>(transform(
-            containers, new Function<String, StorageMetadata>() {
+               containers, new Function<String, StorageMetadata>() {
-               public StorageMetadata apply(String name) {
+                  public StorageMetadata apply(String name) {
-                  MutableStorageMetadata cmd = create();
+                     MutableStorageMetadata cmd = create();
-                  cmd.setName(name);
+                     cmd.setName(name);
-                  cmd.setType(StorageType.CONTAINER);
+                     cmd.setType(StorageType.CONTAINER);
-                  return cmd;
+                     return cmd;
-               }
+                  }
-            }), null));
+               }), null));
    }
 
    protected MutableStorageMetadata create() {
@@ -330,7 +333,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
    @Path("{container}")
    @Override
    public ListenableFuture<Boolean> createContainerInLocation(final Location location,
-         @PathParam("container") @ParamValidators({ FilesystemContainerNameValidator.class }) String name) {
+            @PathParam("container") @ParamValidators( { FilesystemContainerNameValidator.class }) String name) {
       boolean result = storageStrategy.createContainer(name);
       return immediateFuture(result);
    }
@@ -500,7 +503,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
          storageStrategy.writePayloadOnFile(containerName, blobKey, object.getPayload());
       } catch (IOException e) {
          logger.error(e, "An error occurred storing the new object with name [%s] to container [%s].", blobKey,
-               containerName);
+                  containerName);
          Throwables.propagate(e);
       }
       return immediateFuture(eTag);
@@ -547,7 +550,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
             if (blob.getMetadata().getLastModified().before(modifiedSince)) {
                HttpResponse response = new HttpResponse(304, null, null);
                return immediateFailedFuture(new HttpResponseException(String.format("%1$s is before %2$s", blob
-                     .getMetadata().getLastModified(), modifiedSince), null, response));
+                        .getMetadata().getLastModified(), modifiedSince), null, response));
             }
 
          }
@@ -556,7 +559,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
             if (blob.getMetadata().getLastModified().after(unmodifiedSince)) {
                HttpResponse response = new HttpResponse(412, null, null);
                return immediateFailedFuture(new HttpResponseException(String.format("%1$s is after %2$s", blob
-                     .getMetadata().getLastModified(), unmodifiedSince), null, response));
+                        .getMetadata().getLastModified(), unmodifiedSince), null, response));
             }
          }
 

apis/s3/src/main/java/org/jclouds/s3/S3Client.java

@@ -137,7 +137,6 @@ public interface S3Client {
     *           namespace of the object you are deleting
     * @param key
     *           unique key in the s3Bucket identifying the object
-    * @return true if deleted
     * @throws org.jclouds.http.HttpResponseException
     *            if the bucket is not available
     * @see <a href="http://docs.amazonwebservices.com/AmazonS3/2006-03-01/index.html?
@@ -164,7 +163,7 @@ public interface S3Client {
     *           contains the data and metadata to create or overwrite
     * @param options
     *           options for creating the object
-    * @return MD5 hash of the content uploaded
+    * @return ETag of the content uploaded
     * @throws org.jclouds.http.HttpResponseException
     *            if the conditions requested set are not satisfied by the object on the server.
     * @see org.jclouds.s3.domain.CannedAccessPolicy#PRIVATE

apis/s3/src/main/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequest.java

@@ -26,12 +26,11 @@ import javax.inject.Inject;
 import javax.inject.Singleton;
 import javax.ws.rs.core.HttpHeaders;
 
-import org.jclouds.s3.blobstore.functions.ObjectToBlob;
+import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
-import org.jclouds.s3.domain.S3Object;
-import org.jclouds.blobstore.binders.BindUserMetadataToHeadersWithPrefix;
 import org.jclouds.http.HttpRequest;
 import org.jclouds.http.utils.ModifyRequest;
 import org.jclouds.rest.Binder;
+import org.jclouds.s3.domain.S3Object;
 
 /**
  * 
@@ -39,13 +38,11 @@ import org.jclouds.rest.Binder;
  */
 @Singleton
 public class BindS3ObjectMetadataToRequest implements Binder {
-   private final BindUserMetadataToHeadersWithPrefix blobBinder;
+   protected final BindMapToHeadersWithPrefix metadataPrefixer;
-   private final ObjectToBlob object2Blob;
 
    @Inject
-   public BindS3ObjectMetadataToRequest(ObjectToBlob object2Blob, BindUserMetadataToHeadersWithPrefix blobBinder) {
+   public BindS3ObjectMetadataToRequest(BindMapToHeadersWithPrefix metadataPrefixer) {
-      this.blobBinder = checkNotNull(blobBinder, "blobBinder");
+      this.metadataPrefixer = checkNotNull(metadataPrefixer, "metadataPrefixer");
-      this.object2Blob = checkNotNull(object2Blob, "object2Blob");
    }
 
    @Override
@@ -59,7 +56,8 @@ public class BindS3ObjectMetadataToRequest implements Binder {
             "contentLength must be set, streaming not supported");
       checkArgument(s3Object.getPayload().getContentMetadata().getContentLength() <= 5l * 1024 * 1024 * 1024,
             "maximum size for put object is 5GB");
-      request = blobBinder.bindToRequest(request, object2Blob.apply(s3Object));
+      
+      request = metadataPrefixer.bindToRequest(request, s3Object.getMetadata().getUserMetadata());
 
       if (s3Object.getMetadata().getCacheControl() != null) {
          request = ModifyRequest.replaceHeader(request, HttpHeaders.CACHE_CONTROL, s3Object.getMetadata()

apis/s3/src/main/java/org/jclouds/s3/domain/MutableObjectMetadata.java

@@ -22,8 +22,8 @@ package org.jclouds.s3.domain;
 import java.util.Date;
 import java.util.Map;
 
-import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
 import org.jclouds.io.MutableContentMetadata;
+import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
 
 import com.google.inject.ImplementedBy;
 
@@ -69,6 +69,7 @@ public interface MutableObjectMetadata extends ObjectMetadata {
     */
    void setCacheControl(String cacheControl);
 
+   @Override
    MutableContentMetadata getContentMetadata();
 
    void setContentMetadata(MutableContentMetadata md);

apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadata.java

@@ -38,7 +38,7 @@ import org.jclouds.io.ContentMetadata;
 public interface ObjectMetadata extends Comparable<ObjectMetadata> {
 
    public enum StorageClass {
-      STANDARD
+      STANDARD, REDUCED_REDUNDANCY
    }
 
    /**

apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadataBuilder.java

@@ -0,0 +1,114 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.s3.domain;
+
+import java.util.Map;
+
+import org.jclouds.io.ContentMetadataBuilder;
+import org.jclouds.io.payloads.BaseMutableContentMetadata;
+import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
+import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
+
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Allows you to create {@link ObjectMetadata} objects.
+ * 
+ * @author Adrian Cole
+ */
+public class ObjectMetadataBuilder {
+   public static ObjectMetadataBuilder create() {
+      return new ObjectMetadataBuilder();
+   }
+
+   private final ContentMetadataBuilder contentMetadataBuilder = new ContentMetadataBuilder().contentType("binary/octet-stream");
+
+   private String key;
+   private StorageClass storageClass;
+   private String cacheControl;
+   private Map<String, String> userMetadata = ImmutableMap.of();
+ 
+   
+   public ObjectMetadataBuilder key(String key) {
+      this.key = key;
+      return this;
+   }
+
+   public ObjectMetadataBuilder storageClass(StorageClass storageClass) {
+      this.storageClass = storageClass;
+      return this;
+   }
+
+   public ObjectMetadataBuilder cacheControl(String cacheControl) {
+      this.cacheControl = cacheControl;
+      return this;
+   }
+
+   public ObjectMetadataBuilder userMetadata(Map<String, String> userMetadata) {
+      this.userMetadata = ImmutableMap.copyOf(userMetadata);
+      return this;
+   }
+
+   public ObjectMetadataBuilder contentDisposition(String contentDisposition) {
+      contentMetadataBuilder.contentDisposition(contentDisposition);
+      return this;
+   }
+
+   public ObjectMetadataBuilder contentEncoding(String contentEncoding) {
+      contentMetadataBuilder.contentEncoding(contentEncoding);
+      return this;
+
+   }
+
+   public ObjectMetadataBuilder contentLanguage(String contentLanguage) {
+      contentMetadataBuilder.contentLanguage(contentLanguage);
+      return this;
+
+   }
+
+   public ObjectMetadataBuilder contentLength(Long contentLength) {
+      contentMetadataBuilder.contentLength(contentLength);
+      return this;
+
+   }
+
+   public ObjectMetadataBuilder contentMD5(byte[] md5) {
+      contentMetadataBuilder.contentMD5(md5);
+      return this;
+
+   }
+
+   public ObjectMetadataBuilder contentType(String contentType) {
+      contentMetadataBuilder.contentType(contentType);
+      return this;
+
+   }
+
+   public ObjectMetadata build() {
+      MutableObjectMetadataImpl toReturn = new MutableObjectMetadataImpl();
+      toReturn.setContentMetadata(BaseMutableContentMetadata.fromContentMetadata(contentMetadataBuilder.build()));
+      toReturn.setCacheControl(cacheControl);
+      toReturn.setKey(key);
+      toReturn.setStorageClass(storageClass);
+      toReturn.setUserMetadata(userMetadata);
+      return toReturn;
+   }
+
+}

apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java

@@ -78,7 +78,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
    private final String[] firstHeadersToSign = new String[] { HttpHeaders.DATE };
 
    public static Set<String> SPECIAL_QUERIES = ImmutableSet.of("acl", "torrent", "logging", "location",
-         "requestPayment");
+            "requestPayment", "uploads");
    private final SignatureWire signatureWire;
    private final String accessKey;
    private final String secretKey;
@@ -97,10 +97,10 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
 
    @Inject
    public RequestAuthorizeSignature(SignatureWire signatureWire, @Named(PROPERTY_AUTH_TAG) String authTag,
-         @Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
+            @Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
-         @Named(PROPERTY_S3_SERVICE_PATH) String servicePath, @Named(PROPERTY_HEADER_TAG) String headerTag,
+            @Named(PROPERTY_S3_SERVICE_PATH) String servicePath, @Named(PROPERTY_HEADER_TAG) String headerTag,
-         @Named(PROPERTY_IDENTITY) String accessKey, @Named(PROPERTY_CREDENTIAL) String secretKey,
+            @Named(PROPERTY_IDENTITY) String accessKey, @Named(PROPERTY_CREDENTIAL) String secretKey,
-         @TimeStamp Provider<String> timeStampProvider, Crypto crypto, HttpUtils utils) {
+            @TimeStamp Provider<String> timeStampProvider, Crypto crypto, HttpUtils utils) {
       this.isVhostStyle = isVhostStyle;
       this.servicePath = servicePath;
       this.headerTag = headerTag;
@@ -123,7 +123,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
 
    HttpRequest replaceAuthorizationHeader(HttpRequest request, String signature) {
       request = ModifyRequest.replaceHeader(request, HttpHeaders.AUTHORIZATION, authTag + " " + accessKey + ":"
-            + signature);
+               + signature);
       return request;
    }
 
@@ -161,8 +161,8 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
    public String sign(String toSign) {
       String signature;
       try {
-         signature = CryptoStreams.base64(CryptoStreams.mac(InputSuppliers.of(toSign),
+         signature = CryptoStreams.base64(CryptoStreams.mac(InputSuppliers.of(toSign), crypto.hmacSHA1(secretKey
-               crypto.hmacSHA1(secretKey.getBytes())));
+                  .getBytes())));
       } catch (Exception e) {
          throw new HttpException("error signing request", e);
       }
@@ -189,11 +189,11 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
 
    void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {
       buffer.append(
-            utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata()
+               utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata()
-                  .getContentMD5())).append("\n");
+                        .getContentMD5())).append("\n");
       buffer.append(
-            utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata()
+               utils.valueOrEmpty(request.getPayload() == null ? request.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE)
-                  .getContentType())).append("\n");
+                        : request.getPayload().getContentMetadata().getContentType())).append("\n");
    }
 
    void appendHttpHeaders(HttpRequest request, StringBuilder toSign) {
@@ -210,11 +210,11 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
 
       for (int i = 0; i < request.getJavaMethod().getParameterAnnotations().length; i++) {
          if (Iterables.any(Arrays.asList(request.getJavaMethod().getParameterAnnotations()[i]),
-               new Predicate<Annotation>() {
+                  new Predicate<Annotation>() {
-                  public boolean apply(Annotation input) {
+                     public boolean apply(Annotation input) {
-                     return input.annotationType().equals(Bucket.class);
+                        return input.annotationType().equals(Bucket.class);
-                  }
+                     }
-               })) {
+                  })) {
             bucketName = (String) request.getArgs().get(i);
             break;
          }

apis/s3/src/test/java/org/jclouds/s3/BaseS3AsyncClientTest.java

@@ -28,20 +28,17 @@ import org.jclouds.http.HttpRequest;
 import org.jclouds.rest.RestClientTest;
 import org.jclouds.rest.RestContextFactory;
 import org.jclouds.rest.RestContextSpec;
-import org.jclouds.rest.internal.RestAnnotationProcessor;
 import org.jclouds.s3.blobstore.functions.BlobToObject;
 import org.jclouds.s3.filters.RequestAuthorizeSignature;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
-import com.google.inject.TypeLiteral;
-
 /**
  * 
  * @author Adrian Cole
  */
 @Test(groups = "unit")
-public abstract class BaseS3AsyncClientTest extends RestClientTest<S3AsyncClient> {
+public abstract class BaseS3AsyncClientTest<T extends S3AsyncClient> extends RestClientTest<T> {
 
    protected BlobToObject blobToS3Object;
    protected RequestAuthorizeSignature filter;
@@ -52,11 +49,6 @@ public abstract class BaseS3AsyncClientTest extends RestClientTest<S3AsyncClient
       assertEquals(request.getFilters().get(0).getClass(), RequestAuthorizeSignature.class);
    }
 
-   @Override
-   protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
-      return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
-      };
-   }
 
    @BeforeClass
    @Override

apis/s3/src/test/java/org/jclouds/s3/S3AsyncClientTest.java

@@ -80,7 +80,7 @@ import com.google.inject.Module;
  */
 // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
 @Test(groups = "unit", testName = "S3AsyncClientTest")
-public class S3AsyncClientTest extends BaseS3AsyncClientTest {
+public abstract class S3AsyncClientTest<T extends S3AsyncClient> extends BaseS3AsyncClientTest<T> {
 
    protected String url = "s3.amazonaws.com";
 

apis/s3/src/test/java/org/jclouds/s3/binders/BindAsHostPrefixIfConfiguredTest.java

@@ -27,11 +27,15 @@ import java.io.IOException;
 import java.net.URI;
 import java.util.Properties;
 
-import org.jclouds.s3.BaseS3AsyncClientTest;
 import org.jclouds.http.HttpRequest;
+import org.jclouds.rest.internal.RestAnnotationProcessor;
+import org.jclouds.s3.BaseS3AsyncClientTest;
+import org.jclouds.s3.S3AsyncClient;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import com.google.inject.TypeLiteral;
+
 /**
  * Tests behavior of {@code BindAsHostPrefixIfConfigured}
  * 
@@ -39,7 +43,13 @@ import org.testng.annotations.Test;
  */
 // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
 @Test(groups = "unit", testName = "BindAsHostPrefixIfConfiguredTest")
-public class BindAsHostPrefixIfConfiguredTest extends BaseS3AsyncClientTest {
+public class BindAsHostPrefixIfConfiguredTest extends BaseS3AsyncClientTest<S3AsyncClient> {
+
+   @Override
+   protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
+      return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
+      };
+   }
 
    public void testBucket() throws IOException {
 
@@ -64,8 +74,8 @@ public class BindAsHostPrefixIfConfiguredTest extends BaseS3AsyncClientTest {
 
    @DataProvider(name = "objects")
    public Object[][] createData() {
-      return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" },
+      return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" }, { "colon:" },
-            { "colon:" }, { "asteri*k" }, { "quote\"" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
+               { "asteri*k" }, { "quote\"" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
    }
 
    @Override

apis/s3/src/test/java/org/jclouds/s3/binders/BindNoBucketLoggingToXmlPayloadTest.java

@@ -24,10 +24,14 @@ import static org.testng.Assert.assertEquals;
 import java.io.IOException;
 import java.net.URI;
 
-import org.jclouds.s3.BaseS3AsyncClientTest;
 import org.jclouds.http.HttpRequest;
+import org.jclouds.rest.internal.RestAnnotationProcessor;
+import org.jclouds.s3.BaseS3AsyncClientTest;
+import org.jclouds.s3.S3AsyncClient;
 import org.testng.annotations.Test;
 
+import com.google.inject.TypeLiteral;
+
 /**
  * Tests behavior of {@code BindNoBucketLoggingToXmlPayload}
  * 
@@ -35,7 +39,13 @@ import org.testng.annotations.Test;
  */
 // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
 @Test(groups = "unit", testName = "BindNoBucketLoggingToXmlPayloadTest")
-public class BindNoBucketLoggingToXmlPayloadTest extends BaseS3AsyncClientTest {
+public class BindNoBucketLoggingToXmlPayloadTest extends BaseS3AsyncClientTest<S3AsyncClient> {
+
+   @Override
+   protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
+      return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
+      };
+   }
 
    public void testApplyInputStream() throws IOException {
 
@@ -44,7 +54,7 @@ public class BindNoBucketLoggingToXmlPayloadTest extends BaseS3AsyncClientTest {
 
       request = binder.bindToRequest(request, "bucket");
       assertEquals(request.getPayload().getRawContent(),
-            "<BucketLoggingStatus xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>");
+               "<BucketLoggingStatus xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>");
 
    }
 }

apis/s3/src/test/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequestTest.java

@@ -26,15 +26,19 @@ import java.net.URI;
 
 import javax.ws.rs.HttpMethod;
 
-import org.jclouds.s3.BaseS3AsyncClientTest;
-import org.jclouds.s3.domain.S3Object;
 import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
 import org.jclouds.http.HttpRequest;
 import org.jclouds.io.Payload;
 import org.jclouds.io.Payloads;
+import org.jclouds.rest.internal.RestAnnotationProcessor;
+import org.jclouds.s3.BaseS3AsyncClientTest;
+import org.jclouds.s3.S3AsyncClient;
+import org.jclouds.s3.domain.S3Object;
 import org.testng.annotations.Test;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMultimap;
+import com.google.inject.TypeLiteral;
 
 /**
  * Tests behavior of {@code BindS3ObjectMetadataToRequest}
@@ -43,7 +47,13 @@ import com.google.common.collect.ImmutableMultimap;
  */
 // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
 @Test(groups = "unit", testName = "BindS3ObjectMetadataToRequestTest")
-public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest {
+public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest<S3AsyncClient> {
+
+   @Override
+   protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
+      return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
+      };
+   }
 
    @Test
    public void testPassWithMinimumDetailsAndPayload5GB() {
@@ -56,8 +66,8 @@ public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest {
       HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
       BindS3ObjectMetadataToRequest binder = injector.getInstance(BindS3ObjectMetadataToRequest.class);
 
-      assertEquals(binder.bindToRequest(request, object),
+      assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint(
-            HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build());
+               URI.create("http://localhost")).build());
    }
 
    @Test
@@ -68,14 +78,14 @@ public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest {
       object.setPayload(payload);
       object.getMetadata().setKey("foo");
       object.getMetadata().setCacheControl("no-cache");
+      object.getMetadata().setUserMetadata(ImmutableMap.of("foo", "bar"));
 
       HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
       BindS3ObjectMetadataToRequest binder = injector.getInstance(BindS3ObjectMetadataToRequest.class);
 
-      assertEquals(
+      assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint(
-            binder.bindToRequest(request, object),
+               URI.create("http://localhost")).headers(
-            HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost"))
+               ImmutableMultimap.of("Cache-Control", "no-cache", "x-amz-meta-foo", "bar")).build());
-                  .headers(ImmutableMultimap.of("Cache-Control", "no-cache")).build());
    }
 
    @Test(expectedExceptions = IllegalArgumentException.class)

apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobRequestSignerTest.java

@@ -31,6 +31,7 @@ import org.jclouds.http.HttpRequest;
 import org.jclouds.http.RequiresHttp;
 import org.jclouds.io.payloads.PhantomPayload;
 import org.jclouds.rest.ConfiguresRestClient;
+import org.jclouds.rest.internal.RestAnnotationProcessor;
 import org.jclouds.s3.BaseS3AsyncClientTest;
 import org.jclouds.s3.S3AsyncClient;
 import org.jclouds.s3.S3Client;
@@ -40,6 +41,7 @@ import org.testng.annotations.Test;
 
 import com.google.common.base.Supplier;
 import com.google.inject.Module;
+import com.google.inject.TypeLiteral;
 
 /**
  * Tests behavior of {@code S3BlobRequestSigner}
@@ -48,39 +50,45 @@ import com.google.inject.Module;
  */
 // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
 @Test(groups = "unit", testName = "S3BlobRequestSignerTest")
-public class S3BlobRequestSignerTest extends BaseS3AsyncClientTest {
+public class S3BlobRequestSignerTest extends BaseS3AsyncClientTest<S3AsyncClient> {
+
+   @Override
+   protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
+      return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
+      };
+   }
 
    private BlobRequestSigner signer;
    private Factory blobFactory;
 
    public void testSignGetBlob() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
-         NoSuchMethodException, IOException {
+            NoSuchMethodException, IOException {
       HttpRequest request = signer.signGetBlob("container", "name");
 
       assertRequestLineEquals(request, "GET https://container.s3.amazonaws.com/name HTTP/1.1");
       assertNonPayloadHeadersEqual(
-            request,
+               request,
-            "Authorization: AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
+               "Authorization: AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
       assertPayloadEquals(request, null, null, false);
 
       assertEquals(request.getFilters().size(), 0);
    }
 
    public void testSignRemoveBlob() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
-         NoSuchMethodException, IOException {
+            NoSuchMethodException, IOException {
       HttpRequest request = signer.signRemoveBlob("container", "name");
 
       assertRequestLineEquals(request, "DELETE https://container.s3.amazonaws.com/name HTTP/1.1");
       assertNonPayloadHeadersEqual(
-            request,
+               request,
-            "Authorization: AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
+               "Authorization: AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
       assertPayloadEquals(request, null, null, false);
 
       assertEquals(request.getFilters().size(), 0);
    }
 
    public void testSignPutBlob() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
-         NoSuchMethodException, IOException {
+            NoSuchMethodException, IOException {
       Blob blob = blobFactory.create(null);
       blob.getMetadata().setName("name");
       blob.setPayload(new PhantomPayload());
@@ -92,8 +100,8 @@ public class S3BlobRequestSignerTest extends BaseS3AsyncClientTest {
 
       assertRequestLineEquals(request, "PUT https://container.s3.amazonaws.com/name HTTP/1.1");
       assertNonPayloadHeadersEqual(
-            request,
+               request,
-            "Authorization: AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
+               "Authorization: AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
 
       assertContentHeadersEqual(request, "text/plain", null, null, null, (long) 2l, new byte[] { 0, 2, 4, 8 });
 

apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java

@@ -26,18 +26,21 @@ import java.util.Properties;
 
 import javax.ws.rs.core.HttpHeaders;
 
+import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.rest.internal.GeneratedHttpRequest;
+import org.jclouds.rest.internal.RestAnnotationProcessor;
 import org.jclouds.s3.BaseS3AsyncClientTest;
 import org.jclouds.s3.S3AsyncClient;
 import org.jclouds.s3.domain.AccessControlList;
 import org.jclouds.s3.domain.CannedAccessPolicy;
 import org.jclouds.s3.domain.S3Object;
 import org.jclouds.s3.options.PutObjectOptions;
-import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
-import org.jclouds.http.HttpRequest;
-import org.jclouds.rest.internal.GeneratedHttpRequest;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import com.google.inject.TypeLiteral;
+
 /**
  * Tests behavior of {@code RequestAuthorizeSignature}
  * 
@@ -45,7 +48,13 @@ import org.testng.annotations.Test;
  */
 // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
 @Test(groups = "unit", testName = "RequestAuthorizeSignatureTest")
-public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest {
+public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest<S3AsyncClient> {
+
+   @Override
+   protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
+      return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
+      };
+   }
 
    @DataProvider(parallel = true)
    public Object[][] dataProvider() throws NoSuchMethodException {
@@ -70,20 +79,20 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest {
          filter.filter(request);
          if (request.getFirstHeaderOrNull(HttpHeaders.DATE).equals(date))
             assert signature.equals(request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION)) : String.format(
-                  "sig: %s != %s on attempt %s", signature, request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION),
+                     "sig: %s != %s on attempt %s", signature, request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION),
-                  iterations);
+                     iterations);
          else
             iterations++;
 
       }
       System.out.printf("%s: %d iterations before the timestamp updated %n", Thread.currentThread().getName(),
-            iterations);
+               iterations);
    }
 
    @Test
    void testAppendBucketNameHostHeader() throws SecurityException, NoSuchMethodException {
       HttpRequest request = processor.createRequest(S3AsyncClient.class.getMethod("getBucketLocation", String.class),
-            "bucket");
+               "bucket");
       StringBuilder builder = new StringBuilder();
       filter.appendBucketName(request, builder);
       assertEquals(builder.toString(), "/bucket");
@@ -98,12 +107,12 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest {
    }
 
    private GeneratedHttpRequest<S3AsyncClient> putBucketAcl() throws NoSuchMethodException {
-      return processor.createRequest(
+      return processor.createRequest(S3AsyncClient.class.getMethod("putBucketACL", String.class,
-            S3AsyncClient.class.getMethod("putBucketACL", String.class, AccessControlList.class), "bucket",
+               AccessControlList.class), "bucket", AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE,
-            AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE, "1234"));
+               "1234"));
    }
 
-   // "?acl", "?location", "?logging", or "?torrent"
+   // "?acl", "?location", "?logging", "?uploads", or "?torrent"
 
    @Test
    void testAppendBucketNameHostHeaderService() throws SecurityException, NoSuchMethodException {
@@ -129,16 +138,15 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest {
       S3Object object = blobToS3Object.apply(BindBlobToMultipartFormTest.TEST_BLOB);
 
       object.getMetadata().getUserMetadata().put("x-amz-Adrian", "foo");
-      HttpRequest request = processor.createRequest(
+      HttpRequest request = processor.createRequest(S3AsyncClient.class.getMethod("putObject", String.class,
-            S3AsyncClient.class.getMethod("putObject", String.class, S3Object.class, PutObjectOptions[].class),
+               S3Object.class, PutObjectOptions[].class), "bucket", object);
-            "bucket", object);
       return request;
    }
 
    @Test
    void testAppendBucketNameURIHost() throws SecurityException, NoSuchMethodException {
       HttpRequest request = processor.createRequest(S3AsyncClient.class.getMethod("getBucketLocation", String.class),
-            "bucket");
+               "bucket");
       assertEquals(request.getEndpoint().getHost(), "bucket.s3.amazonaws.com");
    }
 

apis/walrus/src/test/java/org/jclouds/walrus/WalrusAsyncClientTestDisabled.java

@@ -19,14 +19,25 @@
 
 package org.jclouds.walrus;
 
+import org.jclouds.rest.internal.RestAnnotationProcessor;
+import org.jclouds.s3.S3AsyncClient;
+import org.jclouds.s3.S3AsyncClientTest;
 import org.testng.annotations.Test;
 
+import com.google.inject.TypeLiteral;
+
 /**
  * @author Adrian Cole
  */
 // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
 @Test(enabled = false, groups = "unit", testName = "WalrusAsyncClientTest")
-public class WalrusAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest {
+public class WalrusAsyncClientTestDisabled extends S3AsyncClientTest<S3AsyncClient> {
+
+   @Override
+   protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
+      return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
+      };
+   }
 
    public WalrusAsyncClientTestDisabled() {
       this.provider = "walrus";

common/aws/src/main/java/org/jclouds/aws/handlers/ParseAWSErrorFromXmlContent.java

@@ -27,6 +27,7 @@ import javax.annotation.Resource;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
+import org.jclouds.aws.AWSResponseException;
 import org.jclouds.aws.domain.AWSError;
 import org.jclouds.aws.util.AWSUtils;
 import org.jclouds.http.HttpCommand;
@@ -71,8 +72,10 @@ public class ParseAWSErrorFromXmlContent implements HttpErrorHandler {
                error = utils.parseAWSErrorFromContent(command.getCurrentRequest(), response);
                if (error != null) {
                   message = error.getMessage();
+                  exception = new AWSResponseException(command, response, error);
+               } else {
+                  exception = new HttpResponseException(command, response, message);
                }
-               exception = new HttpResponseException(command, response, message);
             } else {
                try {
                   message = Strings2.toStringAndClose(response.getPayload().getInput());

core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java

@@ -153,8 +153,8 @@ public abstract class BaseHttpCommandExecutorService<Q> implements HttpCommandEx
                         "After filtering, the request has niether chunked encoding nor content length: " + request);
                logger.debug("Sending request %s: %s", request.hashCode(), request.getRequestLine());
                wirePayloadIfEnabled(wire, request);
-               nativeRequest = convert(request);
                utils.logRequest(headerLog, request, ">>");
+               nativeRequest = convert(request);
                response = invoke(nativeRequest);
 
                logger.debug("Receiving response %s: %s", request.hashCode(), response.getStatusLine());

core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java

@@ -19,6 +19,7 @@
 
 package org.jclouds.http.internal;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Throwables.propagate;
 import static com.google.common.collect.Iterables.getLast;
@@ -66,8 +67,9 @@ import org.jclouds.rest.internal.RestAnnotationProcessor;
 
 import com.google.common.base.Supplier;
 import com.google.common.base.Throwables;
-import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.Multimap;
+import com.google.common.collect.ImmutableMultimap.Builder;
+import com.google.common.io.CountingOutputStream;
 
 /**
  * Basic implementation of a {@link HttpCommandExecutorService}.
@@ -86,11 +88,11 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
 
    @Inject
    public JavaUrlHttpCommandExecutorService(HttpUtils utils,
-         @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor,
+            @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor,
-         DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
+            DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
-         DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
+            DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
-         @Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider) throws SecurityException,
+            @Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider) throws SecurityException,
-         NoSuchFieldException {
+            NoSuchFieldException {
       super(utils, ioWorkerExecutor, retryHandler, ioRetryHandler, errorHandler, wire);
       if (utils.getMaxConnections() > 0)
          System.setProperty("http.maxConnections", String.valueOf(checkNotNull(utils, "utils").getMaxConnections()));
@@ -102,6 +104,7 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
 
    @Override
    protected HttpResponse invoke(HttpURLConnection connection) throws IOException, InterruptedException {
+      HttpResponse.Builder builder = HttpResponse.builder();
       InputStream in = null;
       try {
          in = consumeOnClose(connection.getInputStream());
@@ -113,19 +116,28 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
          assert false : "should have propagated exception";
       }
 
-      if (connection.getResponseCode() == 204) {
+      int responseCode = connection.getResponseCode();
+      if (responseCode == 204) {
          closeQuietly(in);
          in = null;
       }
-      Multimap<String, String> headers = LinkedHashMultimap.create();
+      builder.statusCode(responseCode);
+      builder.message(connection.getResponseMessage());
+
+      Builder<String, String> headerBuilder = ImmutableMultimap.<String, String> builder();
       for (String header : connection.getHeaderFields().keySet()) {
-         headers.putAll(header, connection.getHeaderFields().get(header));
+         // HTTP message comes back as a header without a key
+         if (header != null)
+            headerBuilder.putAll(header, connection.getHeaderFields().get(header));
       }
+      ImmutableMultimap<String, String> headers = headerBuilder.build();
       Payload payload = in != null ? Payloads.newInputStreamPayload(in) : null;
-      if (payload != null)
+      if (payload != null) {
          payload.getContentMetadata().setPropertiesFromHttpHeaders(headers);
-      return new HttpResponse(connection.getResponseCode(), connection.getResponseMessage(), payload,
+         builder.payload(payload);
-            RestAnnotationProcessor.filterOutContentHeaders(headers));
+      }
+      builder.headers(RestAnnotationProcessor.filterOutContentHeaders(headers));
+      return builder.build();
    }
 
    private InputStream bufferAndCloseStream(InputStream inputStream) throws IOException {
@@ -220,14 +232,17 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
          } else {
             Long length = checkNotNull(md.getContentLength(), "payload.getContentLength");
             connection.setRequestProperty(HttpHeaders.CONTENT_LENGTH, length.toString());
+            // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6755625
+            checkArgument(length < Integer.MAX_VALUE,
+                     "JDK 1.6 does not support >2GB chunks. Use chunked encoding, if possible.");
             connection.setFixedLengthStreamingMode(length.intValue());
          }
-         // writeTo will close the output stream
+         CountingOutputStream out = new CountingOutputStream(connection.getOutputStream());
          try {
-            request.getPayload().writeTo(connection.getOutputStream());
+            request.getPayload().writeTo(out);
          } catch (IOException e) {
-            e.printStackTrace();
+            throw new RuntimeException(String.format("error after writing %d/%s bytes to %s", out.getCount(), md
-            throw e;
+                     .getContentLength(), request.getRequestLine()), e);
          }
       } else {
          connection.setRequestProperty(HttpHeaders.CONTENT_LENGTH, "0");

core/src/main/java/org/jclouds/io/ContentMetadata.java

@@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableSet;
  */
 public interface ContentMetadata {
    public static final Set<String> HTTP_HEADERS = ImmutableSet.of(CONTENT_LENGTH, "Content-MD5", CONTENT_TYPE,
-         "Content-Disposition", "Content-Encoding", "Content-Language");
+            "Content-Disposition", "Content-Encoding", "Content-Language");
 
    /**
     * Returns the total size of the payload, or the chunk that's available.
@@ -87,4 +87,6 @@ public interface ContentMetadata {
    @Nullable
    String getContentLanguage();
 
+   ContentMetadataBuilder toBuilder();
+
 }

core/src/main/java/org/jclouds/io/ContentMetadataBuilder.java

@@ -0,0 +1,186 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.io;
+
+import static com.google.common.collect.Iterables.any;
+import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH;
+import static javax.ws.rs.core.HttpHeaders.CONTENT_TYPE;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Map.Entry;
+
+import javax.annotation.Nullable;
+
+import org.jclouds.crypto.CryptoStreams;
+import org.jclouds.io.payloads.BaseImmutableContentMetadata;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Multimap;
+
+/**
+ * @author Adrian Cole
+ */
+public class ContentMetadataBuilder implements Serializable {
+   /** The serialVersionUID */
+   private static final long serialVersionUID = -5279643002875371558L;
+
+   public static ContentMetadataBuilder create() {
+      return new ContentMetadataBuilder();
+   }
+
+   protected String contentType = "application/unknown";
+   protected Long contentLength;
+   protected byte[] contentMD5;
+   protected String contentDisposition;
+   protected String contentLanguage;
+   protected String contentEncoding;
+
+   public ContentMetadataBuilder fromHttpHeaders(Multimap<String, String> headers) {
+      boolean chunked = any(headers.entries(), new Predicate<Entry<String, String>>() {
+         @Override
+         public boolean apply(Entry<String, String> input) {
+            return "Transfer-Encoding".equalsIgnoreCase(input.getKey()) && "chunked".equalsIgnoreCase(input.getValue());
+         }
+      });
+      for (Entry<String, String> header : headers.entries()) {
+         if (!chunked && CONTENT_LENGTH.equalsIgnoreCase(header.getKey())) {
+            contentLength(new Long(header.getValue()));
+         } else if ("Content-MD5".equalsIgnoreCase(header.getKey())) {
+            contentMD5(CryptoStreams.base64(header.getValue()));
+         } else if (CONTENT_TYPE.equalsIgnoreCase(header.getKey())) {
+            contentType(header.getValue());
+         } else if ("Content-Disposition".equalsIgnoreCase(header.getKey())) {
+            contentDisposition(header.getValue());
+         } else if ("Content-Encoding".equalsIgnoreCase(header.getKey())) {
+            contentEncoding(header.getValue());
+         } else if ("Content-Language".equalsIgnoreCase(header.getKey())) {
+            contentLanguage(header.getValue());
+         }
+      }
+      return this;
+   }
+
+   public ContentMetadataBuilder contentLength(@Nullable Long contentLength) {
+      this.contentLength = contentLength;
+      return this;
+   }
+
+   public ContentMetadataBuilder contentMD5(byte[] md5) {
+      if (md5 != null) {
+         byte[] retval = new byte[md5.length];
+         System.arraycopy(md5, 0, retval, 0, md5.length);
+         this.contentMD5 = md5;
+      }
+      return this;
+
+   }
+
+   public ContentMetadataBuilder contentType(@Nullable String contentType) {
+      this.contentType = contentType;
+      return this;
+   }
+
+   public ContentMetadataBuilder contentDisposition(@Nullable String contentDisposition) {
+      this.contentDisposition = contentDisposition;
+      return this;
+
+   }
+
+   public ContentMetadataBuilder contentLanguage(@Nullable String contentLanguage) {
+      this.contentLanguage = contentLanguage;
+      return this;
+   }
+
+   public ContentMetadataBuilder contentEncoding(@Nullable String contentEncoding) {
+      this.contentEncoding = contentEncoding;
+      return this;
+   }
+
+   public ContentMetadata build() {
+      return new BaseImmutableContentMetadata(contentType, contentLength, contentMD5, contentDisposition,
+               contentLanguage, contentEncoding);
+   }
+
+   public static ContentMetadataBuilder fromContentMetadata(ContentMetadata in) {
+      return new ContentMetadataBuilder().contentType(in.getContentType()).contentLength(in.getContentLength())
+               .contentMD5(in.getContentMD5()).contentDisposition(in.getContentDisposition()).contentLanguage(
+                        in.getContentLanguage()).contentEncoding(in.getContentEncoding());
+   }
+
+   @Override
+   public int hashCode() {
+      final int prime = 31;
+      int result = 1;
+      result = prime * result + ((contentDisposition == null) ? 0 : contentDisposition.hashCode());
+      result = prime * result + ((contentEncoding == null) ? 0 : contentEncoding.hashCode());
+      result = prime * result + ((contentLanguage == null) ? 0 : contentLanguage.hashCode());
+      result = prime * result + ((contentLength == null) ? 0 : contentLength.hashCode());
+      result = prime * result + Arrays.hashCode(contentMD5);
+      result = prime * result + ((contentType == null) ? 0 : contentType.hashCode());
+      return result;
+   }
+
+   @Override
+   public boolean equals(Object obj) {
+      if (this == obj)
+         return true;
+      if (obj == null)
+         return false;
+      if (getClass() != obj.getClass())
+         return false;
+      ContentMetadataBuilder other = (ContentMetadataBuilder) obj;
+      if (contentDisposition == null) {
+         if (other.contentDisposition != null)
+            return false;
+      } else if (!contentDisposition.equals(other.contentDisposition))
+         return false;
+      if (contentEncoding == null) {
+         if (other.contentEncoding != null)
+            return false;
+      } else if (!contentEncoding.equals(other.contentEncoding))
+         return false;
+      if (contentLanguage == null) {
+         if (other.contentLanguage != null)
+            return false;
+      } else if (!contentLanguage.equals(other.contentLanguage))
+         return false;
+      if (contentLength == null) {
+         if (other.contentLength != null)
+            return false;
+      } else if (!contentLength.equals(other.contentLength))
+         return false;
+      if (!Arrays.equals(contentMD5, other.contentMD5))
+         return false;
+      if (contentType == null) {
+         if (other.contentType != null)
+            return false;
+      } else if (!contentType.equals(other.contentType))
+         return false;
+      return true;
+   }
+
+   @Override
+   public String toString() {
+      return "[contentDisposition=" + contentDisposition + ", contentEncoding=" + contentEncoding
+               + ", contentLanguage=" + contentLanguage + ", contentLength=" + contentLength + ", contentMD5="
+               + Arrays.toString(contentMD5) + ", contentType=" + contentType + "]";
+   }
+}

core/src/main/java/org/jclouds/io/payloads/BaseImmutableContentMetadata.java

@@ -23,6 +23,7 @@ import java.io.Serializable;
 import java.util.Arrays;
 
 import org.jclouds.io.ContentMetadata;
+import org.jclouds.io.ContentMetadataBuilder;
 
 /**
  * @author Adrian Cole
@@ -161,4 +162,9 @@ public class BaseImmutableContentMetadata implements ContentMetadata, Serializab
       return true;
    }
 
+   @Override
+   public ContentMetadataBuilder toBuilder() {
+      return ContentMetadataBuilder.fromContentMetadata(this);
+   }
+
 }

core/src/main/java/org/jclouds/io/payloads/BaseMutableContentMetadata.java

@@ -19,63 +19,26 @@
 
 package org.jclouds.io.payloads;
 
-import static com.google.common.collect.Iterables.any;
-import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH;
-import static javax.ws.rs.core.HttpHeaders.CONTENT_TYPE;
-
 import java.io.Serializable;
-import java.util.Arrays;
-import java.util.Map.Entry;
 
 import javax.annotation.Nullable;
 
-import org.jclouds.crypto.CryptoStreams;
+import org.jclouds.io.ContentMetadata;
+import org.jclouds.io.ContentMetadataBuilder;
 import org.jclouds.io.MutableContentMetadata;
 
-import com.google.common.base.Predicate;
 import com.google.common.collect.Multimap;
 
 /**
  * @author Adrian Cole
  */
-public class BaseMutableContentMetadata implements MutableContentMetadata, Serializable {
+public class BaseMutableContentMetadata extends ContentMetadataBuilder implements MutableContentMetadata, Serializable {
+   /** The serialVersionUID */
+   private static final long serialVersionUID = 8364286391963469370L;
+
    @Override
    public void setPropertiesFromHttpHeaders(Multimap<String, String> headers) {
-      boolean chunked = any(headers.entries(), new Predicate<Entry<String, String>>() {
+      fromHttpHeaders(headers);
-         @Override
-         public boolean apply(Entry<String, String> input) {
-            return "Transfer-Encoding".equalsIgnoreCase(input.getKey()) && "chunked".equalsIgnoreCase(input.getValue());
-         }
-      });
-      for (Entry<String, String> header : headers.entries()) {
-         if (!chunked && CONTENT_LENGTH.equalsIgnoreCase(header.getKey())) {
-            setContentLength(new Long(header.getValue()));
-         } else if ("Content-MD5".equalsIgnoreCase(header.getKey())) {
-            setContentMD5(CryptoStreams.base64(header.getValue()));
-         } else if (CONTENT_TYPE.equalsIgnoreCase(header.getKey())) {
-            setContentType(header.getValue());
-         } else if ("Content-Disposition".equalsIgnoreCase(header.getKey())) {
-            setContentDisposition(header.getValue());
-         } else if ("Content-Encoding".equalsIgnoreCase(header.getKey())) {
-            setContentEncoding(header.getValue());
-         } else if ("Content-Language".equalsIgnoreCase(header.getKey())) {
-            setContentLanguage(header.getValue());
-         }
-      }
-   }
-
-   /** The serialVersionUID */
-   private static final long serialVersionUID = 4572381435863125873L;
-
-   protected String contentType = "application/unknown";
-   protected Long contentLength;
-   protected byte[] contentMD5;
-   protected String contentDisposition;
-   protected String contentLanguage;
-   protected String contentEncoding;
-
-   public BaseMutableContentMetadata() {
-      super();
    }
 
    /**
@@ -91,7 +54,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
     */
    @Override
    public void setContentLength(@Nullable Long contentLength) {
-      this.contentLength = contentLength;
+      contentLength(contentLength);
    }
 
    /**
@@ -113,11 +76,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
     */
    @Override
    public void setContentMD5(byte[] md5) {
-      if (md5 != null) {
+      contentMD5(md5);
-         byte[] retval = new byte[md5.length];
-         System.arraycopy(md5, 0, retval, 0, md5.length);
-         this.contentMD5 = md5;
-      }
    }
 
    /**
@@ -133,7 +92,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
     */
    @Override
    public void setContentType(@Nullable String contentType) {
-      this.contentType = contentType;
+      contentType(contentType);
    }
 
    /**
@@ -141,7 +100,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
     */
    @Override
    public void setContentDisposition(@Nullable String contentDisposition) {
-      this.contentDisposition = contentDisposition;
+      contentDisposition(contentDisposition);
    }
 
    /**
@@ -157,7 +116,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
     */
    @Override
    public void setContentLanguage(@Nullable String contentLanguage) {
-      this.contentLanguage = contentLanguage;
+      contentLanguage(contentLanguage);
    }
 
    /**
@@ -173,7 +132,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
     */
    @Override
    public void setContentEncoding(@Nullable String contentEncoding) {
-      this.contentEncoding = contentEncoding;
+      contentEncoding(contentEncoding);
    }
 
    /**
@@ -185,62 +144,14 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
    }
 
    @Override
-   public String toString() {
+   public BaseMutableContentMetadata toBuilder() {
-      return "[contentType=" + contentType + ", contentLength=" + contentLength + ", contentDisposition="
+      return BaseMutableContentMetadata.fromContentMetadata(this);
-            + contentDisposition + ", contentEncoding=" + contentEncoding + ", contentLanguage=" + contentLanguage
-            + ", contentMD5=" + Arrays.toString(contentMD5) + "]";
    }
 
-   @Override
+   public static BaseMutableContentMetadata fromContentMetadata(ContentMetadata in) {
-   public int hashCode() {
+      return (BaseMutableContentMetadata) new BaseMutableContentMetadata().contentType(in.getContentType())
-      final int prime = 31;
+               .contentLength(in.getContentLength()).contentMD5(in.getContentMD5()).contentDisposition(
-      int result = 1;
+                        in.getContentDisposition()).contentLanguage(in.getContentLanguage()).contentEncoding(
-      result = prime * result + ((contentDisposition == null) ? 0 : contentDisposition.hashCode());
+                        in.getContentEncoding());
-      result = prime * result + ((contentEncoding == null) ? 0 : contentEncoding.hashCode());
-      result = prime * result + ((contentLanguage == null) ? 0 : contentLanguage.hashCode());
-      result = prime * result + ((contentLength == null) ? 0 : contentLength.hashCode());
-      result = prime * result + Arrays.hashCode(contentMD5);
-      result = prime * result + ((contentType == null) ? 0 : contentType.hashCode());
-      return result;
    }
-
-   @Override
-   public boolean equals(Object obj) {
-      if (this == obj)
-         return true;
-      if (obj == null)
-         return false;
-      if (getClass() != obj.getClass())
-         return false;
-      BaseMutableContentMetadata other = (BaseMutableContentMetadata) obj;
-      if (contentDisposition == null) {
-         if (other.contentDisposition != null)
-            return false;
-      } else if (!contentDisposition.equals(other.contentDisposition))
-         return false;
-      if (contentEncoding == null) {
-         if (other.contentEncoding != null)
-            return false;
-      } else if (!contentEncoding.equals(other.contentEncoding))
-         return false;
-      if (contentLanguage == null) {
-         if (other.contentLanguage != null)
-            return false;
-      } else if (!contentLanguage.equals(other.contentLanguage))
-         return false;
-      if (contentLength == null) {
-         if (other.contentLength != null)
-            return false;
-      } else if (!contentLength.equals(other.contentLength))
-         return false;
-      if (!Arrays.equals(contentMD5, other.contentMD5))
-         return false;
-      if (contentType == null) {
-         if (other.contentType != null)
-            return false;
-      } else if (!contentType.equals(other.contentType))
-         return false;
-      return true;
-   }
-
 }

core/src/main/java/org/jclouds/io/payloads/PhantomPayload.java

@@ -25,6 +25,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
+import org.jclouds.io.ContentMetadata;
 import org.jclouds.io.MutableContentMetadata;
 
 /**
@@ -36,6 +37,10 @@ public class PhantomPayload extends BasePayload<Object> {
       super(Object.class);
    }
 
+   public PhantomPayload(ContentMetadata contentMetadata) {
+      this(BaseMutableContentMetadata.fromContentMetadata(checkNotNull(contentMetadata, "contentMetadata")));
+   }
+
    public PhantomPayload(MutableContentMetadata contentMetadata) {
       super(Object.class, checkNotNull(contentMetadata, "contentMetadata"));
    }

core/src/main/resources/rest.properties

@@ -44,7 +44,7 @@ aws-elb.propertiesbuilder=org.jclouds.aws.elb.AWSELBPropertiesBuilder
 cloudwatch.contextbuilder=org.jclouds.cloudwatch.CloudWatchContextBuilder
 cloudwatch.propertiesbuilder=org.jclouds.cloudwatch.CloudWatchPropertiesBuilder
 
-aws-s3.contextbuilder=org.jclouds.s3.S3ContextBuilder
+aws-s3.contextbuilder=org.jclouds.aws.s3.AWSS3ContextBuilder
 aws-s3.propertiesbuilder=org.jclouds.aws.s3.AWSS3PropertiesBuilder
 
 aws-ec2.contextbuilder=org.jclouds.aws.ec2.AWSEC2ContextBuilder

providers/aws-s3/pom.xml

@@ -70,6 +70,18 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.jclouds.driver</groupId>
+            <artifactId>jclouds-apachehc</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.1</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.jclouds.driver</groupId>
             <artifactId>jclouds-log4j</artifactId>

providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java

@@ -0,0 +1,131 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3;
+
+/**
+ *
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER;
+
+import java.util.Map;
+
+import javax.ws.rs.DELETE;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
+
+import org.jclouds.aws.s3.binders.BindObjectMetadataToRequest;
+import org.jclouds.aws.s3.binders.BindPartIdsAndETagsToRequest;
+import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex;
+import org.jclouds.aws.s3.functions.ObjectMetadataKey;
+import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex;
+import org.jclouds.blobstore.attr.BlobScope;
+import org.jclouds.http.functions.ParseETagHeader;
+import org.jclouds.io.Payload;
+import org.jclouds.rest.annotations.BinderParam;
+import org.jclouds.rest.annotations.ExceptionParser;
+import org.jclouds.rest.annotations.ParamParser;
+import org.jclouds.rest.annotations.ParamValidators;
+import org.jclouds.rest.annotations.QueryParams;
+import org.jclouds.rest.annotations.RequestFilters;
+import org.jclouds.rest.annotations.ResponseParser;
+import org.jclouds.rest.annotations.SkipEncoding;
+import org.jclouds.rest.functions.ReturnVoidOnNotFoundOr404;
+import org.jclouds.s3.Bucket;
+import org.jclouds.s3.S3AsyncClient;
+import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured;
+import org.jclouds.s3.domain.ObjectMetadata;
+import org.jclouds.s3.filters.RequestAuthorizeSignature;
+import org.jclouds.s3.options.PutObjectOptions;
+import org.jclouds.s3.predicates.validators.BucketNameValidator;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+/**
+ * Provides access to amazon-specific S3 features
+ * 
+ * @author Adrian Cole
+ */
+@SkipEncoding('/')
+@RequestFilters(RequestAuthorizeSignature.class)
+@BlobScope(CONTAINER)
+public interface AWSS3AsyncClient extends S3AsyncClient {
+   /**
+    * @see AWSS3Client#initiateMultipartUpload
+    */
+   @POST
+   @QueryParams(keys = "uploads")
+   @Path("/{key}")
+   @ResponseParser(UploadIdFromHttpResponseViaRegex.class)
+   ListenableFuture<String> initiateMultipartUpload(
+            @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName,
+            @PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class) ObjectMetadata objectMetadata,
+            PutObjectOptions... options);
+
+   /**
+    * @see AWSS3Client#abortMultipartUpload
+    */
+   @DELETE
+   @Path("/{key}")
+   @ExceptionParser(ReturnVoidOnNotFoundOr404.class)
+   ListenableFuture<Void> abortMultipartUpload(
+            @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName,
+            @PathParam("key") String key, @QueryParam("uploadId") String uploadId);
+
+   /**
+    * @see AWSS3Client#uploadPart
+    */
+   @PUT
+   @Path("/{key}")
+   @ResponseParser(ParseETagHeader.class)
+   ListenableFuture<String> uploadPart(
+            @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName,
+            @PathParam("key") String key, @QueryParam("partNumber") int partNumber,
+            @QueryParam("uploadId") String uploadId, Payload part);
+
+   /**
+    * @see AWSS3Client#completeMultipartUpload
+    */
+   @POST
+   @Path("/{key}")
+   @ResponseParser(ETagFromHttpResponseViaRegex.class)
+   ListenableFuture<String> completeMultipartUpload(
+            @Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName,
+            @PathParam("key") String key, @QueryParam("uploadId") String uploadId,
+            @BinderParam(BindPartIdsAndETagsToRequest.class) Map<Integer, String> parts);
+
+}

providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java

@@ -0,0 +1,145 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.jclouds.concurrent.Timeout;
+import org.jclouds.io.Payload;
+import org.jclouds.s3.S3Client;
+import org.jclouds.s3.domain.ObjectMetadata;
+import org.jclouds.s3.options.PutObjectOptions;
+
+/**
+ * Provides access to amazon-specific S3 features
+ * 
+ * @author Adrian Cole
+ * @see AWSS3AsyncClient
+ */
+@Timeout(duration = 90, timeUnit = TimeUnit.SECONDS)
+public interface AWSS3Client extends S3Client {
+   /**
+    * This operation initiates a multipart upload and returns an upload ID. This upload ID is used
+    * to associate all the parts in the specific multipart upload. You specify this upload ID in
+    * each of your subsequent upload part requests (see Upload Part). You also include this upload
+    * ID in the final request to either complete or abort the multipart upload request.
+    * 
+    * <h4>Note</h4> If you create an object using the multipart upload APIs, currently you cannot
+    * copy the object between regions.
+    * 
+    * 
+    * @param bucketName
+    *           namespace of the object you are to upload
+    * @param objectMetadata
+    *           metadata around the object you wish to upload
+    * @param options
+    *           controls optional parameters such as canned ACL
+    * @return ID for the initiated multipart upload.
+    */
+   String initiateMultipartUpload(String bucketName, ObjectMetadata objectMetadata, PutObjectOptions... options);
+
+   /**
+    * This operation aborts a multipart upload. After a multipart upload is aborted, no additional
+    * parts can be uploaded using that upload ID. The storage consumed by any previously uploaded
+    * parts will be freed. However, if any part uploads are currently in progress, those part
+    * uploads might or might not succeed. As a result, it might be necessary to abort a given
+    * multipart upload multiple times in order to completely free all storage consumed by all parts.
+    * 
+    * 
+    * @param bucketName
+    *           namespace of the object you are deleting
+    * @param key
+    *           unique key in the s3Bucket identifying the object
+    * @param uploadId
+    *           id of the multipart upload in progress.
+    */
+   void abortMultipartUpload(String bucketName, String key, String uploadId);
+
+   /**
+    * This operation uploads a part in a multipart upload. You must initiate a multipart upload (see
+    * Initiate Multipart Upload) before you can upload any part. In response to your initiate
+    * request. Amazon S3 returns an upload ID, a unique identifier, that you must include in your
+    * upload part request.
+    * 
+    * <p/>
+    * Part numbers can be any number from 1 to 10,000, inclusive. A part number uniquely identifies
+    * a part and also defines its position within the object being created. If you upload a new part
+    * using the same part number that was used with a previous part, the previously uploaded part is
+    * overwritten. Each part must be at least 5 MB in size, except the last part. There is no size
+    * limit on the last part of your multipart upload.
+    * 
+    * <p/>
+    * To ensure that data is not corrupted when traversing the network, specify the Content-MD5
+    * header in the upload part request. Amazon S3 checks the part data against the provided MD5
+    * value. If they do not match, Amazon S3 returns an error.
+    * 
+    * 
+    * @param bucketName
+    *           namespace of the object you are storing
+    * @param key
+    *           unique key in the s3Bucket identifying the object
+    * @param partNumber
+    *           which part is this.
+    * @param uploadId
+    *           id of the multipart upload in progress.
+    * @param part
+    *           contains the data to create or overwrite
+    * @return ETag of the content uploaded
+    * @see <a href="http://docs.amazonwebservices.com/AmazonS3/latest/API/mpUploadUploadPart.html"
+    *      />
+    */
+   @Timeout(duration = 5 * 1024 * 1024 / 128, timeUnit = TimeUnit.SECONDS)
+   String uploadPart(String bucketName, String key, int partNumber, String uploadId, Payload part);
+
+   /**
+    * 
+    This operation completes a multipart upload by assembling previously uploaded parts.
+    * <p/>
+    * You first initiate the multipart upload and then upload all parts using the Upload Parts
+    * operation (see Upload Part). After successfully uploading all relevant parts of an upload, you
+    * call this operation to complete the upload. Upon receiving this request, Amazon S3
+    * concatenates all the parts in ascending order by part number to create a new object. In the
+    * Complete Multipart Upload request, you must provide the parts list. For each part in the list,
+    * you must provide the part number and the ETag header value, returned after that part was
+    * uploaded.
+    * <p/>
+    * Processing of a Complete Multipart Upload request could take several minutes to complete.
+    * After Amazon S3 begins processing the request, it sends an HTTP response header that specifies
+    * a 200 OK response. While processing is in progress, Amazon S3 periodically sends whitespace
+    * characters to keep the connection from timing out. Because a request could fail after the
+    * initial 200 OK response has been sent, it is important that you check the response body to
+    * determine whether the request succeeded.
+    * <p/>
+    * Note that if Complete Multipart Upload fails, applications should be prepared to retry the
+    * failed requests.
+    * 
+    * @param bucketName
+    *           namespace of the object you are deleting
+    * @param key
+    *           unique key in the s3Bucket identifying the object
+    * @param uploadId
+    *           id of the multipart upload in progress.
+    * @param parts
+    *           a map of part id to eTag from the {@link #uploadPart} command.
+    * @return ETag of the content uploaded
+    */
+   String completeMultipartUpload(String bucketName, String key, String uploadId, Map<Integer, String> parts);
+}

providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ContextBuilder.java

@@ -0,0 +1,45 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3;
+
+import java.util.List;
+import java.util.Properties;
+
+import org.jclouds.aws.s3.config.AWSS3RestClientModule;
+import org.jclouds.s3.S3ContextBuilder;
+
+import com.google.inject.Module;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+public class AWSS3ContextBuilder extends S3ContextBuilder {
+
+   public AWSS3ContextBuilder(Properties props) {
+      super(props);
+   }
+
+   @Override
+   protected void addClientModule(List<Module> modules) {
+      modules.add(new AWSS3RestClientModule());
+   }
+
+}

providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequest.java

@@ -0,0 +1,83 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3.binders;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.ws.rs.core.HttpHeaders;
+
+import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.utils.ModifyRequest;
+import org.jclouds.rest.Binder;
+import org.jclouds.s3.domain.ObjectMetadata;
+
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableMultimap.Builder;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class BindObjectMetadataToRequest implements Binder {
+   protected final BindMapToHeadersWithPrefix metadataPrefixer;
+
+   @Inject
+   public BindObjectMetadataToRequest(BindMapToHeadersWithPrefix metadataPrefixer) {
+      this.metadataPrefixer = checkNotNull(metadataPrefixer, "metadataPrefixer");
+   }
+
+   @Override
+   public <R extends HttpRequest> R bindToRequest(R request, Object input) {
+      checkArgument(checkNotNull(input, "input") instanceof ObjectMetadata,
+               "this binder is only valid for ObjectMetadata!");
+      checkNotNull(request, "request");
+
+      ObjectMetadata md = ObjectMetadata.class.cast(input);
+      checkArgument(md.getKey() != null, "objectMetadata.getKey() must be set!");
+
+      request = metadataPrefixer.bindToRequest(request, md.getUserMetadata());
+
+      Builder<String, String> headers = ImmutableMultimap.<String, String> builder();
+      if (md.getCacheControl() != null) {
+         headers.put(HttpHeaders.CACHE_CONTROL, md.getCacheControl());
+      }
+
+      if (md.getContentMetadata().getContentDisposition() != null) {
+         headers.put("Content-Disposition", md.getContentMetadata().getContentDisposition());
+      }
+
+      if (md.getContentMetadata().getContentEncoding() != null) {
+         headers.put("Content-Encoding", md.getContentMetadata().getContentEncoding());
+      }
+      if (md.getContentMetadata().getContentType() != null) {
+         headers.put(HttpHeaders.CONTENT_TYPE, md.getContentMetadata().getContentType());
+      } else {
+         headers.put(HttpHeaders.CONTENT_TYPE, "binary/octet-stream");
+      }
+
+      request = ModifyRequest.replaceHeaders(request, headers.build());
+      return request;
+   }
+}

providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequest.java

@@ -0,0 +1,65 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3.binders;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.inject.Singleton;
+import javax.ws.rs.core.MediaType;
+
+import org.jclouds.http.HttpRequest;
+import org.jclouds.io.Payload;
+import org.jclouds.io.Payloads;
+import org.jclouds.rest.Binder;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class BindPartIdsAndETagsToRequest implements Binder {
+
+   @SuppressWarnings("unchecked")
+   @Override
+   public <R extends HttpRequest> R bindToRequest(R request, Object input) {
+      checkArgument(checkNotNull(input, "input") instanceof Map, "this binder is only valid for Map!");
+      checkNotNull(request, "request");
+
+      Map<Integer, String> map = (Map<Integer, String>) input;
+      checkArgument(map.size() != 0, "Please send parts");
+      StringBuilder content = new StringBuilder();
+      content.append("<CompleteMultipartUpload>");
+      for (Entry<Integer, String> entry : map.entrySet()) {
+         content.append("<Part>");
+         content.append("<PartNumber>").append(entry.getKey()).append("</PartNumber>");
+         content.append("<ETag>").append(entry.getValue()).append("</ETag>");
+         content.append("</Part>");
+      }
+      content.append("</CompleteMultipartUpload>");
+      Payload payload = Payloads.newStringPayload(content.toString());
+      payload.getContentMetadata().setContentType(MediaType.TEXT_XML);
+      request.setPayload(payload);
+      return request;
+   }
+}

providers/aws-s3/src/main/java/org/jclouds/aws/s3/config/AWSS3RestClientModule.java

@@ -0,0 +1,59 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3.config;
+
+import javax.inject.Singleton;
+
+import org.jclouds.aws.s3.AWSS3AsyncClient;
+import org.jclouds.aws.s3.AWSS3Client;
+import org.jclouds.http.RequiresHttp;
+import org.jclouds.rest.ConfiguresRestClient;
+import org.jclouds.s3.S3AsyncClient;
+import org.jclouds.s3.S3Client;
+import org.jclouds.s3.config.S3RestClientModule;
+
+import com.google.inject.Provides;
+
+/**
+ * Configures the S3 connection.
+ * 
+ * @author Adrian Cole
+ */
+@RequiresHttp
+@ConfiguresRestClient
+public class AWSS3RestClientModule extends S3RestClientModule<AWSS3Client, AWSS3AsyncClient> {
+
+   public AWSS3RestClientModule() {
+      super(AWSS3Client.class, AWSS3AsyncClient.class);
+   }
+
+   @Singleton
+   @Provides
+   S3Client provide(AWSS3Client in) {
+      return in;
+   }
+
+   @Singleton
+   @Provides
+   S3AsyncClient provide(AWSS3AsyncClient in) {
+      return in;
+   }
+
+}

providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegex.java

@@ -0,0 +1,60 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3.functions;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.functions.ReturnStringIf2xx;
+
+import com.google.common.base.Function;
+
+/**
+ * @see <a href="http://docs.amazonwebservices.com/AmazonS3/latest/API/mpUploadComplete.html" />
+ * @author Adrian Cole
+ */
+@Singleton
+public class ETagFromHttpResponseViaRegex implements Function<HttpResponse, String> {
+   Pattern pattern = Pattern.compile("<ETag>([\\S&&[^<]]+)</ETag>");
+   private final ReturnStringIf2xx returnStringIf200;
+
+   @Inject
+   ETagFromHttpResponseViaRegex(ReturnStringIf2xx returnStringIf200) {
+      this.returnStringIf200 = returnStringIf200;
+   }
+
+   @Override
+   public String apply(HttpResponse response) {
+      String value = null;
+      String content = returnStringIf200.apply(response);
+      if (content != null) {
+         Matcher matcher = pattern.matcher(content);
+         if (matcher.find()) {
+            value = matcher.group(1);
+         }
+      }
+      return value;
+   }
+
+}

providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ObjectMetadataKey.java

@@ -0,0 +1,39 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3.functions;
+
+import javax.inject.Singleton;
+
+import org.jclouds.s3.domain.ObjectMetadata;
+
+import com.google.common.base.Function;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class ObjectMetadataKey implements Function<Object, String> {
+
+   public String apply(Object from) {
+      return ((ObjectMetadata) from).getKey();
+   }
+
+}

providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegex.java

@@ -0,0 +1,62 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3.functions;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.functions.ReturnStringIf2xx;
+
+import com.google.common.base.Function;
+
+/**
+ * @see <a
+ *      href="http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadInitiate.html"
+ *      />
+ * @author Adrian Cole
+ */
+@Singleton
+public class UploadIdFromHttpResponseViaRegex implements Function<HttpResponse, String> {
+   Pattern pattern = Pattern.compile("<UploadId>([\\S&&[^<]]+)</UploadId>");
+   private final ReturnStringIf2xx returnStringIf200;
+
+   @Inject
+   UploadIdFromHttpResponseViaRegex(ReturnStringIf2xx returnStringIf200) {
+      this.returnStringIf200 = returnStringIf200;
+   }
+
+   @Override
+   public String apply(HttpResponse response) {
+      String value = null;
+      String content = returnStringIf200.apply(response);
+      if (content != null) {
+         Matcher matcher = pattern.matcher(content);
+         if (matcher.find()) {
+            value = matcher.group(1);
+         }
+      }
+      return value;
+   }
+
+}

providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java

@@ -22,43 +22,143 @@ package org.jclouds.aws.s3;
 import java.io.IOException;
 import java.lang.reflect.Array;
 import java.lang.reflect.Method;
+import java.util.Map;
 import java.util.Properties;
 
+import org.jclouds.aws.s3.config.AWSS3RestClientModule;
+import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex;
+import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex;
+import org.jclouds.date.TimeStamp;
 import org.jclouds.http.HttpRequest;
+import org.jclouds.http.RequiresHttp;
+import org.jclouds.http.functions.ParseETagHeader;
+import org.jclouds.http.functions.ReleasePayloadAndReturn;
 import org.jclouds.http.functions.ReturnTrueIf2xx;
+import org.jclouds.io.Payload;
+import org.jclouds.io.Payloads;
+import org.jclouds.rest.ConfiguresRestClient;
 import org.jclouds.rest.RestContextFactory;
-import org.jclouds.s3.S3AsyncClient;
+import org.jclouds.rest.functions.MapHttp4xxCodesToExceptions;
+import org.jclouds.rest.functions.ReturnVoidOnNotFoundOr404;
+import org.jclouds.rest.internal.RestAnnotationProcessor;
+import org.jclouds.s3.domain.ObjectMetadata;
+import org.jclouds.s3.domain.ObjectMetadataBuilder;
 import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState;
 import org.jclouds.s3.options.PutBucketOptions;
+import org.jclouds.s3.options.PutObjectOptions;
 import org.testng.annotations.Test;
 
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Module;
+import com.google.inject.TypeLiteral;
+
 /**
  * @author Adrian Cole
  */
 // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
 @Test(groups = "unit", testName = "AWSS3AsyncClientTest")
-public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest {
+public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest<AWSS3AsyncClient> {
 
    public AWSS3AsyncClientTest() {
       this.provider = "aws-s3";
    }
 
+   @Override
+   protected TypeLiteral<RestAnnotationProcessor<AWSS3AsyncClient>> createTypeLiteral() {
+      return new TypeLiteral<RestAnnotationProcessor<AWSS3AsyncClient>>() {
+      };
+   }
+
    @Override
    protected Properties getProperties() {
       return RestContextFactory.getPropertiesFromResource("/rest.properties");
    }
 
+   public void testInitiateMultipartUpload() throws SecurityException, NegativeArraySizeException,
+            NoSuchMethodException {
+      Method method = AWSS3AsyncClient.class.getMethod("initiateMultipartUpload", String.class, ObjectMetadata.class,
+               PutObjectOptions[].class);
+      HttpRequest request = processor
+               .createRequest(method, "bucket", ObjectMetadataBuilder.create().key("foo").build());
+
+      assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1");
+      assertNonPayloadHeadersEqual(request, "Content-Type: binary/octet-stream\nHost: bucket." + url + "\n");
+      assertPayloadEquals(request, null, null, false);
+
+      assertResponseParserClassEquals(method, request, UploadIdFromHttpResponseViaRegex.class);
+      assertSaxResponseParserClassEquals(method, null);
+      assertExceptionParserClassEquals(method, MapHttp4xxCodesToExceptions.class);
+
+      checkFilters(request);
+   }
+
+   public void testAbortMultipartUpload() throws SecurityException, NegativeArraySizeException, NoSuchMethodException {
+      Method method = AWSS3AsyncClient.class
+               .getMethod("abortMultipartUpload", String.class, String.class, String.class);
+      HttpRequest request = processor.createRequest(method, "bucket", "foo", "asdsadasdas", 1, Payloads
+               .newStringPayload(""));
+
+      assertRequestLineEquals(request, "DELETE https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1");
+      assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
+      assertPayloadEquals(request, "", "application/unknown", false);
+
+      assertResponseParserClassEquals(method, request, ReleasePayloadAndReturn.class);
+      assertSaxResponseParserClassEquals(method, null);
+      assertExceptionParserClassEquals(method, ReturnVoidOnNotFoundOr404.class);
+
+      checkFilters(request);
+   }
+
+   public void testUploadPart() throws SecurityException, NegativeArraySizeException, NoSuchMethodException {
+      Method method = AWSS3AsyncClient.class.getMethod("uploadPart", String.class, String.class, int.class,
+               String.class, Payload.class);
+      HttpRequest request = processor.createRequest(method, "bucket", "foo", 1, "asdsadasdas", Payloads
+               .newStringPayload(""));
+
+      assertRequestLineEquals(request, "PUT https://bucket." + url + "/foo?partNumber=1&uploadId=asdsadasdas HTTP/1.1");
+      assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
+      assertPayloadEquals(request, "", "application/unknown", false);
+
+      assertResponseParserClassEquals(method, request, ParseETagHeader.class);
+      assertSaxResponseParserClassEquals(method, null);
+      assertExceptionParserClassEquals(method, MapHttp4xxCodesToExceptions.class);
+
+      checkFilters(request);
+   }
+
+   public void testCompleteMultipartUpload() throws SecurityException, NegativeArraySizeException,
+            NoSuchMethodException {
+      Method method = AWSS3AsyncClient.class.getMethod("completeMultipartUpload", String.class, String.class,
+               String.class, Map.class);
+      HttpRequest request = processor.createRequest(method, "bucket", "foo", "asdsadasdas", ImmutableMap
+               .<Integer, String> of(1, "\"a54357aff0632cce46d942af68356b38\""));
+
+      assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1");
+      assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
+      assertPayloadEquals(
+               request,
+               "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>\"a54357aff0632cce46d942af68356b38\"</ETag></Part></CompleteMultipartUpload>",
+               "text/xml", false);
+
+      assertResponseParserClassEquals(method, request, ETagFromHttpResponseViaRegex.class);
+      assertSaxResponseParserClassEquals(method, null);
+      assertExceptionParserClassEquals(method, MapHttp4xxCodesToExceptions.class);
+
+      checkFilters(request);
+   }
+
    public void testPutBucketEu() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
-         NoSuchMethodException, IOException {
+            NoSuchMethodException, IOException {
-      Method method = S3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class,
+      Method method = AWSS3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class, Array
-            Array.newInstance(PutBucketOptions.class, 0).getClass());
+               .newInstance(PutBucketOptions.class, 0).getClass());
       HttpRequest request = processor.createRequest(method, "EU", "bucket");
 
       assertRequestLineEquals(request, "PUT https://bucket." + url + "/ HTTP/1.1");
       assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
       assertPayloadEquals(request,
-            "<CreateBucketConfiguration><LocationConstraint>EU</LocationConstraint></CreateBucketConfiguration>",
+               "<CreateBucketConfiguration><LocationConstraint>EU</LocationConstraint></CreateBucketConfiguration>",
-            "text/xml", false);
+               "text/xml", false);
 
       assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class);
       assertSaxResponseParserClassEquals(method, null);
@@ -66,4 +166,24 @@ public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest {
 
       checkFilters(request);
    }
+
+   @RequiresHttp
+   @ConfiguresRestClient
+   private static final class TestAWSS3RestClientModule extends AWSS3RestClientModule {
+
+      public TestAWSS3RestClientModule() {
+         super();
+      }
+
+      @Override
+      protected String provideTimeStamp(@TimeStamp Supplier<String> cache) {
+         return "2009-11-08T15:54:08.897Z";
+      }
+   }
+
+   @Override
+   protected Module createModule() {
+      return new TestAWSS3RestClientModule();
+   }
+
 }

providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java

@@ -19,9 +19,29 @@
 
 package org.jclouds.aws.s3;
 
+import static org.testng.Assert.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.GZIPInputStream;
+
+import org.jclouds.crypto.CryptoStreams;
+import org.jclouds.http.BaseJettyTest;
+import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule;
+import org.jclouds.io.Payload;
+import org.jclouds.io.Payloads;
 import org.jclouds.s3.S3ClientLiveTest;
+import org.jclouds.s3.domain.ObjectMetadataBuilder;
+import org.testng.ITestContext;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.InputSupplier;
+import com.google.inject.Module;
+
 /**
  * Tests behavior of {@code S3Client}
  * 
@@ -29,5 +49,59 @@ import org.testng.annotations.Test;
  */
 @Test(groups = "live", sequential = true, testName = "AWSS3ClientLiveTest")
 public class AWSS3ClientLiveTest extends S3ClientLiveTest {
+   private InputSupplier<InputStream> oneHundredOneConstitutions;
+   private byte[] oneHundredOneConstitutionsMD5;
+   private static long oneHundredOneConstitutionsLength;
+
+   @Override
+   public AWSS3Client getApi() {
+      return (AWSS3Client) context.getProviderSpecificContext().getApi();
+   }
+
+   @BeforeClass(groups = { "integration", "live" })
+   @Override
+   public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
+      super.setUpResourcesOnThisThread(testContext);
+      oneHundredOneConstitutions = getTestDataSupplier();
+      oneHundredOneConstitutionsMD5 = CryptoStreams.md5(oneHundredOneConstitutions);
+   }
+
+   @SuppressWarnings("unchecked")
+   public static InputSupplier<InputStream> getTestDataSupplier() throws IOException {
+      byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class
+               .getResourceAsStream("/const.txt.gz")));
+      InputSupplier<ByteArrayInputStream> constitutionSupplier = ByteStreams.newInputStreamSupplier(oneConstitution);
+
+      InputSupplier<InputStream> temp = ByteStreams.join(constitutionSupplier);
+      // we have to go beyond 5MB per part
+      for (oneHundredOneConstitutionsLength = oneConstitution.length; oneHundredOneConstitutionsLength < 5 * 1024 * 1024; oneHundredOneConstitutionsLength += oneConstitution.length) {
+         temp = ByteStreams.join(temp, constitutionSupplier);
+      }
+      return temp;
+   }
+
+   public void testMultipartSynchronously() throws InterruptedException, IOException {
+      String containerName = getContainerName();
+      try {
+         String key = "constitution.txt";
+         String uploadId = getApi().initiateMultipartUpload(containerName,
+                  ObjectMetadataBuilder.create().key(key).build());
+
+         byte[] buffer = ByteStreams.toByteArray(oneHundredOneConstitutions.getInput());
+         Payload part1 = Payloads.newByteArrayPayload(buffer);
+         part1.getContentMetadata().setContentLength((long) buffer.length);
+         part1.getContentMetadata().setContentMD5(oneHundredOneConstitutionsMD5);
+
+         // failure here looks very similar to http://java.net/jira/browse/GLASSFISH-15773
+         String eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
+
+         String eTag = getApi().completeMultipartUpload(containerName, key, uploadId, ImmutableMap.of(1, eTagOf1));
+
+         assertEquals(eTagOf1, eTag);
+
+      } finally {
+         returnContainer(containerName);
+      }
+   }
 
 }

providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequestTest.java

@@ -0,0 +1,104 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3.binders;
+
+import static org.testng.Assert.assertEquals;
+
+import java.io.File;
+import java.net.URI;
+
+import javax.ws.rs.HttpMethod;
+
+import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.rest.internal.RestAnnotationProcessor;
+import org.jclouds.s3.BaseS3AsyncClientTest;
+import org.jclouds.s3.S3AsyncClient;
+import org.jclouds.s3.domain.ObjectMetadata;
+import org.jclouds.s3.domain.ObjectMetadataBuilder;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.inject.TypeLiteral;
+
+/**
+ * Tests behavior of {@code BindObjectMetadataToRequest}
+ * 
+ * @author Adrian Cole
+ */
+// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
+@Test(groups = "unit", testName = "BindObjectMetadataToRequestTest")
+public class BindObjectMetadataToRequestTest extends BaseS3AsyncClientTest<S3AsyncClient> {
+
+   @Override
+   protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
+      return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
+      };
+   }
+
+   @Test
+   public void testPassWithMinimumDetailsAndPayload5GB() {
+      ObjectMetadata md = ObjectMetadataBuilder.create().key("foo").build();
+
+      HttpRequest request = HttpRequest.builder().method("POST").endpoint(URI.create("http://localhost")).build();
+      BindObjectMetadataToRequest binder = injector.getInstance(BindObjectMetadataToRequest.class);
+
+      assertEquals(binder.bindToRequest(request, md), HttpRequest.builder().method("POST").endpoint(
+               URI.create("http://localhost")).headers(ImmutableMultimap.of("Content-Type", "binary/octet-stream"))
+               .build());
+   }
+
+   @Test
+   public void testExtendedPropertiesBind() {
+      ObjectMetadata md = ObjectMetadataBuilder.create().key("foo").cacheControl("no-cache").userMetadata(
+               ImmutableMap.of("foo", "bar")).build();
+
+      HttpRequest request = HttpRequest.builder().method("POST").endpoint(URI.create("http://localhost")).build();
+      BindObjectMetadataToRequest binder = injector.getInstance(BindObjectMetadataToRequest.class);
+
+      assertEquals(binder.bindToRequest(request, md), HttpRequest.builder().method("POST").endpoint(
+               URI.create("http://localhost")).headers(
+               ImmutableMultimap.of("Cache-Control", "no-cache", "x-amz-meta-foo", "bar", "Content-Type",
+                        "binary/octet-stream")).build());
+   }
+
+   @Test(expectedExceptions = IllegalArgumentException.class)
+   public void testNoKeyIsBad() {
+      ObjectMetadata md = ObjectMetadataBuilder.create().build();
+
+      HttpRequest request = HttpRequest.builder().method("POST").endpoint(URI.create("http://localhost")).build();
+      BindObjectMetadataToRequest binder = injector.getInstance(BindObjectMetadataToRequest.class);
+      binder.bindToRequest(request, md);
+   }
+
+   @Test(expectedExceptions = IllegalArgumentException.class)
+   public void testMustBeObjectMetadata() {
+      HttpRequest request = new HttpRequest(HttpMethod.POST, URI.create("http://localhost"));
+      injector.getInstance(BindObjectMetadataToRequest.class).bindToRequest(request, new File("foo"));
+   }
+
+   @Test(expectedExceptions = { NullPointerException.class, IllegalStateException.class })
+   public void testNullIsBad() {
+      BindMapToHeadersWithPrefix binder = new BindMapToHeadersWithPrefix("prefix:");
+      HttpRequest request = HttpRequest.builder().method("GET").endpoint(URI.create("http://momma")).build();
+      binder.bindToRequest(request, null);
+   }
+}

providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequestTest.java

@@ -0,0 +1,72 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3.binders;
+
+import static org.testng.Assert.assertEquals;
+
+import java.net.URI;
+
+import javax.ws.rs.core.MediaType;
+
+import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.io.Payload;
+import org.jclouds.io.Payloads;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Tests behavior of {@code BindPartIdsAndETagsToRequest}
+ * 
+ * @author Adrian Cole
+ */
+// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
+@Test(groups = "unit", testName = "BindPartIdsAndETagsToRequestTest")
+public class BindPartIdsAndETagsToRequestTest {
+   BindPartIdsAndETagsToRequest binder = new BindPartIdsAndETagsToRequest();
+
+   @Test
+   public void testPassWithMinimumDetailsAndPayload5GB() {
+      HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
+      Payload payload = Payloads
+               .newStringPayload("<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>\"a54357aff0632cce46d942af68356b38\"</ETag></Part></CompleteMultipartUpload>");
+      payload.getContentMetadata().setContentType(MediaType.TEXT_XML);
+      request = binder.bindToRequest(request, ImmutableMap.<Integer, String> of(1,
+               "\"a54357aff0632cce46d942af68356b38\""));
+      assertEquals(request.getPayload().getRawContent(), payload.getRawContent());
+      assertEquals(request, HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).payload(
+               payload).build());
+   }
+
+   @Test(expectedExceptions = IllegalArgumentException.class)
+   public void testEmptyIsBad() {
+
+      HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
+      binder.bindToRequest(request, ImmutableMap.<Integer, String> of());
+   }
+
+   @Test(expectedExceptions = { NullPointerException.class, IllegalStateException.class })
+   public void testNullIsBad() {
+      BindMapToHeadersWithPrefix binder = new BindMapToHeadersWithPrefix("prefix:");
+      HttpRequest request = HttpRequest.builder().method("GET").endpoint(URI.create("http://momma")).build();
+      binder.bindToRequest(request, null);
+   }
+}

providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegexTest.java

@@ -0,0 +1,49 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3.functions;
+
+import static org.testng.Assert.assertEquals;
+
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.functions.ReturnStringIf2xx;
+import org.jclouds.io.Payloads;
+import org.testng.annotations.Test;
+
+/**
+ * Tests behavior of {@code ETagFromHttpResponseViaRegex}
+ * 
+ * @author Adrian Cole
+ */
+// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
+@Test(groups = "unit", testName = "ETagFromHttpResponseViaRegexTest")
+public class ETagFromHttpResponseViaRegexTest {
+
+   @Test
+   public void test() {
+
+      HttpResponse response = HttpResponse.builder().statusCode(200).payload(
+               Payloads.newInputStreamPayload(getClass().getResourceAsStream("/complete-multipart-upload.xml")))
+               .build();
+      ETagFromHttpResponseViaRegex parser = new ETagFromHttpResponseViaRegex(new ReturnStringIf2xx());
+
+      assertEquals(parser.apply(response), "\"3858f62230ac3c915f300c664312c11f-9\"");
+   }
+
+}

providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegexTest.java

@@ -0,0 +1,49 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.aws.s3.functions;
+
+import static org.testng.Assert.assertEquals;
+
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.functions.ReturnStringIf2xx;
+import org.jclouds.io.Payloads;
+import org.testng.annotations.Test;
+
+/**
+ * Tests behavior of {@code UploadIdFromHttpResponseViaRegex}
+ * 
+ * @author Adrian Cole
+ */
+// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
+@Test(groups = "unit", testName = "UploadIdFromHttpResponseViaRegexTest")
+public class UploadIdFromHttpResponseViaRegexTest {
+
+   @Test
+   public void test() {
+
+      HttpResponse response = HttpResponse.builder().statusCode(200).payload(
+               Payloads.newInputStreamPayload(getClass().getResourceAsStream("/initiate-multipart-upload.xml")))
+               .build();
+      UploadIdFromHttpResponseViaRegex parser = new UploadIdFromHttpResponseViaRegex(new ReturnStringIf2xx());
+
+      assertEquals(parser.apply(response), "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA");
+   }
+
+}

providers/aws-s3/src/test/resources/complete-multipart-upload.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Location>http://Example-Bucket.s3.amazonaws.com/Example-Object</Location>
+  <Bucket>Example-Bucket</Bucket>
+  <Key>Example-Object</Key>
+  <ETag>"3858f62230ac3c915f300c664312c11f-9"</ETag>
+</CompleteMultipartUploadResult>

providers/aws-s3/src/test/resources/initiate-multipart-upload.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Bucket>example-bucket</Bucket>
+  <Key>example-object</Key>
+  <UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId>
+</InitiateMultipartUploadResult>

providers/aws-s3/src/test/resources/log4j.xml

@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+    <!--
+
+
+        Copyright (C) 2010 Cloud Conscious, LLC.
+        <info@cloudconscious.com>
+
+        ====================================================================
+        Licensed under the Apache License, Version 2.0 (the "License");
+        you may not use this file except in compliance with the License.
+        You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0 Unless required by
+        applicable law or agreed to in writing, software distributed
+        under the License is distributed on an "AS IS" BASIS, WITHOUT
+        WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+        See the License for the specific language governing permissions
+        and limitations under the License.
+        ====================================================================
+    -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+    <!--
+        For more configuration infromation and examples see the Apache
+        Log4j website: http://logging.apache.org/log4j/
+    -->
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
+    debug="false">
+
+    <!-- A time/date based rolling appender -->
+    <appender name="WIREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="target/test-data/jclouds-wire.log" />
+        <param name="Append" value="true" />
+
+        <!-- Rollover at midnight each day -->
+        <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+        <param name="Threshold" value="TRACE" />
+
+        <layout class="org.apache.log4j.PatternLayout">
+            <!-- The default pattern: Date Priority [Category] Message\n -->
+            <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+            <!--
+                The full pattern: Date MS Priority [Category]
+                (Thread:NDC) Message\n <param name="ConversionPattern"
+                value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+            -->
+        </layout>
+    </appender>
+
+    <!-- A time/date based rolling appender -->
+    <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="target/test-data/jclouds.log" />
+        <param name="Append" value="true" />
+
+        <!-- Rollover at midnight each day -->
+        <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+        <param name="Threshold" value="TRACE" />
+
+        <layout class="org.apache.log4j.PatternLayout">
+            <!-- The default pattern: Date Priority [Category] Message\n -->
+            <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+            <!--
+                The full pattern: Date MS Priority [Category]
+                (Thread:NDC) Message\n <param name="ConversionPattern"
+                value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+            -->
+        </layout>
+    </appender>
+    <!-- A time/date based rolling appender -->
+    <appender name="BLOBSTOREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="target/test-data/jclouds-blobstore.log" />
+        <param name="Append" value="true" />
+        <param name="DatePattern" value="'.'yyyy-MM-dd" />
+        <param name="Threshold" value="TRACE" />
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+        </layout>
+    </appender>
+
+    <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+        <appender-ref ref="FILE" />
+    </appender>
+
+    <appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
+        <appender-ref ref="WIREFILE" />
+    </appender>
+
+    <appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
+        <appender-ref ref="BLOBSTOREFILE" />
+    </appender>
+    <!-- ================ -->
+    <!-- Limit categories -->
+    <!-- ================ -->
+
+    <category name="org.jclouds">
+        <priority value="DEBUG" />
+        <appender-ref ref="ASYNC" />
+    </category>
+
+    <category name="jclouds.headers">
+        <priority value="DEBUG" />
+        <appender-ref ref="ASYNCWIRE" />
+    </category><!--
+    <category name="jclouds.wire">
+        <priority value="DEBUG" />
+        <appender-ref ref="ASYNCWIRE" />
+    </category>
+    --><category name="jclouds.blobstore">
+        <priority value="DEBUG" />
+        <appender-ref ref="ASYNCBLOBSTORE" />
+    </category>
+    <!-- ======================= -->
+    <!-- Setup the Root category -->
+    <!-- ======================= -->
+
+    <root>
+        <priority value="WARN" />
+    </root>
+
+</log4j:configuration>

providers/googlestorage/src/main/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequest.java

@@ -26,11 +26,10 @@ import javax.inject.Inject;
 import javax.inject.Singleton;
 import javax.ws.rs.core.HttpHeaders;
 
-import org.jclouds.blobstore.binders.BindUserMetadataToHeadersWithPrefix;
+import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
 import org.jclouds.http.HttpRequest;
 import org.jclouds.http.utils.ModifyRequest;
 import org.jclouds.s3.binders.BindS3ObjectMetadataToRequest;
-import org.jclouds.s3.blobstore.functions.ObjectToBlob;
 import org.jclouds.s3.domain.S3Object;
 
 /**
@@ -39,15 +38,9 @@ import org.jclouds.s3.domain.S3Object;
  */
 @Singleton
 public class BindGoogleStorageObjectMetadataToRequest extends BindS3ObjectMetadataToRequest {
-   private final BindUserMetadataToHeadersWithPrefix blobBinder;
-   private final ObjectToBlob object2Blob;
-
    @Inject
-   public BindGoogleStorageObjectMetadataToRequest(ObjectToBlob object2Blob,
+   public BindGoogleStorageObjectMetadataToRequest(BindMapToHeadersWithPrefix metadataPrefixer) {
-            BindUserMetadataToHeadersWithPrefix blobBinder) {
+      super(metadataPrefixer);
-      super(object2Blob, blobBinder);
-      this.blobBinder = checkNotNull(blobBinder, "blobBinder");
-      this.object2Blob = checkNotNull(object2Blob, "object2Blob");
    }
 
    @Override
@@ -67,7 +60,7 @@ public class BindGoogleStorageObjectMetadataToRequest extends BindS3ObjectMetada
          request = ModifyRequest.replaceHeader(request, "Transfer-Encoding", "chunked");
       }
 
-      request = blobBinder.bindToRequest(request, object2Blob.apply(s3Object));
+      request = metadataPrefixer.bindToRequest(request, s3Object.getMetadata().getUserMetadata());
 
       if (s3Object.getMetadata().getCacheControl() != null) {
          request = ModifyRequest.replaceHeader(request, HttpHeaders.CACHE_CONTROL, s3Object.getMetadata()

providers/googlestorage/src/test/java/org/jclouds/googlestorage/GoogleStorageAsyncClientTestDisabled.java

@@ -19,19 +19,29 @@
 
 package org.jclouds.googlestorage;
 
+import org.jclouds.rest.internal.RestAnnotationProcessor;
+import org.jclouds.s3.S3AsyncClient;
 import org.testng.annotations.Test;
 
+import com.google.inject.TypeLiteral;
+
 /**
  * @author Adrian Cole
  */
 // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
 @Test(enabled = false, groups = "unit", testName = "GoogleStorageAsyncClientTest")
-public class GoogleStorageAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest {
+public class GoogleStorageAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest<S3AsyncClient> {
 
    public GoogleStorageAsyncClientTestDisabled() {
       this.provider = "googlestorage";
       this.url = "commondatastorage.googleapis.com";
    }
 
+   @Override
+   protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
+      return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
+      };
+   }
+
    // TODO parameterize this test so that it can pass
 }

providers/googlestorage/src/test/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequestTest.java

@@ -10,12 +10,15 @@ import javax.ws.rs.HttpMethod;
 import org.jclouds.http.HttpRequest;
 import org.jclouds.io.Payload;
 import org.jclouds.io.Payloads;
+import org.jclouds.rest.internal.RestAnnotationProcessor;
 import org.jclouds.s3.BaseS3AsyncClientTest;
+import org.jclouds.s3.S3AsyncClient;
 import org.jclouds.s3.domain.S3Object;
 import org.testng.annotations.Test;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMultimap;
+import com.google.inject.TypeLiteral;
 
 /**
  * Tests behavior of {@code BindGoogleStorageObjectMetadataToRequest}
@@ -24,8 +27,13 @@ import com.google.common.collect.ImmutableMultimap;
  */
 // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
 @Test(groups = "unit", testName = "BindGoogleStorageObjectMetadataToRequestTest")
-public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncClientTest {
+public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncClientTest<S3AsyncClient> {
 
+   @Override
+   protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
+      return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
+      };
+   }
 
    @Test
    public void testPassWithMinimumDetailsAndPayload5GB() {
@@ -36,7 +44,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
       object.getMetadata().setKey("foo");
 
       HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
-      BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
+      BindGoogleStorageObjectMetadataToRequest binder = injector
+               .getInstance(BindGoogleStorageObjectMetadataToRequest.class);
 
       assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint(
                URI.create("http://localhost")).build());
@@ -52,7 +61,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
       object.getMetadata().getUserMetadata().putAll(ImmutableMap.of("foo", "bar"));
 
       HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
-      BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
+      BindGoogleStorageObjectMetadataToRequest binder = injector
+               .getInstance(BindGoogleStorageObjectMetadataToRequest.class);
 
       assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint(
                URI.create("http://localhost")).headers(ImmutableMultimap.of("x-amz-meta-foo", "bar")).build());
@@ -66,7 +76,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
       object.getMetadata().setKey("foo");
 
       HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
-      BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
+      BindGoogleStorageObjectMetadataToRequest binder = injector
+               .getInstance(BindGoogleStorageObjectMetadataToRequest.class);
 
       assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint(
                URI.create("http://localhost")).headers(ImmutableMultimap.of("Transfer-Encoding", "chunked")).build());
@@ -80,7 +91,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
       object.setPayload(payload);
 
       HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
-      BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
+      BindGoogleStorageObjectMetadataToRequest binder = injector
+               .getInstance(BindGoogleStorageObjectMetadataToRequest.class);
       binder.bindToRequest(request, object);
    }
 
@@ -93,7 +105,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
       object.getMetadata().setKey("foo");
 
       HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
-      BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
+      BindGoogleStorageObjectMetadataToRequest binder = injector
+               .getInstance(BindGoogleStorageObjectMetadataToRequest.class);
       binder.bindToRequest(request, object);
    }
 
@@ -105,7 +118,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
 
    @Test(expectedExceptions = { NullPointerException.class, IllegalStateException.class })
    public void testNullIsBad() {
-      BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
+      BindGoogleStorageObjectMetadataToRequest binder = injector
+               .getInstance(BindGoogleStorageObjectMetadataToRequest.class);
       HttpRequest request = HttpRequest.builder().method("GET").endpoint(URI.create("http://momma")).build();
       binder.bindToRequest(request, null);
    }

providers/scaleup-storage/src/test/java/org/jclouds/scaleup/storage/ScaleUpStorageAsyncClientTestDisabled.java

@@ -19,14 +19,25 @@
 
 package org.jclouds.scaleup.storage;
 
+import org.jclouds.rest.internal.RestAnnotationProcessor;
+import org.jclouds.s3.S3AsyncClient;
+import org.jclouds.s3.S3AsyncClientTest;
 import org.testng.annotations.Test;
 
+import com.google.inject.TypeLiteral;
+
 /**
  * @author Adrian Cole
  */
 // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
 @Test(enabled = false, groups = "unit", testName = "ScaleUpStorageAsyncClientTest")
-public class ScaleUpStorageAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest {
+public class ScaleUpStorageAsyncClientTestDisabled extends S3AsyncClientTest<S3AsyncClient> {
+
+   @Override
+   protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
+      return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
+      };
+   }
 
    public ScaleUpStorageAsyncClientTestDisabled() {
       this.provider = "scaleup-storage";

sandbox-apis/cloudstack/src/test/java/org/jclouds/cloudstack/features/VirtualMachineClientLiveTest.java

@@ -21,12 +21,14 @@ package org.jclouds.cloudstack.features;
 
 import static com.google.common.base.Predicates.equalTo;
 import static com.google.common.base.Predicates.or;
+import static com.google.common.collect.Iterables.filter;
 import static com.google.common.collect.Iterables.find;
 import static com.google.common.collect.Iterables.get;
 import static com.google.common.collect.Iterables.getOnlyElement;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 
+import java.util.NoSuchElementException;
 import java.util.Set;
 
 import org.jclouds.cloudstack.CloudStackClient;
@@ -72,15 +74,30 @@ public class VirtualMachineClientLiveTest extends BaseCloudStackClientLiveTest {
 
       long serviceOfferingId = DEFAULT_SIZE_ORDERING.min(client.getOfferingClient().listServiceOfferings()).getId();
 
-      long templateId = find(client.getTemplateClient().listTemplates(), new Predicate<Template>() {
+      Iterable<Template> templates = filter(client.getTemplateClient().listTemplates(), new Predicate<Template>() {
 
          @Override
          public boolean apply(Template arg0) {
             return arg0.getZoneId() == zone.getId() && arg0.isFeatured() && arg0.isReady()
-                     && or(equalTo("Ubuntu 10.04 (64-bit)"), equalTo("CentOS 5.3 (32-bit)")).apply(arg0.getOSType());
+                     && or(equalTo("Ubuntu 10.04 (64-bit)"), equalTo("CentOS 5.3 (64-bit)")).apply(arg0.getOSType());
          }
 
-      }).getId();
+      });
+
+      long templateId;
+      try {
+         // prefer password enabled
+         templateId = find(templates, new Predicate<Template>() {
+
+            @Override
+            public boolean apply(Template arg0) {
+               return arg0.isPasswordEnabled();
+            }
+
+         }).getId();
+      } catch (NoSuchElementException e) {
+         templateId = get(templates, 0).getId();
+      }
 
       DeployVirtualMachineOptions options = new DeployVirtualMachineOptions();
       if (zone.getNetworkType() == NetworkType.ADVANCED) {