enhanced multi-part test to look at md5

providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequest.java

@@ -21,6 +21,7 @@ package org.jclouds.aws.s3.binders;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static org.jclouds.crypto.CryptoStreams.base64;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
@@ -77,6 +78,10 @@ public class BindObjectMetadataToRequest implements Binder {
          headers.put(HttpHeaders.CONTENT_TYPE, "binary/octet-stream");
       }
 
+      if (md.getContentMetadata().getContentMD5() != null) {
+         headers.put("Content-MD5", base64(md.getContentMetadata().getContentMD5()));
+      }
+
       request = ModifyRequest.replaceHeaders(request, headers.build());
       return request;
    }

providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java

@@ -79,11 +79,23 @@ public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest<AWSS3
             NoSuchMethodException {
       Method method = AWSS3AsyncClient.class.getMethod("initiateMultipartUpload", String.class, ObjectMetadata.class,
                PutObjectOptions[].class);
-      HttpRequest request = processor
+      HttpRequest request = processor.createRequest(method, "bucket", ObjectMetadataBuilder.create().key("foo")
-               .createRequest(method, "bucket", ObjectMetadataBuilder.create().key("foo").build());
+               .contentMD5(new byte[] { 1, 2, 3, 4 }).build());
 
       assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1");
-      assertNonPayloadHeadersEqual(request, "Content-Type: binary/octet-stream\nHost: bucket." + url + "\n");
+      assertNonPayloadHeadersEqual(request, "Content-MD5: AQIDBA==\nContent-Type: binary/octet-stream\nHost: bucket."
+               + url + "\n");
+      assertPayloadEquals(request, null, null, false);
+
+      // as this is a payload-related command, but with no payload, be careful that we check
+      // filtering and do not ignore if this fails later.
+      request = request.getFilters().get(0).filter(request);
+
+      assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1");
+      assertNonPayloadHeadersEqual(request,
+               "Authorization: AWS identity:Sp1FX4svL9P2u2bFJwroaYpSANo=\nContent-MD5: AQIDBA==\n"
+                        + "Content-Type: binary/octet-stream\nDate: 2009-11-08T15:54:08.897Z\nHost: bucket." + url
+                        + "\n");
       assertPayloadEquals(request, null, null, false);
 
       assertResponseParserClassEquals(method, request, UploadIdFromHttpResponseViaRegex.class);

providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java

@@ -19,27 +19,30 @@
 
 package org.jclouds.aws.s3;
 
-import static org.testng.Assert.assertTrue;
+import static com.google.common.io.ByteStreams.join;
-
+import static com.google.common.io.ByteStreams.newInputStreamSupplier;
+import static com.google.common.io.ByteStreams.toByteArray;
+import static org.jclouds.crypto.CryptoStreams.md5;
+import static org.jclouds.io.Payloads.newByteArrayPayload;
+import static org.testng.Assert.assertEquals;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.zip.GZIPInputStream;
 
-import org.jclouds.crypto.CryptoStreams;
+import org.jclouds.blobstore.KeyNotFoundException;
 import org.jclouds.http.BaseJettyTest;
 import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule;
 import org.jclouds.io.Payload;
-import org.jclouds.io.Payloads;
 import org.jclouds.s3.S3ClientLiveTest;
 import org.jclouds.s3.domain.ObjectMetadataBuilder;
+import org.jclouds.s3.domain.S3Object;
 import org.testng.ITestContext;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import com.google.common.collect.ImmutableMap;
-import com.google.common.io.ByteStreams;
 import com.google.common.io.InputSupplier;
 import com.google.inject.Module;
 
@@ -58,7 +61,7 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
    public AWSS3Client getApi() {
       return (AWSS3Client) context.getProviderSpecificContext().getApi();
    }
-   
+
    @Override
    protected Module createHttpModule() {
       // in order to be able to debug the wire protocol I use ApacheHC...
@@ -70,44 +73,65 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
    public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
       super.setUpResourcesOnThisThread(testContext);
       oneHundredOneConstitutions = getTestDataSupplier();
-      oneHundredOneConstitutionsMD5 = CryptoStreams.md5(oneHundredOneConstitutions);
+      oneHundredOneConstitutionsMD5 = md5(oneHundredOneConstitutions);
    }
 
    @SuppressWarnings("unchecked")
    public static InputSupplier<InputStream> getTestDataSupplier() throws IOException {
-      byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class
+      byte[] oneConstitution = toByteArray(new GZIPInputStream(BaseJettyTest.class.getResourceAsStream("/const.txt.gz")));
-               .getResourceAsStream("/const.txt.gz")));
+      InputSupplier<ByteArrayInputStream> constitutionSupplier = newInputStreamSupplier(oneConstitution);
-      InputSupplier<ByteArrayInputStream> constitutionSupplier = ByteStreams.newInputStreamSupplier(oneConstitution);
 
-      InputSupplier<InputStream> temp = ByteStreams.join(constitutionSupplier);
+      InputSupplier<InputStream> temp = join(constitutionSupplier);
       // we have to go beyond 5MB per part
       for (oneHundredOneConstitutionsLength = oneConstitution.length; oneHundredOneConstitutionsLength < 5 * 1024 * 1024; oneHundredOneConstitutionsLength += oneConstitution.length) {
-         temp = ByteStreams.join(temp, constitutionSupplier);
+         temp = join(temp, constitutionSupplier);
       }
       return temp;
    }
 
    public void testMultipartSynchronously() throws InterruptedException, IOException {
       String containerName = getContainerName();
+      S3Object object = null;
       try {
          String key = "constitution.txt";
          String uploadId = getApi().initiateMultipartUpload(containerName,
-                  ObjectMetadataBuilder.create().key(key).build());
+                  ObjectMetadataBuilder.create().key(key).contentMD5(oneHundredOneConstitutionsMD5).build());
+         byte[] buffer = toByteArray(oneHundredOneConstitutions.getInput());
+         assertEquals(oneHundredOneConstitutionsLength, (long) buffer.length);
 
-         byte[] buffer = ByteStreams.toByteArray(oneHundredOneConstitutions.getInput());
+         Payload part1 = newByteArrayPayload(buffer);
-         Payload part1 = Payloads.newByteArrayPayload(buffer);
          part1.getContentMetadata().setContentLength((long) buffer.length);
          part1.getContentMetadata().setContentMD5(oneHundredOneConstitutionsMD5);
 
-         String eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
+         String eTagOf1 = null;
+         try {
+            eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
+         } catch (KeyNotFoundException e) {
+            // note that because of eventual consistency, the upload id may not be present yet
+            // we may wish to add this condition to the retry handler
+
+            // we may also choose to implement ListParts and wait for the uploadId to become
+            // available there.
+            eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
+         }
 
          String eTag = getApi().completeMultipartUpload(containerName, key, uploadId, ImmutableMap.of(1, eTagOf1));
 
-         assertTrue(true);
+         assert !eTagOf1.equals(eTag);
+
+         object = getApi().getObject(containerName, key);
+         assertEquals(toByteArray(object.getPayload()), buffer);
+
+         // noticing amazon does not return content-md5 header or a parsable ETag after a multi-part
+         // upload is complete:
+         // https://forums.aws.amazon.com/thread.jspa?threadID=61344
+         assertEquals(object.getPayload().getContentMetadata().getContentMD5(), null);
+         assertEquals(getApi().headObject(containerName, key).getContentMetadata().getContentMD5(), null);
 
       } finally {
+         if (object != null)
+            object.getPayload().close();
          returnContainer(containerName);
       }
    }
-
 }