Issue 353: updated and tested through s3, azureblob, cloudfiles, and synaptic

atmos/src/main/java/org/jclouds/atmosonline/saas/binders/BindMetadataToHeaders.java

@@ -19,6 +19,8 @@
 
 package org.jclouds.atmosonline.saas.binders;
 
+import static com.google.common.base.Preconditions.checkArgument;
+
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
@@ -32,13 +34,14 @@ public class BindMetadataToHeaders implements Binder {
    private final BindUserMetadataToHeaders metaBinder;
 
    @Inject
-   protected BindMetadataToHeaders(BindUserMetadataToHeaders metaBinder,
+   protected BindMetadataToHeaders(BindUserMetadataToHeaders metaBinder, Crypto crypto) {
-            Crypto crypto) {
       this.metaBinder = metaBinder;
    }
 
    public void bindToRequest(HttpRequest request, Object payload) {
       AtmosObject object = (AtmosObject) payload;
+      checkArgument(object.getPayload().getContentMetadata().getContentLength() != null,
+               "contentLength must be set, streaming not supported");
       metaBinder.bindToRequest(request, object.getUserMetadata());
    }
 }

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageIntegrationLiveTest.java

@@ -19,6 +19,13 @@
 
 package org.jclouds.atmosonline.saas.blobstore.integration;
 
+import static org.testng.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobMetadata;
 import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
@@ -73,6 +80,39 @@ public class AtmosStorageIntegrationLiveTest extends BaseBlobIntegrationTest {
       // not supported
    }
 
+   // not supported
+   @Override
+   protected void checkContentDisposition(Blob blob, String contentDisposition) {
+      assert blob.getPayload().getContentMetadata().getContentDisposition() == null;
+      assert blob.getMetadata().getContentMetadata().getContentDisposition() == null;
+   }
+
+   // not supported
+   @Override
+   protected void checkContentEncoding(Blob blob, String contentEncoding) {
+      assert blob.getPayload().getContentMetadata().getContentEncoding() == null;
+      assert blob.getMetadata().getContentMetadata().getContentEncoding() == null;
+   }
+
+   // not supported
+   @Override
+   protected void checkContentLanguage(Blob blob, String contentLanguage) {
+      assert blob.getPayload().getContentMetadata().getContentLanguage() == null;
+      assert blob.getMetadata().getContentMetadata().getContentLanguage() == null;
+   }
+
+   @Override
+   @Test(expectedExceptions = IllegalArgumentException.class)
+   public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException {
+      super.testPutObjectStream();
+   }
+
+   // not supported
+   @Override
+   protected void checkMD5(BlobMetadata metadata) throws IOException {
+      assertEquals(metadata.getContentMetadata().getContentMD5(), null);
+   }
+
    @Test(enabled = false)
    // problem with the stub and md5, live is fine
    public void testMetadata() {

aws/core/src/main/java/org/jclouds/aws/s3/functions/ParseObjectFromHeadersAndHttpContent.java

@@ -35,8 +35,7 @@ import com.google.common.base.Function;
  * @see ParseMetadataFromHeaders
  * @author Adrian Cole
  */
-public class ParseObjectFromHeadersAndHttpContent implements Function<HttpResponse, S3Object>,
+public class ParseObjectFromHeadersAndHttpContent implements Function<HttpResponse, S3Object>, InvocationContext {
-         InvocationContext {
 
    private final ParseObjectMetadataFromHeaders metadataParser;
    private final S3Object.Factory objectProvider;
@@ -53,6 +52,7 @@ public class ParseObjectFromHeadersAndHttpContent implements Function<HttpRespon
       S3Object object = objectProvider.create(metadata);
       object.getAllHeaders().putAll(from.getHeaders());
       object.setPayload(from.getPayload());
+
       return object;
    }
 

aws/core/src/main/java/org/jclouds/aws/s3/functions/ParseObjectMetadataFromHeaders.java

@@ -29,6 +29,7 @@ import org.jclouds.aws.s3.blobstore.functions.BlobToObjectMetadata;
 import org.jclouds.aws.s3.domain.MutableObjectMetadata;
 import org.jclouds.blobstore.domain.BlobMetadata;
 import org.jclouds.blobstore.functions.ParseSystemAndUserMetadataFromHeaders;
+import org.jclouds.crypto.CryptoStreams;
 import org.jclouds.http.HttpRequest;
 import org.jclouds.http.HttpResponse;
 import org.jclouds.rest.InvocationContext;
@@ -64,6 +65,12 @@ public class ParseObjectMetadataFromHeaders implements Function<HttpResponse, Mu
       BlobMetadata base = blobMetadataParser.apply(from);
       MutableObjectMetadata to = blobToObjectMetadata.apply(base);
       addETagTo(from, to);
+      if (to.getContentMetadata().getContentMD5() == null && to.getETag() != null) {
+         byte[] md5 = CryptoStreams.hex(to.getETag().replaceAll("\"", ""));
+         // it is possible others will look at the http payload directly
+         from.getPayload().getContentMetadata().setContentMD5(md5);
+         to.getContentMetadata().setContentMD5(md5);
+      }
       to.setCacheControl(from.getFirstHeaderOrNull(HttpHeaders.CACHE_CONTROL));
       return to;
    }

aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3BlobIntegrationLiveTest.java

@@ -19,6 +19,9 @@
 
 package org.jclouds.aws.s3.blobstore.integration;
 
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
 import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
 import org.testng.annotations.Test;
 
@@ -30,4 +33,10 @@ import org.testng.annotations.Test;
 @Test(groups = { "live" }, testName = "s3.S3BlobIntegrationTest")
 public class S3BlobIntegrationLiveTest extends BaseBlobIntegrationTest {
 
+   @Override
+   @Test(expectedExceptions = IllegalArgumentException.class)
+   public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException {
+      super.testPutObjectStream();
+   }
+
 }

azure/src/main/java/org/jclouds/azure/storage/blob/binders/BindAzureBlobToPayload.java

@@ -46,7 +46,8 @@ public class BindAzureBlobToPayload implements Binder {
 
    public void bindToRequest(HttpRequest request, Object payload) {
       AzureBlob blob = (AzureBlob) payload;
-      checkArgument(blob.getPayload().getContentMetadata().getContentLength() >= 0, "size must be set");
+      checkArgument(blob.getPayload().getContentMetadata().getContentLength() != null
+               && blob.getPayload().getContentMetadata().getContentLength() >= 0, "size must be set");
       request.getHeaders().put("x-ms-blob-type", blob.getProperties().getType().toString());
 
       switch (blob.getProperties().getType()) {

azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobIntegrationLiveTest.java

@@ -19,8 +19,11 @@
 
 package org.jclouds.azure.storage.blob.blobstore.integration;
 
+import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.util.concurrent.ExecutionException;
 
+import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
 import org.testng.annotations.Test;
 
@@ -35,4 +38,18 @@ public class AzureBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
    public void testGetIfMatch() throws InterruptedException, UnsupportedEncodingException {
       // this currently fails
    }
+
+   @Override
+   @Test(expectedExceptions = IllegalArgumentException.class)
+   public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException {
+      super.testPutObjectStream();
+   }
+
+   // according to docs, content disposition is not persisted
+   // http://msdn.microsoft.com/en-us/library/dd179440.aspx
+   @Override
+   protected void checkContentDisposition(Blob blob, String contentDisposition) {
+      assert blob.getPayload().getContentMetadata().getContentDisposition() == null;
+      assert blob.getMetadata().getContentMetadata().getContentDisposition() == null;
+   }
 }

blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java

@@ -45,7 +45,6 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.zip.GZIPInputStream;
 
-import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.MediaType;
 
 import org.jclouds.blobstore.ContainerNotFoundException;
@@ -85,6 +84,7 @@ import com.google.common.io.InputSupplier;
 public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
    private InputSupplier<InputStream> oneHundredOneConstitutions;
    private byte[] oneHundredOneConstitutionsMD5;
+   private static long oneHundredOneConstitutionsLength;
 
    @BeforeClass(groups = { "integration", "live" })
    @Override
@@ -105,6 +105,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
       for (int i = 0; i < 100; i++) {
          temp = ByteStreams.join(temp, constitutionSupplier);
       }
+      oneHundredOneConstitutionsLength = oneConstitution.length * 101l;
       return temp;
    }
 
@@ -150,6 +151,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
       sourceObject.setPayload(oneHundredOneConstitutions.getInput());
       sourceObject.getMetadata().getContentMetadata().setContentType("text/plain");
       sourceObject.getMetadata().getContentMetadata().setContentMD5(oneHundredOneConstitutionsMD5);
+      sourceObject.getMetadata().getContentMetadata().setContentLength(oneHundredOneConstitutionsLength);
       sourceObject.getMetadata().getContentMetadata().setContentDisposition(contentDisposition);
       context.getBlobStore().putBlob(containerName, sourceObject);
    }
@@ -497,8 +499,6 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
                .getContentMetadata().getContentType();
       assert blob.getMetadata().getContentMetadata().getContentType().startsWith(contentType) : blob.getMetadata()
                .getContentMetadata().getContentType();
-      assert Iterables.getOnlyElement(blob.getAllHeaders().get(HttpHeaders.CONTENT_TYPE)).startsWith(contentType) : blob
-               .getAllHeaders().get(HttpHeaders.CONTENT_TYPE);
    }
 
    protected void checkContentDisposition(Blob blob, String contentDisposition) {
@@ -578,6 +578,10 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
                .getContentType();
       assertEquals(metadata.getContentMetadata().getContentLength(), new Long(TEST_STRING.length()));
       assertEquals(metadata.getUserMetadata().get("adrian"), "powderpuff");
+      checkMD5(metadata);
+   }
+
+   protected void checkMD5(BlobMetadata metadata) throws IOException {
       assertEquals(metadata.getContentMetadata().getContentMD5(), CryptoStreams.md5(InputSuppliers.of(TEST_STRING)));
    }
 

rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java

@@ -19,6 +19,7 @@
 
 package org.jclouds.rackspace.cloudfiles.blobstore.integration;
 
+import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
 import org.testng.annotations.Test;
 
@@ -36,4 +37,18 @@ public class CloudFilesBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
       // not supported in cloud files
    }
 
+   // not supported
+   @Override
+   protected void checkContentDisposition(Blob blob, String contentDisposition) {
+      assert blob.getPayload().getContentMetadata().getContentDisposition() == null;
+      assert blob.getMetadata().getContentMetadata().getContentDisposition() == null;
+   }
+
+   // not supported
+   @Override
+   protected void checkContentLanguage(Blob blob, String contentLanguage) {
+      assert blob.getPayload().getContentMetadata().getContentLanguage() == null;
+      assert blob.getMetadata().getContentMetadata().getContentLanguage() == null;
+   }
+
 }

rackspace/src/test/resources/log4j.xml

@@ -161,12 +161,12 @@
     <category name="jclouds.ssh">
         <priority value="DEBUG" />
         <appender-ref ref="ASYNCSSH" />
-    </category>
+    </category><!--
     <category name="jclouds.wire">
         <priority value="DEBUG" />
         <appender-ref ref="ASYNCWIRE" />
     </category>
-    <category name="jclouds.blobstore">
+    --><category name="jclouds.blobstore">
         <priority value="DEBUG" />
         <appender-ref ref="ASYNCBLOBSTORE" />
     </category>