added md5OutputStream to encryptionService and refactored blobstore.clj to use it

blobstore/src/main/clojure/org/jclouds/blobstore.clj

@@ -25,7 +25,8 @@ See http://code.google.com/p/jclouds for details."
             AsyncBlobStore BlobStore BlobStoreContext BlobStoreContextFactory
             domain.BlobMetadata domain.StorageMetadata domain.Blob
             options.ListContainerOptions]
-           [java.security DigestOutputStream MessageDigest]
+           [org.jclouds.encryption.internal JCEEncryptionService]
+           [java.util Arrays]
            [com.google.common.collect ImmutableSet]))
 
 (defn blobstore
@@ -68,6 +69,8 @@ Options can also be specified for extension modules
 
 (def *blobstore*)
 
+(def *encryption-service* (JCEEncryptionService.)) ;; TODO: use guice
+
 (def *max-retries* 3)
 
 (defmacro with-blobstore [[& blobstore-or-args] & body]
@@ -240,7 +243,7 @@ example:
   "Create an blob representing text data:
 container, name, string -> etag
 "
-  ([container-name name data]
+  ([container-name name data] ;; TODO: allow payload to be a stream
      (create-blob *blobstore* container-name name data))
   ([blobstore container-name name data]
      (put-blob blobstore container-name
@@ -262,12 +265,12 @@ container, name, string -> etag
 (defmethod download-blob OutputStream [blobstore container-name name target
                                        & [retries]]
   (let [blob (get-blob blobstore container-name name)
-        digest-stream (DigestOutputStream. ;; TODO: not all clouds use MD5
+        digest-stream (.md5OutputStream ;; TODO: not all clouds use MD5
-                       target (MessageDigest/getInstance "MD5"))]
+                       *encryption-service* target)]
     (copy (.getContent blob) digest-stream)
-    (let [digest (.digest (.getMessageDigest digest-stream))
+    (let [digest (.getMD5 digest-stream)
           metadata-digest (.getContentMD5 (.getMetadata blob))]
-      (when-not (MessageDigest/isEqual digest metadata-digest)
+      (when-not (Arrays/equals digest metadata-digest)
         (if (<= (or retries 0) *max-retries*)
           (recur blobstore container-name name target [(inc (or retries 1))])
           (throw (Exception. (format "Download failed for %s/%s"

core/src/main/java/org/jclouds/encryption/EncryptionService.java

@@ -21,7 +21,9 @@ package org.jclouds.encryption;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.io.FilterOutputStream;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
@@ -64,6 +66,16 @@ public interface EncryptionService {
 
    MD5InputStreamResult generateMD5Result(InputStream toEncode);
 
+   MD5OutputStream md5OutputStream(OutputStream out);
+
+   public static abstract class MD5OutputStream extends FilterOutputStream {
+      public MD5OutputStream(OutputStream out) {
+         super(out);
+      }
+
+      public abstract byte[] getMD5();
+   }
+
    public static class MD5InputStreamResult {
       public final byte[] data;
       public final byte[] md5;

core/src/main/java/org/jclouds/encryption/internal/JCEEncryptionService.java

@@ -21,6 +21,8 @@ package org.jclouds.encryption.internal;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.DigestOutputStream;
 import java.security.InvalidKeyException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@@ -122,7 +124,7 @@ public class JCEEncryptionService extends BaseEncryptionService {
       return new MD5InputStreamResult(out.toByteArray(), eTag.digest(), length);
    }
 
-   private MessageDigest getDigest() {
+   private static MessageDigest getDigest() {
       MessageDigest eTag;
       try {
          eTag = MessageDigest.getInstance("MD5");
@@ -136,4 +138,20 @@ public class JCEEncryptionService extends BaseEncryptionService {
       return Base64.decode(encoded);
    }
 
+   @Override
+   public MD5OutputStream md5OutputStream(OutputStream out) {
+      return new JCEMD5OutputStream(out);
+   }
+
+   public static class JCEMD5OutputStream extends MD5OutputStream {
+      public JCEMD5OutputStream(OutputStream out) {
+         super(new DigestOutputStream(out, getDigest()));
+      }
+
+      @Override
+      public byte[] getMD5() {
+         MessageDigest digest = ((DigestOutputStream) out).getMessageDigest();
+         return digest.digest();
+      }
+   }
 }

core/src/test/java/org/jclouds/encryption/EncryptionServiceTest.java

@@ -20,6 +20,8 @@ package org.jclouds.encryption;
 
 import static org.testng.Assert.assertEquals;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
@@ -30,12 +32,13 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorCompletionService;
 
 import org.jclouds.PerformanceTest;
-import org.jclouds.encryption.EncryptionService;
+import org.jclouds.encryption.EncryptionService.MD5OutputStream;
 import org.jclouds.encryption.internal.Base64;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import com.google.common.io.ByteStreams;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 
@@ -48,51 +51,51 @@ import com.google.inject.Injector;
 public class EncryptionServiceTest extends PerformanceTest {
 
    protected EncryptionService encryptionService;
+
    @BeforeTest
    protected void createEncryptionService() {
       Injector i = Guice.createInjector();
       encryptionService = i.getInstance(EncryptionService.class);
    }
-   
+
    public final static Object[][] base64KeyMessageDigest = {
-      { Base64.decode("CwsLCwsLCwsLCwsLCwsLCwsLCws="), "Hi There",
+            { Base64.decode("CwsLCwsLCwsLCwsLCwsLCwsLCws="), "Hi There",
-               "thcxhlUFcmTii8C2+zeMjvFGvgA=" },
+                     "thcxhlUFcmTii8C2+zeMjvFGvgA=" },
-      { Base64.decode("SmVmZQ=="), "what do ya want for nothing?",
+            { Base64.decode("SmVmZQ=="), "what do ya want for nothing?",
-               "7/zfauXrL6LSdBbV8YTfnCWafHk=" },
+                     "7/zfauXrL6LSdBbV8YTfnCWafHk=" },
-      { Base64.decode("DAwMDAwMDAwMDAwMDAwMDAwMDAw="), "Test With Truncation",
+            { Base64.decode("DAwMDAwMDAwMDAwMDAwMDAwMDAw="), "Test With Truncation",
-               "TBoDQktV4H/n8nvh1Yu5MkqaWgQ=" },
+                     "TBoDQktV4H/n8nvh1Yu5MkqaWgQ=" },
-      {
+            {
-               Base64
+                     Base64
-                        .decode("qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqo="),
+                              .decode("qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqo="),
-               "Test Using Larger Than Block-Size Key - Hash Key First",
+                     "Test Using Larger Than Block-Size Key - Hash Key First",
-               "qkrl4VJy0A6VcFY3zoo7Ve1AIRI=" },
+                     "qkrl4VJy0A6VcFY3zoo7Ve1AIRI=" },
-      {
+            {
-               Base64
+                     Base64
-                        .decode("qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqo="),
+                              .decode("qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqo="),
-               "Test Using Larger Than Block-Size Key and Larger Than One Block-Size Data",
+                     "Test Using Larger Than Block-Size Key and Larger Than One Block-Size Data",
-               "6OmdD0UjfXhta7qnllx4CLv/GpE=" } };
+                     "6OmdD0UjfXhta7qnllx4CLv/GpE=" } };
-   
 
    @DataProvider(name = "hmacsha1")
    public Object[][] createData1() {
       return base64KeyMessageDigest;
    }
 
-   @Test(dataProvider = "hmacsha1", enabled = false)
+   @Test(dataProvider = "hmacsha1")
    public void testHmacSha1Base64(byte[] key, String message, String base64Digest)
             throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException {
       String b64 = encryptionService.hmacSha1Base64(message, key);
       assertEquals(b64, base64Digest);
    }
 
-   @Test(dataProvider = "hmacsha1", enabled = false)
+   @Test(dataProvider = "hmacsha1")
    void testDigestSerialResponseTime(byte[] key, String message, String base64Digest)
             throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException {
       for (int i = 0; i < 10000; i++)
          testHmacSha1Base64(key, message, base64Digest);
    }
 
-   @Test(dataProvider = "hmacsha1", enabled = false)
+   @Test(dataProvider = "hmacsha1")
    void testDigestParallelResponseTime(final byte[] key, final String message,
             final String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException,
             InvalidKeyException, InterruptedException, ExecutionException {
@@ -110,22 +113,28 @@ public class EncryptionServiceTest extends PerformanceTest {
 
    @DataProvider(name = "eTag")
    public Object[][] createMD5Data() {
-      return base64MD5MessageDigest;
+      return hexMD5MessageDigest;
    }
 
-   public final static Object[][] base64MD5MessageDigest = {
+   public final static Object[][] hexMD5MessageDigest = {
             { "apple", "1f3870be274f6c49b3e31a0c6728957f" },
             { "bear", "893b56e3cfe153fb770a120b83bac20c" },
             { "candy", "c48ba993d35c3abe0380f91738fe2a34" },
             { "dogma", "95eb470e4faee302e9cd3063b1923dab" },
             { "emma", "00a809937eddc44521da9521269e75c6" } };
 
-
+   @Test(dataProvider = "eTag")
-   @Test(dataProvider = "eTag", enabled = false)
+   public void testMD5Digest(String message, String hexMD5Digest) throws NoSuchProviderException,
-   public void testMD5Digest(String message, String base64Digest) throws NoSuchProviderException,
+            NoSuchAlgorithmException, InvalidKeyException, IOException {
-            NoSuchAlgorithmException, InvalidKeyException, UnsupportedEncodingException {
       String b64 = encryptionService.md5Hex(message.getBytes());
-      assertEquals(base64Digest, b64);
+      assertEquals(hexMD5Digest, b64);
+
+      MD5OutputStream outputStream = encryptionService.md5OutputStream(new ByteArrayOutputStream());
+      ByteStreams.copy(ByteStreams.newInputStreamSupplier(message.getBytes()).getInput(),
+               outputStream);
+
+      assertEquals(encryptionService.fromHexString(hexMD5Digest), outputStream.getMD5());
+
    }
 
    byte[] bytes = { 0, 1, 2, 4, 8, 16, 32, 64 };

extensions/bouncycastle/src/main/java/org/jclouds/encryption/bouncycastle/BouncyCastleEncryptionService.java

@@ -21,6 +21,7 @@ package org.jclouds.encryption.bouncycastle;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.security.NoSuchProviderException;
@@ -31,6 +32,7 @@ import org.bouncycastle.crypto.Digest;
 import org.bouncycastle.crypto.digests.MD5Digest;
 import org.bouncycastle.crypto.digests.SHA1Digest;
 import org.bouncycastle.crypto.digests.SHA256Digest;
+import org.bouncycastle.crypto.io.DigestOutputStream;
 import org.bouncycastle.crypto.macs.HMac;
 import org.bouncycastle.crypto.params.KeyParameter;
 import org.bouncycastle.util.encoders.Base64;
@@ -136,4 +138,22 @@ public class BouncyCastleEncryptionService extends BaseEncryptionService {
       return Base64.decode(encoded);
    }
 
+   @Override
+   public MD5OutputStream md5OutputStream(OutputStream out) {
+      return new BouncyCastleMD5OutputStream(out);
+   }
+
+   public static class BouncyCastleMD5OutputStream extends MD5OutputStream {
+      public BouncyCastleMD5OutputStream(OutputStream out) {
+         super(new DigestOutputStream(out, new MD5Digest()));
+      }
+
+      @Override
+      public byte[] getMD5() {
+         MD5Digest digest = (MD5Digest) ((DigestOutputStream) out).getDigest();
+         byte[] resBuf = new byte[digest.getDigestSize()];
+         digest.doFinal(resBuf, 0);
+         return resBuf;
+      }
+   }
 }

project/pom.xml

@@ -283,9 +283,9 @@
             </execution>
           </executions>
           <configuration>
-            <testNamespaces>
+            <clojureOptions>-Xms128m -Xmx512m</clojureOptions>
-              <testNamespace>!clojure.*</testNamespace>
+            <compileDeclaredNamespaceOnly>true</compileDeclaredNamespaceOnly>
-            </testNamespaces>
+            <testDeclaredNamespaceOnly>true</testDeclaredNamespaceOnly>
           </configuration>
         </plugin>
             <plugin>