From 574af762f605588238bd5007e3ade146f8e9a4ca Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Sun, 21 Mar 2010 21:04:10 -0700 Subject: [PATCH] added md5OutputStream to encryptionService and refactored blobstore.clj to use it --- .../main/clojure/org/jclouds/blobstore.clj | 15 +++-- .../jclouds/encryption/EncryptionService.java | 12 ++++ .../internal/JCEEncryptionService.java | 20 +++++- .../encryption/EncryptionServiceTest.java | 67 +++++++++++-------- .../BouncyCastleEncryptionService.java | 20 ++++++ project/pom.xml | 6 +- 6 files changed, 101 insertions(+), 39 deletions(-) diff --git a/blobstore/src/main/clojure/org/jclouds/blobstore.clj b/blobstore/src/main/clojure/org/jclouds/blobstore.clj index 29eac52516..cf19ffcb85 100644 --- a/blobstore/src/main/clojure/org/jclouds/blobstore.clj +++ b/blobstore/src/main/clojure/org/jclouds/blobstore.clj @@ -25,7 +25,8 @@ See http://code.google.com/p/jclouds for details." AsyncBlobStore BlobStore BlobStoreContext BlobStoreContextFactory domain.BlobMetadata domain.StorageMetadata domain.Blob options.ListContainerOptions] - [java.security DigestOutputStream MessageDigest] + [org.jclouds.encryption.internal JCEEncryptionService] + [java.util Arrays] [com.google.common.collect ImmutableSet])) (defn blobstore @@ -68,6 +69,8 @@ Options can also be specified for extension modules (def *blobstore*) +(def *encryption-service* (JCEEncryptionService.)) ;; TODO: use guice + (def *max-retries* 3) (defmacro with-blobstore [[& blobstore-or-args] & body] @@ -240,7 +243,7 @@ example: "Create an blob representing text data: container, name, string -> etag " - ([container-name name data] + ([container-name name data] ;; TODO: allow payload to be a stream (create-blob *blobstore* container-name name data)) ([blobstore container-name name data] (put-blob blobstore container-name @@ -262,12 +265,12 @@ container, name, string -> etag (defmethod download-blob OutputStream [blobstore container-name name target & [retries]] (let [blob (get-blob blobstore container-name name) - digest-stream (DigestOutputStream. ;; TODO: not all clouds use MD5 - target (MessageDigest/getInstance "MD5"))] + digest-stream (.md5OutputStream ;; TODO: not all clouds use MD5 + *encryption-service* target)] (copy (.getContent blob) digest-stream) - (let [digest (.digest (.getMessageDigest digest-stream)) + (let [digest (.getMD5 digest-stream) metadata-digest (.getContentMD5 (.getMetadata blob))] - (when-not (MessageDigest/isEqual digest metadata-digest) + (when-not (Arrays/equals digest metadata-digest) (if (<= (or retries 0) *max-retries*) (recur blobstore container-name name target [(inc (or retries 1))]) (throw (Exception. (format "Download failed for %s/%s" diff --git a/core/src/main/java/org/jclouds/encryption/EncryptionService.java b/core/src/main/java/org/jclouds/encryption/EncryptionService.java index d63e3ecb8f..5dc2519903 100644 --- a/core/src/main/java/org/jclouds/encryption/EncryptionService.java +++ b/core/src/main/java/org/jclouds/encryption/EncryptionService.java @@ -21,7 +21,9 @@ package org.jclouds.encryption; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import java.io.FilterOutputStream; import java.io.InputStream; +import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -64,6 +66,16 @@ public interface EncryptionService { MD5InputStreamResult generateMD5Result(InputStream toEncode); + MD5OutputStream md5OutputStream(OutputStream out); + + public static abstract class MD5OutputStream extends FilterOutputStream { + public MD5OutputStream(OutputStream out) { + super(out); + } + + public abstract byte[] getMD5(); + } + public static class MD5InputStreamResult { public final byte[] data; public final byte[] md5; diff --git a/core/src/main/java/org/jclouds/encryption/internal/JCEEncryptionService.java b/core/src/main/java/org/jclouds/encryption/internal/JCEEncryptionService.java index 26ffa1f8de..668202c8c7 100644 --- a/core/src/main/java/org/jclouds/encryption/internal/JCEEncryptionService.java +++ b/core/src/main/java/org/jclouds/encryption/internal/JCEEncryptionService.java @@ -21,6 +21,8 @@ package org.jclouds.encryption.internal; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.security.DigestOutputStream; import java.security.InvalidKeyException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -122,7 +124,7 @@ public class JCEEncryptionService extends BaseEncryptionService { return new MD5InputStreamResult(out.toByteArray(), eTag.digest(), length); } - private MessageDigest getDigest() { + private static MessageDigest getDigest() { MessageDigest eTag; try { eTag = MessageDigest.getInstance("MD5"); @@ -136,4 +138,20 @@ public class JCEEncryptionService extends BaseEncryptionService { return Base64.decode(encoded); } + @Override + public MD5OutputStream md5OutputStream(OutputStream out) { + return new JCEMD5OutputStream(out); + } + + public static class JCEMD5OutputStream extends MD5OutputStream { + public JCEMD5OutputStream(OutputStream out) { + super(new DigestOutputStream(out, getDigest())); + } + + @Override + public byte[] getMD5() { + MessageDigest digest = ((DigestOutputStream) out).getMessageDigest(); + return digest.digest(); + } + } } diff --git a/core/src/test/java/org/jclouds/encryption/EncryptionServiceTest.java b/core/src/test/java/org/jclouds/encryption/EncryptionServiceTest.java index af7317cf39..1ea3e5cf87 100644 --- a/core/src/test/java/org/jclouds/encryption/EncryptionServiceTest.java +++ b/core/src/test/java/org/jclouds/encryption/EncryptionServiceTest.java @@ -20,6 +20,8 @@ package org.jclouds.encryption; import static org.testng.Assert.assertEquals; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -30,12 +32,13 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; import org.jclouds.PerformanceTest; -import org.jclouds.encryption.EncryptionService; +import org.jclouds.encryption.EncryptionService.MD5OutputStream; import org.jclouds.encryption.internal.Base64; import org.testng.annotations.BeforeTest; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import com.google.common.io.ByteStreams; import com.google.inject.Guice; import com.google.inject.Injector; @@ -48,51 +51,51 @@ import com.google.inject.Injector; public class EncryptionServiceTest extends PerformanceTest { protected EncryptionService encryptionService; + @BeforeTest protected void createEncryptionService() { Injector i = Guice.createInjector(); encryptionService = i.getInstance(EncryptionService.class); } - + public final static Object[][] base64KeyMessageDigest = { - { Base64.decode("CwsLCwsLCwsLCwsLCwsLCwsLCws="), "Hi There", - "thcxhlUFcmTii8C2+zeMjvFGvgA=" }, - { Base64.decode("SmVmZQ=="), "what do ya want for nothing?", - "7/zfauXrL6LSdBbV8YTfnCWafHk=" }, - { Base64.decode("DAwMDAwMDAwMDAwMDAwMDAwMDAw="), "Test With Truncation", - "TBoDQktV4H/n8nvh1Yu5MkqaWgQ=" }, - { - Base64 - .decode("qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqo="), - "Test Using Larger Than Block-Size Key - Hash Key First", - "qkrl4VJy0A6VcFY3zoo7Ve1AIRI=" }, - { - Base64 - .decode("qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqo="), - "Test Using Larger Than Block-Size Key and Larger Than One Block-Size Data", - "6OmdD0UjfXhta7qnllx4CLv/GpE=" } }; - + { Base64.decode("CwsLCwsLCwsLCwsLCwsLCwsLCws="), "Hi There", + "thcxhlUFcmTii8C2+zeMjvFGvgA=" }, + { Base64.decode("SmVmZQ=="), "what do ya want for nothing?", + "7/zfauXrL6LSdBbV8YTfnCWafHk=" }, + { Base64.decode("DAwMDAwMDAwMDAwMDAwMDAwMDAw="), "Test With Truncation", + "TBoDQktV4H/n8nvh1Yu5MkqaWgQ=" }, + { + Base64 + .decode("qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqo="), + "Test Using Larger Than Block-Size Key - Hash Key First", + "qkrl4VJy0A6VcFY3zoo7Ve1AIRI=" }, + { + Base64 + .decode("qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqo="), + "Test Using Larger Than Block-Size Key and Larger Than One Block-Size Data", + "6OmdD0UjfXhta7qnllx4CLv/GpE=" } }; @DataProvider(name = "hmacsha1") public Object[][] createData1() { return base64KeyMessageDigest; } - @Test(dataProvider = "hmacsha1", enabled = false) + @Test(dataProvider = "hmacsha1") public void testHmacSha1Base64(byte[] key, String message, String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException { String b64 = encryptionService.hmacSha1Base64(message, key); assertEquals(b64, base64Digest); } - @Test(dataProvider = "hmacsha1", enabled = false) + @Test(dataProvider = "hmacsha1") void testDigestSerialResponseTime(byte[] key, String message, String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException { for (int i = 0; i < 10000; i++) testHmacSha1Base64(key, message, base64Digest); } - @Test(dataProvider = "hmacsha1", enabled = false) + @Test(dataProvider = "hmacsha1") void testDigestParallelResponseTime(final byte[] key, final String message, final String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException, InterruptedException, ExecutionException { @@ -110,22 +113,28 @@ public class EncryptionServiceTest extends PerformanceTest { @DataProvider(name = "eTag") public Object[][] createMD5Data() { - return base64MD5MessageDigest; + return hexMD5MessageDigest; } - public final static Object[][] base64MD5MessageDigest = { + public final static Object[][] hexMD5MessageDigest = { { "apple", "1f3870be274f6c49b3e31a0c6728957f" }, { "bear", "893b56e3cfe153fb770a120b83bac20c" }, { "candy", "c48ba993d35c3abe0380f91738fe2a34" }, { "dogma", "95eb470e4faee302e9cd3063b1923dab" }, { "emma", "00a809937eddc44521da9521269e75c6" } }; - - @Test(dataProvider = "eTag", enabled = false) - public void testMD5Digest(String message, String base64Digest) throws NoSuchProviderException, - NoSuchAlgorithmException, InvalidKeyException, UnsupportedEncodingException { + @Test(dataProvider = "eTag") + public void testMD5Digest(String message, String hexMD5Digest) throws NoSuchProviderException, + NoSuchAlgorithmException, InvalidKeyException, IOException { String b64 = encryptionService.md5Hex(message.getBytes()); - assertEquals(base64Digest, b64); + assertEquals(hexMD5Digest, b64); + + MD5OutputStream outputStream = encryptionService.md5OutputStream(new ByteArrayOutputStream()); + ByteStreams.copy(ByteStreams.newInputStreamSupplier(message.getBytes()).getInput(), + outputStream); + + assertEquals(encryptionService.fromHexString(hexMD5Digest), outputStream.getMD5()); + } byte[] bytes = { 0, 1, 2, 4, 8, 16, 32, 64 }; diff --git a/extensions/bouncycastle/src/main/java/org/jclouds/encryption/bouncycastle/BouncyCastleEncryptionService.java b/extensions/bouncycastle/src/main/java/org/jclouds/encryption/bouncycastle/BouncyCastleEncryptionService.java index 0c5554e097..fdb95051f0 100644 --- a/extensions/bouncycastle/src/main/java/org/jclouds/encryption/bouncycastle/BouncyCastleEncryptionService.java +++ b/extensions/bouncycastle/src/main/java/org/jclouds/encryption/bouncycastle/BouncyCastleEncryptionService.java @@ -21,6 +21,7 @@ package org.jclouds.encryption.bouncycastle; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.NoSuchProviderException; @@ -31,6 +32,7 @@ import org.bouncycastle.crypto.Digest; import org.bouncycastle.crypto.digests.MD5Digest; import org.bouncycastle.crypto.digests.SHA1Digest; import org.bouncycastle.crypto.digests.SHA256Digest; +import org.bouncycastle.crypto.io.DigestOutputStream; import org.bouncycastle.crypto.macs.HMac; import org.bouncycastle.crypto.params.KeyParameter; import org.bouncycastle.util.encoders.Base64; @@ -136,4 +138,22 @@ public class BouncyCastleEncryptionService extends BaseEncryptionService { return Base64.decode(encoded); } + @Override + public MD5OutputStream md5OutputStream(OutputStream out) { + return new BouncyCastleMD5OutputStream(out); + } + + public static class BouncyCastleMD5OutputStream extends MD5OutputStream { + public BouncyCastleMD5OutputStream(OutputStream out) { + super(new DigestOutputStream(out, new MD5Digest())); + } + + @Override + public byte[] getMD5() { + MD5Digest digest = (MD5Digest) ((DigestOutputStream) out).getDigest(); + byte[] resBuf = new byte[digest.getDigestSize()]; + digest.doFinal(resBuf, 0); + return resBuf; + } + } } diff --git a/project/pom.xml b/project/pom.xml index ca8b6adc4e..f76822f9d5 100644 --- a/project/pom.xml +++ b/project/pom.xml @@ -283,9 +283,9 @@ - - !clojure.* - + -Xms128m -Xmx512m + true + true