From 05a9c79242047955cd2e2582364aa16a38444937 Mon Sep 17 00:00:00 2001 From: Ka-Hing Cheung Date: Tue, 9 Jun 2015 17:25:18 -0700 Subject: [PATCH] fix multipart put of a blob with content md5 previously the parts will carry the content md5 of the entire object, because unsetting the md5 of a ContentMetadata actually didn't do anything --- .../internal/BaseBlobIntegrationTest.java | 24 ++++++++++++++----- .../jclouds/io/ContentMetadataBuilder.java | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java index 23de2468e9..35f7bcdee7 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java @@ -47,6 +47,7 @@ import java.util.concurrent.atomic.AtomicInteger; import javax.ws.rs.core.MediaType; +import com.google.common.hash.Hashing; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.attr.ConsistencyModel; @@ -59,6 +60,7 @@ import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.domain.StorageType; import org.jclouds.blobstore.options.CopyOptions; import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.blobstore.strategy.internal.MultipartUploadSlicingAlgorithm; import org.jclouds.crypto.Crypto; import org.jclouds.encryption.internal.JCECrypto; import org.jclouds.http.HttpResponseException; @@ -555,7 +557,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { long length = 42; ByteSource byteSource = TestUtils.randomByteSource().slice(0, length); Payload payload = new ByteSourcePayload(byteSource); - testPut(payload, payload, length, new PutOptions()); + testPut(payload, null, payload, length, new PutOptions()); } @Test(groups = { "integration", "live" }) @@ -563,15 +565,22 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { long length = 42; ByteSource byteSource = TestUtils.randomByteSource().slice(0, length); Payload payload = new InputStreamPayload(byteSource.openStream()); - testPut(payload, new ByteSourcePayload(byteSource), length, new PutOptions()); + testPut(payload, null, new ByteSourcePayload(byteSource), length, new PutOptions()); } @Test(groups = { "integration", "live" }) public void testPutMultipartByteSource() throws Exception { - long length = getMinimumMultipartBlobSize(); + long length = 32 * 1024 * 1024 + 1; // MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE + 1 + BlobStore blobStore = view.getBlobStore(); + MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( + blobStore.getMinimumMultipartPartSize(), blobStore.getMaximumMultipartPartSize(), + blobStore.getMaximumNumberOfParts()); + // make sure that we are creating multiple parts + assertThat(algorithm.calculateChunkSize(length)).isLessThan(length); ByteSource byteSource = TestUtils.randomByteSource().slice(0, length); Payload payload = new ByteSourcePayload(byteSource); - testPut(payload, payload, length, new PutOptions().multipart(true)); + HashCode hashCode = byteSource.hash(Hashing.md5()); + testPut(payload, hashCode, payload, length, new PutOptions().multipart(true)); } @Test(groups = { "integration", "live" }) @@ -579,7 +588,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { long length = getMinimumMultipartBlobSize(); ByteSource byteSource = TestUtils.randomByteSource().slice(0, length); Payload payload = new InputStreamPayload(byteSource.openStream()); - testPut(payload, new ByteSourcePayload(byteSource), length, new PutOptions().multipart(true)); + testPut(payload, null, new ByteSourcePayload(byteSource), length, new PutOptions().multipart(true)); } @Test(groups = { "integration", "live" }) @@ -606,7 +615,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { assertThat(userMetadata1).isEqualTo(userMetadata2); } - private void testPut(Payload payload, Payload expectedPayload, long length, PutOptions options) + private void testPut(Payload payload, HashCode hashCode, Payload expectedPayload, long length, PutOptions options) throws IOException, InterruptedException { BlobStore blobStore = view.getBlobStore(); String blobName = "multipart-upload"; @@ -616,6 +625,9 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { .payload(payload) .contentLength(length); addContentMetadata(blobBuilder); + if (hashCode != null) { + blobBuilder.contentMD5(payload.getContentMetadata().getContentMD5AsHashCode()); + } String container = getContainerName(); try { diff --git a/core/src/main/java/org/jclouds/io/ContentMetadataBuilder.java b/core/src/main/java/org/jclouds/io/ContentMetadataBuilder.java index 84cca64663..48d3effb7b 100644 --- a/core/src/main/java/org/jclouds/io/ContentMetadataBuilder.java +++ b/core/src/main/java/org/jclouds/io/ContentMetadataBuilder.java @@ -53,8 +53,8 @@ public class ContentMetadataBuilder { public ContentMetadataBuilder contentMD5(@Nullable HashCode contentMD5) { if (contentMD5 != null) { Preconditions.checkArgument(contentMD5.bits() == 128, "MD5 hash must have 128 bits, was: %s", contentMD5.bits()); - this.contentMD5 = contentMD5; } + this.contentMD5 = contentMD5; return this; }