diff --git a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java index 9c1b483e9e..cda138c24a 100644 --- a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java +++ b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java @@ -19,23 +19,31 @@ package org.jclouds.filesystem.integration; import static org.jclouds.filesystem.util.Utils.isMacOSX; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Properties; -import static java.nio.charset.StandardCharsets.US_ASCII; import static org.assertj.core.api.Assertions.assertThat; +import com.google.common.collect.ImmutableList; +import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; +import com.google.common.io.BaseEncoding; import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobBuilder; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.MultipartPart; +import org.jclouds.blobstore.domain.MultipartUpload; import org.jclouds.blobstore.domain.Tier; +import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest; import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest; import org.jclouds.blobstore.options.PutOptions; import org.jclouds.filesystem.reference.FilesystemConstants; import org.jclouds.filesystem.utils.TestUtils; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; import org.testng.annotations.Test; import org.testng.SkipException; @@ -113,12 +121,12 @@ public class FilesystemBlobIntegrationTest extends BaseBlobIntegrationTest { @Override protected void checkMPUParts(Blob blob, List partsList) { assertThat(blob.getMetadata().getETag()).endsWith(String.format("-%d\"", partsList.size())); - StringBuilder eTags = new StringBuilder(); + Hasher eTagHasher = Hashing.md5().newHasher(); for (MultipartPart part : partsList) { - eTags.append(part.partETag()); + eTagHasher.putBytes(BaseEncoding.base16().lowerCase().decode(part.partETag())); } String expectedETag = new StringBuilder("\"") - .append(Hashing.md5().hashString(eTags.toString(), US_ASCII)) + .append(eTagHasher.hash()) .append("-") .append(partsList.size()) .append("\"") @@ -126,6 +134,36 @@ public class FilesystemBlobIntegrationTest extends BaseBlobIntegrationTest { assertThat(blob.getMetadata().getETag()).isEqualTo(expectedETag); } + @Test(groups = { "integration", "live" }) + public void testMultipartUploadMultiplePartsKnownETag() throws Exception { + BlobStore blobStore = view.getBlobStore(); + String container = getContainerName(); + // Pre-computed ETag returned by AWS S3 for the MPU consisting of two 5MB parts filled with 'b' + String expectedETag = "\"84462a16f6a60478d50148808aa609c1-2\""; + int partSize = 5 * 1024 * 1024; + try { + String name = "blob-name"; + BlobBuilder blobBuilder = blobStore.blobBuilder(name); + Blob blob = blobBuilder.build(); + MultipartUpload mpu = blobStore.initiateMultipartUpload(container, blob.getMetadata(), new PutOptions()); + + byte[] content = new byte[partSize]; + Arrays.fill(content, (byte) 'b'); + Payload payload = Payloads.newByteArrayPayload(content); + + payload.getContentMetadata().setContentLength((long) partSize); + + MultipartPart part1 = blobStore.uploadMultipartPart(mpu, 1, payload); + MultipartPart part2 = blobStore.uploadMultipartPart(mpu, 2, payload); + blobStore.completeMultipartUpload(mpu, ImmutableList.of(part1, part2)); + + BlobMetadata newBlobMetadata = blobStore.blobMetadata(container, name); + assertThat(newBlobMetadata.getETag()).isEqualTo(expectedETag); + } finally { + returnContainer(container); + } + } + protected void checkExtendedAttributesSupport() { if (isMacOSX()) { throw new SkipException("filesystem does not support extended attributes in Mac OSX"); diff --git a/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java index f938b358ba..3717ed4da2 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java @@ -25,7 +25,6 @@ import static com.google.common.collect.Iterables.transform; import static com.google.common.collect.Iterables.tryFind; import static com.google.common.collect.Sets.filter; import static com.google.common.collect.Sets.newTreeSet; -import static java.nio.charset.StandardCharsets.US_ASCII; import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive; import java.io.File; @@ -46,6 +45,7 @@ import javax.annotation.Resource; import javax.inject.Inject; import javax.inject.Singleton; +import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; @@ -101,6 +101,7 @@ import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; +import com.google.common.io.BaseEncoding; import com.google.common.io.ByteSource; import com.google.common.net.HttpHeaders; @@ -826,7 +827,7 @@ public final class LocalBlobStore implements BlobStore { public String completeMultipartUpload(MultipartUpload mpu, List parts) { ImmutableList.Builder streams = ImmutableList.builder(); long contentLength = 0; - StringBuilder partHashes = new StringBuilder(); + Hasher md5Hasher = Hashing.md5().newHasher(); for (MultipartPart part : parts) { Blob blobPart = getBlob(mpu.containerName(), MULTIPART_PREFIX + mpu.id() + "-" + mpu.blobName() + "-" + part.partNumber()); @@ -838,10 +839,10 @@ public final class LocalBlobStore implements BlobStore { throw propagate(ioe); } streams.add(is); - partHashes.append(blobPart.getMetadata().getETag()); + md5Hasher.putBytes(BaseEncoding.base16().lowerCase().decode(blobPart.getMetadata().getETag())); } String mpuETag = new StringBuilder("\"") - .append(Hashing.md5().hashString(partHashes.toString(), US_ASCII).toString()) + .append(md5Hasher.hash()) .append("-") .append(parts.size()) .append("\"")