Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Filesystem: Fix the MPU ETags to match S3. 

Prior commit introduced a bug in the computation of the MPU ETag value,
where it was concatenating strings, rather than operating on the bytes
of the integer value.
This commit is contained in:
Timur Alperovich 2018-10-24 18:34:39 -07:00 committed by Andrew Gaul
parent 22ce5484a4
commit 896e99df09
2 changed files with 47 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java
View File

@ -19,23 +19,31 @@ package org.jclouds.filesystem.integration;
import static org.jclouds.filesystem.util.Utils.isMacOSX;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static org.assertj.core.api.Assertions.assertThat;
import com.google.common.collect.ImmutableList;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobBuilder;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.MultipartPart;
import org.jclouds.blobstore.domain.MultipartUpload;
import org.jclouds.blobstore.domain.Tier;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.filesystem.reference.FilesystemConstants;
import org.jclouds.filesystem.utils.TestUtils;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.testng.annotations.Test;
import org.testng.SkipException;
@ -113,12 +121,12 @@ public class FilesystemBlobIntegrationTest extends BaseBlobIntegrationTest {
@Override
protected void checkMPUParts(Blob blob, List<MultipartPart> partsList) {
assertThat(blob.getMetadata().getETag()).endsWith(String.format("-%d\"", partsList.size()));
StringBuilder eTags = new StringBuilder();
Hasher eTagHasher = Hashing.md5().newHasher();
for (MultipartPart part : partsList) {
eTags.append(part.partETag());
eTagHasher.putBytes(BaseEncoding.base16().lowerCase().decode(part.partETag()));
}
String expectedETag = new StringBuilder("\"")
.append(Hashing.md5().hashString(eTags.toString(), US_ASCII))
.append(eTagHasher.hash())
.append("-")
.append(partsList.size())
.append("\"")
@ -126,6 +134,36 @@ public class FilesystemBlobIntegrationTest extends BaseBlobIntegrationTest {
assertThat(blob.getMetadata().getETag()).isEqualTo(expectedETag);
}
@Test(groups = { "integration", "live" })
public void testMultipartUploadMultiplePartsKnownETag() throws Exception {
BlobStore blobStore = view.getBlobStore();
String container = getContainerName();
// Pre-computed ETag returned by AWS S3 for the MPU consisting of two 5MB parts filled with 'b'
String expectedETag = "\"84462a16f6a60478d50148808aa609c1-2\"";
int partSize = 5 * 1024 * 1024;
try {
String name = "blob-name";
BlobBuilder blobBuilder = blobStore.blobBuilder(name);
Blob blob = blobBuilder.build();
MultipartUpload mpu = blobStore.initiateMultipartUpload(container, blob.getMetadata(), new PutOptions());
byte[] content = new byte[partSize];
Arrays.fill(content, (byte) 'b');
Payload payload = Payloads.newByteArrayPayload(content);
payload.getContentMetadata().setContentLength((long) partSize);
MultipartPart part1 = blobStore.uploadMultipartPart(mpu, 1, payload);
MultipartPart part2 = blobStore.uploadMultipartPart(mpu, 2, payload);
blobStore.completeMultipartUpload(mpu, ImmutableList.of(part1, part2));
BlobMetadata newBlobMetadata = blobStore.blobMetadata(container, name);
assertThat(newBlobMetadata.getETag()).isEqualTo(expectedETag);
} finally {
returnContainer(container);
}
}
protected void checkExtendedAttributesSupport() {
if (isMacOSX()) {
throw new SkipException("filesystem does not support extended attributes in Mac OSX");

9
blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java
View File

@ -25,7 +25,6 @@ import static com.google.common.collect.Iterables.transform;
import static com.google.common.collect.Iterables.tryFind;
import static com.google.common.collect.Sets.filter;
import static com.google.common.collect.Sets.newTreeSet;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive;
import java.io.File;
@ -46,6 +45,7 @@ import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
@ -101,6 +101,7 @@ import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteSource;
import com.google.common.net.HttpHeaders;
@ -826,7 +827,7 @@ public final class LocalBlobStore implements BlobStore {
public String completeMultipartUpload(MultipartUpload mpu, List<MultipartPart> parts) {
ImmutableList.Builder<InputStream> streams = ImmutableList.builder();
long contentLength = 0;
StringBuilder partHashes = new StringBuilder();
Hasher md5Hasher = Hashing.md5().newHasher();
for (MultipartPart part : parts) {
Blob blobPart = getBlob(mpu.containerName(), MULTIPART_PREFIX + mpu.id() + "-" + mpu.blobName() + "-" + part.partNumber());
@ -838,10 +839,10 @@ public final class LocalBlobStore implements BlobStore {
throw propagate(ioe);
}
streams.add(is);
partHashes.append(blobPart.getMetadata().getETag());
md5Hasher.putBytes(BaseEncoding.base16().lowerCase().decode(blobPart.getMetadata().getETag()));
}
String mpuETag = new StringBuilder("\"")
.append(Hashing.md5().hashString(partHashes.toString(), US_ASCII).toString())
.append(md5Hasher.hash())
.append("-")
.append(parts.size())
.append("\"")