mirror of https://github.com/apache/jclouds.git
Enforce correct MD5 for local blobstores
Matches behavior of real blobstores.
This commit is contained in:
parent
1d218b1705
commit
f4eca0422d
|
@ -47,6 +47,7 @@ import com.google.common.base.Throwables;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.Iterables;
|
import com.google.common.collect.Iterables;
|
||||||
import com.google.common.collect.Sets;
|
import com.google.common.collect.Sets;
|
||||||
|
import com.google.common.hash.HashCode;
|
||||||
import com.google.common.hash.Hashing;
|
import com.google.common.hash.Hashing;
|
||||||
import com.google.common.hash.HashingInputStream;
|
import com.google.common.hash.HashingInputStream;
|
||||||
import com.google.common.io.ByteSource;
|
import com.google.common.io.ByteSource;
|
||||||
|
@ -201,9 +202,15 @@ public class FilesystemStorageStrategyImpl implements LocalStorageStrategy {
|
||||||
Files.createParentDirs(outputFile);
|
Files.createParentDirs(outputFile);
|
||||||
his = new HashingInputStream(Hashing.md5(), payload.openStream());
|
his = new HashingInputStream(Hashing.md5(), payload.openStream());
|
||||||
Files.asByteSink(outputFile).writeFrom(his);
|
Files.asByteSink(outputFile).writeFrom(his);
|
||||||
payload.getContentMetadata().setContentMD5(his.hash());
|
HashCode actualHashCode = his.hash();
|
||||||
String eTag = base16().lowerCase().encode(payload.getContentMetadata().getContentMD5());
|
HashCode expectedHashCode = payload.getContentMetadata().getContentMD5AsHashCode();
|
||||||
return eTag;
|
if (expectedHashCode != null && !actualHashCode.equals(expectedHashCode)) {
|
||||||
|
throw new IOException("MD5 hash code mismatch, actual: " + actualHashCode +
|
||||||
|
" expected: " + expectedHashCode);
|
||||||
|
}
|
||||||
|
payload.getContentMetadata().setContentMD5(actualHashCode);
|
||||||
|
// TODO: store metadata in extended attributes when moving to Java 7
|
||||||
|
return base16().lowerCase().encode(actualHashCode.asBytes());
|
||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
if (outputFile != null) {
|
if (outputFile != null) {
|
||||||
if (!outputFile.delete()) {
|
if (!outputFile.delete()) {
|
||||||
|
|
|
@ -77,9 +77,4 @@ public class FilesystemBlobIntegrationTest extends BaseBlobIntegrationTest {
|
||||||
public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException {
|
public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException {
|
||||||
throw new SkipException("not yet implemented");
|
throw new SkipException("not yet implemented");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public void testPutIncorrectContentMD5() throws InterruptedException, IOException {
|
|
||||||
throw new SkipException("not yet implemented");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -392,6 +392,11 @@ public class LocalAsyncBlobStore extends BaseAsyncBlobStore {
|
||||||
try {
|
try {
|
||||||
return immediateFuture(storageStrategy.putBlob(containerName, blob));
|
return immediateFuture(storageStrategy.putBlob(containerName, blob));
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
|
if (e.getMessage().startsWith("MD5 hash code mismatch")) {
|
||||||
|
HttpResponseException exception = returnResponseException(400);
|
||||||
|
exception.initCause(e);
|
||||||
|
throw exception;
|
||||||
|
}
|
||||||
logger.error(e, "An error occurred storing the new blob with name [%s] to container [%s].", blobKey,
|
logger.error(e, "An error occurred storing the new blob with name [%s] to container [%s].", blobKey,
|
||||||
containerName);
|
containerName);
|
||||||
throw Throwables.propagate(e);
|
throw Throwables.propagate(e);
|
||||||
|
|
|
@ -39,14 +39,15 @@ import org.jclouds.io.ContentMetadataCodec;
|
||||||
import org.jclouds.io.MutableContentMetadata;
|
import org.jclouds.io.MutableContentMetadata;
|
||||||
import org.jclouds.io.Payload;
|
import org.jclouds.io.Payload;
|
||||||
import org.jclouds.io.Payloads;
|
import org.jclouds.io.Payloads;
|
||||||
import org.jclouds.io.payloads.ByteArrayPayload;
|
import org.jclouds.util.Closeables2;
|
||||||
import org.jclouds.io.payloads.DelegatingPayload;
|
|
||||||
|
|
||||||
import com.google.common.base.Supplier;
|
import com.google.common.base.Supplier;
|
||||||
import com.google.common.base.Throwables;
|
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.Multimaps;
|
import com.google.common.collect.Multimaps;
|
||||||
|
import com.google.common.hash.HashCode;
|
||||||
import com.google.common.hash.Hashing;
|
import com.google.common.hash.Hashing;
|
||||||
|
import com.google.common.hash.HashingInputStream;
|
||||||
|
import com.google.common.io.ByteSource;
|
||||||
import com.google.common.io.ByteStreams;
|
import com.google.common.io.ByteStreams;
|
||||||
import com.google.common.net.HttpHeaders;
|
import com.google.common.net.HttpHeaders;
|
||||||
|
|
||||||
|
@ -123,10 +124,25 @@ public class TransientStorageStrategy implements LocalStorageStrategy {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String putBlob(final String containerName, final Blob blob) throws IOException {
|
public String putBlob(final String containerName, final Blob blob) throws IOException {
|
||||||
Blob newBlob = createUpdatedCopyOfBlobInContainer(containerName, blob);
|
byte[] payload;
|
||||||
|
HashCode actualHashCode;
|
||||||
|
HashingInputStream input = new HashingInputStream(Hashing.md5(), blob.getPayload().openStream());
|
||||||
|
try {
|
||||||
|
payload = ByteStreams.toByteArray(input);
|
||||||
|
actualHashCode = input.hash();
|
||||||
|
HashCode expectedHashCode = blob.getPayload().getContentMetadata().getContentMD5AsHashCode();
|
||||||
|
if (expectedHashCode != null && !actualHashCode.equals(expectedHashCode)) {
|
||||||
|
throw new IOException("MD5 hash code mismatch, actual: " + actualHashCode +
|
||||||
|
" expected: " + expectedHashCode);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
Closeables2.closeQuietly(input);
|
||||||
|
}
|
||||||
|
|
||||||
|
Blob newBlob = createUpdatedCopyOfBlobInContainer(containerName, blob, payload, actualHashCode);
|
||||||
Map<String, Blob> map = containerToBlobs.get(containerName);
|
Map<String, Blob> map = containerToBlobs.get(containerName);
|
||||||
map.put(newBlob.getMetadata().getName(), newBlob);
|
map.put(newBlob.getMetadata().getName(), newBlob);
|
||||||
return base16().lowerCase().encode(newBlob.getPayload().getContentMetadata().getContentMD5());
|
return base16().lowerCase().encode(actualHashCode.asBytes());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -146,37 +162,22 @@ public class TransientStorageStrategy implements LocalStorageStrategy {
|
||||||
return "/";
|
return "/";
|
||||||
}
|
}
|
||||||
|
|
||||||
private Blob createUpdatedCopyOfBlobInContainer(String containerName, Blob in) {
|
private Blob createUpdatedCopyOfBlobInContainer(String containerName, Blob in, byte[] input, HashCode contentMd5) {
|
||||||
|
checkNotNull(containerName, "containerName");
|
||||||
checkNotNull(in, "blob");
|
checkNotNull(in, "blob");
|
||||||
checkNotNull(in.getPayload(), "blob.payload");
|
checkNotNull(input, "input");
|
||||||
ByteArrayPayload payload = (in.getPayload() instanceof ByteArrayPayload) ? ByteArrayPayload.class.cast(in
|
checkNotNull(contentMd5, "contentMd5");
|
||||||
.getPayload()) : null;
|
Payload payload = Payloads.newByteSourcePayload(ByteSource.wrap(input));
|
||||||
if (payload == null)
|
|
||||||
payload = (in.getPayload() instanceof DelegatingPayload) ? (DelegatingPayload.class.cast(in.getPayload())
|
|
||||||
.getDelegate() instanceof ByteArrayPayload) ? ByteArrayPayload.class.cast(DelegatingPayload.class
|
|
||||||
.cast(in.getPayload()).getDelegate()) : null : null;
|
|
||||||
try {
|
|
||||||
if (payload == null || !(payload instanceof ByteArrayPayload)) {
|
|
||||||
MutableContentMetadata oldMd = in.getPayload().getContentMetadata();
|
MutableContentMetadata oldMd = in.getPayload().getContentMetadata();
|
||||||
byte[] out = ByteStreams.toByteArray(in.getPayload());
|
|
||||||
payload = Payloads.newByteArrayPayload(out);
|
|
||||||
HttpUtils.copy(oldMd, payload.getContentMetadata());
|
HttpUtils.copy(oldMd, payload.getContentMetadata());
|
||||||
payload.getContentMetadata().setContentMD5(Hashing.md5().hashBytes(out));
|
payload.getContentMetadata().setContentMD5(contentMd5);
|
||||||
} else {
|
|
||||||
if (payload.getContentMetadata().getContentMD5() == null) {
|
|
||||||
payload.getContentMetadata().setContentMD5(ByteStreams.hash(payload, Hashing.md5()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (IOException e) {
|
|
||||||
Throwables.propagate(e);
|
|
||||||
}
|
|
||||||
Blob blob = blobFactory.create(BlobStoreUtils.copy(in.getMetadata()));
|
Blob blob = blobFactory.create(BlobStoreUtils.copy(in.getMetadata()));
|
||||||
blob.setPayload(payload);
|
blob.setPayload(payload);
|
||||||
blob.getMetadata().setContainer(containerName);
|
blob.getMetadata().setContainer(containerName);
|
||||||
blob.getMetadata().setUri(
|
blob.getMetadata().setUri(
|
||||||
uriBuilder(new StringBuilder("mem://").append(containerName)).path(in.getMetadata().getName()).build());
|
uriBuilder(new StringBuilder("mem://").append(containerName)).path(in.getMetadata().getName()).build());
|
||||||
blob.getMetadata().setLastModified(new Date());
|
blob.getMetadata().setLastModified(new Date());
|
||||||
String eTag = base16().lowerCase().encode(payload.getContentMetadata().getContentMD5());
|
String eTag = base16().lowerCase().encode(contentMd5.asBytes());
|
||||||
blob.getMetadata().setETag(eTag);
|
blob.getMetadata().setETag(eTag);
|
||||||
// Set HTTP headers to match metadata
|
// Set HTTP headers to match metadata
|
||||||
blob.getAllHeaders().replaceValues(HttpHeaders.LAST_MODIFIED,
|
blob.getAllHeaders().replaceValues(HttpHeaders.LAST_MODIFIED,
|
||||||
|
|
|
@ -27,9 +27,4 @@ public class TransientBlobIntegrationTest extends BaseBlobIntegrationTest {
|
||||||
public TransientBlobIntegrationTest() {
|
public TransientBlobIntegrationTest() {
|
||||||
provider = "transient";
|
provider = "transient";
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public void testPutIncorrectContentMD5() throws InterruptedException, IOException {
|
|
||||||
throw new SkipException("not yet implemented");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue