Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhanced multi-part test to look at md5

This commit is contained in:
Adrian Cole 2011-02-26 21:56:31 -08:00
parent 01ef7140db
commit bedebd53dc
3 changed files with 62 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequest.java
View File

@ -21,6 +21,7 @@ package org.jclouds.aws.s3.binders;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.crypto.CryptoStreams.base64;
import javax.inject.Inject;
import javax.inject.Singleton;
@ -77,6 +78,10 @@ public class BindObjectMetadataToRequest implements Binder {
headers.put(HttpHeaders.CONTENT_TYPE, "binary/octet-stream");
}
if (md.getContentMetadata().getContentMD5() != null) {
headers.put("Content-MD5", base64(md.getContentMetadata().getContentMD5()));
}
request = ModifyRequest.replaceHeaders(request, headers.build());
return request;
}

18
providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java
View File

@ -79,11 +79,23 @@ public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest<AWSS3
NoSuchMethodException {
Method method = AWSS3AsyncClient.class.getMethod("initiateMultipartUpload", String.class, ObjectMetadata.class,
PutObjectOptions[].class);
HttpRequest request = processor
.createRequest(method, "bucket", ObjectMetadataBuilder.create().key("foo").build());
HttpRequest request = processor.createRequest(method, "bucket", ObjectMetadataBuilder.create().key("foo")
.contentMD5(new byte[] { 1, 2, 3, 4 }).build());
assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1");
assertNonPayloadHeadersEqual(request, "Content-Type: binary/octet-stream\nHost: bucket." + url + "\n");
assertNonPayloadHeadersEqual(request, "Content-MD5: AQIDBA==\nContent-Type: binary/octet-stream\nHost: bucket."
+ url + "\n");
assertPayloadEquals(request, null, null, false);
// as this is a payload-related command, but with no payload, be careful that we check
// filtering and do not ignore if this fails later.
request = request.getFilters().get(0).filter(request);
assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1");
assertNonPayloadHeadersEqual(request,
"Authorization: AWS identity:Sp1FX4svL9P2u2bFJwroaYpSANo=\nContent-MD5: AQIDBA==\n"
+ "Content-Type: binary/octet-stream\nDate: 2009-11-08T15:54:08.897Z\nHost: bucket." + url
+ "\n");
assertPayloadEquals(request, null, null, false);
assertResponseParserClassEquals(method, request, UploadIdFromHttpResponseViaRegex.class);

60
providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java
View File

@ -19,27 +19,30 @@
package org.jclouds.aws.s3;
import static org.testng.Assert.assertTrue;
import static com.google.common.io.ByteStreams.join;
import static com.google.common.io.ByteStreams.newInputStreamSupplier;
import static com.google.common.io.ByteStreams.toByteArray;
import static org.jclouds.crypto.CryptoStreams.md5;
import static org.jclouds.io.Payloads.newByteArrayPayload;
import static org.testng.Assert.assertEquals;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPInputStream;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.blobstore.KeyNotFoundException;
import org.jclouds.http.BaseJettyTest;
import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.s3.S3ClientLiveTest;
import org.jclouds.s3.domain.ObjectMetadataBuilder;
import org.jclouds.s3.domain.S3Object;
import org.testng.ITestContext;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.ByteStreams;
import com.google.common.io.InputSupplier;
import com.google.inject.Module;
@ -58,7 +61,7 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
public AWSS3Client getApi() {
return (AWSS3Client) context.getProviderSpecificContext().getApi();
}
@Override
protected Module createHttpModule() {
// in order to be able to debug the wire protocol I use ApacheHC...
@ -70,44 +73,65 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext);
oneHundredOneConstitutions = getTestDataSupplier();
oneHundredOneConstitutionsMD5 = CryptoStreams.md5(oneHundredOneConstitutions);
oneHundredOneConstitutionsMD5 = md5(oneHundredOneConstitutions);
}
@SuppressWarnings("unchecked")
public static InputSupplier<InputStream> getTestDataSupplier() throws IOException {
byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class
.getResourceAsStream("/const.txt.gz")));
InputSupplier<ByteArrayInputStream> constitutionSupplier = ByteStreams.newInputStreamSupplier(oneConstitution);
byte[] oneConstitution = toByteArray(new GZIPInputStream(BaseJettyTest.class.getResourceAsStream("/const.txt.gz")));
InputSupplier<ByteArrayInputStream> constitutionSupplier = newInputStreamSupplier(oneConstitution);
InputSupplier<InputStream> temp = ByteStreams.join(constitutionSupplier);
InputSupplier<InputStream> temp = join(constitutionSupplier);
// we have to go beyond 5MB per part
for (oneHundredOneConstitutionsLength = oneConstitution.length; oneHundredOneConstitutionsLength < 5 * 1024 * 1024; oneHundredOneConstitutionsLength += oneConstitution.length) {
temp = ByteStreams.join(temp, constitutionSupplier);
temp = join(temp, constitutionSupplier);
}
return temp;
}
public void testMultipartSynchronously() throws InterruptedException, IOException {
String containerName = getContainerName();
S3Object object = null;
try {
String key = "constitution.txt";
String uploadId = getApi().initiateMultipartUpload(containerName,
ObjectMetadataBuilder.create().key(key).build());
ObjectMetadataBuilder.create().key(key).contentMD5(oneHundredOneConstitutionsMD5).build());
byte[] buffer = toByteArray(oneHundredOneConstitutions.getInput());
assertEquals(oneHundredOneConstitutionsLength, (long) buffer.length);
byte[] buffer = ByteStreams.toByteArray(oneHundredOneConstitutions.getInput());
Payload part1 = Payloads.newByteArrayPayload(buffer);
Payload part1 = newByteArrayPayload(buffer);
part1.getContentMetadata().setContentLength((long) buffer.length);
part1.getContentMetadata().setContentMD5(oneHundredOneConstitutionsMD5);
String eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
String eTagOf1 = null;
try {
eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
} catch (KeyNotFoundException e) {
// note that because of eventual consistency, the upload id may not be present yet
// we may wish to add this condition to the retry handler
// we may also choose to implement ListParts and wait for the uploadId to become
// available there.
eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
}
String eTag = getApi().completeMultipartUpload(containerName, key, uploadId, ImmutableMap.of(1, eTagOf1));
assertTrue(true);
assert !eTagOf1.equals(eTag);
object = getApi().getObject(containerName, key);
assertEquals(toByteArray(object.getPayload()), buffer);
// noticing amazon does not return content-md5 header or a parsable ETag after a multi-part
// upload is complete:
// https://forums.aws.amazon.com/thread.jspa?threadID=61344
assertEquals(object.getPayload().getContentMetadata().getContentMD5(), null);
assertEquals(getApi().headObject(containerName, key).getContentMetadata().getContentMD5(), null);
} finally {
if (object != null)
object.getPayload().close();
returnContainer(containerName);
}
}
}