refactored aws-s3 MPU test

This commit is contained in:
adriancole 2013-04-08 18:03:37 -07:00
parent 33745cdeb1
commit 413538c453
7 changed files with 212 additions and 262 deletions

View File

@ -111,6 +111,11 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.mockwebserver</groupId>
<artifactId>mockwebserver</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>

View File

@ -19,7 +19,6 @@
package org.jclouds.aws.s3;
import java.util.Map;
import java.util.Set;
import org.jclouds.aws.s3.domain.DeleteResult;
import org.jclouds.io.Payload;
import org.jclouds.s3.S3Client;

View File

@ -88,7 +88,7 @@ public class AWSS3BlobStore extends S3BlobStore {
public String putBlob(String container, Blob blob, PutOptions options) {
if (options.isMultipart()) {
// need to use a provider if the strategy object is stateful
return multipartUploadStrategy.get().execute(container, blob, options);
return multipartUploadStrategy.get().execute(container, blob);
} else if ((options instanceof AWSS3PutOptions) &&
(((AWSS3PutOptions) options).getStorageClass() == REDUCED_REDUNDANCY)) {

View File

@ -20,7 +20,6 @@ package org.jclouds.aws.s3.blobstore.strategy;
import org.jclouds.aws.s3.blobstore.strategy.internal.SequentialMultipartUploadStrategy;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.options.PutOptions;
import com.google.inject.ImplementedBy;
@ -32,5 +31,5 @@ import com.google.inject.ImplementedBy;
@ImplementedBy(SequentialMultipartUploadStrategy.class)
public interface MultipartUploadStrategy extends MultipartUpload {
String execute(String container, Blob blob, PutOptions options);
String execute(String container, Blob blob);
}

View File

@ -25,19 +25,16 @@ import java.util.SortedMap;
import javax.annotation.Resource;
import javax.inject.Named;
import org.jclouds.aws.s3.AWSS3ApiMetadata;
import org.jclouds.aws.s3.AWSS3Client;
import org.jclouds.aws.s3.blobstore.AWSS3BlobStore;
import org.jclouds.aws.s3.blobstore.strategy.MultipartUploadStrategy;
import org.jclouds.blobstore.KeyNotFoundException;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.blobstore.reference.BlobStoreConstants;
import org.jclouds.io.Payload;
import org.jclouds.io.PayloadSlicer;
import org.jclouds.logging.Logger;
import org.jclouds.s3.blobstore.functions.BlobToObject;
import org.jclouds.s3.domain.ObjectMetadataBuilder;
import org.jclouds.util.Throwables2;
import com.google.common.collect.Maps;
import com.google.inject.Inject;
@ -47,86 +44,83 @@ import com.google.inject.Inject;
*
* The file partitioning algorithm:
*
* The default partition size we choose is 32mb. A multiple of this default partition size is used.
* The number of parts first grows to a chosen magnitude (for example 100 parts), then it grows the
* partition size instead of number of partitions. When we reached the maximum part size, then again
* it starts to grow the number of partitions.
* The default partition size we choose is 32mb. A multiple of this default
* partition size is used. The number of partCount first grows to a chosen magnitude
* (for example 100 partCount), then it grows the partition size instead of number
* of partitions. When we reached the maximum part size, then again it starts to
* grow the number of partitions.
*
* @author Tibor Kiss
* @author Tibor Kiss, Adrian Cole
*/
public class SequentialMultipartUploadStrategy implements MultipartUploadStrategy {
@Resource
@Named(BlobStoreConstants.BLOBSTORE_LOGGER)
protected Logger logger = Logger.NULL;
private Logger logger = Logger.NULL;
protected final AWSS3BlobStore ablobstore;
protected final PayloadSlicer slicer;
private final AWSS3Client client;
private final BlobToObject blobToObject;
private final MultipartUploadSlicingAlgorithm algorithm;
private final PayloadSlicer slicer;
@Inject
public SequentialMultipartUploadStrategy(AWSS3BlobStore ablobstore, PayloadSlicer slicer) {
this.ablobstore = checkNotNull(ablobstore, "ablobstore");
public SequentialMultipartUploadStrategy(AWSS3Client client, BlobToObject blobToObject,
MultipartUploadSlicingAlgorithm algorithm, PayloadSlicer slicer) {
this.client = checkNotNull(client, "client");
this.blobToObject = checkNotNull(blobToObject, "blobToObject");
this.algorithm = checkNotNull(algorithm, "algorithm");
this.slicer = checkNotNull(slicer, "slicer");
}
protected void prepareUploadPart(String container, String key, String uploadId, int part,
Payload payload, long offset, long size, SortedMap<Integer, String> etags) {
AWSS3Client client = ablobstore.getContext().unwrap(AWSS3ApiMetadata.CONTEXT_TOKEN).getApi();
@Override
public String execute(String container, Blob blob) {
String key = blob.getMetadata().getName();
Payload payload = blob.getPayload();
Long length = payload.getContentMetadata().getContentLength();
checkNotNull(length,
"please invoke payload.getContentMetadata().setContentLength(length) prior to multipart upload");
long chunkSize = algorithm.calculateChunkSize(length);
int partCount = algorithm.getParts();
if (partCount > 0) {
String uploadId = client.initiateMultipartUpload(container, ObjectMetadataBuilder.create().key(key).build());
try {
SortedMap<Integer, String> etags = Maps.newTreeMap();
int part;
while ((part = algorithm.getNextPart()) <= partCount) {
prepareUploadPart(container, key, uploadId, part, payload, algorithm.getNextChunkOffset(), chunkSize,
etags);
}
long remaining = algorithm.getRemaining();
if (remaining > 0) {
prepareUploadPart(container, key, uploadId, part, payload, algorithm.getNextChunkOffset(), remaining,
etags);
}
return client.completeMultipartUpload(container, key, uploadId, etags);
} catch (RuntimeException ex) {
client.abortMultipartUpload(container, key, uploadId);
throw ex;
}
} else {
// TODO: find a way to disable multipart. if we pass the original
// options, it goes into a stack overflow
return client.putObject(container, blobToObject.apply(blob));
}
}
private void prepareUploadPart(String container, String key, String uploadId, int part, Payload payload,
long offset, long size, SortedMap<Integer, String> etags) {
Payload chunkedPart = slicer.slice(payload, offset, size);
String eTag = null;
try {
eTag = client.uploadPart(container, key, part, uploadId, chunkedPart);
etags.put(Integer.valueOf(part), eTag);
} catch (KeyNotFoundException e) {
// note that because of eventual consistency, the upload id may not be present yet
// we may wish to add this condition to the retry handler
// note that because of eventual consistency, the upload id may not be
// present yet we may wish to add this condition to the retry handler
// we may also choose to implement ListParts and wait for the uploadId to become
// available there.
// we may also choose to implement ListParts and wait for the uploadId
// to become available there.
eTag = client.uploadPart(container, key, part, uploadId, chunkedPart);
etags.put(Integer.valueOf(part), eTag);
}
}
@Override
public String execute(String container, Blob blob, PutOptions options) {
String key = blob.getMetadata().getName();
Payload payload = blob.getPayload();
MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm();
algorithm
.calculateChunkSize(checkNotNull(
payload.getContentMetadata().getContentLength(),
"contentLength required on all uploads to amazon s3; please invoke payload.getContentMetadata().setContentLength(length) first"));
int parts = algorithm.getParts();
long chunkSize = algorithm.getChunkSize();
if (parts > 0) {
AWSS3Client client = ablobstore.getContext()
.unwrap(AWSS3ApiMetadata.CONTEXT_TOKEN).getApi();
String uploadId = client.initiateMultipartUpload(container,
ObjectMetadataBuilder.create().key(key).build()); // TODO md5
try {
SortedMap<Integer, String> etags = Maps.newTreeMap();
int part;
while ((part = algorithm.getNextPart()) <= parts) {
prepareUploadPart(container, key, uploadId, part, payload,
algorithm.getNextChunkOffset(), chunkSize, etags);
}
long remaining = algorithm.getRemaining();
if (remaining > 0) {
prepareUploadPart(container, key, uploadId, part, payload,
algorithm.getNextChunkOffset(), remaining, etags);
}
return client.completeMultipartUpload(container, key, uploadId, etags);
} catch (Exception ex) {
RuntimeException rtex = Throwables2.getFirstThrowableOfType(ex, RuntimeException.class);
if (rtex == null) {
rtex = new RuntimeException(ex);
}
client.abortMultipartUpload(container, key, uploadId);
throw rtex;
}
} else {
// TODO: find a way to disable multipart. if we pass the original options, it goes into a stack overflow
return ablobstore.putBlob(container, blob, PutOptions.NONE);
}
}
}

View File

@ -0,0 +1,146 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.aws.s3.blobstore.strategy.internal;
import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor;
import static org.jclouds.Constants.PROPERTY_MAX_RETRIES;
import static org.jclouds.Constants.PROPERTY_SO_TIMEOUT;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.util.Properties;
import java.util.Set;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.domain.internal.BlobBuilderImpl;
import org.jclouds.concurrent.config.ExecutorServiceModule;
import org.jclouds.http.HttpResponseException;
import org.testng.annotations.Test;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Module;
import com.google.mockwebserver.MockResponse;
import com.google.mockwebserver.MockWebServer;
import com.google.mockwebserver.RecordedRequest;
/**
*
* @author Adrian Cole
*/
@Test(singleThreaded = true)
public class SequentialMultipartUploadStrategyMockTest {
@Test
public void testMPUDoesMultipart() throws IOException, InterruptedException {
MockWebServer server = new MockWebServer();
server.enqueue(new MockResponse().setResponseCode(200).setBody("<UploadId>upload-id</UploadId>"));
server.enqueue(new MockResponse().setResponseCode(200).addHeader("ETag", "a00"));
server.enqueue(new MockResponse().setResponseCode(200).addHeader("ETag", "b00"));
server.enqueue(new MockResponse().setResponseCode(200).setBody("<ETag>fff</ETag>"));
server.play();
byte[] bytes = "0123456789abcdef".getBytes(Charsets.US_ASCII);
int partSize = bytes.length / 2;
SequentialMultipartUploadStrategy api = mockSequentialMultipartUploadStrategy(server.getUrl("/").toString(),
partSize);
try {
assertEquals(api.execute("container", new BlobBuilderImpl().name("foo").payload(bytes).build()), "fff");
} finally {
RecordedRequest initiate = server.takeRequest();
assertEquals(initiate.getRequestLine(), "POST /container/foo?uploads HTTP/1.1");
assertEquals(initiate.getHeader("Content-Length"), "0");
RecordedRequest part1 = server.takeRequest();
assertEquals(part1.getRequestLine(), "PUT /container/foo?partNumber=1&uploadId=upload-id HTTP/1.1");
assertEquals(part1.getHeader("Content-Length"), String.valueOf(partSize));
assertEquals(new String(part1.getBody()), "01234567");
RecordedRequest part2 = server.takeRequest();
assertEquals(part2.getRequestLine(), "PUT /container/foo?partNumber=2&uploadId=upload-id HTTP/1.1");
assertEquals(part2.getHeader("Content-Length"), String.valueOf(partSize));
assertEquals(new String(part2.getBody()), "89abcdef");
RecordedRequest manifest = server.takeRequest();
assertEquals(manifest.getRequestLine(), "POST /container/foo?uploadId=upload-id HTTP/1.1");
assertEquals(manifest.getHeader("Content-Length"), "161");
assertEquals(
new String(manifest.getBody()),
"<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>a00</ETag></Part><Part><PartNumber>2</PartNumber><ETag>b00</ETag></Part></CompleteMultipartUpload>");
server.shutdown();
}
}
@Test(expectedExceptions = HttpResponseException.class)
public void testMPUAbortsOnProblem() throws IOException, InterruptedException {
MockWebServer server = new MockWebServer();
server.enqueue(new MockResponse().setResponseCode(200).setBody("<UploadId>upload-id</UploadId>"));
server.enqueue(new MockResponse().setResponseCode(400));
server.enqueue(new MockResponse().setResponseCode(200));
server.play();
byte[] bytes = "0123456789abcdef".getBytes(Charsets.US_ASCII);
int partSize = bytes.length / 2;
SequentialMultipartUploadStrategy api = mockSequentialMultipartUploadStrategy(server.getUrl("/").toString(),
partSize);
try {
assertEquals(api.execute("container", new BlobBuilderImpl().name("foo").payload(bytes).build()), "fff");
} finally {
RecordedRequest initiate = server.takeRequest();
assertEquals(initiate.getRequestLine(), "POST /container/foo?uploads HTTP/1.1");
assertEquals(initiate.getHeader("Content-Length"), "0");
RecordedRequest part1 = server.takeRequest();
assertEquals(part1.getRequestLine(), "PUT /container/foo?partNumber=1&uploadId=upload-id HTTP/1.1");
assertEquals(part1.getHeader("Content-Length"), String.valueOf(partSize));
assertEquals(new String(part1.getBody()), "01234567");
RecordedRequest abort = server.takeRequest();
assertEquals(abort.getRequestLine(), "DELETE /container/foo?uploadId=upload-id HTTP/1.1");
server.shutdown();
}
}
private static final Set<Module> modules = ImmutableSet.<Module> of(new ExecutorServiceModule(sameThreadExecutor(),
sameThreadExecutor()));
static SequentialMultipartUploadStrategy mockSequentialMultipartUploadStrategy(String uri, int partSize) {
Properties overrides = new Properties();
overrides.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "false");
// prevent expect-100 bug http://code.google.com/p/mockwebserver/issues/detail?id=6
overrides.setProperty(PROPERTY_SO_TIMEOUT, "0");
overrides.setProperty(PROPERTY_MAX_RETRIES, "1");
overrides.setProperty("jclouds.mpu.parts.size", String.valueOf(partSize));
return ContextBuilder.newBuilder("aws-s3")
.credentials("accessKey", "secretKey")
.endpoint(uri)
.overrides(overrides)
.modules(modules)
.buildInjector().getInstance(SequentialMultipartUploadStrategy.class);
}
}

View File

@ -1,193 +0,0 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.aws.s3.blobstore.strategy.internal;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import static org.testng.Assert.fail;
import java.util.SortedMap;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.AWSS3ApiMetadata;
import org.jclouds.aws.s3.AWSS3AsyncClient;
import org.jclouds.aws.s3.AWSS3Client;
import org.jclouds.aws.s3.blobstore.AWSS3BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.MutableBlobMetadata;
import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.io.MutableContentMetadata;
import org.jclouds.io.Payload;
import org.jclouds.io.PayloadSlicer;
import org.jclouds.rest.RestContext;
import org.jclouds.rest.internal.RestContextImpl;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.ObjectMetadataBuilder;
import org.jclouds.util.Throwables2;
import org.testng.annotations.Test;
import com.google.common.collect.Maps;
/**
* Tests behavior of {@code SequentialMultipartUploadStrategy}
*
* @author Tibor Kiss
*/
@Test(groups = "unit")
public class SequentialMultipartUploadStrategyTest {
@SuppressWarnings("unchecked")
@Test
public void testWithTwoParts() {
AWSS3BlobStore ablobStore = createMock(AWSS3BlobStore.class);
PayloadSlicer slicer = createMock(PayloadSlicer.class);
String container = "container";
String key = "mpu-test";
Blob blob = createMock(Blob.class);
MutableBlobMetadata blobMeta = createMock(MutableBlobMetadata.class);
Payload payload = createMock(Payload.class);
MutableContentMetadata contentMeta = createMock(MutableContentMetadata.class);
BlobStoreContext context = createMock(BlobStoreContext.class);
RestContext<AWSS3Client, AWSS3AsyncClient> psc = createMock(RestContextImpl.class);
AWSS3Client client = createMock(AWSS3Client.class);
ObjectMetadata ometa = createMock(ObjectMetadata.class);
String uploadId = "uploadId";
long chunkSize = MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE;
long remaining = 100L;
SortedMap<Integer, String> etags = Maps.newTreeMap();
etags.put(Integer.valueOf(1), "eTag1");
etags.put(Integer.valueOf(2), "eTag2");
expect(blob.getMetadata()).andReturn(blobMeta).atLeastOnce();
expect(blobMeta.getName()).andReturn(key).atLeastOnce();
expect(blob.getPayload()).andReturn(payload).atLeastOnce();
expect(payload.getContentMetadata()).andReturn(contentMeta).atLeastOnce();
expect(contentMeta.getContentLength()).andReturn(Long.valueOf(chunkSize + remaining));
expect(ablobStore.getContext()).andReturn(context).atLeastOnce();
expect(context.unwrap(AWSS3ApiMetadata.CONTEXT_TOKEN)).andReturn(psc).atLeastOnce();
expect(psc.getApi()).andReturn(client).atLeastOnce();
expect(client.initiateMultipartUpload(container, new ObjectMetadataBuilder().key(key).build())).andReturn("uploadId").atLeastOnce();
expect(slicer.slice(payload, 0, chunkSize)).andReturn(payload).atLeastOnce();
expect(client.uploadPart(container, key, 1, uploadId, payload)).andReturn("eTag1").atLeastOnce();
expect(slicer.slice(payload, chunkSize, remaining)).andReturn(payload).atLeastOnce();
expect(client.uploadPart(container, key, 2, uploadId, payload)).andReturn("eTag2").atLeastOnce();
expect(client.completeMultipartUpload(container, key, uploadId, etags)).andReturn("eTag").atLeastOnce();
replay(ablobStore);
replay(slicer);
replay(blob);
replay(blobMeta);
replay(payload);
replay(contentMeta);
replay(context);
replay(psc);
replay(client);
replay(ometa);
SequentialMultipartUploadStrategy strategy = new SequentialMultipartUploadStrategy(ablobStore, slicer);
strategy.execute(container, blob, PutOptions.NONE);
verify(ablobStore);
verify(slicer);
verify(blob);
verify(blobMeta);
verify(payload);
verify(contentMeta);
verify(context);
verify(psc);
verify(client);
verify(ometa);
}
@SuppressWarnings("unchecked")
@Test
public void testWithTimeout() {
AWSS3BlobStore ablobStore = createMock(AWSS3BlobStore.class);
PayloadSlicer slicer = createMock(PayloadSlicer.class);
String container = "container";
String key = "mpu-test";
Blob blob = createMock(Blob.class);
MutableBlobMetadata blobMeta = createMock(MutableBlobMetadata.class);
Payload payload = createMock(Payload.class);
MutableContentMetadata contentMeta = createMock(MutableContentMetadata.class);
BlobStoreContext context = createMock(BlobStoreContext.class);
RestContext<AWSS3Client, AWSS3AsyncClient> psc = createMock(RestContextImpl.class);
AWSS3Client client = createMock(AWSS3Client.class);
ObjectMetadata ometa = createMock(ObjectMetadata.class);
String uploadId = "uploadId";
long chunkSize = MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE;
long remaining = 100L;
SortedMap<Integer, String> etags = Maps.newTreeMap();
etags.put(Integer.valueOf(1), "eTag1");
etags.put(Integer.valueOf(2), "eTag2");
expect(blob.getMetadata()).andReturn(blobMeta).atLeastOnce();
expect(blobMeta.getName()).andReturn(key).atLeastOnce();
expect(blob.getPayload()).andReturn(payload).atLeastOnce();
expect(payload.getContentMetadata()).andReturn(contentMeta).atLeastOnce();
expect(contentMeta.getContentLength()).andReturn(Long.valueOf(chunkSize + remaining));
expect(ablobStore.getContext()).andReturn(context).atLeastOnce();
expect(context.unwrap(AWSS3ApiMetadata.CONTEXT_TOKEN)).andReturn(psc).atLeastOnce();
expect(psc.getApi()).andReturn(client).atLeastOnce();
expect(client.initiateMultipartUpload(container, new ObjectMetadataBuilder().key(key).build())).andReturn("uploadId").atLeastOnce();
expect(slicer.slice(payload, 0, chunkSize)).andReturn(payload).atLeastOnce();
expect(client.uploadPart(container, key, 1, uploadId, payload)).andReturn("eTag1").atLeastOnce();
expect(slicer.slice(payload, chunkSize, remaining)).andReturn(payload).atLeastOnce();
expect(client.uploadPart(container, key, 2, uploadId, payload)).andThrow(new RuntimeException(new TimeoutException()));
client.abortMultipartUpload(container, key, uploadId);
expectLastCall().atLeastOnce();
replay(ablobStore);
replay(slicer);
replay(blob);
replay(blobMeta);
replay(payload);
replay(contentMeta);
replay(context);
replay(psc);
replay(client);
replay(ometa);
SequentialMultipartUploadStrategy strategy = new SequentialMultipartUploadStrategy(ablobStore, slicer);
try {
strategy.execute(container, blob, PutOptions.NONE);
fail("Should throw RuntimeException with TimeoutException cause!");
} catch (RuntimeException rtex) {
TimeoutException timeout = Throwables2.getFirstThrowableOfType(rtex, TimeoutException.class);
if (timeout == null) {
throw rtex;
}
}
verify(ablobStore);
verify(slicer);
verify(blob);
verify(blobMeta);
verify(payload);
verify(contentMeta);
verify(context);
verify(psc);
verify(client);
verify(ometa);
}
}