mirror of https://github.com/apache/jclouds.git
AWS S3 sequential Multipart Upload strategy
This commit is contained in:
parent
ebd4d1e432
commit
ed27cbc7c0
|
@ -207,6 +207,16 @@ public class AtmosBlobStore extends BaseBlobStore {
|
|||
return AtmosUtils.putBlob(sync, crypto, blob2Object, container, blob);
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation invokes {@link AtmosClient#createFile}
|
||||
* <p/>
|
||||
* Since there is no etag support in atmos, we just return the path.
|
||||
*/
|
||||
@Override
|
||||
public String putBlobMultipart(String container, Blob blob) {
|
||||
return putBlob(container, blob);
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation invokes {@link AtmosClient#deletePath}
|
||||
*/
|
||||
|
|
|
@ -83,7 +83,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
|
|||
private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
|
||||
|
||||
@Inject
|
||||
S3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils,
|
||||
protected S3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils,
|
||||
@Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier<Location> defaultLocation,
|
||||
@Memoized Supplier<Set<? extends Location>> locations, S3AsyncClient async, S3Client sync,
|
||||
BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions,
|
||||
|
|
|
@ -74,7 +74,7 @@ public class S3BlobStore extends BaseBlobStore {
|
|||
private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
|
||||
|
||||
@Inject
|
||||
S3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
|
||||
protected S3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
|
||||
@Memoized Supplier<Set<? extends Location>> locations, S3Client sync,
|
||||
BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions,
|
||||
BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob,
|
||||
|
@ -225,6 +225,19 @@ public class S3BlobStore extends BaseBlobStore {
|
|||
return sync.putObject(container, blob2Object.apply(blob));
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation invokes {@link S3Client#putObject}
|
||||
*
|
||||
* @param container
|
||||
* bucket name
|
||||
* @param blob
|
||||
* object
|
||||
*/
|
||||
@Override
|
||||
public String putBlobMultipart(String container, Blob blob) {
|
||||
return sync.putObject(container, blob2Object.apply(blob));
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation invokes {@link S3Client#deleteObject}
|
||||
*
|
||||
|
|
|
@ -195,6 +195,19 @@ public class SwiftBlobStore extends BaseBlobStore {
|
|||
return sync.putObject(container, blob2Object.apply(blob));
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation invokes {@link CommonSwiftClient#putObject}
|
||||
*
|
||||
* @param container
|
||||
* container name
|
||||
* @param blob
|
||||
* object
|
||||
*/
|
||||
@Override
|
||||
public String putBlobMultipart(String container, Blob blob) {
|
||||
return putBlob(container, blob);
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation invokes {@link CommonSwiftClient#removeObject}
|
||||
*
|
||||
|
|
|
@ -199,6 +199,22 @@ public interface BlobStore {
|
|||
* if the container doesn't exist
|
||||
*/
|
||||
String putBlob(String container, Blob blob);
|
||||
|
||||
/**
|
||||
* Adds a {@code Blob} representing the data at location {@code container/blob.metadata.name}
|
||||
* using multipart strategies.
|
||||
*
|
||||
* @param container
|
||||
* container to place the blob.
|
||||
* @param blob
|
||||
* fully qualified name relative to the container.
|
||||
* @param options
|
||||
* byte range or condition options
|
||||
* @return etag of the blob you uploaded, possibly null where etags are unsupported
|
||||
* @throws ContainerNotFoundException
|
||||
* if the container doesn't exist
|
||||
*/
|
||||
String putBlobMultipart(String container, Blob blob);
|
||||
|
||||
/**
|
||||
* Retrieves the metadata of a {@code Blob} at location {@code container/name}
|
||||
|
|
|
@ -119,6 +119,11 @@
|
|||
<artifactId>jsr305</artifactId>
|
||||
<version>1.3.9</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.jboss.netty</groupId>
|
||||
<artifactId>netty</artifactId>
|
||||
<version>3.2.4.Final</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<profiles>
|
||||
|
|
|
@ -35,6 +35,7 @@ import javax.annotation.Nullable;
|
|||
|
||||
import org.jclouds.crypto.CryptoStreams;
|
||||
import org.jclouds.io.payloads.ByteArrayPayload;
|
||||
import org.jclouds.io.payloads.ChunkedFilePayload;
|
||||
import org.jclouds.io.payloads.FilePayload;
|
||||
import org.jclouds.io.payloads.InputStreamPayload;
|
||||
import org.jclouds.io.payloads.StringPayload;
|
||||
|
@ -83,6 +84,10 @@ public class Payloads {
|
|||
public static FilePayload newFilePayload(File data) {
|
||||
return new FilePayload(checkNotNull(data, "data"));
|
||||
}
|
||||
|
||||
public static ChunkedFilePayload newChunkedFilePayload(File data, int part, long chunkOffset, long chunkSize) {
|
||||
return new ChunkedFilePayload(checkNotNull(data, "data"), part, chunkOffset, chunkSize);
|
||||
}
|
||||
|
||||
public static UrlEncodedFormPayload newUrlEncodedFormPayload(Multimap<String, String> formParams, char... skips) {
|
||||
return new UrlEncodedFormPayload(formParams, skips);
|
||||
|
|
|
@ -22,8 +22,10 @@ package org.jclouds.aws.s3;
|
|||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.jclouds.aws.s3.blobstore.config.AWSS3BlobStoreContextModule;
|
||||
import org.jclouds.aws.s3.config.AWSS3RestClientModule;
|
||||
import org.jclouds.s3.S3ContextBuilder;
|
||||
import org.jclouds.s3.blobstore.config.S3BlobStoreContextModule;
|
||||
|
||||
import com.google.inject.Module;
|
||||
|
||||
|
@ -37,6 +39,11 @@ public class AWSS3ContextBuilder extends S3ContextBuilder {
|
|||
super(props);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void addContextModule(List<Module> modules) {
|
||||
modules.add(new AWSS3BlobStoreContextModule());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void addClientModule(List<Module> modules) {
|
||||
modules.add(new AWSS3RestClientModule());
|
||||
|
|
|
@ -27,11 +27,15 @@ import static org.jclouds.io.Payloads.newByteArrayPayload;
|
|||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import org.jclouds.blobstore.BlobStore;
|
||||
import org.jclouds.blobstore.KeyNotFoundException;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
import org.jclouds.http.BaseJettyTest;
|
||||
import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule;
|
||||
import org.jclouds.io.Payload;
|
||||
|
@ -43,6 +47,7 @@ import org.testng.annotations.BeforeClass;
|
|||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import com.google.common.io.InputSupplier;
|
||||
import com.google.inject.Module;
|
||||
|
||||
|
@ -134,4 +139,23 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
|
|||
returnContainer(containerName);
|
||||
}
|
||||
}
|
||||
|
||||
public void testMultipartChunkedFileStream() throws IOException, InterruptedException {
|
||||
|
||||
FileOutputStream fous = new FileOutputStream(new File("target/const.txt"));
|
||||
ByteStreams.copy(oneHundredOneConstitutions.getInput(), fous);
|
||||
fous.flush();
|
||||
fous.close();
|
||||
String containerName = getContainerName();
|
||||
|
||||
try {
|
||||
BlobStore blobStore = context.getBlobStore();
|
||||
blobStore.createContainerInLocation(null, containerName);
|
||||
Blob blob = blobStore.blobBuilder("const.txt")
|
||||
.payload(new File("target/const.txt")).build();
|
||||
blobStore.putBlobMultipart(containerName, blob);
|
||||
} finally {
|
||||
returnContainer(containerName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -192,6 +192,19 @@ public class AzureBlobStore extends BaseBlobStore {
|
|||
return sync.putBlob(container, blob2AzureBlob.apply(blob));
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation invokes {@link AzureBlobClient#putObject}
|
||||
*
|
||||
* @param container
|
||||
* container name
|
||||
* @param blob
|
||||
* object
|
||||
*/
|
||||
@Override
|
||||
public String putBlobMultipart(String container, Blob blob) {
|
||||
return putBlob(container, blob);
|
||||
}
|
||||
|
||||
/**
|
||||
* This implementation invokes {@link AzureBlobClient#deleteObject}
|
||||
*
|
||||
|
|
Loading…
Reference in New Issue