mirror of https://github.com/apache/jclouds.git
JCLOUDS-1337: S3 putBlob portable storage tiers
Also promote hacky and limited storage class support from aws-s3 provider to s3 api.
This commit is contained in:
parent
14c41ea133
commit
89053d9a8b
|
@ -81,6 +81,11 @@ public class BindObjectMetadataToRequest implements Binder {
|
||||||
headers.put("Content-MD5", base64().encode(md.getContentMetadata().getContentMD5()));
|
headers.put("Content-MD5", base64().encode(md.getContentMetadata().getContentMD5()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ObjectMetadata.StorageClass storageClass = md.getStorageClass();
|
||||||
|
if (storageClass != ObjectMetadata.StorageClass.STANDARD) {
|
||||||
|
headers.put("x-amz-storage-class", storageClass.toString());
|
||||||
|
}
|
||||||
|
|
||||||
return (R) request.toBuilder().replaceHeaders(headers.build()).build();
|
return (R) request.toBuilder().replaceHeaders(headers.build()).build();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,6 +25,7 @@ import javax.inject.Singleton;
|
||||||
import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
|
import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
|
||||||
import org.jclouds.http.HttpRequest;
|
import org.jclouds.http.HttpRequest;
|
||||||
import org.jclouds.rest.Binder;
|
import org.jclouds.rest.Binder;
|
||||||
|
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
|
||||||
import org.jclouds.s3.domain.S3Object;
|
import org.jclouds.s3.domain.S3Object;
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
|
@ -48,7 +49,14 @@ public class BindS3ObjectMetadataToRequest implements Binder {
|
||||||
"contentLength must be set, streaming not supported");
|
"contentLength must be set, streaming not supported");
|
||||||
checkArgument(s3Object.getPayload().getContentMetadata().getContentLength() <= 5L * 1024 * 1024 * 1024,
|
checkArgument(s3Object.getPayload().getContentMetadata().getContentLength() <= 5L * 1024 * 1024 * 1024,
|
||||||
"maximum size for put object is 5GB");
|
"maximum size for put object is 5GB");
|
||||||
|
|
||||||
|
StorageClass storageClass = s3Object.getMetadata().getStorageClass();
|
||||||
|
if (storageClass != StorageClass.STANDARD) {
|
||||||
|
request = (R) request.toBuilder()
|
||||||
|
.replaceHeader("x-amz-storage-class", storageClass.toString())
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
request = metadataPrefixer.bindToRequest(request, s3Object.getMetadata().getUserMetadata());
|
request = metadataPrefixer.bindToRequest(request, s3Object.getMetadata().getUserMetadata());
|
||||||
|
|
||||||
return request;
|
return request;
|
||||||
|
|
|
@ -266,6 +266,8 @@ public class S3BlobStore extends BaseBlobStore {
|
||||||
if (overrides.getBlobAccess() == BlobAccess.PUBLIC_READ) {
|
if (overrides.getBlobAccess() == BlobAccess.PUBLIC_READ) {
|
||||||
options = options.withAcl(CannedAccessPolicy.PUBLIC_READ);
|
options = options.withAcl(CannedAccessPolicy.PUBLIC_READ);
|
||||||
}
|
}
|
||||||
|
// TODO: S3 does not allow putObject if Tier.ARCHIVE. Instead, copyBlob
|
||||||
|
// after putBlob when the former supports tiers.
|
||||||
return sync.putObject(container, blob2Object.apply(blob), options);
|
return sync.putObject(container, blob2Object.apply(blob), options);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -26,6 +26,7 @@ import org.jclouds.http.HttpUtils;
|
||||||
import org.jclouds.rest.InvocationContext;
|
import org.jclouds.rest.InvocationContext;
|
||||||
import org.jclouds.rest.internal.GeneratedHttpRequest;
|
import org.jclouds.rest.internal.GeneratedHttpRequest;
|
||||||
import org.jclouds.s3.domain.MutableObjectMetadata;
|
import org.jclouds.s3.domain.MutableObjectMetadata;
|
||||||
|
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
|
||||||
import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
|
import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
|
||||||
|
|
||||||
import com.google.common.base.Function;
|
import com.google.common.base.Function;
|
||||||
|
@ -48,6 +49,7 @@ public class BlobToObjectMetadata implements Function<BlobMetadata, MutableObjec
|
||||||
for (Entry<String, String> entry : from.getUserMetadata().entrySet())
|
for (Entry<String, String> entry : from.getUserMetadata().entrySet())
|
||||||
to.getUserMetadata().put(entry.getKey().toLowerCase(), entry.getValue());
|
to.getUserMetadata().put(entry.getKey().toLowerCase(), entry.getValue());
|
||||||
}
|
}
|
||||||
|
to.setStorageClass(StorageClass.fromTier(from.getTier()));
|
||||||
return to;
|
return to;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -51,6 +51,7 @@ public class ObjectToBlobMetadata implements Function<ObjectMetadata, MutableBlo
|
||||||
to.setLocation(locationOfBucket.apply(from.getBucket()));
|
to.setLocation(locationOfBucket.apply(from.getBucket()));
|
||||||
to.setType(StorageType.BLOB);
|
to.setType(StorageType.BLOB);
|
||||||
to.setSize(from.getContentMetadata().getContentLength());
|
to.setSize(from.getContentMetadata().getContentLength());
|
||||||
|
to.setTier((from.getStorageClass() == null ? ObjectMetadata.StorageClass.STANDARD : from.getStorageClass()).toTier());
|
||||||
return to;
|
return to;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,6 +21,7 @@ import java.util.Date;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import org.jclouds.io.MutableContentMetadata;
|
import org.jclouds.io.MutableContentMetadata;
|
||||||
|
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
|
||||||
import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
|
import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
|
||||||
|
|
||||||
import com.google.inject.ImplementedBy;
|
import com.google.inject.ImplementedBy;
|
||||||
|
|
|
@ -16,10 +16,13 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.s3.domain;
|
package org.jclouds.s3.domain;
|
||||||
|
|
||||||
|
import static com.google.common.base.Preconditions.checkNotNull;
|
||||||
|
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.jclouds.blobstore.domain.Tier;
|
||||||
import org.jclouds.io.ContentMetadata;
|
import org.jclouds.io.ContentMetadata;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -31,7 +34,29 @@ import org.jclouds.io.ContentMetadata;
|
||||||
public interface ObjectMetadata extends Comparable<ObjectMetadata> {
|
public interface ObjectMetadata extends Comparable<ObjectMetadata> {
|
||||||
|
|
||||||
public enum StorageClass {
|
public enum StorageClass {
|
||||||
STANDARD, STANDARD_IA, REDUCED_REDUNDANCY
|
STANDARD(Tier.STANDARD),
|
||||||
|
STANDARD_IA(Tier.INFREQUENT),
|
||||||
|
REDUCED_REDUNDANCY(Tier.STANDARD),
|
||||||
|
GLACIER(Tier.ARCHIVE);
|
||||||
|
|
||||||
|
private final Tier tier;
|
||||||
|
|
||||||
|
private StorageClass(Tier tier) {
|
||||||
|
this.tier = checkNotNull(tier, "tier");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static StorageClass fromTier(Tier tier) {
|
||||||
|
switch (tier) {
|
||||||
|
case STANDARD: return StorageClass.STANDARD;
|
||||||
|
case INFREQUENT: return StorageClass.STANDARD_IA;
|
||||||
|
case ARCHIVE: return StorageClass.GLACIER;
|
||||||
|
}
|
||||||
|
throw new IllegalArgumentException("invalid tier: " + tier);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Tier toTier() {
|
||||||
|
return tier;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -32,6 +32,7 @@ import org.jclouds.http.HttpResponse;
|
||||||
import org.jclouds.rest.InvocationContext;
|
import org.jclouds.rest.InvocationContext;
|
||||||
import org.jclouds.s3.blobstore.functions.BlobToObjectMetadata;
|
import org.jclouds.s3.blobstore.functions.BlobToObjectMetadata;
|
||||||
import org.jclouds.s3.domain.MutableObjectMetadata;
|
import org.jclouds.s3.domain.MutableObjectMetadata;
|
||||||
|
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import com.google.common.base.Function;
|
import com.google.common.base.Function;
|
||||||
|
@ -75,6 +76,10 @@ public class ParseObjectMetadataFromHeaders implements Function<HttpResponse, Mu
|
||||||
// amz has an etag, but matches syntax for usermetadata
|
// amz has an etag, but matches syntax for usermetadata
|
||||||
to.getUserMetadata().remove("object-etag");
|
to.getUserMetadata().remove("object-etag");
|
||||||
to.setCacheControl(from.getFirstHeaderOrNull(HttpHeaders.CACHE_CONTROL));
|
to.setCacheControl(from.getFirstHeaderOrNull(HttpHeaders.CACHE_CONTROL));
|
||||||
|
String storageClass = from.getFirstHeaderOrNull("x-amz-storage-class");
|
||||||
|
if (storageClass != null) {
|
||||||
|
to.setStorageClass(StorageClass.valueOf(storageClass));
|
||||||
|
}
|
||||||
return to;
|
return to;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -16,13 +16,17 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.s3.blobstore.integration;
|
package org.jclouds.s3.blobstore.integration;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.concurrent.ExecutionException;
|
||||||
|
|
||||||
import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
|
import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
|
||||||
import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
|
import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
|
||||||
|
import org.jclouds.http.HttpResponseException;
|
||||||
import org.jclouds.s3.blobstore.strategy.MultipartUpload;
|
import org.jclouds.s3.blobstore.strategy.MultipartUpload;
|
||||||
|
import org.testng.SkipException;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
|
|
||||||
@Test(groups = "live", testName = "S3BlobIntegrationLiveTest")
|
@Test(groups = "live", testName = "S3BlobIntegrationLiveTest")
|
||||||
|
@ -51,4 +55,23 @@ public class S3BlobIntegrationLiveTest extends BaseBlobIntegrationTest {
|
||||||
super.testPutObjectStream();
|
super.testPutObjectStream();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void testPutBlobTierArchive() throws Exception {
|
||||||
|
try {
|
||||||
|
super.testPutBlobTierArchive();
|
||||||
|
failBecauseExceptionWasNotThrown(HttpResponseException.class);
|
||||||
|
} catch (HttpResponseException hre) {
|
||||||
|
throw new SkipException("S3 does not allow setting Glacier storage class on putBlob", hre);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void testPutBlobTierArchiveMultipart() throws Exception {
|
||||||
|
try {
|
||||||
|
super.testPutBlobTierArchiveMultipart();
|
||||||
|
failBecauseExceptionWasNotThrown(HttpResponseException.class);
|
||||||
|
} catch (HttpResponseException hre) {
|
||||||
|
throw new SkipException("S3 does not allow setting Glacier storage class on putBlob", hre);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,8 +16,6 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.aws.s3.blobstore;
|
package org.jclouds.aws.s3.blobstore;
|
||||||
|
|
||||||
import static org.jclouds.s3.domain.ObjectMetadata.StorageClass.REDUCED_REDUNDANCY;
|
|
||||||
|
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import javax.inject.Inject;
|
import javax.inject.Inject;
|
||||||
|
@ -25,15 +23,11 @@ import javax.inject.Provider;
|
||||||
|
|
||||||
import org.jclouds.aws.domain.Region;
|
import org.jclouds.aws.domain.Region;
|
||||||
import org.jclouds.aws.s3.AWSS3Client;
|
import org.jclouds.aws.s3.AWSS3Client;
|
||||||
import org.jclouds.aws.s3.blobstore.options.AWSS3PutObjectOptions;
|
|
||||||
import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions;
|
|
||||||
import org.jclouds.blobstore.BlobStoreContext;
|
import org.jclouds.blobstore.BlobStoreContext;
|
||||||
import org.jclouds.blobstore.domain.Blob;
|
|
||||||
import org.jclouds.blobstore.domain.PageSet;
|
import org.jclouds.blobstore.domain.PageSet;
|
||||||
import org.jclouds.blobstore.domain.StorageMetadata;
|
import org.jclouds.blobstore.domain.StorageMetadata;
|
||||||
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
|
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
|
||||||
import org.jclouds.blobstore.options.CreateContainerOptions;
|
import org.jclouds.blobstore.options.CreateContainerOptions;
|
||||||
import org.jclouds.blobstore.options.PutOptions;
|
|
||||||
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
|
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
|
||||||
import org.jclouds.blobstore.util.BlobUtils;
|
import org.jclouds.blobstore.util.BlobUtils;
|
||||||
import org.jclouds.collect.Memoized;
|
import org.jclouds.collect.Memoized;
|
||||||
|
@ -47,7 +41,6 @@ import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions;
|
||||||
import org.jclouds.s3.blobstore.functions.ObjectToBlob;
|
import org.jclouds.s3.blobstore.functions.ObjectToBlob;
|
||||||
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
|
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
|
||||||
import org.jclouds.s3.domain.BucketMetadata;
|
import org.jclouds.s3.domain.BucketMetadata;
|
||||||
import org.jclouds.s3.domain.ObjectMetadata;
|
|
||||||
|
|
||||||
import com.google.common.base.Function;
|
import com.google.common.base.Function;
|
||||||
import com.google.common.base.Supplier;
|
import com.google.common.base.Supplier;
|
||||||
|
@ -73,25 +66,6 @@ public class AWSS3BlobStore extends S3BlobStore {
|
||||||
this.blob2Object = blob2Object;
|
this.blob2Object = blob2Object;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String putBlob(String container, Blob blob, PutOptions options) {
|
|
||||||
if (options.isMultipart()) {
|
|
||||||
return putMultipartBlob(container, blob, options);
|
|
||||||
} else if ((options instanceof AWSS3PutOptions) &&
|
|
||||||
(((AWSS3PutOptions) options).getStorageClass() == REDUCED_REDUNDANCY)) {
|
|
||||||
return putBlobWithReducedRedundancy(container, blob);
|
|
||||||
|
|
||||||
} else {
|
|
||||||
return super.putBlob(container, blob, options);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String putBlobWithReducedRedundancy(String container, Blob blob) {
|
|
||||||
AWSS3PutObjectOptions options = new AWSS3PutObjectOptions();
|
|
||||||
options.storageClass(ObjectMetadata.StorageClass.REDUCED_REDUNDANCY);
|
|
||||||
return getContext().unwrapApi(AWSS3Client.class).putObject(container, blob2Object.apply(blob), options);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean createContainerInLocation(Location location, String container,
|
public boolean createContainerInLocation(Location location, String container,
|
||||||
CreateContainerOptions options) {
|
CreateContainerOptions options) {
|
||||||
|
|
|
@ -33,6 +33,7 @@ public class AWSS3PutObjectOptions extends PutObjectOptions {
|
||||||
/**
|
/**
|
||||||
* @see AWSS3PutObjectOptions#storageClass
|
* @see AWSS3PutObjectOptions#storageClass
|
||||||
*/
|
*/
|
||||||
|
@Deprecated
|
||||||
public static AWSS3PutObjectOptions storageClass(ObjectMetadata.StorageClass storageClass) {
|
public static AWSS3PutObjectOptions storageClass(ObjectMetadata.StorageClass storageClass) {
|
||||||
AWSS3PutObjectOptions options = new AWSS3PutObjectOptions();
|
AWSS3PutObjectOptions options = new AWSS3PutObjectOptions();
|
||||||
return options.storageClass(storageClass);
|
return options.storageClass(storageClass);
|
||||||
|
@ -49,6 +50,7 @@ public class AWSS3PutObjectOptions extends PutObjectOptions {
|
||||||
|
|
||||||
private ObjectMetadata.StorageClass storageClass = ObjectMetadata.StorageClass.STANDARD;
|
private ObjectMetadata.StorageClass storageClass = ObjectMetadata.StorageClass.STANDARD;
|
||||||
|
|
||||||
|
@Deprecated
|
||||||
public AWSS3PutObjectOptions storageClass(ObjectMetadata.StorageClass storageClass) {
|
public AWSS3PutObjectOptions storageClass(ObjectMetadata.StorageClass storageClass) {
|
||||||
this.storageClass = storageClass;
|
this.storageClass = storageClass;
|
||||||
if (storageClass != ObjectMetadata.StorageClass.STANDARD) {
|
if (storageClass != ObjectMetadata.StorageClass.STANDARD) {
|
||||||
|
@ -57,6 +59,7 @@ public class AWSS3PutObjectOptions extends PutObjectOptions {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Deprecated
|
||||||
public ObjectMetadata.StorageClass getStorageClass() {
|
public ObjectMetadata.StorageClass getStorageClass() {
|
||||||
return storageClass;
|
return storageClass;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue