Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-1337: S3 putBlob portable storage tiers 

Also promote hacky and limited storage class support from aws-s3
provider to s3 api.
This commit is contained in:
Andrew Gaul 2017-10-10 18:53:52 -07:00
parent 14c41ea133
commit 89053d9a8b
11 changed files with 77 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apis/s3/src/main/java/org/jclouds/s3/binders/BindObjectMetadataToRequest.java
View File

@ -81,6 +81,11 @@ public class BindObjectMetadataToRequest implements Binder {
headers.put("Content-MD5", base64().encode(md.getContentMetadata().getContentMD5()));
}
ObjectMetadata.StorageClass storageClass = md.getStorageClass();
if (storageClass != ObjectMetadata.StorageClass.STANDARD) {
headers.put("x-amz-storage-class", storageClass.toString());
}
return (R) request.toBuilder().replaceHeaders(headers.build()).build();
}
}

10
apis/s3/src/main/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequest.java
View File

@ -25,6 +25,7 @@ import javax.inject.Singleton;
import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.Binder;
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
import org.jclouds.s3.domain.S3Object;
@Singleton
@ -48,7 +49,14 @@ public class BindS3ObjectMetadataToRequest implements Binder {
"contentLength must be set, streaming not supported");
checkArgument(s3Object.getPayload().getContentMetadata().getContentLength() <= 5L * 1024 * 1024 * 1024,
"maximum size for put object is 5GB");
StorageClass storageClass = s3Object.getMetadata().getStorageClass();
if (storageClass != StorageClass.STANDARD) {
request = (R) request.toBuilder()
.replaceHeader("x-amz-storage-class", storageClass.toString())
.build();
}
request = metadataPrefixer.bindToRequest(request, s3Object.getMetadata().getUserMetadata());
return request;

2
apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java
View File

@ -266,6 +266,8 @@ public class S3BlobStore extends BaseBlobStore {
if (overrides.getBlobAccess() == BlobAccess.PUBLIC_READ) {
options = options.withAcl(CannedAccessPolicy.PUBLIC_READ);
}
// TODO: S3 does not allow putObject if Tier.ARCHIVE. Instead, copyBlob
// after putBlob when the former supports tiers.
return sync.putObject(container, blob2Object.apply(blob), options);
}

2
apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BlobToObjectMetadata.java
View File

@ -26,6 +26,7 @@ import org.jclouds.http.HttpUtils;
import org.jclouds.rest.InvocationContext;
import org.jclouds.rest.internal.GeneratedHttpRequest;
import org.jclouds.s3.domain.MutableObjectMetadata;
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
import com.google.common.base.Function;
@ -48,6 +49,7 @@ public class BlobToObjectMetadata implements Function<BlobMetadata, MutableObjec
for (Entry<String, String> entry : from.getUserMetadata().entrySet())
to.getUserMetadata().put(entry.getKey().toLowerCase(), entry.getValue());
}
to.setStorageClass(StorageClass.fromTier(from.getTier()));
return to;
}

1
apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java
View File

@ -51,6 +51,7 @@ public class ObjectToBlobMetadata implements Function<ObjectMetadata, MutableBlo
to.setLocation(locationOfBucket.apply(from.getBucket()));
to.setType(StorageType.BLOB);
to.setSize(from.getContentMetadata().getContentLength());
to.setTier((from.getStorageClass() == null ? ObjectMetadata.StorageClass.STANDARD : from.getStorageClass()).toTier());
return to;
}
}

1
apis/s3/src/main/java/org/jclouds/s3/domain/MutableObjectMetadata.java
View File

@ -21,6 +21,7 @@ import java.util.Date;
import java.util.Map;
import org.jclouds.io.MutableContentMetadata;
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
import com.google.inject.ImplementedBy;

27
apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadata.java
View File

@ -16,10 +16,13 @@
*/
package org.jclouds.s3.domain;
import static com.google.common.base.Preconditions.checkNotNull;
import java.net.URI;
import java.util.Date;
import java.util.Map;
import org.jclouds.blobstore.domain.Tier;
import org.jclouds.io.ContentMetadata;
/**
@ -31,7 +34,29 @@ import org.jclouds.io.ContentMetadata;
public interface ObjectMetadata extends Comparable<ObjectMetadata> {
public enum StorageClass {
STANDARD, STANDARD_IA, REDUCED_REDUNDANCY
STANDARD(Tier.STANDARD),
STANDARD_IA(Tier.INFREQUENT),
REDUCED_REDUNDANCY(Tier.STANDARD),
GLACIER(Tier.ARCHIVE);
private final Tier tier;
private StorageClass(Tier tier) {
this.tier = checkNotNull(tier, "tier");
}
public static StorageClass fromTier(Tier tier) {
switch (tier) {
case STANDARD: return StorageClass.STANDARD;
case INFREQUENT: return StorageClass.STANDARD_IA;
case ARCHIVE: return StorageClass.GLACIER;
}
throw new IllegalArgumentException("invalid tier: " + tier);
}
public Tier toTier() {
return tier;
}
}
/**

5
apis/s3/src/main/java/org/jclouds/s3/functions/ParseObjectMetadataFromHeaders.java
View File

@ -32,6 +32,7 @@ import org.jclouds.http.HttpResponse;
import org.jclouds.rest.InvocationContext;
import org.jclouds.s3.blobstore.functions.BlobToObjectMetadata;
import org.jclouds.s3.domain.MutableObjectMetadata;
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
@ -75,6 +76,10 @@ public class ParseObjectMetadataFromHeaders implements Function<HttpResponse, Mu
// amz has an etag, but matches syntax for usermetadata
to.getUserMetadata().remove("object-etag");
to.setCacheControl(from.getFirstHeaderOrNull(HttpHeaders.CACHE_CONTROL));
String storageClass = from.getFirstHeaderOrNull("x-amz-storage-class");
if (storageClass != null) {
to.setStorageClass(StorageClass.valueOf(storageClass));
}
return to;
}

23
apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3BlobIntegrationLiveTest.java
View File

@ -16,13 +16,17 @@
*/
package org.jclouds.s3.blobstore.integration;
import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;
import java.io.IOException;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
import org.jclouds.http.HttpResponseException;
import org.jclouds.s3.blobstore.strategy.MultipartUpload;
import org.testng.SkipException;
import org.testng.annotations.Test;
@Test(groups = "live", testName = "S3BlobIntegrationLiveTest")
@ -51,4 +55,23 @@ public class S3BlobIntegrationLiveTest extends BaseBlobIntegrationTest {
super.testPutObjectStream();
}
@Override
public void testPutBlobTierArchive() throws Exception {
try {
super.testPutBlobTierArchive();
failBecauseExceptionWasNotThrown(HttpResponseException.class);
} catch (HttpResponseException hre) {
throw new SkipException("S3 does not allow setting Glacier storage class on putBlob", hre);
}
}
@Override
public void testPutBlobTierArchiveMultipart() throws Exception {
try {
super.testPutBlobTierArchiveMultipart();
failBecauseExceptionWasNotThrown(HttpResponseException.class);
} catch (HttpResponseException hre) {
throw new SkipException("S3 does not allow setting Glacier storage class on putBlob", hre);
}
}
}

26
providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java
View File

@ -16,8 +16,6 @@
*/
package org.jclouds.aws.s3.blobstore;
import static org.jclouds.s3.domain.ObjectMetadata.StorageClass.REDUCED_REDUNDANCY;
import java.util.Set;
import javax.inject.Inject;
@ -25,15 +23,11 @@ import javax.inject.Provider;
import org.jclouds.aws.domain.Region;
import org.jclouds.aws.s3.AWSS3Client;
import org.jclouds.aws.s3.blobstore.options.AWSS3PutObjectOptions;
import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.options.CreateContainerOptions;
import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.collect.Memoized;
@ -47,7 +41,6 @@ import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions;
import org.jclouds.s3.blobstore.functions.ObjectToBlob;
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.domain.ObjectMetadata;
import com.google.common.base.Function;
import com.google.common.base.Supplier;
@ -73,25 +66,6 @@ public class AWSS3BlobStore extends S3BlobStore {
this.blob2Object = blob2Object;
}
@Override
public String putBlob(String container, Blob blob, PutOptions options) {
if (options.isMultipart()) {
return putMultipartBlob(container, blob, options);
} else if ((options instanceof AWSS3PutOptions) &&
(((AWSS3PutOptions) options).getStorageClass() == REDUCED_REDUNDANCY)) {
return putBlobWithReducedRedundancy(container, blob);
} else {
return super.putBlob(container, blob, options);
}
}
private String putBlobWithReducedRedundancy(String container, Blob blob) {
AWSS3PutObjectOptions options = new AWSS3PutObjectOptions();
options.storageClass(ObjectMetadata.StorageClass.REDUCED_REDUNDANCY);
return getContext().unwrapApi(AWSS3Client.class).putObject(container, blob2Object.apply(blob), options);
}
@Override
public boolean createContainerInLocation(Location location, String container,
CreateContainerOptions options) {

3
providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/options/AWSS3PutObjectOptions.java
View File

@ -33,6 +33,7 @@ public class AWSS3PutObjectOptions extends PutObjectOptions {
/**
* @see AWSS3PutObjectOptions#storageClass
*/
@Deprecated
public static AWSS3PutObjectOptions storageClass(ObjectMetadata.StorageClass storageClass) {
AWSS3PutObjectOptions options = new AWSS3PutObjectOptions();
return options.storageClass(storageClass);
@ -49,6 +50,7 @@ public class AWSS3PutObjectOptions extends PutObjectOptions {
private ObjectMetadata.StorageClass storageClass = ObjectMetadata.StorageClass.STANDARD;
@Deprecated
public AWSS3PutObjectOptions storageClass(ObjectMetadata.StorageClass storageClass) {
this.storageClass = storageClass;
if (storageClass != ObjectMetadata.StorageClass.STANDARD) {
@ -57,6 +59,7 @@ public class AWSS3PutObjectOptions extends PutObjectOptions {
return this;
}
@Deprecated
public ObjectMetadata.StorageClass getStorageClass() {
return storageClass;
}