Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 430:large blob support on aws-s3

This commit is contained in:
Adrian Cole 2011-02-24 21:46:14 -08:00
parent 84004b8986
commit 95a19b99fd
44 changed files with 1707 additions and 270 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apis/atmos/src/main/java/org/jclouds/atmos/domain/internal/DelegatingMutableContentMetadata.java
View File

@ -20,6 +20,7 @@
package org.jclouds.atmos.domain.internal;
import org.jclouds.atmos.domain.MutableContentMetadata;
import org.jclouds.io.ContentMetadataBuilder;
import org.jclouds.io.payloads.BaseMutableContentMetadata;
import com.google.common.collect.Multimap;
@ -157,4 +158,9 @@ public class DelegatingMutableContentMetadata implements MutableContentMetadata
delegate.setPropertiesFromHttpHeaders(headers);
}
@Override
public ContentMetadataBuilder toBuilder() {
return delegate.toBuilder();
}
}

89
apis/filesystem/src/main/java/org/jclouds/filesystem/FilesystemAsyncBlobStore.java
View File

@ -49,10 +49,10 @@ import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.Map.Entry;
import java.util.concurrent.ExecutorService;
import javax.annotation.Nullable;
@ -67,13 +67,13 @@ import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.ContainerNotFoundException;
import org.jclouds.blobstore.KeyNotFoundException;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.Blob.Factory;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.MutableBlobMetadata;
import org.jclouds.blobstore.domain.MutableStorageMetadata;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.StorageType;
import org.jclouds.blobstore.domain.Blob.Factory;
import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl;
import org.jclouds.blobstore.domain.internal.MutableStorageMetadataImpl;
import org.jclouds.blobstore.domain.internal.PageSetImpl;
@ -97,6 +97,7 @@ import org.jclouds.http.HttpResponseException;
import org.jclouds.http.options.HttpRequestOptions;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.io.payloads.BaseMutableContentMetadata;
import org.jclouds.io.payloads.FilePayload;
import org.jclouds.logging.Logger;
import org.jclouds.rest.annotations.ParamValidators;
@ -129,10 +130,10 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
@Inject
protected FilesystemAsyncBlobStore(BlobStoreContext context, DateService dateService, Crypto crypto,
HttpGetOptionsListToGetOptions httpGetOptionsConverter, IfDirectoryReturnNameStrategy ifDirectoryReturnName,
Factory blobFactory, BlobUtils blobUtils, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service,
Supplier<Location> defaultLocation, @Memoized Supplier<Set<? extends Location>> locations,
FilesystemStorageStrategy storageStrategy) {
HttpGetOptionsListToGetOptions httpGetOptionsConverter,
IfDirectoryReturnNameStrategy ifDirectoryReturnName, Factory blobFactory, BlobUtils blobUtils,
@Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations, FilesystemStorageStrategy storageStrategy) {
super(context, blobUtils, service, defaultLocation, locations);
// super(context, blobUtils, service, null, null);
this.blobFactory = blobFactory;
@ -164,22 +165,22 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
}
SortedSet<StorageMetadata> contents = newTreeSet(transform(blobBelongingToContainer,
new Function<String, StorageMetadata>() {
public StorageMetadata apply(String key) {
Blob oldBlob = loadFileBlob(container, key);
new Function<String, StorageMetadata>() {
public StorageMetadata apply(String key) {
Blob oldBlob = loadFileBlob(container, key);
checkState(oldBlob != null, "blob " + key + " is not present although it was in the list of "
+ container);
checkState(oldBlob.getMetadata() != null, "blob " + container + "/" + key + " has no metadata");
MutableBlobMetadata md = copy(oldBlob.getMetadata());
String directoryName = ifDirectoryReturnName.execute(md);
if (directoryName != null) {
md.setName(directoryName);
md.setType(StorageType.RELATIVE_PATH);
checkState(oldBlob != null, "blob " + key + " is not present although it was in the list of "
+ container);
checkState(oldBlob.getMetadata() != null, "blob " + container + "/" + key + " has no metadata");
MutableBlobMetadata md = copy(oldBlob.getMetadata());
String directoryName = ifDirectoryReturnName.execute(md);
if (directoryName != null) {
md.setName(directoryName);
md.setType(StorageType.RELATIVE_PATH);
}
return md;
}
return md;
}
}));
}));
String marker = null;
if (options != null) {
@ -219,21 +220,21 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
if (delimiter != null) {
SortedSet<String> commonPrefixes = null;
Iterable<String> iterable = transform(contents, new CommonPrefixes(prefix != null ? prefix : null,
delimiter));
delimiter));
commonPrefixes = iterable != null ? newTreeSet(iterable) : new TreeSet<String>();
commonPrefixes.remove(CommonPrefixes.NO_PREFIX);
contents = newTreeSet(filter(contents, new DelimiterFilter(prefix != null ? prefix : null, delimiter)));
Iterables.<StorageMetadata> addAll(contents,
transform(commonPrefixes, new Function<String, StorageMetadata>() {
public StorageMetadata apply(String o) {
MutableStorageMetadata md = new MutableStorageMetadataImpl();
md.setType(StorageType.RELATIVE_PATH);
md.setName(o);
return md;
}
}));
Iterables.<StorageMetadata> addAll(contents, transform(commonPrefixes,
new Function<String, StorageMetadata>() {
public StorageMetadata apply(String o) {
MutableStorageMetadata md = new MutableStorageMetadataImpl();
md.setType(StorageType.RELATIVE_PATH);
md.setName(o);
return md;
}
}));
}
// trim metadata, if the response isn't supposed to be detailed.
@ -245,7 +246,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
}
return Futures.<PageSet<? extends StorageMetadata>> immediateFuture(new PageSetImpl<StorageMetadata>(contents,
marker));
marker));
}
@ -262,6 +263,8 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
ObjectInput is = new ObjectInputStream(new ByteArrayInputStream(bout.toByteArray()));
MutableBlobMetadata metadata = (MutableBlobMetadata) is.readObject();
convertUserMetadataKeysToLowercase(metadata);
metadata.setContentMetadata(BaseMutableContentMetadata.fromContentMetadata(in.getContentMetadata().toBuilder()
.build()));
return metadata;
} catch (Exception e) {
propagate(e);
@ -310,14 +313,14 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
Iterable<String> containers = storageStrategy.getAllContainerNames();
return Futures.<PageSet<? extends StorageMetadata>> immediateFuture(new PageSetImpl<StorageMetadata>(transform(
containers, new Function<String, StorageMetadata>() {
public StorageMetadata apply(String name) {
MutableStorageMetadata cmd = create();
cmd.setName(name);
cmd.setType(StorageType.CONTAINER);
return cmd;
}
}), null));
containers, new Function<String, StorageMetadata>() {
public StorageMetadata apply(String name) {
MutableStorageMetadata cmd = create();
cmd.setName(name);
cmd.setType(StorageType.CONTAINER);
return cmd;
}
}), null));
}
protected MutableStorageMetadata create() {
@ -330,7 +333,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
@Path("{container}")
@Override
public ListenableFuture<Boolean> createContainerInLocation(final Location location,
@PathParam("container") @ParamValidators({ FilesystemContainerNameValidator.class }) String name) {
@PathParam("container") @ParamValidators( { FilesystemContainerNameValidator.class }) String name) {
boolean result = storageStrategy.createContainer(name);
return immediateFuture(result);
}
@ -500,7 +503,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
storageStrategy.writePayloadOnFile(containerName, blobKey, object.getPayload());
} catch (IOException e) {
logger.error(e, "An error occurred storing the new object with name [%s] to container [%s].", blobKey,
containerName);
containerName);
Throwables.propagate(e);
}
return immediateFuture(eTag);
@ -547,7 +550,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
if (blob.getMetadata().getLastModified().before(modifiedSince)) {
HttpResponse response = new HttpResponse(304, null, null);
return immediateFailedFuture(new HttpResponseException(String.format("%1$s is before %2$s", blob
.getMetadata().getLastModified(), modifiedSince), null, response));
.getMetadata().getLastModified(), modifiedSince), null, response));
}
}
@ -556,7 +559,7 @@ public class FilesystemAsyncBlobStore extends BaseAsyncBlobStore {
if (blob.getMetadata().getLastModified().after(unmodifiedSince)) {
HttpResponse response = new HttpResponse(412, null, null);
return immediateFailedFuture(new HttpResponseException(String.format("%1$s is after %2$s", blob
.getMetadata().getLastModified(), unmodifiedSince), null, response));
.getMetadata().getLastModified(), unmodifiedSince), null, response));
}
}

3
apis/s3/src/main/java/org/jclouds/s3/S3Client.java
View File

@ -137,7 +137,6 @@ public interface S3Client {
* namespace of the object you are deleting
* @param key
* unique key in the s3Bucket identifying the object
* @return true if deleted
* @throws org.jclouds.http.HttpResponseException
* if the bucket is not available
* @see <a href="http://docs.amazonwebservices.com/AmazonS3/2006-03-01/index.html?
@ -164,7 +163,7 @@ public interface S3Client {
* contains the data and metadata to create or overwrite
* @param options
* options for creating the object
* @return MD5 hash of the content uploaded
* @return ETag of the content uploaded
* @throws org.jclouds.http.HttpResponseException
* if the conditions requested set are not satisfied by the object on the server.
* @see org.jclouds.s3.domain.CannedAccessPolicy#PRIVATE

16
apis/s3/src/main/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequest.java
View File

@ -26,12 +26,11 @@ import javax.inject.Inject;
import javax.inject.Singleton;
import javax.ws.rs.core.HttpHeaders;
import org.jclouds.s3.blobstore.functions.ObjectToBlob;
import org.jclouds.s3.domain.S3Object;
import org.jclouds.blobstore.binders.BindUserMetadataToHeadersWithPrefix;
import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.utils.ModifyRequest;
import org.jclouds.rest.Binder;
import org.jclouds.s3.domain.S3Object;
/**
*
@ -39,13 +38,11 @@ import org.jclouds.rest.Binder;
*/
@Singleton
public class BindS3ObjectMetadataToRequest implements Binder {
private final BindUserMetadataToHeadersWithPrefix blobBinder;
private final ObjectToBlob object2Blob;
protected final BindMapToHeadersWithPrefix metadataPrefixer;
@Inject
public BindS3ObjectMetadataToRequest(ObjectToBlob object2Blob, BindUserMetadataToHeadersWithPrefix blobBinder) {
this.blobBinder = checkNotNull(blobBinder, "blobBinder");
this.object2Blob = checkNotNull(object2Blob, "object2Blob");
public BindS3ObjectMetadataToRequest(BindMapToHeadersWithPrefix metadataPrefixer) {
this.metadataPrefixer = checkNotNull(metadataPrefixer, "metadataPrefixer");
}
@Override
@ -59,7 +56,8 @@ public class BindS3ObjectMetadataToRequest implements Binder {
"contentLength must be set, streaming not supported");
checkArgument(s3Object.getPayload().getContentMetadata().getContentLength() <= 5l * 1024 * 1024 * 1024,
"maximum size for put object is 5GB");
request = blobBinder.bindToRequest(request, object2Blob.apply(s3Object));
request = metadataPrefixer.bindToRequest(request, s3Object.getMetadata().getUserMetadata());
if (s3Object.getMetadata().getCacheControl() != null) {
request = ModifyRequest.replaceHeader(request, HttpHeaders.CACHE_CONTROL, s3Object.getMetadata()

3
apis/s3/src/main/java/org/jclouds/s3/domain/MutableObjectMetadata.java
View File

@ -22,8 +22,8 @@ package org.jclouds.s3.domain;
import java.util.Date;
import java.util.Map;
import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
import org.jclouds.io.MutableContentMetadata;
import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
import com.google.inject.ImplementedBy;
@ -69,6 +69,7 @@ public interface MutableObjectMetadata extends ObjectMetadata {
*/
void setCacheControl(String cacheControl);
@Override
MutableContentMetadata getContentMetadata();
void setContentMetadata(MutableContentMetadata md);

2
apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadata.java
View File

@ -38,7 +38,7 @@ import org.jclouds.io.ContentMetadata;
public interface ObjectMetadata extends Comparable<ObjectMetadata> {
public enum StorageClass {
STANDARD
STANDARD, REDUCED_REDUNDANCY
}
/**

114
apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadataBuilder.java Normal file
View File

@ -0,0 +1,114 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.s3.domain;
import java.util.Map;
import org.jclouds.io.ContentMetadataBuilder;
import org.jclouds.io.payloads.BaseMutableContentMetadata;
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl;
import com.google.common.collect.ImmutableMap;
/**
* Allows you to create {@link ObjectMetadata} objects.
*
* @author Adrian Cole
*/
public class ObjectMetadataBuilder {
public static ObjectMetadataBuilder create() {
return new ObjectMetadataBuilder();
}
private final ContentMetadataBuilder contentMetadataBuilder = new ContentMetadataBuilder().contentType("binary/octet-stream");
private String key;
private StorageClass storageClass;
private String cacheControl;
private Map<String, String> userMetadata = ImmutableMap.of();
public ObjectMetadataBuilder key(String key) {
this.key = key;
return this;
}
public ObjectMetadataBuilder storageClass(StorageClass storageClass) {
this.storageClass = storageClass;
return this;
}
public ObjectMetadataBuilder cacheControl(String cacheControl) {
this.cacheControl = cacheControl;
return this;
}
public ObjectMetadataBuilder userMetadata(Map<String, String> userMetadata) {
this.userMetadata = ImmutableMap.copyOf(userMetadata);
return this;
}
public ObjectMetadataBuilder contentDisposition(String contentDisposition) {
contentMetadataBuilder.contentDisposition(contentDisposition);
return this;
}
public ObjectMetadataBuilder contentEncoding(String contentEncoding) {
contentMetadataBuilder.contentEncoding(contentEncoding);
return this;
}
public ObjectMetadataBuilder contentLanguage(String contentLanguage) {
contentMetadataBuilder.contentLanguage(contentLanguage);
return this;
}
public ObjectMetadataBuilder contentLength(Long contentLength) {
contentMetadataBuilder.contentLength(contentLength);
return this;
}
public ObjectMetadataBuilder contentMD5(byte[] md5) {
contentMetadataBuilder.contentMD5(md5);
return this;
}
public ObjectMetadataBuilder contentType(String contentType) {
contentMetadataBuilder.contentType(contentType);
return this;
}
public ObjectMetadata build() {
MutableObjectMetadataImpl toReturn = new MutableObjectMetadataImpl();
toReturn.setContentMetadata(BaseMutableContentMetadata.fromContentMetadata(contentMetadataBuilder.build()));
toReturn.setCacheControl(cacheControl);
toReturn.setKey(key);
toReturn.setStorageClass(storageClass);
toReturn.setUserMetadata(userMetadata);
return toReturn;
}
}

34
apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java
View File

@ -78,7 +78,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
private final String[] firstHeadersToSign = new String[] { HttpHeaders.DATE };
public static Set<String> SPECIAL_QUERIES = ImmutableSet.of("acl", "torrent", "logging", "location",
"requestPayment");
"requestPayment", "uploads");
private final SignatureWire signatureWire;
private final String accessKey;
private final String secretKey;
@ -97,10 +97,10 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
@Inject
public RequestAuthorizeSignature(SignatureWire signatureWire, @Named(PROPERTY_AUTH_TAG) String authTag,
@Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
@Named(PROPERTY_S3_SERVICE_PATH) String servicePath, @Named(PROPERTY_HEADER_TAG) String headerTag,
@Named(PROPERTY_IDENTITY) String accessKey, @Named(PROPERTY_CREDENTIAL) String secretKey,
@TimeStamp Provider<String> timeStampProvider, Crypto crypto, HttpUtils utils) {
@Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
@Named(PROPERTY_S3_SERVICE_PATH) String servicePath, @Named(PROPERTY_HEADER_TAG) String headerTag,
@Named(PROPERTY_IDENTITY) String accessKey, @Named(PROPERTY_CREDENTIAL) String secretKey,
@TimeStamp Provider<String> timeStampProvider, Crypto crypto, HttpUtils utils) {
this.isVhostStyle = isVhostStyle;
this.servicePath = servicePath;
this.headerTag = headerTag;
@ -123,7 +123,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
HttpRequest replaceAuthorizationHeader(HttpRequest request, String signature) {
request = ModifyRequest.replaceHeader(request, HttpHeaders.AUTHORIZATION, authTag + " " + accessKey + ":"
+ signature);
+ signature);
return request;
}
@ -161,8 +161,8 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
public String sign(String toSign) {
String signature;
try {
signature = CryptoStreams.base64(CryptoStreams.mac(InputSuppliers.of(toSign),
crypto.hmacSHA1(secretKey.getBytes())));
signature = CryptoStreams.base64(CryptoStreams.mac(InputSuppliers.of(toSign), crypto.hmacSHA1(secretKey
.getBytes())));
} catch (Exception e) {
throw new HttpException("error signing request", e);
}
@ -189,11 +189,11 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {
buffer.append(
utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata()
.getContentMD5())).append("\n");
utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata()
.getContentMD5())).append("\n");
buffer.append(
utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata()
.getContentType())).append("\n");
utils.valueOrEmpty(request.getPayload() == null ? request.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE)
: request.getPayload().getContentMetadata().getContentType())).append("\n");
}
void appendHttpHeaders(HttpRequest request, StringBuilder toSign) {
@ -210,11 +210,11 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
for (int i = 0; i < request.getJavaMethod().getParameterAnnotations().length; i++) {
if (Iterables.any(Arrays.asList(request.getJavaMethod().getParameterAnnotations()[i]),
new Predicate<Annotation>() {
public boolean apply(Annotation input) {
return input.annotationType().equals(Bucket.class);
}
})) {
new Predicate<Annotation>() {
public boolean apply(Annotation input) {
return input.annotationType().equals(Bucket.class);
}
})) {
bucketName = (String) request.getArgs().get(i);
break;
}

10
apis/s3/src/test/java/org/jclouds/s3/BaseS3AsyncClientTest.java
View File

@ -28,20 +28,17 @@ import org.jclouds.http.HttpRequest;
import org.jclouds.rest.RestClientTest;
import org.jclouds.rest.RestContextFactory;
import org.jclouds.rest.RestContextSpec;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.blobstore.functions.BlobToObject;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import com.google.inject.TypeLiteral;
/**
*
* @author Adrian Cole
*/
@Test(groups = "unit")
public abstract class BaseS3AsyncClientTest extends RestClientTest<S3AsyncClient> {
public abstract class BaseS3AsyncClientTest<T extends S3AsyncClient> extends RestClientTest<T> {
protected BlobToObject blobToS3Object;
protected RequestAuthorizeSignature filter;
@ -52,11 +49,6 @@ public abstract class BaseS3AsyncClientTest extends RestClientTest<S3AsyncClient
assertEquals(request.getFilters().get(0).getClass(), RequestAuthorizeSignature.class);
}
@Override
protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
};
}
@BeforeClass
@Override

4
apis/s3/src/test/java/org/jclouds/s3/S3AsyncClientTest.java
View File

@ -80,7 +80,7 @@ import com.google.inject.Module;
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "S3AsyncClientTest")
public class S3AsyncClientTest extends BaseS3AsyncClientTest {
public abstract class S3AsyncClientTest<T extends S3AsyncClient> extends BaseS3AsyncClientTest<T> {
protected String url = "s3.amazonaws.com";
@ -388,7 +388,7 @@ public class S3AsyncClientTest extends BaseS3AsyncClientTest {
checkFilters(request);
}
public void testPutObject() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
NoSuchMethodException, IOException {

18
apis/s3/src/test/java/org/jclouds/s3/binders/BindAsHostPrefixIfConfiguredTest.java
View File

@ -27,11 +27,15 @@ import java.io.IOException;
import java.net.URI;
import java.util.Properties;
import org.jclouds.s3.BaseS3AsyncClientTest;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.BaseS3AsyncClientTest;
import org.jclouds.s3.S3AsyncClient;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import com.google.inject.TypeLiteral;
/**
* Tests behavior of {@code BindAsHostPrefixIfConfigured}
*
@ -39,7 +43,13 @@ import org.testng.annotations.Test;
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "BindAsHostPrefixIfConfiguredTest")
public class BindAsHostPrefixIfConfiguredTest extends BaseS3AsyncClientTest {
public class BindAsHostPrefixIfConfiguredTest extends BaseS3AsyncClientTest<S3AsyncClient> {
@Override
protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
};
}
public void testBucket() throws IOException {
@ -64,8 +74,8 @@ public class BindAsHostPrefixIfConfiguredTest extends BaseS3AsyncClientTest {
@DataProvider(name = "objects")
public Object[][] createData() {
return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" },
{ "colon:" }, { "asteri*k" }, { "quote\"" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" }, { "colon:" },
{ "asteri*k" }, { "quote\"" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
}
@Override

16
apis/s3/src/test/java/org/jclouds/s3/binders/BindNoBucketLoggingToXmlPayloadTest.java
View File

@ -24,10 +24,14 @@ import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.net.URI;
import org.jclouds.s3.BaseS3AsyncClientTest;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.BaseS3AsyncClientTest;
import org.jclouds.s3.S3AsyncClient;
import org.testng.annotations.Test;
import com.google.inject.TypeLiteral;
/**
* Tests behavior of {@code BindNoBucketLoggingToXmlPayload}
*
@ -35,7 +39,13 @@ import org.testng.annotations.Test;
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "BindNoBucketLoggingToXmlPayloadTest")
public class BindNoBucketLoggingToXmlPayloadTest extends BaseS3AsyncClientTest {
public class BindNoBucketLoggingToXmlPayloadTest extends BaseS3AsyncClientTest<S3AsyncClient> {
@Override
protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
};
}
public void testApplyInputStream() throws IOException {
@ -44,7 +54,7 @@ public class BindNoBucketLoggingToXmlPayloadTest extends BaseS3AsyncClientTest {
request = binder.bindToRequest(request, "bucket");
assertEquals(request.getPayload().getRawContent(),
"<BucketLoggingStatus xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>");
"<BucketLoggingStatus xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>");
}
}

28
apis/s3/src/test/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequestTest.java
View File

@ -26,15 +26,19 @@ import java.net.URI;
import javax.ws.rs.HttpMethod;
import org.jclouds.s3.BaseS3AsyncClientTest;
import org.jclouds.s3.domain.S3Object;
import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
import org.jclouds.http.HttpRequest;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.BaseS3AsyncClientTest;
import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.domain.S3Object;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.inject.TypeLiteral;
/**
* Tests behavior of {@code BindS3ObjectMetadataToRequest}
@ -43,7 +47,13 @@ import com.google.common.collect.ImmutableMultimap;
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "BindS3ObjectMetadataToRequestTest")
public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest {
public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest<S3AsyncClient> {
@Override
protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
};
}
@Test
public void testPassWithMinimumDetailsAndPayload5GB() {
@ -56,8 +66,8 @@ public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest {
HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
BindS3ObjectMetadataToRequest binder = injector.getInstance(BindS3ObjectMetadataToRequest.class);
assertEquals(binder.bindToRequest(request, object),
HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build());
assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint(
URI.create("http://localhost")).build());
}
@Test
@ -68,14 +78,14 @@ public class BindS3ObjectMetadataToRequestTest extends BaseS3AsyncClientTest {
object.setPayload(payload);
object.getMetadata().setKey("foo");
object.getMetadata().setCacheControl("no-cache");
object.getMetadata().setUserMetadata(ImmutableMap.of("foo", "bar"));
HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
BindS3ObjectMetadataToRequest binder = injector.getInstance(BindS3ObjectMetadataToRequest.class);
assertEquals(
binder.bindToRequest(request, object),
HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost"))
.headers(ImmutableMultimap.of("Cache-Control", "no-cache")).build());
assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint(
URI.create("http://localhost")).headers(
ImmutableMultimap.of("Cache-Control", "no-cache", "x-amz-meta-foo", "bar")).build());
}
@Test(expectedExceptions = IllegalArgumentException.class)

28
apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobRequestSignerTest.java
View File

@ -31,6 +31,7 @@ import org.jclouds.http.HttpRequest;
import org.jclouds.http.RequiresHttp;
import org.jclouds.io.payloads.PhantomPayload;
import org.jclouds.rest.ConfiguresRestClient;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.BaseS3AsyncClientTest;
import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.S3Client;
@ -40,6 +41,7 @@ import org.testng.annotations.Test;
import com.google.common.base.Supplier;
import com.google.inject.Module;
import com.google.inject.TypeLiteral;
/**
* Tests behavior of {@code S3BlobRequestSigner}
@ -48,39 +50,45 @@ import com.google.inject.Module;
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "S3BlobRequestSignerTest")
public class S3BlobRequestSignerTest extends BaseS3AsyncClientTest {
public class S3BlobRequestSignerTest extends BaseS3AsyncClientTest<S3AsyncClient> {
@Override
protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
};
}
private BlobRequestSigner signer;
private Factory blobFactory;
public void testSignGetBlob() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
NoSuchMethodException, IOException {
NoSuchMethodException, IOException {
HttpRequest request = signer.signGetBlob("container", "name");
assertRequestLineEquals(request, "GET https://container.s3.amazonaws.com/name HTTP/1.1");
assertNonPayloadHeadersEqual(
request,
"Authorization: AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
request,
"Authorization: AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
assertPayloadEquals(request, null, null, false);
assertEquals(request.getFilters().size(), 0);
}
public void testSignRemoveBlob() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
NoSuchMethodException, IOException {
NoSuchMethodException, IOException {
HttpRequest request = signer.signRemoveBlob("container", "name");
assertRequestLineEquals(request, "DELETE https://container.s3.amazonaws.com/name HTTP/1.1");
assertNonPayloadHeadersEqual(
request,
"Authorization: AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
request,
"Authorization: AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
assertPayloadEquals(request, null, null, false);
assertEquals(request.getFilters().size(), 0);
}
public void testSignPutBlob() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
NoSuchMethodException, IOException {
NoSuchMethodException, IOException {
Blob blob = blobFactory.create(null);
blob.getMetadata().setName("name");
blob.setPayload(new PhantomPayload());
@ -92,8 +100,8 @@ public class S3BlobRequestSignerTest extends BaseS3AsyncClientTest {
assertRequestLineEquals(request, "PUT https://container.s3.amazonaws.com/name HTTP/1.1");
assertNonPayloadHeadersEqual(
request,
"Authorization: AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
request,
"Authorization: AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nHost: container.s3.amazonaws.com\n");
assertContentHeadersEqual(request, "text/plain", null, null, null, (long) 2l, new byte[] { 0, 2, 4, 8 });

40
apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java
View File

@ -26,18 +26,21 @@ import java.util.Properties;
import javax.ws.rs.core.HttpHeaders;
import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.internal.GeneratedHttpRequest;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.BaseS3AsyncClientTest;
import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.domain.AccessControlList;
import org.jclouds.s3.domain.CannedAccessPolicy;
import org.jclouds.s3.domain.S3Object;
import org.jclouds.s3.options.PutObjectOptions;
import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.internal.GeneratedHttpRequest;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import com.google.inject.TypeLiteral;
/**
* Tests behavior of {@code RequestAuthorizeSignature}
*
@ -45,7 +48,13 @@ import org.testng.annotations.Test;
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "RequestAuthorizeSignatureTest")
public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest {
public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest<S3AsyncClient> {
@Override
protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
};
}
@DataProvider(parallel = true)
public Object[][] dataProvider() throws NoSuchMethodException {
@ -70,20 +79,20 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest {
filter.filter(request);
if (request.getFirstHeaderOrNull(HttpHeaders.DATE).equals(date))
assert signature.equals(request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION)) : String.format(
"sig: %s != %s on attempt %s", signature, request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION),
iterations);
"sig: %s != %s on attempt %s", signature, request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION),
iterations);
else
iterations++;
}
System.out.printf("%s: %d iterations before the timestamp updated %n", Thread.currentThread().getName(),
iterations);
iterations);
}
@Test
void testAppendBucketNameHostHeader() throws SecurityException, NoSuchMethodException {
HttpRequest request = processor.createRequest(S3AsyncClient.class.getMethod("getBucketLocation", String.class),
"bucket");
"bucket");
StringBuilder builder = new StringBuilder();
filter.appendBucketName(request, builder);
assertEquals(builder.toString(), "/bucket");
@ -98,12 +107,12 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest {
}
private GeneratedHttpRequest<S3AsyncClient> putBucketAcl() throws NoSuchMethodException {
return processor.createRequest(
S3AsyncClient.class.getMethod("putBucketACL", String.class, AccessControlList.class), "bucket",
AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE, "1234"));
return processor.createRequest(S3AsyncClient.class.getMethod("putBucketACL", String.class,
AccessControlList.class), "bucket", AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE,
"1234"));
}
// "?acl", "?location", "?logging", or "?torrent"
// "?acl", "?location", "?logging", "?uploads", or "?torrent"
@Test
void testAppendBucketNameHostHeaderService() throws SecurityException, NoSuchMethodException {
@ -129,16 +138,15 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest {
S3Object object = blobToS3Object.apply(BindBlobToMultipartFormTest.TEST_BLOB);
object.getMetadata().getUserMetadata().put("x-amz-Adrian", "foo");
HttpRequest request = processor.createRequest(
S3AsyncClient.class.getMethod("putObject", String.class, S3Object.class, PutObjectOptions[].class),
"bucket", object);
HttpRequest request = processor.createRequest(S3AsyncClient.class.getMethod("putObject", String.class,
S3Object.class, PutObjectOptions[].class), "bucket", object);
return request;
}
@Test
void testAppendBucketNameURIHost() throws SecurityException, NoSuchMethodException {
HttpRequest request = processor.createRequest(S3AsyncClient.class.getMethod("getBucketLocation", String.class),
"bucket");
"bucket");
assertEquals(request.getEndpoint().getHost(), "bucket.s3.amazonaws.com");
}

13
apis/walrus/src/test/java/org/jclouds/walrus/WalrusAsyncClientTestDisabled.java
View File

@ -19,14 +19,25 @@
package org.jclouds.walrus;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.S3AsyncClientTest;
import org.testng.annotations.Test;
import com.google.inject.TypeLiteral;
/**
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(enabled = false, groups = "unit", testName = "WalrusAsyncClientTest")
public class WalrusAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest {
public class WalrusAsyncClientTestDisabled extends S3AsyncClientTest<S3AsyncClient> {
@Override
protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
};
}
public WalrusAsyncClientTestDisabled() {
this.provider = "walrus";

5
common/aws/src/main/java/org/jclouds/aws/handlers/ParseAWSErrorFromXmlContent.java
View File

@ -27,6 +27,7 @@ import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.aws.AWSResponseException;
import org.jclouds.aws.domain.AWSError;
import org.jclouds.aws.util.AWSUtils;
import org.jclouds.http.HttpCommand;
@ -71,8 +72,10 @@ public class ParseAWSErrorFromXmlContent implements HttpErrorHandler {
error = utils.parseAWSErrorFromContent(command.getCurrentRequest(), response);
if (error != null) {
message = error.getMessage();
exception = new AWSResponseException(command, response, error);
} else {
exception = new HttpResponseException(command, response, message);
}
exception = new HttpResponseException(command, response, message);
} else {
try {
message = Strings2.toStringAndClose(response.getPayload().getInput());

4
core/src/main/java/org/jclouds/io/ContentMetadata.java
View File

@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableSet;
*/
public interface ContentMetadata {
public static final Set<String> HTTP_HEADERS = ImmutableSet.of(CONTENT_LENGTH, "Content-MD5", CONTENT_TYPE,
"Content-Disposition", "Content-Encoding", "Content-Language");
"Content-Disposition", "Content-Encoding", "Content-Language");
/**
* Returns the total size of the payload, or the chunk that's available.
@ -87,4 +87,6 @@ public interface ContentMetadata {
@Nullable
String getContentLanguage();
ContentMetadataBuilder toBuilder();
}

186
core/src/main/java/org/jclouds/io/ContentMetadataBuilder.java Normal file
View File

@ -0,0 +1,186 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.io;
import static com.google.common.collect.Iterables.any;
import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH;
import static javax.ws.rs.core.HttpHeaders.CONTENT_TYPE;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Map.Entry;
import javax.annotation.Nullable;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.io.payloads.BaseImmutableContentMetadata;
import com.google.common.base.Predicate;
import com.google.common.collect.Multimap;
/**
* @author Adrian Cole
*/
public class ContentMetadataBuilder implements Serializable {
/** The serialVersionUID */
private static final long serialVersionUID = -5279643002875371558L;
public static ContentMetadataBuilder create() {
return new ContentMetadataBuilder();
}
protected String contentType = "application/unknown";
protected Long contentLength;
protected byte[] contentMD5;
protected String contentDisposition;
protected String contentLanguage;
protected String contentEncoding;
public ContentMetadataBuilder fromHttpHeaders(Multimap<String, String> headers) {
boolean chunked = any(headers.entries(), new Predicate<Entry<String, String>>() {
@Override
public boolean apply(Entry<String, String> input) {
return "Transfer-Encoding".equalsIgnoreCase(input.getKey()) && "chunked".equalsIgnoreCase(input.getValue());
}
});
for (Entry<String, String> header : headers.entries()) {
if (!chunked && CONTENT_LENGTH.equalsIgnoreCase(header.getKey())) {
contentLength(new Long(header.getValue()));
} else if ("Content-MD5".equalsIgnoreCase(header.getKey())) {
contentMD5(CryptoStreams.base64(header.getValue()));
} else if (CONTENT_TYPE.equalsIgnoreCase(header.getKey())) {
contentType(header.getValue());
} else if ("Content-Disposition".equalsIgnoreCase(header.getKey())) {
contentDisposition(header.getValue());
} else if ("Content-Encoding".equalsIgnoreCase(header.getKey())) {
contentEncoding(header.getValue());
} else if ("Content-Language".equalsIgnoreCase(header.getKey())) {
contentLanguage(header.getValue());
}
}
return this;
}
public ContentMetadataBuilder contentLength(@Nullable Long contentLength) {
this.contentLength = contentLength;
return this;
}
public ContentMetadataBuilder contentMD5(byte[] md5) {
if (md5 != null) {
byte[] retval = new byte[md5.length];
System.arraycopy(md5, 0, retval, 0, md5.length);
this.contentMD5 = md5;
}
return this;
}
public ContentMetadataBuilder contentType(@Nullable String contentType) {
this.contentType = contentType;
return this;
}
public ContentMetadataBuilder contentDisposition(@Nullable String contentDisposition) {
this.contentDisposition = contentDisposition;
return this;
}
public ContentMetadataBuilder contentLanguage(@Nullable String contentLanguage) {
this.contentLanguage = contentLanguage;
return this;
}
public ContentMetadataBuilder contentEncoding(@Nullable String contentEncoding) {
this.contentEncoding = contentEncoding;
return this;
}
public ContentMetadata build() {
return new BaseImmutableContentMetadata(contentType, contentLength, contentMD5, contentDisposition,
contentLanguage, contentEncoding);
}
public static ContentMetadataBuilder fromContentMetadata(ContentMetadata in) {
return new ContentMetadataBuilder().contentType(in.getContentType()).contentLength(in.getContentLength())
.contentMD5(in.getContentMD5()).contentDisposition(in.getContentDisposition()).contentLanguage(
in.getContentLanguage()).contentEncoding(in.getContentEncoding());
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((contentDisposition == null) ? 0 : contentDisposition.hashCode());
result = prime * result + ((contentEncoding == null) ? 0 : contentEncoding.hashCode());
result = prime * result + ((contentLanguage == null) ? 0 : contentLanguage.hashCode());
result = prime * result + ((contentLength == null) ? 0 : contentLength.hashCode());
result = prime * result + Arrays.hashCode(contentMD5);
result = prime * result + ((contentType == null) ? 0 : contentType.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
ContentMetadataBuilder other = (ContentMetadataBuilder) obj;
if (contentDisposition == null) {
if (other.contentDisposition != null)
return false;
} else if (!contentDisposition.equals(other.contentDisposition))
return false;
if (contentEncoding == null) {
if (other.contentEncoding != null)
return false;
} else if (!contentEncoding.equals(other.contentEncoding))
return false;
if (contentLanguage == null) {
if (other.contentLanguage != null)
return false;
} else if (!contentLanguage.equals(other.contentLanguage))
return false;
if (contentLength == null) {
if (other.contentLength != null)
return false;
} else if (!contentLength.equals(other.contentLength))
return false;
if (!Arrays.equals(contentMD5, other.contentMD5))
return false;
if (contentType == null) {
if (other.contentType != null)
return false;
} else if (!contentType.equals(other.contentType))
return false;
return true;
}
@Override
public String toString() {
return "[contentDisposition=" + contentDisposition + ", contentEncoding=" + contentEncoding
+ ", contentLanguage=" + contentLanguage + ", contentLength=" + contentLength + ", contentMD5="
+ Arrays.toString(contentMD5) + ", contentType=" + contentType + "]";
}
}

6
core/src/main/java/org/jclouds/io/payloads/BaseImmutableContentMetadata.java
View File

@ -23,6 +23,7 @@ import java.io.Serializable;
import java.util.Arrays;
import org.jclouds.io.ContentMetadata;
import org.jclouds.io.ContentMetadataBuilder;
/**
* @author Adrian Cole
@ -161,4 +162,9 @@ public class BaseImmutableContentMetadata implements ContentMetadata, Serializab
return true;
}
@Override
public ContentMetadataBuilder toBuilder() {
return ContentMetadataBuilder.fromContentMetadata(this);
}
}

129
core/src/main/java/org/jclouds/io/payloads/BaseMutableContentMetadata.java
View File

@ -19,63 +19,26 @@
package org.jclouds.io.payloads;
import static com.google.common.collect.Iterables.any;
import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH;
import static javax.ws.rs.core.HttpHeaders.CONTENT_TYPE;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Map.Entry;
import javax.annotation.Nullable;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.io.ContentMetadata;
import org.jclouds.io.ContentMetadataBuilder;
import org.jclouds.io.MutableContentMetadata;
import com.google.common.base.Predicate;
import com.google.common.collect.Multimap;
/**
* @author Adrian Cole
*/
public class BaseMutableContentMetadata implements MutableContentMetadata, Serializable {
public class BaseMutableContentMetadata extends ContentMetadataBuilder implements MutableContentMetadata, Serializable {
/** The serialVersionUID */
private static final long serialVersionUID = 8364286391963469370L;
@Override
public void setPropertiesFromHttpHeaders(Multimap<String, String> headers) {
boolean chunked = any(headers.entries(), new Predicate<Entry<String, String>>() {
@Override
public boolean apply(Entry<String, String> input) {
return "Transfer-Encoding".equalsIgnoreCase(input.getKey()) && "chunked".equalsIgnoreCase(input.getValue());
}
});
for (Entry<String, String> header : headers.entries()) {
if (!chunked && CONTENT_LENGTH.equalsIgnoreCase(header.getKey())) {
setContentLength(new Long(header.getValue()));
} else if ("Content-MD5".equalsIgnoreCase(header.getKey())) {
setContentMD5(CryptoStreams.base64(header.getValue()));
} else if (CONTENT_TYPE.equalsIgnoreCase(header.getKey())) {
setContentType(header.getValue());
} else if ("Content-Disposition".equalsIgnoreCase(header.getKey())) {
setContentDisposition(header.getValue());
} else if ("Content-Encoding".equalsIgnoreCase(header.getKey())) {
setContentEncoding(header.getValue());
} else if ("Content-Language".equalsIgnoreCase(header.getKey())) {
setContentLanguage(header.getValue());
}
}
}
/** The serialVersionUID */
private static final long serialVersionUID = 4572381435863125873L;
protected String contentType = "application/unknown";
protected Long contentLength;
protected byte[] contentMD5;
protected String contentDisposition;
protected String contentLanguage;
protected String contentEncoding;
public BaseMutableContentMetadata() {
super();
fromHttpHeaders(headers);
}
/**
@ -91,7 +54,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
*/
@Override
public void setContentLength(@Nullable Long contentLength) {
this.contentLength = contentLength;
contentLength(contentLength);
}
/**
@ -113,11 +76,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
*/
@Override
public void setContentMD5(byte[] md5) {
if (md5 != null) {
byte[] retval = new byte[md5.length];
System.arraycopy(md5, 0, retval, 0, md5.length);
this.contentMD5 = md5;
}
contentMD5(md5);
}
/**
@ -133,7 +92,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
*/
@Override
public void setContentType(@Nullable String contentType) {
this.contentType = contentType;
contentType(contentType);
}
/**
@ -141,7 +100,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
*/
@Override
public void setContentDisposition(@Nullable String contentDisposition) {
this.contentDisposition = contentDisposition;
contentDisposition(contentDisposition);
}
/**
@ -157,7 +116,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
*/
@Override
public void setContentLanguage(@Nullable String contentLanguage) {
this.contentLanguage = contentLanguage;
contentLanguage(contentLanguage);
}
/**
@ -173,7 +132,7 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
*/
@Override
public void setContentEncoding(@Nullable String contentEncoding) {
this.contentEncoding = contentEncoding;
contentEncoding(contentEncoding);
}
/**
@ -185,62 +144,14 @@ public class BaseMutableContentMetadata implements MutableContentMetadata, Seria
}
@Override
public String toString() {
return "[contentType=" + contentType + ", contentLength=" + contentLength + ", contentDisposition="
+ contentDisposition + ", contentEncoding=" + contentEncoding + ", contentLanguage=" + contentLanguage
+ ", contentMD5=" + Arrays.toString(contentMD5) + "]";
public BaseMutableContentMetadata toBuilder() {
return BaseMutableContentMetadata.fromContentMetadata(this);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((contentDisposition == null) ? 0 : contentDisposition.hashCode());
result = prime * result + ((contentEncoding == null) ? 0 : contentEncoding.hashCode());
result = prime * result + ((contentLanguage == null) ? 0 : contentLanguage.hashCode());
result = prime * result + ((contentLength == null) ? 0 : contentLength.hashCode());
result = prime * result + Arrays.hashCode(contentMD5);
result = prime * result + ((contentType == null) ? 0 : contentType.hashCode());
return result;
public static BaseMutableContentMetadata fromContentMetadata(ContentMetadata in) {
return (BaseMutableContentMetadata) new BaseMutableContentMetadata().contentType(in.getContentType())
.contentLength(in.getContentLength()).contentMD5(in.getContentMD5()).contentDisposition(
in.getContentDisposition()).contentLanguage(in.getContentLanguage()).contentEncoding(
in.getContentEncoding());
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
BaseMutableContentMetadata other = (BaseMutableContentMetadata) obj;
if (contentDisposition == null) {
if (other.contentDisposition != null)
return false;
} else if (!contentDisposition.equals(other.contentDisposition))
return false;
if (contentEncoding == null) {
if (other.contentEncoding != null)
return false;
} else if (!contentEncoding.equals(other.contentEncoding))
return false;
if (contentLanguage == null) {
if (other.contentLanguage != null)
return false;
} else if (!contentLanguage.equals(other.contentLanguage))
return false;
if (contentLength == null) {
if (other.contentLength != null)
return false;
} else if (!contentLength.equals(other.contentLength))
return false;
if (!Arrays.equals(contentMD5, other.contentMD5))
return false;
if (contentType == null) {
if (other.contentType != null)
return false;
} else if (!contentType.equals(other.contentType))
return false;
return true;
}
}

5
core/src/main/java/org/jclouds/io/payloads/PhantomPayload.java
View File

@ -25,6 +25,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.jclouds.io.ContentMetadata;
import org.jclouds.io.MutableContentMetadata;
/**
@ -36,6 +37,10 @@ public class PhantomPayload extends BasePayload<Object> {
super(Object.class);
}
public PhantomPayload(ContentMetadata contentMetadata) {
this(BaseMutableContentMetadata.fromContentMetadata(checkNotNull(contentMetadata, "contentMetadata")));
}
public PhantomPayload(MutableContentMetadata contentMetadata) {
super(Object.class, checkNotNull(contentMetadata, "contentMetadata"));
}

12
providers/aws-s3/pom.xml
View File

@ -70,6 +70,18 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-apachehc</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-log4j</artifactId>

76
providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3AsyncClient.java
View File

@ -18,6 +18,7 @@
*/
package org.jclouds.aws.s3;
/**
*
*
@ -38,11 +39,41 @@ package org.jclouds.aws.s3;
import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER;
import java.util.Map;
import javax.ws.rs.DELETE;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import org.jclouds.aws.s3.binders.BindObjectMetadataToRequest;
import org.jclouds.aws.s3.binders.BindPartIdsAndETagsToRequest;
import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex;
import org.jclouds.aws.s3.functions.ObjectMetadataKey;
import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex;
import org.jclouds.blobstore.attr.BlobScope;
import org.jclouds.http.functions.ParseETagHeader;
import org.jclouds.io.Payload;
import org.jclouds.rest.annotations.BinderParam;
import org.jclouds.rest.annotations.ExceptionParser;
import org.jclouds.rest.annotations.ParamParser;
import org.jclouds.rest.annotations.ParamValidators;
import org.jclouds.rest.annotations.QueryParams;
import org.jclouds.rest.annotations.RequestFilters;
import org.jclouds.rest.annotations.ResponseParser;
import org.jclouds.rest.annotations.SkipEncoding;
import org.jclouds.rest.functions.ReturnVoidOnNotFoundOr404;
import org.jclouds.s3.Bucket;
import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
import org.jclouds.s3.options.PutObjectOptions;
import org.jclouds.s3.predicates.validators.BucketNameValidator;
import com.google.common.util.concurrent.ListenableFuture;
/**
* Provides access to amazon-specific S3 features
@ -53,5 +84,48 @@ import org.jclouds.s3.filters.RequestAuthorizeSignature;
@RequestFilters(RequestAuthorizeSignature.class)
@BlobScope(CONTAINER)
public interface AWSS3AsyncClient extends S3AsyncClient {
/**
* @see AWSS3Client#initiateMultipartUpload
*/
@POST
@QueryParams(keys = "uploads")
@Path("/{key}")
@ResponseParser(UploadIdFromHttpResponseViaRegex.class)
ListenableFuture<String> initiateMultipartUpload(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName,
@PathParam("key") @ParamParser(ObjectMetadataKey.class) @BinderParam(BindObjectMetadataToRequest.class) ObjectMetadata objectMetadata,
PutObjectOptions... options);
/**
* @see AWSS3Client#abortMultipartUpload
*/
@DELETE
@Path("/{key}")
@ExceptionParser(ReturnVoidOnNotFoundOr404.class)
ListenableFuture<Void> abortMultipartUpload(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName,
@PathParam("key") String key, @QueryParam("uploadId") String uploadId);
/**
* @see AWSS3Client#uploadPart
*/
@PUT
@Path("/{key}")
@ResponseParser(ParseETagHeader.class)
ListenableFuture<String> uploadPart(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName,
@PathParam("key") String key, @QueryParam("partNumber") int partNumber,
@QueryParam("uploadId") String uploadId, Payload part);
/**
* @see AWSS3Client#completeMultipartUpload
*/
@POST
@Path("/{key}")
@ResponseParser(ETagFromHttpResponseViaRegex.class)
ListenableFuture<String> completeMultipartUpload(
@Bucket @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators( { BucketNameValidator.class }) String bucketName,
@PathParam("key") String key, @QueryParam("uploadId") String uploadId,
@BinderParam(BindPartIdsAndETagsToRequest.class) Map<Integer, String> parts);
}

109
providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java
View File

@ -19,10 +19,14 @@
package org.jclouds.aws.s3;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.jclouds.concurrent.Timeout;
import org.jclouds.io.Payload;
import org.jclouds.s3.S3Client;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.options.PutObjectOptions;
/**
* Provides access to amazon-specific S3 features
@ -32,5 +36,110 @@ import org.jclouds.s3.S3Client;
*/
@Timeout(duration = 90, timeUnit = TimeUnit.SECONDS)
public interface AWSS3Client extends S3Client {
/**
* This operation initiates a multipart upload and returns an upload ID. This upload ID is used
* to associate all the parts in the specific multipart upload. You specify this upload ID in
* each of your subsequent upload part requests (see Upload Part). You also include this upload
* ID in the final request to either complete or abort the multipart upload request.
*
* <h4>Note</h4> If you create an object using the multipart upload APIs, currently you cannot
* copy the object between regions.
*
*
* @param bucketName
* namespace of the object you are to upload
* @param objectMetadata
* metadata around the object you wish to upload
* @param options
* controls optional parameters such as canned ACL
* @return ID for the initiated multipart upload.
*/
String initiateMultipartUpload(String bucketName, ObjectMetadata objectMetadata, PutObjectOptions... options);
/**
* This operation aborts a multipart upload. After a multipart upload is aborted, no additional
* parts can be uploaded using that upload ID. The storage consumed by any previously uploaded
* parts will be freed. However, if any part uploads are currently in progress, those part
* uploads might or might not succeed. As a result, it might be necessary to abort a given
* multipart upload multiple times in order to completely free all storage consumed by all parts.
*
*
* @param bucketName
* namespace of the object you are deleting
* @param key
* unique key in the s3Bucket identifying the object
* @param uploadId
* id of the multipart upload in progress.
*/
void abortMultipartUpload(String bucketName, String key, String uploadId);
/**
* This operation uploads a part in a multipart upload. You must initiate a multipart upload (see
* Initiate Multipart Upload) before you can upload any part. In response to your initiate
* request. Amazon S3 returns an upload ID, a unique identifier, that you must include in your
* upload part request.
*
* <p/>
* Part numbers can be any number from 1 to 10,000, inclusive. A part number uniquely identifies
* a part and also defines its position within the object being created. If you upload a new part
* using the same part number that was used with a previous part, the previously uploaded part is
* overwritten. Each part must be at least 5 MB in size, except the last part. There is no size
* limit on the last part of your multipart upload.
*
* <p/>
* To ensure that data is not corrupted when traversing the network, specify the Content-MD5
* header in the upload part request. Amazon S3 checks the part data against the provided MD5
* value. If they do not match, Amazon S3 returns an error.
*
*
* @param bucketName
* namespace of the object you are storing
* @param key
* unique key in the s3Bucket identifying the object
* @param partNumber
* which part is this.
* @param uploadId
* id of the multipart upload in progress.
* @param part
* contains the data to create or overwrite
* @return ETag of the content uploaded
* @see <a href="http://docs.amazonwebservices.com/AmazonS3/latest/API/mpUploadUploadPart.html"
* />
*/
@Timeout(duration = 5 * 1024 * 1024 / 128, timeUnit = TimeUnit.SECONDS)
String uploadPart(String bucketName, String key, int partNumber, String uploadId, Payload part);
/**
*
This operation completes a multipart upload by assembling previously uploaded parts.
* <p/>
* You first initiate the multipart upload and then upload all parts using the Upload Parts
* operation (see Upload Part). After successfully uploading all relevant parts of an upload, you
* call this operation to complete the upload. Upon receiving this request, Amazon S3
* concatenates all the parts in ascending order by part number to create a new object. In the
* Complete Multipart Upload request, you must provide the parts list. For each part in the list,
* you must provide the part number and the ETag header value, returned after that part was
* uploaded.
* <p/>
* Processing of a Complete Multipart Upload request could take several minutes to complete.
* After Amazon S3 begins processing the request, it sends an HTTP response header that specifies
* a 200 OK response. While processing is in progress, Amazon S3 periodically sends whitespace
* characters to keep the connection from timing out. Because a request could fail after the
* initial 200 OK response has been sent, it is important that you check the response body to
* determine whether the request succeeded.
* <p/>
* Note that if Complete Multipart Upload fails, applications should be prepared to retry the
* failed requests.
*
* @param bucketName
* namespace of the object you are deleting
* @param key
* unique key in the s3Bucket identifying the object
* @param uploadId
* id of the multipart upload in progress.
* @param parts
* a map of part id to eTag from the {@link #uploadPart} command.
* @return ETag of the content uploaded
*/
String completeMultipartUpload(String bucketName, String key, String uploadId, Map<Integer, String> parts);
}

83
providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequest.java Normal file
View File

@ -0,0 +1,83 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.binders;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.ws.rs.core.HttpHeaders;
import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.utils.ModifyRequest;
import org.jclouds.rest.Binder;
import org.jclouds.s3.domain.ObjectMetadata;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableMultimap.Builder;
/**
*
* @author Adrian Cole
*/
@Singleton
public class BindObjectMetadataToRequest implements Binder {
protected final BindMapToHeadersWithPrefix metadataPrefixer;
@Inject
public BindObjectMetadataToRequest(BindMapToHeadersWithPrefix metadataPrefixer) {
this.metadataPrefixer = checkNotNull(metadataPrefixer, "metadataPrefixer");
}
@Override
public <R extends HttpRequest> R bindToRequest(R request, Object input) {
checkArgument(checkNotNull(input, "input") instanceof ObjectMetadata,
"this binder is only valid for ObjectMetadata!");
checkNotNull(request, "request");
ObjectMetadata md = ObjectMetadata.class.cast(input);
checkArgument(md.getKey() != null, "objectMetadata.getKey() must be set!");
request = metadataPrefixer.bindToRequest(request, md.getUserMetadata());
Builder<String, String> headers = ImmutableMultimap.<String, String> builder();
if (md.getCacheControl() != null) {
headers.put(HttpHeaders.CACHE_CONTROL, md.getCacheControl());
}
if (md.getContentMetadata().getContentDisposition() != null) {
headers.put("Content-Disposition", md.getContentMetadata().getContentDisposition());
}
if (md.getContentMetadata().getContentEncoding() != null) {
headers.put("Content-Encoding", md.getContentMetadata().getContentEncoding());
}
if (md.getContentMetadata().getContentType() != null) {
headers.put(HttpHeaders.CONTENT_TYPE, md.getContentMetadata().getContentType());
} else {
headers.put(HttpHeaders.CONTENT_TYPE, "binary/octet-stream");
}
request = ModifyRequest.replaceHeaders(request, headers.build());
return request;
}
}

65
providers/aws-s3/src/main/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequest.java Normal file
View File

@ -0,0 +1,65 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.binders;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import java.util.Map;
import java.util.Map.Entry;
import javax.inject.Singleton;
import javax.ws.rs.core.MediaType;
import org.jclouds.http.HttpRequest;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.rest.Binder;
/**
*
* @author Adrian Cole
*/
@Singleton
public class BindPartIdsAndETagsToRequest implements Binder {
@SuppressWarnings("unchecked")
@Override
public <R extends HttpRequest> R bindToRequest(R request, Object input) {
checkArgument(checkNotNull(input, "input") instanceof Map, "this binder is only valid for Map!");
checkNotNull(request, "request");
Map<Integer, String> map = (Map<Integer, String>) input;
checkArgument(map.size() != 0, "Please send parts");
StringBuilder content = new StringBuilder();
content.append("<CompleteMultipartUpload>");
for (Entry<Integer, String> entry : map.entrySet()) {
content.append("<Part>");
content.append("<PartNumber>").append(entry.getKey()).append("</PartNumber>");
content.append("<ETag>").append(entry.getValue()).append("</ETag>");
content.append("</Part>");
}
content.append("</CompleteMultipartUpload>");
Payload payload = Payloads.newStringPayload(content.toString());
payload.getContentMetadata().setContentType(MediaType.TEXT_XML);
request.setPayload(payload);
return request;
}
}

60
providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegex.java Normal file
View File

@ -0,0 +1,60 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.functions;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.functions.ReturnStringIf2xx;
import com.google.common.base.Function;
/**
* @see <a href="http://docs.amazonwebservices.com/AmazonS3/latest/API/mpUploadComplete.html" />
* @author Adrian Cole
*/
@Singleton
public class ETagFromHttpResponseViaRegex implements Function<HttpResponse, String> {
Pattern pattern = Pattern.compile("<ETag>([\\S&&[^<]]+)</ETag>");
private final ReturnStringIf2xx returnStringIf200;
@Inject
ETagFromHttpResponseViaRegex(ReturnStringIf2xx returnStringIf200) {
this.returnStringIf200 = returnStringIf200;
}
@Override
public String apply(HttpResponse response) {
String value = null;
String content = returnStringIf200.apply(response);
if (content != null) {
Matcher matcher = pattern.matcher(content);
if (matcher.find()) {
value = matcher.group(1);
}
}
return value;
}
}

39
providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ObjectMetadataKey.java Normal file
View File

@ -0,0 +1,39 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.functions;
import javax.inject.Singleton;
import org.jclouds.s3.domain.ObjectMetadata;
import com.google.common.base.Function;
/**
*
* @author Adrian Cole
*/
@Singleton
public class ObjectMetadataKey implements Function<Object, String> {
public String apply(Object from) {
return ((ObjectMetadata) from).getKey();
}
}

62
providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegex.java Normal file
View File

@ -0,0 +1,62 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.functions;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.functions.ReturnStringIf2xx;
import com.google.common.base.Function;
/**
* @see <a
* href="http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadInitiate.html"
* />
* @author Adrian Cole
*/
@Singleton
public class UploadIdFromHttpResponseViaRegex implements Function<HttpResponse, String> {
Pattern pattern = Pattern.compile("<UploadId>([\\S&&[^<]]+)</UploadId>");
private final ReturnStringIf2xx returnStringIf200;
@Inject
UploadIdFromHttpResponseViaRegex(ReturnStringIf2xx returnStringIf200) {
this.returnStringIf200 = returnStringIf200;
}
@Override
public String apply(HttpResponse response) {
String value = null;
String content = returnStringIf200.apply(response);
if (content != null) {
Matcher matcher = pattern.matcher(content);
if (matcher.find()) {
value = matcher.group(1);
}
}
return value;
}
}

134
providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3AsyncClientTest.java
View File

@ -22,43 +22,143 @@ package org.jclouds.aws.s3;
import java.io.IOException;
import java.lang.reflect.Array;
import java.lang.reflect.Method;
import java.util.Map;
import java.util.Properties;
import org.jclouds.aws.s3.config.AWSS3RestClientModule;
import org.jclouds.aws.s3.functions.ETagFromHttpResponseViaRegex;
import org.jclouds.aws.s3.functions.UploadIdFromHttpResponseViaRegex;
import org.jclouds.date.TimeStamp;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.RequiresHttp;
import org.jclouds.http.functions.ParseETagHeader;
import org.jclouds.http.functions.ReleasePayloadAndReturn;
import org.jclouds.http.functions.ReturnTrueIf2xx;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.rest.ConfiguresRestClient;
import org.jclouds.rest.RestContextFactory;
import org.jclouds.s3.S3AsyncClient;
import org.jclouds.rest.functions.MapHttp4xxCodesToExceptions;
import org.jclouds.rest.functions.ReturnVoidOnNotFoundOr404;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.ObjectMetadataBuilder;
import org.jclouds.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYouOrIllegalState;
import org.jclouds.s3.options.PutBucketOptions;
import org.jclouds.s3.options.PutObjectOptions;
import org.testng.annotations.Test;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Module;
import com.google.inject.TypeLiteral;
/**
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "AWSS3AsyncClientTest")
public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest {
public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest<AWSS3AsyncClient> {
public AWSS3AsyncClientTest() {
this.provider = "aws-s3";
}
@Override
protected TypeLiteral<RestAnnotationProcessor<AWSS3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<AWSS3AsyncClient>>() {
};
}
@Override
protected Properties getProperties() {
return RestContextFactory.getPropertiesFromResource("/rest.properties");
}
public void testInitiateMultipartUpload() throws SecurityException, NegativeArraySizeException,
NoSuchMethodException {
Method method = AWSS3AsyncClient.class.getMethod("initiateMultipartUpload", String.class, ObjectMetadata.class,
PutObjectOptions[].class);
HttpRequest request = processor
.createRequest(method, "bucket", ObjectMetadataBuilder.create().key("foo").build());
assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploads HTTP/1.1");
assertNonPayloadHeadersEqual(request, "Content-Type: binary/octet-stream\nHost: bucket." + url + "\n");
assertPayloadEquals(request, null, null, false);
assertResponseParserClassEquals(method, request, UploadIdFromHttpResponseViaRegex.class);
assertSaxResponseParserClassEquals(method, null);
assertExceptionParserClassEquals(method, MapHttp4xxCodesToExceptions.class);
checkFilters(request);
}
public void testAbortMultipartUpload() throws SecurityException, NegativeArraySizeException, NoSuchMethodException {
Method method = AWSS3AsyncClient.class
.getMethod("abortMultipartUpload", String.class, String.class, String.class);
HttpRequest request = processor.createRequest(method, "bucket", "foo", "asdsadasdas", 1, Payloads
.newStringPayload(""));
assertRequestLineEquals(request, "DELETE https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1");
assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
assertPayloadEquals(request, "", "application/unknown", false);
assertResponseParserClassEquals(method, request, ReleasePayloadAndReturn.class);
assertSaxResponseParserClassEquals(method, null);
assertExceptionParserClassEquals(method, ReturnVoidOnNotFoundOr404.class);
checkFilters(request);
}
public void testUploadPart() throws SecurityException, NegativeArraySizeException, NoSuchMethodException {
Method method = AWSS3AsyncClient.class.getMethod("uploadPart", String.class, String.class, int.class,
String.class, Payload.class);
HttpRequest request = processor.createRequest(method, "bucket", "foo", 1, "asdsadasdas", Payloads
.newStringPayload(""));
assertRequestLineEquals(request, "PUT https://bucket." + url + "/foo?partNumber=1&uploadId=asdsadasdas HTTP/1.1");
assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
assertPayloadEquals(request, "", "application/unknown", false);
assertResponseParserClassEquals(method, request, ParseETagHeader.class);
assertSaxResponseParserClassEquals(method, null);
assertExceptionParserClassEquals(method, MapHttp4xxCodesToExceptions.class);
checkFilters(request);
}
public void testCompleteMultipartUpload() throws SecurityException, NegativeArraySizeException,
NoSuchMethodException {
Method method = AWSS3AsyncClient.class.getMethod("completeMultipartUpload", String.class, String.class,
String.class, Map.class);
HttpRequest request = processor.createRequest(method, "bucket", "foo", "asdsadasdas", ImmutableMap
.<Integer, String> of(1, "\"a54357aff0632cce46d942af68356b38\""));
assertRequestLineEquals(request, "POST https://bucket." + url + "/foo?uploadId=asdsadasdas HTTP/1.1");
assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
assertPayloadEquals(
request,
"<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>\"a54357aff0632cce46d942af68356b38\"</ETag></Part></CompleteMultipartUpload>",
"text/xml", false);
assertResponseParserClassEquals(method, request, ETagFromHttpResponseViaRegex.class);
assertSaxResponseParserClassEquals(method, null);
assertExceptionParserClassEquals(method, MapHttp4xxCodesToExceptions.class);
checkFilters(request);
}
public void testPutBucketEu() throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
NoSuchMethodException, IOException {
Method method = S3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class,
Array.newInstance(PutBucketOptions.class, 0).getClass());
NoSuchMethodException, IOException {
Method method = AWSS3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class, Array
.newInstance(PutBucketOptions.class, 0).getClass());
HttpRequest request = processor.createRequest(method, "EU", "bucket");
assertRequestLineEquals(request, "PUT https://bucket." + url + "/ HTTP/1.1");
assertNonPayloadHeadersEqual(request, "Host: bucket." + url + "\n");
assertPayloadEquals(request,
"<CreateBucketConfiguration><LocationConstraint>EU</LocationConstraint></CreateBucketConfiguration>",
"text/xml", false);
"<CreateBucketConfiguration><LocationConstraint>EU</LocationConstraint></CreateBucketConfiguration>",
"text/xml", false);
assertResponseParserClassEquals(method, request, ReturnTrueIf2xx.class);
assertSaxResponseParserClassEquals(method, null);
@ -66,4 +166,24 @@ public class AWSS3AsyncClientTest extends org.jclouds.s3.S3AsyncClientTest {
checkFilters(request);
}
@RequiresHttp
@ConfiguresRestClient
private static final class TestAWSS3RestClientModule extends AWSS3RestClientModule {
public TestAWSS3RestClientModule() {
super();
}
@Override
protected String provideTimeStamp(@TimeStamp Supplier<String> cache) {
return "2009-11-08T15:54:08.897Z";
}
}
@Override
protected Module createModule() {
return new TestAWSS3RestClientModule();
}
}

74
providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java
View File

@ -19,9 +19,29 @@
package org.jclouds.aws.s3;
import static org.testng.Assert.assertEquals;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPInputStream;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.BaseJettyTest;
import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.s3.S3ClientLiveTest;
import org.jclouds.s3.domain.ObjectMetadataBuilder;
import org.testng.ITestContext;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.ByteStreams;
import com.google.common.io.InputSupplier;
import com.google.inject.Module;
/**
* Tests behavior of {@code S3Client}
*
@ -29,5 +49,59 @@ import org.testng.annotations.Test;
*/
@Test(groups = "live", sequential = true, testName = "AWSS3ClientLiveTest")
public class AWSS3ClientLiveTest extends S3ClientLiveTest {
private InputSupplier<InputStream> oneHundredOneConstitutions;
private byte[] oneHundredOneConstitutionsMD5;
private static long oneHundredOneConstitutionsLength;
@Override
public AWSS3Client getApi() {
return (AWSS3Client) context.getProviderSpecificContext().getApi();
}
@BeforeClass(groups = { "integration", "live" })
@Override
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext);
oneHundredOneConstitutions = getTestDataSupplier();
oneHundredOneConstitutionsMD5 = CryptoStreams.md5(oneHundredOneConstitutions);
}
@SuppressWarnings("unchecked")
public static InputSupplier<InputStream> getTestDataSupplier() throws IOException {
byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class
.getResourceAsStream("/const.txt.gz")));
InputSupplier<ByteArrayInputStream> constitutionSupplier = ByteStreams.newInputStreamSupplier(oneConstitution);
InputSupplier<InputStream> temp = ByteStreams.join(constitutionSupplier);
// we have to go beyond 5MB per part
for (oneHundredOneConstitutionsLength = oneConstitution.length; oneHundredOneConstitutionsLength < 5 * 1024 * 1024; oneHundredOneConstitutionsLength += oneConstitution.length) {
temp = ByteStreams.join(temp, constitutionSupplier);
}
return temp;
}
public void testMultipartSynchronously() throws InterruptedException, IOException {
String containerName = getContainerName();
try {
String key = "constitution.txt";
String uploadId = getApi().initiateMultipartUpload(containerName,
ObjectMetadataBuilder.create().key(key).build());
byte[] buffer = ByteStreams.toByteArray(oneHundredOneConstitutions.getInput());
Payload part1 = Payloads.newByteArrayPayload(buffer);
part1.getContentMetadata().setContentLength((long) buffer.length);
part1.getContentMetadata().setContentMD5(oneHundredOneConstitutionsMD5);
// failure here looks very similar to http://java.net/jira/browse/GLASSFISH-15773
String eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
String eTag = getApi().completeMultipartUpload(containerName, key, uploadId, ImmutableMap.of(1, eTagOf1));
assertEquals(eTagOf1, eTag);
} finally {
returnContainer(containerName);
}
}
}

104
providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindObjectMetadataToRequestTest.java Normal file
View File

@ -0,0 +1,104 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.binders;
import static org.testng.Assert.assertEquals;
import java.io.File;
import java.net.URI;
import javax.ws.rs.HttpMethod;
import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.BaseS3AsyncClientTest;
import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.ObjectMetadataBuilder;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.inject.TypeLiteral;
/**
* Tests behavior of {@code BindObjectMetadataToRequest}
*
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "BindObjectMetadataToRequestTest")
public class BindObjectMetadataToRequestTest extends BaseS3AsyncClientTest<S3AsyncClient> {
@Override
protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
};
}
@Test
public void testPassWithMinimumDetailsAndPayload5GB() {
ObjectMetadata md = ObjectMetadataBuilder.create().key("foo").build();
HttpRequest request = HttpRequest.builder().method("POST").endpoint(URI.create("http://localhost")).build();
BindObjectMetadataToRequest binder = injector.getInstance(BindObjectMetadataToRequest.class);
assertEquals(binder.bindToRequest(request, md), HttpRequest.builder().method("POST").endpoint(
URI.create("http://localhost")).headers(ImmutableMultimap.of("Content-Type", "binary/octet-stream"))
.build());
}
@Test
public void testExtendedPropertiesBind() {
ObjectMetadata md = ObjectMetadataBuilder.create().key("foo").cacheControl("no-cache").userMetadata(
ImmutableMap.of("foo", "bar")).build();
HttpRequest request = HttpRequest.builder().method("POST").endpoint(URI.create("http://localhost")).build();
BindObjectMetadataToRequest binder = injector.getInstance(BindObjectMetadataToRequest.class);
assertEquals(binder.bindToRequest(request, md), HttpRequest.builder().method("POST").endpoint(
URI.create("http://localhost")).headers(
ImmutableMultimap.of("Cache-Control", "no-cache", "x-amz-meta-foo", "bar", "Content-Type",
"binary/octet-stream")).build());
}
@Test(expectedExceptions = IllegalArgumentException.class)
public void testNoKeyIsBad() {
ObjectMetadata md = ObjectMetadataBuilder.create().build();
HttpRequest request = HttpRequest.builder().method("POST").endpoint(URI.create("http://localhost")).build();
BindObjectMetadataToRequest binder = injector.getInstance(BindObjectMetadataToRequest.class);
binder.bindToRequest(request, md);
}
@Test(expectedExceptions = IllegalArgumentException.class)
public void testMustBeObjectMetadata() {
HttpRequest request = new HttpRequest(HttpMethod.POST, URI.create("http://localhost"));
injector.getInstance(BindObjectMetadataToRequest.class).bindToRequest(request, new File("foo"));
}
@Test(expectedExceptions = { NullPointerException.class, IllegalStateException.class })
public void testNullIsBad() {
BindMapToHeadersWithPrefix binder = new BindMapToHeadersWithPrefix("prefix:");
HttpRequest request = HttpRequest.builder().method("GET").endpoint(URI.create("http://momma")).build();
binder.bindToRequest(request, null);
}
}

72
providers/aws-s3/src/test/java/org/jclouds/aws/s3/binders/BindPartIdsAndETagsToRequestTest.java Normal file
View File

@ -0,0 +1,72 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.binders;
import static org.testng.Assert.assertEquals;
import java.net.URI;
import javax.ws.rs.core.MediaType;
import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
import org.jclouds.http.HttpRequest;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableMap;
/**
* Tests behavior of {@code BindPartIdsAndETagsToRequest}
*
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "BindPartIdsAndETagsToRequestTest")
public class BindPartIdsAndETagsToRequestTest {
BindPartIdsAndETagsToRequest binder = new BindPartIdsAndETagsToRequest();
@Test
public void testPassWithMinimumDetailsAndPayload5GB() {
HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
Payload payload = Payloads
.newStringPayload("<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>\"a54357aff0632cce46d942af68356b38\"</ETag></Part></CompleteMultipartUpload>");
payload.getContentMetadata().setContentType(MediaType.TEXT_XML);
request = binder.bindToRequest(request, ImmutableMap.<Integer, String> of(1,
"\"a54357aff0632cce46d942af68356b38\""));
assertEquals(request.getPayload().getRawContent(), payload.getRawContent());
assertEquals(request, HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).payload(
payload).build());
}
@Test(expectedExceptions = IllegalArgumentException.class)
public void testEmptyIsBad() {
HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
binder.bindToRequest(request, ImmutableMap.<Integer, String> of());
}
@Test(expectedExceptions = { NullPointerException.class, IllegalStateException.class })
public void testNullIsBad() {
BindMapToHeadersWithPrefix binder = new BindMapToHeadersWithPrefix("prefix:");
HttpRequest request = HttpRequest.builder().method("GET").endpoint(URI.create("http://momma")).build();
binder.bindToRequest(request, null);
}
}

49
providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegexTest.java Normal file
View File

@ -0,0 +1,49 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.functions;
import static org.testng.Assert.assertEquals;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.functions.ReturnStringIf2xx;
import org.jclouds.io.Payloads;
import org.testng.annotations.Test;
/**
* Tests behavior of {@code ETagFromHttpResponseViaRegex}
*
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "ETagFromHttpResponseViaRegexTest")
public class ETagFromHttpResponseViaRegexTest {
@Test
public void test() {
HttpResponse response = HttpResponse.builder().statusCode(200).payload(
Payloads.newInputStreamPayload(getClass().getResourceAsStream("/complete-multipart-upload.xml")))
.build();
ETagFromHttpResponseViaRegex parser = new ETagFromHttpResponseViaRegex(new ReturnStringIf2xx());
assertEquals(parser.apply(response), "\"3858f62230ac3c915f300c664312c11f-9\"");
}
}

49
providers/aws-s3/src/test/java/org/jclouds/aws/s3/functions/UploadIdFromHttpResponseViaRegexTest.java Normal file
View File

@ -0,0 +1,49 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.functions;
import static org.testng.Assert.assertEquals;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.functions.ReturnStringIf2xx;
import org.jclouds.io.Payloads;
import org.testng.annotations.Test;
/**
* Tests behavior of {@code UploadIdFromHttpResponseViaRegex}
*
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "UploadIdFromHttpResponseViaRegexTest")
public class UploadIdFromHttpResponseViaRegexTest {
@Test
public void test() {
HttpResponse response = HttpResponse.builder().statusCode(200).payload(
Payloads.newInputStreamPayload(getClass().getResourceAsStream("/initiate-multipart-upload.xml")))
.build();
UploadIdFromHttpResponseViaRegex parser = new UploadIdFromHttpResponseViaRegex(new ReturnStringIf2xx());
assertEquals(parser.apply(response), "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA");
}
}

7
providers/aws-s3/src/test/resources/complete-multipart-upload.xml Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Location>http://Example-Bucket.s3.amazonaws.com/Example-Object</Location>
<Bucket>Example-Bucket</Bucket>
<Key>Example-Object</Key>
<ETag>"3858f62230ac3c915f300c664312c11f-9"</ETag>
</CompleteMultipartUploadResult>

6
providers/aws-s3/src/test/resources/initiate-multipart-upload.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>example-bucket</Bucket>
<Key>example-object</Key>
<UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId>
</InitiateMultipartUploadResult>

124
providers/aws-s3/src/test/resources/log4j.xml Normal file
View File

@ -0,0 +1,124 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2010 Cloud Conscious, LLC.
<info@cloudconscious.com>
====================================================================
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0 Unless required by
applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions
and limitations under the License.
====================================================================
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<!--
For more configuration infromation and examples see the Apache
Log4j website: http://logging.apache.org/log4j/
-->
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
debug="false">
<!-- A time/date based rolling appender -->
<appender name="WIREFILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="target/test-data/jclouds-wire.log" />
<param name="Append" value="true" />
<!-- Rollover at midnight each day -->
<param name="DatePattern" value="'.'yyyy-MM-dd" />
<param name="Threshold" value="TRACE" />
<layout class="org.apache.log4j.PatternLayout">
<!-- The default pattern: Date Priority [Category] Message\n -->
<param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
<!--
The full pattern: Date MS Priority [Category]
(Thread:NDC) Message\n <param name="ConversionPattern"
value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
-->
</layout>
</appender>
<!-- A time/date based rolling appender -->
<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="target/test-data/jclouds.log" />
<param name="Append" value="true" />
<!-- Rollover at midnight each day -->
<param name="DatePattern" value="'.'yyyy-MM-dd" />
<param name="Threshold" value="TRACE" />
<layout class="org.apache.log4j.PatternLayout">
<!-- The default pattern: Date Priority [Category] Message\n -->
<param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
<!--
The full pattern: Date MS Priority [Category]
(Thread:NDC) Message\n <param name="ConversionPattern"
value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
-->
</layout>
</appender>
<!-- A time/date based rolling appender -->
<appender name="BLOBSTOREFILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="target/test-data/jclouds-blobstore.log" />
<param name="Append" value="true" />
<param name="DatePattern" value="'.'yyyy-MM-dd" />
<param name="Threshold" value="TRACE" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
</layout>
</appender>
<appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
<appender-ref ref="FILE" />
</appender>
<appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
<appender-ref ref="WIREFILE" />
</appender>
<appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
<appender-ref ref="BLOBSTOREFILE" />
</appender>
<!-- ================ -->
<!-- Limit categories -->
<!-- ================ -->
<category name="org.jclouds">
<priority value="DEBUG" />
<appender-ref ref="ASYNC" />
</category>
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
</category><!--
<category name="jclouds.wire">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
</category>
--><category name="jclouds.blobstore">
<priority value="DEBUG" />
<appender-ref ref="ASYNCBLOBSTORE" />
</category>
<!-- ======================= -->
<!-- Setup the Root category -->
<!-- ======================= -->
<root>
<priority value="WARN" />
</root>
</log4j:configuration>

15
providers/googlestorage/src/main/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequest.java
View File

@ -26,11 +26,10 @@ import javax.inject.Inject;
import javax.inject.Singleton;
import javax.ws.rs.core.HttpHeaders;
import org.jclouds.blobstore.binders.BindUserMetadataToHeadersWithPrefix;
import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.utils.ModifyRequest;
import org.jclouds.s3.binders.BindS3ObjectMetadataToRequest;
import org.jclouds.s3.blobstore.functions.ObjectToBlob;
import org.jclouds.s3.domain.S3Object;
/**
@ -39,15 +38,9 @@ import org.jclouds.s3.domain.S3Object;
*/
@Singleton
public class BindGoogleStorageObjectMetadataToRequest extends BindS3ObjectMetadataToRequest {
private final BindUserMetadataToHeadersWithPrefix blobBinder;
private final ObjectToBlob object2Blob;
@Inject
public BindGoogleStorageObjectMetadataToRequest(ObjectToBlob object2Blob,
BindUserMetadataToHeadersWithPrefix blobBinder) {
super(object2Blob, blobBinder);
this.blobBinder = checkNotNull(blobBinder, "blobBinder");
this.object2Blob = checkNotNull(object2Blob, "object2Blob");
public BindGoogleStorageObjectMetadataToRequest(BindMapToHeadersWithPrefix metadataPrefixer) {
super(metadataPrefixer);
}
@Override
@ -67,7 +60,7 @@ public class BindGoogleStorageObjectMetadataToRequest extends BindS3ObjectMetada
request = ModifyRequest.replaceHeader(request, "Transfer-Encoding", "chunked");
}
request = blobBinder.bindToRequest(request, object2Blob.apply(s3Object));
request = metadataPrefixer.bindToRequest(request, s3Object.getMetadata().getUserMetadata());
if (s3Object.getMetadata().getCacheControl() != null) {
request = ModifyRequest.replaceHeader(request, HttpHeaders.CACHE_CONTROL, s3Object.getMetadata()

12
providers/googlestorage/src/test/java/org/jclouds/googlestorage/GoogleStorageAsyncClientTestDisabled.java
View File

@ -19,19 +19,29 @@
package org.jclouds.googlestorage;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.S3AsyncClient;
import org.testng.annotations.Test;
import com.google.inject.TypeLiteral;
/**
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(enabled = false, groups = "unit", testName = "GoogleStorageAsyncClientTest")
public class GoogleStorageAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest {
public class GoogleStorageAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest<S3AsyncClient> {
public GoogleStorageAsyncClientTestDisabled() {
this.provider = "googlestorage";
this.url = "commondatastorage.googleapis.com";
}
@Override
protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
};
}
// TODO parameterize this test so that it can pass
}

30
providers/googlestorage/src/test/java/org/jclouds/googlestorage/binders/BindGoogleStorageObjectMetadataToRequestTest.java
View File

@ -10,12 +10,15 @@ import javax.ws.rs.HttpMethod;
import org.jclouds.http.HttpRequest;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.BaseS3AsyncClientTest;
import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.domain.S3Object;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.inject.TypeLiteral;
/**
* Tests behavior of {@code BindGoogleStorageObjectMetadataToRequest}
@ -24,8 +27,13 @@ import com.google.common.collect.ImmutableMultimap;
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "BindGoogleStorageObjectMetadataToRequestTest")
public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncClientTest {
public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncClientTest<S3AsyncClient> {
@Override
protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
};
}
@Test
public void testPassWithMinimumDetailsAndPayload5GB() {
@ -36,7 +44,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
object.getMetadata().setKey("foo");
HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
BindGoogleStorageObjectMetadataToRequest binder = injector
.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint(
URI.create("http://localhost")).build());
@ -52,7 +61,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
object.getMetadata().getUserMetadata().putAll(ImmutableMap.of("foo", "bar"));
HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
BindGoogleStorageObjectMetadataToRequest binder = injector
.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint(
URI.create("http://localhost")).headers(ImmutableMultimap.of("x-amz-meta-foo", "bar")).build());
@ -66,7 +76,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
object.getMetadata().setKey("foo");
HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
BindGoogleStorageObjectMetadataToRequest binder = injector
.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
assertEquals(binder.bindToRequest(request, object), HttpRequest.builder().method("PUT").endpoint(
URI.create("http://localhost")).headers(ImmutableMultimap.of("Transfer-Encoding", "chunked")).build());
@ -80,7 +91,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
object.setPayload(payload);
HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
BindGoogleStorageObjectMetadataToRequest binder = injector
.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
binder.bindToRequest(request, object);
}
@ -93,7 +105,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
object.getMetadata().setKey("foo");
HttpRequest request = HttpRequest.builder().method("PUT").endpoint(URI.create("http://localhost")).build();
BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
BindGoogleStorageObjectMetadataToRequest binder = injector
.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
binder.bindToRequest(request, object);
}
@ -105,7 +118,8 @@ public class BindGoogleStorageObjectMetadataToRequestTest extends BaseS3AsyncCli
@Test(expectedExceptions = { NullPointerException.class, IllegalStateException.class })
public void testNullIsBad() {
BindGoogleStorageObjectMetadataToRequest binder = injector.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
BindGoogleStorageObjectMetadataToRequest binder = injector
.getInstance(BindGoogleStorageObjectMetadataToRequest.class);
HttpRequest request = HttpRequest.builder().method("GET").endpoint(URI.create("http://momma")).build();
binder.bindToRequest(request, null);
}

13
providers/scaleup-storage/src/test/java/org/jclouds/scaleup/storage/ScaleUpStorageAsyncClientTestDisabled.java
View File

@ -19,14 +19,25 @@
package org.jclouds.scaleup.storage;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.S3AsyncClientTest;
import org.testng.annotations.Test;
import com.google.inject.TypeLiteral;
/**
* @author Adrian Cole
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(enabled = false, groups = "unit", testName = "ScaleUpStorageAsyncClientTest")
public class ScaleUpStorageAsyncClientTestDisabled extends org.jclouds.s3.S3AsyncClientTest {
public class ScaleUpStorageAsyncClientTestDisabled extends S3AsyncClientTest<S3AsyncClient> {
@Override
protected TypeLiteral<RestAnnotationProcessor<S3AsyncClient>> createTypeLiteral() {
return new TypeLiteral<RestAnnotationProcessor<S3AsyncClient>>() {
};
}
public ScaleUpStorageAsyncClientTestDisabled() {
this.provider = "scaleup-storage";

23
sandbox-apis/cloudstack/src/test/java/org/jclouds/cloudstack/features/VirtualMachineClientLiveTest.java
View File

@ -21,12 +21,14 @@ package org.jclouds.cloudstack.features;
import static com.google.common.base.Predicates.equalTo;
import static com.google.common.base.Predicates.or;
import static com.google.common.collect.Iterables.filter;
import static com.google.common.collect.Iterables.find;
import static com.google.common.collect.Iterables.get;
import static com.google.common.collect.Iterables.getOnlyElement;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import java.util.NoSuchElementException;
import java.util.Set;
import org.jclouds.cloudstack.CloudStackClient;
@ -72,15 +74,30 @@ public class VirtualMachineClientLiveTest extends BaseCloudStackClientLiveTest {
long serviceOfferingId = DEFAULT_SIZE_ORDERING.min(client.getOfferingClient().listServiceOfferings()).getId();
long templateId = find(client.getTemplateClient().listTemplates(), new Predicate<Template>() {
Iterable<Template> templates = filter(client.getTemplateClient().listTemplates(), new Predicate<Template>() {
@Override
public boolean apply(Template arg0) {
return arg0.getZoneId() == zone.getId() && arg0.isFeatured() && arg0.isReady()
&& or(equalTo("Ubuntu 10.04 (64-bit)"), equalTo("CentOS 5.3 (32-bit)")).apply(arg0.getOSType());
&& or(equalTo("Ubuntu 10.04 (64-bit)"), equalTo("CentOS 5.3 (64-bit)")).apply(arg0.getOSType());
}
}).getId();
});
long templateId;
try {
// prefer password enabled
templateId = find(templates, new Predicate<Template>() {
@Override
public boolean apply(Template arg0) {
return arg0.isPasswordEnabled();
}
}).getId();
} catch (NoSuchElementException e) {
templateId = get(templates, 0).getId();
}
DeployVirtualMachineOptions options = new DeployVirtualMachineOptions();
if (zone.getNetworkType() == NetworkType.ADVANCED) {