From 518b453c6dc7b6c48fe99253ad422bf5be2ed655 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Tue, 15 Jun 2010 17:04:01 -0400 Subject: [PATCH] Issue 271 updated to support google storage --- .../java/org/jclouds/aws/domain/Region.java | 47 +++--- .../CreateSecurityGroupIfNeeded.java | 41 +++-- .../s3/GoogleStoragePropertiesBuilder.java | 48 ++++++ .../jclouds/aws/s3/S3PropertiesBuilder.java | 24 ++- .../aws/s3/WalrusPropertiesBuilder.java | 4 +- .../functions/BucketToResourceMetadata.java | 28 ++- .../aws/s3/config/S3RestClientModule.java | 17 +- .../s3/filters/RequestAuthorizeSignature.java | 63 ++++--- .../s3/functions/BindRegionToXmlPayload.java | 47 +++--- .../jclouds/aws/s3/reference/S3Constants.java | 5 + .../compute/EC2ComputeServiceLiveTest.java | 2 +- .../org/jclouds/aws/s3/S3AsyncClientTest.java | 4 +- .../aws/s3/config/S3StubClientModule.java | 2 +- .../jclouds/aws/sqs/SQSAsyncClientTest.java | 159 ++++++++++-------- .../src/main/resources/blobstore.properties | 2 + 15 files changed, 323 insertions(+), 170 deletions(-) create mode 100644 aws/core/src/main/java/org/jclouds/aws/s3/GoogleStoragePropertiesBuilder.java diff --git a/aws/core/src/main/java/org/jclouds/aws/domain/Region.java b/aws/core/src/main/java/org/jclouds/aws/domain/Region.java index 5f4019b887..6288c510b8 100644 --- a/aws/core/src/main/java/org/jclouds/aws/domain/Region.java +++ b/aws/core/src/main/java/org/jclouds/aws/domain/Region.java @@ -27,8 +27,8 @@ import com.google.common.collect.ImmutableSet; * Regions used for all aws commands. * * @author Adrian Cole - * @see + * @see * */ public class Region { @@ -37,8 +37,9 @@ public class Region { *

*

S3

*

- * In Amazon S3, the EU (Ireland) Region provides read-after-write consistency for PUTS of new - * objects in your Amazon S3 bucket and eventual consistency for overwrite PUTS and DELETES. + * In Amazon S3, the EU (Ireland) Region provides read-after-write + * consistency for PUTS of new objects in your Amazon S3 bucket and eventual + * consistency for overwrite PUTS and DELETES. */ public static final String EU_WEST_1 = "eu-west-1"; @@ -48,11 +49,13 @@ public class Region { *

*

S3

*

- * This is the default Region. All requests sent to s3.amazonaws.com go to this Region unless you - * specify a LocationConstraint on a bucket. The US Standard Region automatically places your - * data in either Amazon's east or west coast data centers depending on what will provide you - * with the lowest latency. To use this region, do not set the LocationConstraint bucket - * parameter. The US Standard Region provides eventual consistency for all requests. + * This is the default Region. All requests sent to s3.amazonaws.com go to + * this Region unless you specify a LocationConstraint on a bucket. The US + * Standard Region automatically places your data in either Amazon's east or + * west coast data centers depending on what will provide you with the lowest + * latency. To use this region, do not set the LocationConstraint bucket + * parameter. The US Standard Region provides eventual consistency for all + * requests. */ public static final String US_STANDARD = "us-standard"; @@ -62,25 +65,27 @@ public class Region { public static final String US_EAST_1 = "us-east-1"; /** - * US-West (Northern California)

S3

Uses Amazon S3 servers in Northern California + * US-West (Northern California)

S3

Uses Amazon S3 servers in + * Northern California *

- * Optionally, use the endpoint s3-us-west-1.amazonaws.com on all requests to this bucket to - * reduce the latency you might experience after the first hour of creating a bucket in this - * Region. + * Optionally, use the endpoint s3-us-west-1.amazonaws.com on all requests to + * this bucket to reduce the latency you might experience after the first + * hour of creating a bucket in this Region. *

- * In Amazon S3, the US-West (Northern California) Region provides read-after-write consistency - * for PUTS of new objects in your Amazon S3 bucket and eventual consistency for overwrite PUTS - * and DELETES. + * In Amazon S3, the US-West (Northern California) Region provides + * read-after-write consistency for PUTS of new objects in your Amazon S3 + * bucket and eventual consistency for overwrite PUTS and DELETES. */ public static final String US_WEST_1 = "us-west-1"; /** - * Region in Singapore, launched April 28, 2010. This region improves latency for Asia-based - * users + * Region in Singapore, launched April 28, 2010. This region improves latency + * for Asia-based users */ public static final String AP_SOUTHEAST_1 = "ap-southeast-1"; - public static Set ALL = ImmutableSet.of(EU_WEST_1, US_STANDARD, US_EAST_1, US_WEST_1, - AP_SOUTHEAST_1); - + public static Set ALL_S3 = ImmutableSet.of("EU", US_STANDARD, + US_EAST_1, US_WEST_1, AP_SOUTHEAST_1); + public static Set ALL_SQS = ImmutableSet.of(EU_WEST_1, US_STANDARD, + US_EAST_1, US_WEST_1, AP_SOUTHEAST_1); } \ No newline at end of file diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/functions/CreateSecurityGroupIfNeeded.java b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/functions/CreateSecurityGroupIfNeeded.java index 5f2a2dbeb8..00cf36dab9 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/functions/CreateSecurityGroupIfNeeded.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/functions/CreateSecurityGroupIfNeeded.java @@ -41,7 +41,8 @@ import com.google.common.collect.Iterables; * @author Adrian Cole */ @Singleton -public class CreateSecurityGroupIfNeeded implements Function { +public class CreateSecurityGroupIfNeeded implements + Function { @Resource @Named(ComputeServiceConstants.COMPUTE_LOGGER) protected Logger logger = Logger.NULL; @@ -54,16 +55,20 @@ public class CreateSecurityGroupIfNeeded implements Function> creating securityGroup region(%s) name(%s)", region, name); + logger.debug(">> creating securityGroup region(%s) name(%s)", region, + name); try { - ec2Client.getSecurityGroupServices().createSecurityGroupInRegion(region, name, name); + ec2Client.getSecurityGroupServices().createSecurityGroupInRegion( + region, name, name); logger.debug("<< created securityGroup(%s)", name); for (int port : ports) { createIngressRuleForTCPPort(region, name, port); @@ -72,7 +77,8 @@ public class CreateSecurityGroupIfNeeded implements Function> authorizing securityGroup region(%s) name(%s) port(%s)", region, name, port); - ec2Client.getSecurityGroupServices().authorizeSecurityGroupIngressInRegion(region, name, - IpProtocol.TCP, port, port, "0.0.0.0/0"); + logger.debug(">> authorizing securityGroup region(%s) name(%s) port(%s)", + region, name, port); + ec2Client.getSecurityGroupServices() + .authorizeSecurityGroupIngressInRegion(region, name, + IpProtocol.TCP, port, port, "0.0.0.0/0"); logger.debug("<< authorized securityGroup(%s)", name); } private void authorizeGroupToItself(String region, String name) { - logger.debug(">> authorizing securityGroup region(%s) name(%s) permission to itself", region, - name); + logger + .debug( + ">> authorizing securityGroup region(%s) name(%s) permission to itself", + region, name); String myOwnerId = Iterables.get( - ec2Client.getSecurityGroupServices().describeSecurityGroupsInRegion(region), 0) - .getOwnerId(); - ec2Client.getSecurityGroupServices().authorizeSecurityGroupIngressInRegion(region, name, - new UserIdGroupPair(myOwnerId, name)); + ec2Client.getSecurityGroupServices() + .describeSecurityGroupsInRegion(region), 0).getOwnerId(); + ec2Client.getSecurityGroupServices() + .authorizeSecurityGroupIngressInRegion(region, name, + new UserIdGroupPair(myOwnerId, name)); logger.debug("<< authorized securityGroup(%s)", name); } diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/GoogleStoragePropertiesBuilder.java b/aws/core/src/main/java/org/jclouds/aws/s3/GoogleStoragePropertiesBuilder.java new file mode 100644 index 0000000000..92f99a2eda --- /dev/null +++ b/aws/core/src/main/java/org/jclouds/aws/s3/GoogleStoragePropertiesBuilder.java @@ -0,0 +1,48 @@ +package org.jclouds.aws.s3; + +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_AUTH_TAG; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_DEFAULT_REGIONS; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_ENDPOINT; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_HEADER_TAG; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_REGIONS; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_SERVICE_EXPR; +import static org.jclouds.blobstore.reference.BlobStoreConstants.PROPERTY_USER_METADATA_PREFIX; + +import java.util.Properties; + +/** + * Builds properties used in Walrus Clients + * + * @author Adrian Cole + */ +public class GoogleStoragePropertiesBuilder extends S3PropertiesBuilder { + @Override + protected Properties defaultProperties() { + Properties properties = super.defaultProperties(); + properties.setProperty(PROPERTY_USER_METADATA_PREFIX, "x-goog-meta-"); + properties.setProperty(PROPERTY_S3_AUTH_TAG, "GOOG1"); + properties.setProperty(PROPERTY_S3_HEADER_TAG, "goog"); + properties.setProperty(PROPERTY_S3_SERVICE_EXPR, + "\\.commondatastorage\\.googleapis\\.com"); + return properties; + } + + @Override + protected Properties addEndpoints(Properties properties) { + properties.setProperty(PROPERTY_S3_REGIONS, "GoogleStorage"); + properties.setProperty(PROPERTY_S3_DEFAULT_REGIONS, "GoogleStorage"); + properties.setProperty(PROPERTY_S3_ENDPOINT, + "https://commondatastorage.googleapis.com"); + properties.setProperty(PROPERTY_S3_ENDPOINT + ".GoogleStorage", + "https://commondatastorage.googleapis.com"); + return properties; + } + + public GoogleStoragePropertiesBuilder(Properties properties) { + super(properties); + } + + public GoogleStoragePropertiesBuilder(String id, String secret) { + super(id, secret); + } +} diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/S3PropertiesBuilder.java b/aws/core/src/main/java/org/jclouds/aws/s3/S3PropertiesBuilder.java index 9e2f640301..467515fe8f 100644 --- a/aws/core/src/main/java/org/jclouds/aws/s3/S3PropertiesBuilder.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/S3PropertiesBuilder.java @@ -22,8 +22,12 @@ import static com.google.common.base.Preconditions.checkNotNull; import static org.jclouds.Constants.PROPERTY_RELAX_HOSTNAME; import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AWS_ACCESSKEYID; import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AWS_SECRETACCESSKEY; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_AUTH_TAG; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_DEFAULT_REGIONS; import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_ENDPOINT; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_HEADER_TAG; import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_REGIONS; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_SERVICE_EXPR; import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_SESSIONINTERVAL; import static org.jclouds.blobstore.reference.BlobStoreConstants.DIRECTORY_SUFFIX_FOLDER; import static org.jclouds.blobstore.reference.BlobStoreConstants.PROPERTY_BLOBSTORE_DIRECTORY_SUFFIX; @@ -47,26 +51,34 @@ public class S3PropertiesBuilder extends PropertiesBuilder { @Override protected Properties defaultProperties() { Properties properties = super.defaultProperties(); - properties.setProperty(PROPERTY_RELAX_HOSTNAME, "true"); properties.setProperty(PROPERTY_USER_METADATA_PREFIX, "x-amz-meta-"); - addEndpointProperties(properties); + properties.setProperty(PROPERTY_S3_AUTH_TAG, "AWS"); + properties.setProperty(PROPERTY_S3_HEADER_TAG, "aws"); + properties.setProperty(PROPERTY_S3_SERVICE_EXPR, + "\\.s3[^.]*\\.amazonaws\\.com"); + properties.setProperty(PROPERTY_RELAX_HOSTNAME, "true"); + addEndpoints(properties); properties.setProperty(PROPERTY_S3_SESSIONINTERVAL, "60"); properties.setProperty(PROPERTY_BLOBSTORE_DIRECTORY_SUFFIX, DIRECTORY_SUFFIX_FOLDER); + return properties; } - protected Properties addEndpointProperties(Properties properties) { + protected Properties addEndpoints(Properties properties) { properties.setProperty(PROPERTY_S3_REGIONS, Joiner.on(',').join( - Region.US_STANDARD, Region.US_WEST_1, Region.EU_WEST_1, + Region.US_STANDARD, Region.US_EAST_1, Region.US_WEST_1, "EU", Region.AP_SOUTHEAST_1)); - + properties.setProperty(PROPERTY_S3_DEFAULT_REGIONS, Joiner.on(',').join( + Region.US_STANDARD, Region.US_EAST_1)); properties.setProperty(PROPERTY_S3_ENDPOINT, "https://s3.amazonaws.com"); properties.setProperty(PROPERTY_S3_ENDPOINT + "." + Region.US_STANDARD, "https://s3.amazonaws.com"); + properties.setProperty(PROPERTY_S3_ENDPOINT + "." + Region.US_EAST_1, + "https://s3.amazonaws.com"); properties.setProperty(PROPERTY_S3_ENDPOINT + "." + Region.US_WEST_1, "https://s3-us-west-1.amazonaws.com"); - properties.setProperty(PROPERTY_S3_ENDPOINT + "." + Region.EU_WEST_1, + properties.setProperty(PROPERTY_S3_ENDPOINT + "." + "EU", "https://s3-eu-west-1.amazonaws.com"); properties.setProperty( PROPERTY_S3_ENDPOINT + "." + Region.AP_SOUTHEAST_1, diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/WalrusPropertiesBuilder.java b/aws/core/src/main/java/org/jclouds/aws/s3/WalrusPropertiesBuilder.java index ef2823c53d..86ec123627 100644 --- a/aws/core/src/main/java/org/jclouds/aws/s3/WalrusPropertiesBuilder.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/WalrusPropertiesBuilder.java @@ -1,5 +1,6 @@ package org.jclouds.aws.s3; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_DEFAULT_REGIONS; import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_ENDPOINT; import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_REGIONS; @@ -12,8 +13,9 @@ import java.util.Properties; */ public class WalrusPropertiesBuilder extends S3PropertiesBuilder { @Override - protected Properties addEndpointProperties(Properties properties) { + protected Properties addEndpoints(Properties properties) { properties.setProperty(PROPERTY_S3_REGIONS, "Walrus"); + properties.setProperty(PROPERTY_S3_DEFAULT_REGIONS, "Walrus"); properties.setProperty(PROPERTY_S3_ENDPOINT, "http://173.205.188.130:8773/services/Walrus"); properties.setProperty(PROPERTY_S3_ENDPOINT + ".Walrus", diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/BucketToResourceMetadata.java b/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/BucketToResourceMetadata.java index c17eb6f535..8539096de1 100644 --- a/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/BucketToResourceMetadata.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/BucketToResourceMetadata.java @@ -43,8 +43,10 @@ import com.google.common.collect.Iterables; * @author Adrian Cole */ @Singleton -public class BucketToResourceMetadata implements Function { +public class BucketToResourceMetadata implements + Function { private final S3Client client; + private final Location onlyLocation; private final Set locations; @Resource @@ -53,6 +55,8 @@ public class BucketToResourceMetadata implements Function locations) { this.client = client; + this.onlyLocation = locations.size() == 1 ? Iterables.get(locations, 0) + : null; this.locations = locations; } @@ -60,29 +64,37 @@ public class BucketToResourceMetadata implements Function() { + return Iterables.find(locations, new Predicate() { @Override public boolean apply(Location input) { return input.getId().equals(region.toString()); } - })); + }); } catch (NoSuchElementException e) { - logger.error("could not get location for region %s in %s", region, locations); + logger.error("could not get location for region %s in %s", + region, locations); } } else { logger.error("could not get region for %s", from.getName()); } } catch (ContainerNotFoundException e) { - logger.error(e, - "could not get region for %s, as service suggests the bucket doesn't exist", from - .getName()); + logger + .error( + e, + "could not get region for %s, as service suggests the bucket doesn't exist", + from.getName()); } - return to; + return null; } } \ No newline at end of file diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/config/S3RestClientModule.java b/aws/core/src/main/java/org/jclouds/aws/s3/config/S3RestClientModule.java index 0bfb5cddcd..33825a0ab3 100755 --- a/aws/core/src/main/java/org/jclouds/aws/s3/config/S3RestClientModule.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/config/S3RestClientModule.java @@ -21,6 +21,7 @@ package org.jclouds.aws.s3.config; import java.net.URI; import java.util.Map; import java.util.Set; +import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import javax.inject.Named; @@ -47,9 +48,10 @@ import org.jclouds.rest.ConfiguresRestClient; import org.jclouds.rest.RequestSigner; import org.jclouds.rest.config.RestClientModule; +import com.google.common.base.Predicate; import com.google.common.base.Splitter; import com.google.common.base.Supplier; -import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.inject.Injector; import com.google.inject.Key; @@ -137,9 +139,16 @@ public class S3RestClientModule extends @Provides @Singleton @S3 - String getDefaultRegion(@S3 URI uri, @S3 Map map) { - return ImmutableBiMap. builder().putAll(map).build() - .inverse().get(uri); + String getDefaultRegion(@S3 final URI uri, @S3 Map map) { + return Iterables.find(map.entrySet(), + new Predicate>() { + + @Override + public boolean apply(Entry input) { + return input.getValue().equals(uri); + } + + }).getKey(); } @Override diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java b/aws/core/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java index 585654ee93..a7e8247602 100755 --- a/aws/core/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java @@ -52,17 +52,20 @@ import com.google.common.collect.ImmutableSet; /** * Signs the S3 request. * - * @see + * @see * @author Adrian Cole * */ @Singleton -public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSigner { +public class RequestAuthorizeSignature implements HttpRequestFilter, + RequestSigner { private final String[] firstHeadersToSign = new String[] { "Content-MD5", - HttpHeaders.CONTENT_TYPE, HttpHeaders.DATE }; + HttpHeaders.CONTENT_TYPE, HttpHeaders.DATE }; - public static Set SPECIAL_QUERIES = ImmutableSet.of("acl", "torrent", "logging", - "location", "requestPayment"); + public static Set SPECIAL_QUERIES = ImmutableSet.of("acl", + "torrent", "logging", "location", "requestPayment"); private final SignatureWire signatureWire; private final String accessKey; private final String secretKey; @@ -73,11 +76,22 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign @Named(Constants.LOGGER_SIGNATURE) Logger signatureLog = Logger.NULL; + private final String authTag; + private final String headerTag; + private final String srvExpr; + @Inject public RequestAuthorizeSignature(SignatureWire signatureWire, - @Named(S3Constants.PROPERTY_AWS_ACCESSKEYID) String accessKey, - @Named(S3Constants.PROPERTY_AWS_SECRETACCESSKEY) String secretKey, - @TimeStamp Provider timeStampProvider, EncryptionService encryptionService) { + @Named(S3Constants.PROPERTY_S3_AUTH_TAG) String authTag, + @Named(S3Constants.PROPERTY_S3_SERVICE_EXPR) String srvExpr, + @Named(S3Constants.PROPERTY_S3_HEADER_TAG) String headerTag, + @Named(S3Constants.PROPERTY_AWS_ACCESSKEYID) String accessKey, + @Named(S3Constants.PROPERTY_AWS_SECRETACCESSKEY) String secretKey, + @TimeStamp Provider timeStampProvider, + EncryptionService encryptionService) { + this.srvExpr = srvExpr; + this.headerTag = headerTag; + this.authTag = authTag; this.signatureWire = signatureWire; this.accessKey = accessKey; this.secretKey = secretKey; @@ -107,18 +121,21 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign } private void calculateAndReplaceAuthHeader(HttpRequest request, String toSign) - throws HttpException { + throws HttpException { String signature = sign(toSign); if (signatureWire.enabled()) signatureWire.input(Utils.toInputStream(signature)); - request.getHeaders().replaceValues(HttpHeaders.AUTHORIZATION, - Collections.singletonList("AWS " + accessKey + ":" + signature)); + request.getHeaders().replaceValues( + HttpHeaders.AUTHORIZATION, + Collections.singletonList(authTag + " " + accessKey + ":" + + signature)); } public String sign(String toSign) { String signature; try { - signature = encryptionService.hmacSha1Base64(toSign, secretKey.getBytes()); + signature = encryptionService.hmacSha1Base64(toSign, secretKey + .getBytes()); } catch (Exception e) { throw new HttpException("error signing request", e); } @@ -131,16 +148,17 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign private void replaceDateHeader(HttpRequest request) { request.getHeaders().replaceValues(HttpHeaders.DATE, - Collections.singletonList(timeStampProvider.get())); + Collections.singletonList(timeStampProvider.get())); } private void appendAmzHeaders(HttpRequest request, StringBuilder toSign) { Set headers = new TreeSet(request.getHeaders().keySet()); for (String header : headers) { - if (header.startsWith("x-amz-")) { + if (header.startsWith("x-" + headerTag + "-")) { toSign.append(header.toLowerCase()).append(":"); for (String value : request.getHeaders().get(header)) { - toSign.append(Utils.replaceAll(value, NEWLINE_PATTERN, "")).append(","); + toSign.append(Utils.replaceAll(value, NEWLINE_PATTERN, "")) + .append(","); } toSign.deleteCharAt(toSign.lastIndexOf(",")); toSign.append("\n"); @@ -150,7 +168,8 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign private void appendHttpHeaders(HttpRequest request, StringBuilder toSign) { for (String header : firstHeadersToSign) - toSign.append(valueOrEmpty(request.getHeaders().get(header))).append("\n"); + toSign.append(valueOrEmpty(request.getHeaders().get(header))).append( + "\n"); } @VisibleForTesting @@ -158,9 +177,9 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign String hostHeader = request.getFirstHeaderOrNull(HttpHeaders.HOST); if (hostHeader == null) hostHeader = checkNotNull(request.getEndpoint().getHost(), - "request.getEndPoint().getHost()"); - if (hostHeader.endsWith(".amazonaws.com") && !hostHeader.equals("s3.amazonaws.com")) - toSign.append("/").append(hostHeader.substring(0, hostHeader.lastIndexOf(".s3"))); + "request.getEndPoint().getHost()"); + if (hostHeader.matches(".*" + srvExpr)) + toSign.append("/").append(hostHeader.replaceAll(srvExpr, "")); } @VisibleForTesting @@ -168,7 +187,8 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign toSign.append(request.getEndpoint().getRawPath()); - // ...however, there are a few exceptions that must be included in the signed URI. + // ...however, there are a few exceptions that must be included in the + // signed URI. if (request.getEndpoint().getQuery() != null) { StringBuilder paramsToSign = new StringBuilder("?"); @@ -188,6 +208,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign } private String valueOrEmpty(Collection collection) { - return (collection != null && collection.size() >= 1) ? collection.iterator().next() : ""; + return (collection != null && collection.size() >= 1) ? collection + .iterator().next() : ""; } } \ No newline at end of file diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/functions/BindRegionToXmlPayload.java b/aws/core/src/main/java/org/jclouds/aws/s3/functions/BindRegionToXmlPayload.java index 5b1090e746..0b56b087fe 100644 --- a/aws/core/src/main/java/org/jclouds/aws/s3/functions/BindRegionToXmlPayload.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/functions/BindRegionToXmlPayload.java @@ -20,19 +20,23 @@ package org.jclouds.aws.s3.functions; import static com.google.common.base.Preconditions.checkArgument; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_DEFAULT_REGIONS; +import static org.jclouds.aws.s3.reference.S3Constants.PROPERTY_S3_REGIONS; import javax.inject.Inject; +import javax.inject.Named; import javax.inject.Singleton; -import org.jclouds.aws.domain.Region; -import org.jclouds.aws.s3.S3; import org.jclouds.http.HttpRequest; import org.jclouds.rest.binders.BindToStringPayload; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; + /** * - * Depending on your latency and legal requirements, you can specify a location constraint that will - * affect where your data physically resides. + * Depending on your latency and legal requirements, you can specify a location + * constraint that will affect where your data physically resides. * * @author Adrian Cole * @@ -40,35 +44,36 @@ import org.jclouds.rest.binders.BindToStringPayload; @Singleton public class BindRegionToXmlPayload extends BindToStringPayload { - private final String defaultRegion; + private final Iterable defaultRegions; + private final Iterable regions; @Inject - BindRegionToXmlPayload(@S3 String defaultRegion) { - this.defaultRegion = defaultRegion; + BindRegionToXmlPayload( + @Named(PROPERTY_S3_DEFAULT_REGIONS) String defaultRegions, + @Named(PROPERTY_S3_REGIONS) String regions) { + this.defaultRegions = Splitter.on(',').split(defaultRegions); + this.regions = Splitter.on(',').split(regions); } @Override public void bindToRequest(HttpRequest request, Object input) { - input = input == null ? defaultRegion : input; - checkArgument(input instanceof String, "this binder is only valid for Region!"); + input = input == null ? Iterables.get(defaultRegions, 0) : input; + checkArgument(input instanceof String, + "this binder is only valid for Region!"); String constraint = (String) input; String value = null; - if (Region.US_STANDARD.equals(constraint) || Region.US_EAST_1.equals(constraint)) { + if (Iterables.contains(defaultRegions, constraint)) { // nothing to bind as this is default. return; - } else if (Region.EU_WEST_1.equals(constraint)) - value = "EU"; - else if (Region.US_WEST_1.equals(constraint)) - value = "us-west-1"; - else if (Region.AP_SOUTHEAST_1.equals(constraint)) - value = "ap-southeast-1"; - else { - throw new IllegalStateException("unimplemented location: " + this); + } else if (Iterables.contains(regions, constraint)) { + value = constraint; + } else { + throw new IllegalStateException("unimplemented location: " + constraint); } String payload = String - .format( - "%s", - value); + .format( + "%s", + value); super.bindToRequest(request, payload); } diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/reference/S3Constants.java b/aws/core/src/main/java/org/jclouds/aws/s3/reference/S3Constants.java index 5acb648fab..47ab7ff498 100755 --- a/aws/core/src/main/java/org/jclouds/aws/s3/reference/S3Constants.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/reference/S3Constants.java @@ -37,9 +37,14 @@ public interface S3Constants extends AWSConstants, S3Headers { public static final String DELIMITER = "delimiter"; public static final String PROPERTY_S3_ENDPOINT = "jclouds.s3.endpoint"; public static final String PROPERTY_S3_REGIONS = "jclouds.s3.regions"; + public static final String PROPERTY_S3_DEFAULT_REGIONS = "jclouds.s3.default-regions"; + + public static final String PROPERTY_S3_SERVICE_EXPR = "jclouds.service.expr"; /** * how long do we wait before obtaining a new timestamp for requests. */ public static final String PROPERTY_S3_SESSIONINTERVAL = "jclouds.s3.sessioninterval"; + public static final String PROPERTY_S3_AUTH_TAG = "jclouds.s3.auth.tag"; + public static final String PROPERTY_S3_HEADER_TAG = "jclouds.s3.header.tag"; } diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java index 7ec15bfa48..3d7e5e3977 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java @@ -52,7 +52,7 @@ public class EC2ComputeServiceLiveTest extends BaseComputeServiceLiveTest { @BeforeClass @Override public void setServiceDefaults() { - service = "ec2"; + service = "eucalyptus"; } @Override diff --git a/aws/core/src/test/java/org/jclouds/aws/s3/S3AsyncClientTest.java b/aws/core/src/test/java/org/jclouds/aws/s3/S3AsyncClientTest.java index cdc4a3cb0e..f52d2f3ec3 100644 --- a/aws/core/src/test/java/org/jclouds/aws/s3/S3AsyncClientTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/s3/S3AsyncClientTest.java @@ -90,7 +90,7 @@ public class S3AsyncClientTest extends RestClientTest { public void testAllRegions() throws SecurityException, NoSuchMethodException, IOException { Method method = S3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class, Array.newInstance(PutBucketOptions.class, 0).getClass()); - for (String region : Region.ALL) { + for (String region : Region.ALL_S3) { processor.createRequest(method, region, "bucket-name"); } } @@ -416,7 +416,7 @@ public class S3AsyncClientTest extends RestClientTest { Method method = S3AsyncClient.class.getMethod("putBucketInRegion", String.class, String.class, Array.newInstance(PutBucketOptions.class, 0).getClass()); GeneratedHttpRequest httpMethod = processor.createRequest(method, - Region.EU_WEST_1, "bucket"); + "EU", "bucket"); assertRequestLineEquals(httpMethod, "PUT https://bucket.s3.amazonaws.com/ HTTP/1.1"); assertHeadersEqual(httpMethod, diff --git a/aws/core/src/test/java/org/jclouds/aws/s3/config/S3StubClientModule.java b/aws/core/src/test/java/org/jclouds/aws/s3/config/S3StubClientModule.java index 71973bdf9a..a0c635b691 100755 --- a/aws/core/src/test/java/org/jclouds/aws/s3/config/S3StubClientModule.java +++ b/aws/core/src/test/java/org/jclouds/aws/s3/config/S3StubClientModule.java @@ -53,7 +53,7 @@ public class S3StubClientModule extends RestClientModule>() { - }).annotatedWith(S3.class).toInstance(Region.ALL); + }).annotatedWith(S3.class).toInstance(Region.ALL_S3); } @Override diff --git a/aws/core/src/test/java/org/jclouds/aws/sqs/SQSAsyncClientTest.java b/aws/core/src/test/java/org/jclouds/aws/sqs/SQSAsyncClientTest.java index 5cc7892aae..9c36834f73 100644 --- a/aws/core/src/test/java/org/jclouds/aws/sqs/SQSAsyncClientTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/sqs/SQSAsyncClientTest.java @@ -57,91 +57,111 @@ import com.google.inject.TypeLiteral; @Test(groups = "unit", testName = "sqs.SQSAsyncClientTest") public class SQSAsyncClientTest extends RestClientTest { - public void testListQueuesInRegion() throws SecurityException, NoSuchMethodException, - IOException { - Method method = SQSAsyncClient.class.getMethod("listQueuesInRegion", String.class, Array - .newInstance(ListQueuesOptions.class, 0).getClass()); - GeneratedHttpRequest httpMethod = processor.createRequest(method, - (String) null); + public void testListQueuesInRegion() throws SecurityException, + NoSuchMethodException, IOException { + Method method = SQSAsyncClient.class.getMethod("listQueuesInRegion", + String.class, Array.newInstance(ListQueuesOptions.class, 0) + .getClass()); + GeneratedHttpRequest httpMethod = processor + .createRequest(method, (String) null); - assertRequestLineEquals(httpMethod, "POST https://sqs.us-east-1.amazonaws.com/ HTTP/1.1"); + assertRequestLineEquals(httpMethod, + "POST https://sqs.us-east-1.amazonaws.com/ HTTP/1.1"); assertHeadersEqual( - httpMethod, - "Content-Length: 36\nContent-Type: application/x-www-form-urlencoded\nHost: sqs.us-east-1.amazonaws.com\n"); + httpMethod, + "Content-Length: 36\nContent-Type: application/x-www-form-urlencoded\nHost: sqs.us-east-1.amazonaws.com\n"); assertPayloadEquals(httpMethod, "Version=2009-02-01&Action=ListQueues"); - assertResponseParserClassEquals(method, httpMethod, RegexListQueuesResponseHandler.class); + assertResponseParserClassEquals(method, httpMethod, + RegexListQueuesResponseHandler.class); assertSaxResponseParserClassEquals(method, null); assertExceptionParserClassEquals(method, null); checkFilters(httpMethod); } - public void testListQueuesInRegionOptions() throws SecurityException, NoSuchMethodException, - IOException { - Method method = SQSAsyncClient.class.getMethod("listQueuesInRegion", String.class, Array - .newInstance(ListQueuesOptions.class, 0).getClass()); - GeneratedHttpRequest httpMethod = processor.createRequest(method, null, - ListQueuesOptions.Builder.queuePrefix("prefix")); + public void testListQueuesInRegionOptions() throws SecurityException, + NoSuchMethodException, IOException { + Method method = SQSAsyncClient.class.getMethod("listQueuesInRegion", + String.class, Array.newInstance(ListQueuesOptions.class, 0) + .getClass()); + GeneratedHttpRequest httpMethod = processor + .createRequest(method, null, ListQueuesOptions.Builder + .queuePrefix("prefix")); - assertRequestLineEquals(httpMethod, "POST https://sqs.us-east-1.amazonaws.com/ HTTP/1.1"); + assertRequestLineEquals(httpMethod, + "POST https://sqs.us-east-1.amazonaws.com/ HTTP/1.1"); assertHeadersEqual( - httpMethod, - "Content-Length: 59\nContent-Type: application/x-www-form-urlencoded\nHost: sqs.us-east-1.amazonaws.com\n"); - assertPayloadEquals(httpMethod, "Version=2009-02-01&Action=ListQueues&QueueNamePrefix=prefix"); - - assertResponseParserClassEquals(method, httpMethod, RegexListQueuesResponseHandler.class); - assertSaxResponseParserClassEquals(method, null); - assertExceptionParserClassEquals(method, null); - - checkFilters(httpMethod); - } - - public void testCreateQueueInRegion() throws SecurityException, NoSuchMethodException, - IOException { - Method method = SQSAsyncClient.class.getMethod("createQueueInRegion", String.class, - String.class, Array.newInstance(CreateQueueOptions.class, 0).getClass()); - GeneratedHttpRequest httpMethod = processor.createRequest(method, null, - "queueName"); - - assertRequestLineEquals(httpMethod, "POST https://sqs.us-east-1.amazonaws.com/ HTTP/1.1"); - assertHeadersEqual( - httpMethod, - "Content-Length: 57\nContent-Type: application/x-www-form-urlencoded\nHost: sqs.us-east-1.amazonaws.com\n"); - assertPayloadEquals(httpMethod, "Version=2009-02-01&Action=CreateQueue&QueueName=queueName"); - - assertResponseParserClassEquals(method, httpMethod, RegexQueueHandler.class); - assertSaxResponseParserClassEquals(method, null); - assertExceptionParserClassEquals(method, null); - - checkFilters(httpMethod); - } - - public void testCreateQueueInRegionOptions() throws SecurityException, NoSuchMethodException, - IOException { - Method method = SQSAsyncClient.class.getMethod("createQueueInRegion", String.class, - String.class, Array.newInstance(CreateQueueOptions.class, 0).getClass()); - GeneratedHttpRequest httpMethod = processor.createRequest(method, null, - "queueName", CreateQueueOptions.Builder.defaultVisibilityTimeout(45)); - - assertRequestLineEquals(httpMethod, "POST https://sqs.us-east-1.amazonaws.com/ HTTP/1.1"); - assertHeadersEqual( - httpMethod, - "Content-Length: 85\nContent-Type: application/x-www-form-urlencoded\nHost: sqs.us-east-1.amazonaws.com\n"); + httpMethod, + "Content-Length: 59\nContent-Type: application/x-www-form-urlencoded\nHost: sqs.us-east-1.amazonaws.com\n"); assertPayloadEquals(httpMethod, - "Version=2009-02-01&Action=CreateQueue&QueueName=queueName&DefaultVisibilityTimeout=45"); + "Version=2009-02-01&Action=ListQueues&QueueNamePrefix=prefix"); - assertResponseParserClassEquals(method, httpMethod, RegexQueueHandler.class); + assertResponseParserClassEquals(method, httpMethod, + RegexListQueuesResponseHandler.class); assertSaxResponseParserClassEquals(method, null); assertExceptionParserClassEquals(method, null); checkFilters(httpMethod); } - public void testAllRegions() throws SecurityException, NoSuchMethodException, IOException { - Method method = SQSAsyncClient.class.getMethod("createQueueInRegion", String.class, - String.class, Array.newInstance(CreateQueueOptions.class, 0).getClass()); - for (String region : Iterables.filter(Region.ALL, not(equalTo("us-standard")))) { + public void testCreateQueueInRegion() throws SecurityException, + NoSuchMethodException, IOException { + Method method = SQSAsyncClient.class.getMethod("createQueueInRegion", + String.class, String.class, Array.newInstance( + CreateQueueOptions.class, 0).getClass()); + GeneratedHttpRequest httpMethod = processor + .createRequest(method, null, "queueName"); + + assertRequestLineEquals(httpMethod, + "POST https://sqs.us-east-1.amazonaws.com/ HTTP/1.1"); + assertHeadersEqual( + httpMethod, + "Content-Length: 57\nContent-Type: application/x-www-form-urlencoded\nHost: sqs.us-east-1.amazonaws.com\n"); + assertPayloadEquals(httpMethod, + "Version=2009-02-01&Action=CreateQueue&QueueName=queueName"); + + assertResponseParserClassEquals(method, httpMethod, + RegexQueueHandler.class); + assertSaxResponseParserClassEquals(method, null); + assertExceptionParserClassEquals(method, null); + + checkFilters(httpMethod); + } + + public void testCreateQueueInRegionOptions() throws SecurityException, + NoSuchMethodException, IOException { + Method method = SQSAsyncClient.class.getMethod("createQueueInRegion", + String.class, String.class, Array.newInstance( + CreateQueueOptions.class, 0).getClass()); + GeneratedHttpRequest httpMethod = processor + .createRequest(method, null, "queueName", + CreateQueueOptions.Builder.defaultVisibilityTimeout(45)); + + assertRequestLineEquals(httpMethod, + "POST https://sqs.us-east-1.amazonaws.com/ HTTP/1.1"); + assertHeadersEqual( + httpMethod, + "Content-Length: 85\nContent-Type: application/x-www-form-urlencoded\nHost: sqs.us-east-1.amazonaws.com\n"); + assertPayloadEquals( + httpMethod, + "Version=2009-02-01&Action=CreateQueue&QueueName=queueName&DefaultVisibilityTimeout=45"); + + assertResponseParserClassEquals(method, httpMethod, + RegexQueueHandler.class); + assertSaxResponseParserClassEquals(method, null); + assertExceptionParserClassEquals(method, null); + + checkFilters(httpMethod); + } + + public void testAllRegions() throws SecurityException, + NoSuchMethodException, IOException { + Method method = SQSAsyncClient.class.getMethod("createQueueInRegion", + String.class, String.class, Array.newInstance( + CreateQueueOptions.class, 0).getClass()); + for (String region : Iterables.filter(Region.ALL_SQS, + not(equalTo("us-standard")))) { processor.createRequest(method, region, "queueName"); } } @@ -163,15 +183,16 @@ public class SQSAsyncClientTest extends RestClientTest { return new SQSRestClientModule() { @Override protected void configure() { - Names.bindProperties(binder(), new SQSPropertiesBuilder(new Properties()) - .withCredentials("user", "key").build()); + Names.bindProperties(binder(), new SQSPropertiesBuilder( + new Properties()).withCredentials("user", "key").build()); install(new NullLoggingModule()); super.configure(); } @Override - protected String provideTimeStamp(final DateService dateService, - @Named(SQSConstants.PROPERTY_AWS_EXPIREINTERVAL) final int expiration) { + protected String provideTimeStamp( + final DateService dateService, + @Named(SQSConstants.PROPERTY_AWS_EXPIREINTERVAL) final int expiration) { return "2009-11-08T15:54:08.897Z"; } }; diff --git a/blobstore/src/main/resources/blobstore.properties b/blobstore/src/main/resources/blobstore.properties index be66b76917..a6e84b21e8 100644 --- a/blobstore/src/main/resources/blobstore.properties +++ b/blobstore/src/main/resources/blobstore.properties @@ -29,5 +29,7 @@ s3.contextbuilder=org.jclouds.aws.s3.S3ContextBuilder s3.propertiesbuilder=org.jclouds.aws.s3.S3PropertiesBuilder walrus.contextbuilder=org.jclouds.aws.s3.S3ContextBuilder walrus.propertiesbuilder=org.jclouds.aws.s3.WalrusPropertiesBuilder +googlestorage.contextbuilder=org.jclouds.aws.s3.S3ContextBuilder +googlestorage.propertiesbuilder=org.jclouds.aws.s3.GoogleStoragePropertiesBuilder transient.contextbuilder=org.jclouds.blobstore.TransientBlobStoreContextBuilder transient.propertiesbuilder=org.jclouds.blobstore.TransientBlobStorePropertiesBuilder