From 06b8cd7ac5420b0e3668bca6116ad5a3a29bbca8 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 26 Aug 2013 23:26:15 -0700 Subject: [PATCH] JCLOUDS-305. Disable S3 virtual host buckets for generic S3 Not all S3-compatible providers support virtual host buckets and thus we should disable this feature by default. Continue to enable virtual host buckets for AWS-S3 which supports this although this feature suffers from DNS settling issues. Ran ran integration tests against AWS-S3 and Scality using its S3 API. --- .../java/org/jclouds/s3/S3ApiMetadata.java | 2 +- .../org/jclouds/s3/S3ClientExpectTest.java | 5 +-- .../java/org/jclouds/s3/S3ClientMockTest.java | 1 - .../s3/blobstore/S3BlobSignerExpectTest.java | 18 +++------ .../RequestAuthorizeSignatureTest.java | 14 ++++--- ...zeSignatureWithSessionCredentialsTest.java | 6 +-- .../S3RedirectionRetryHandlerExpectTest.java | 12 +++--- .../rest/internal/BaseRestApiExpectTest.java | 4 +- .../org/jclouds/aws/s3/AWSS3ApiMetadata.java | 2 + .../blobstore/AWSS3BlobSignerExpectTest.java | 39 +++++++++++++++++++ 10 files changed, 70 insertions(+), 33 deletions(-) diff --git a/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java index 92ec4272cb..281d97a489 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java @@ -85,7 +85,7 @@ public class S3ApiMetadata extends BaseRestApiMetadata { properties.setProperty(PROPERTY_AUTH_TAG, "AWS"); properties.setProperty(PROPERTY_HEADER_TAG, S3Headers.DEFAULT_AMAZON_HEADERTAG); properties.setProperty(PROPERTY_S3_SERVICE_PATH, "/"); - properties.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "true"); + properties.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "false"); properties.setProperty(PROPERTY_RELAX_HOSTNAME, "true"); properties.setProperty(PROPERTY_BLOBSTORE_DIRECTORY_SUFFIX, "/"); properties.setProperty(PROPERTY_USER_METADATA_PREFIX, String.format("x-${%s}-meta-", PROPERTY_HEADER_TAG)); diff --git a/apis/s3/src/test/java/org/jclouds/s3/S3ClientExpectTest.java b/apis/s3/src/test/java/org/jclouds/s3/S3ClientExpectTest.java index e932cd9e50..9203394058 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/S3ClientExpectTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/S3ClientExpectTest.java @@ -36,11 +36,10 @@ public class S3ClientExpectTest extends BaseS3ClientExpectTest { public void testBucketExistsReturnsTrueOn200AndFalseOn404() { HttpRequest bucketFooExists = HttpRequest.builder().method("GET").endpoint( - URI.create("https://foo.s3.amazonaws.com/?max-keys=0")).headers( + URI.create("https://s3.amazonaws.com/foo?max-keys=0")).headers( ImmutableMultimap. builder() - .put("Host", "foo.s3.amazonaws.com") .put("Date", CONSTANT_DATE) - .put("Authorization", "AWS identity:ZWVz2v/jGB+ZMmijoyfH9mFMPo0=") + .put("Authorization", "AWS identity:p32RsBr2inawMBeCkkiA228BT2w=") .build()).build(); S3Client clientWhenBucketExists = requestSendsResponse(bucketFooExists, HttpResponse.builder().statusCode(200).build()); diff --git a/apis/s3/src/test/java/org/jclouds/s3/S3ClientMockTest.java b/apis/s3/src/test/java/org/jclouds/s3/S3ClientMockTest.java index 748a6b451f..ccdf326e93 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/S3ClientMockTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/S3ClientMockTest.java @@ -54,7 +54,6 @@ public class S3ClientMockTest { static S3Client getS3Client(URL server) { Properties overrides = new Properties(); - overrides.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "false"); // prevent expect-100 bug http://code.google.com/p/mockwebserver/issues/detail?id=6 overrides.setProperty(PROPERTY_SO_TIMEOUT, "0"); overrides.setProperty(PROPERTY_MAX_RETRIES, "1"); diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobSignerExpectTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobSignerExpectTest.java index 18f046c85e..522381c84e 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobSignerExpectTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobSignerExpectTest.java @@ -44,8 +44,7 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest getBlob() { return HttpRequest.builder().method("GET") - .endpoint("https://container.s3.amazonaws.com/name") - .addHeader("Host", "container.s3.amazonaws.com") + .endpoint("https://s3.amazonaws.com/container/name") .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build(); } @@ -60,8 +59,7 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest getBlobWithTime() { return HttpRequest.builder().method("GET") - .endpoint("https://container.s3.amazonaws.com/name") - .addHeader("Host", "container.s3.amazonaws.com") + .endpoint("https://s3.amazonaws.com/container/name") .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build(); } @@ -69,9 +67,8 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest getBlobWithOptions() { return HttpRequest.builder().method("GET") - .endpoint("https://container.s3.amazonaws.com/name") + .endpoint("https://s3.amazonaws.com/container/name") .addHeader("Range", "bytes=0-1") - .addHeader("Host", "container.s3.amazonaws.com") .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build(); } @@ -79,9 +76,8 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest putBlob() { return HttpRequest.builder().method("PUT") - .endpoint("https://container.s3.amazonaws.com/name") + .endpoint("https://s3.amazonaws.com/container/name") .addHeader("Expect", "100-continue") - .addHeader("Host", "container.s3.amazonaws.com") .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=").build(); } @@ -96,9 +92,8 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest putBlobWithTime() { return HttpRequest.builder().method("PUT") - .endpoint("https://container.s3.amazonaws.com/name") + .endpoint("https://s3.amazonaws.com/container/name") .addHeader("Expect", "100-continue") - .addHeader("Host", "container.s3.amazonaws.com") .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build(); } @@ -106,8 +101,7 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest { @Override protected HttpRequest removeBlob() { return HttpRequest.builder().method("DELETE") - .endpoint("https://container.s3.amazonaws.com/name") - .addHeader("Host", "container.s3.amazonaws.com") + .endpoint("https://s3.amazonaws.com/container/name") .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT") .addHeader("Authorization", "AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=").build(); } diff --git a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java index 129800321f..a680103cbb 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java @@ -20,6 +20,7 @@ import static org.jclouds.Constants.PROPERTY_SESSION_INTERVAL; import static org.jclouds.reflect.Reflection2.method; import static org.testng.Assert.assertEquals; +import java.net.URI; import java.util.Properties; import javax.ws.rs.core.HttpHeaders; @@ -48,6 +49,7 @@ import com.google.common.collect.TreeMultimap; // NOTE:without testName, this will not call @Before* and fail w/NPE during surefire @Test(groups = "unit", testName = "RequestAuthorizeSignatureTest") public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest { + String bucketName = "bucket"; @DataProvider(parallel = true) public Object[][] dataProvider() throws NoSuchMethodException { @@ -89,7 +91,7 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest of("bucket")); StringBuilder builder = new StringBuilder(); filter.appendBucketName(request, builder); - assertEquals(builder.toString(), "/bucket"); + assertEquals(builder.toString(), ""); } @Test @@ -97,7 +99,7 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest of("bucket")); - assertEquals(request.getEndpoint().getHost(), "bucket.s3.amazonaws.com"); + ImmutableList. of(bucketName)); + URI uri = request.getEndpoint(); + assertEquals(uri.getHost(), "s3.amazonaws.com"); + assertEquals(uri.getPath(), "/" + bucketName); } @Override diff --git a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureWithSessionCredentialsTest.java b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureWithSessionCredentialsTest.java index 267eec3833..d80394f977 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureWithSessionCredentialsTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureWithSessionCredentialsTest.java @@ -79,14 +79,14 @@ public class RequestAuthorizeSignatureWithSessionCredentialsTest { HttpRequest bucketFooExists = GeneratedHttpRequest.builder().method("GET") .invocation(invocation) - .endpoint("https://foo.s3.amazonaws.com/?max-keys=0") - .addHeader("Host", "foo.s3.amazonaws.com").build(); + .endpoint("https://s3.amazonaws.com/foo?max-keys=0") + .build(); @Test void testAddsSecurityToken() { HttpRequest filtered = filter(temporaryCredentials).filter(bucketFooExists); assertEquals(filtered.getFirstHeaderOrNull("Authorization"), - "AWS AKIAIOSFODNN7EXAMPLE:0fUhWTaRBcIvIAndg2C+5eLfE24="); + "AWS AKIAIOSFODNN7EXAMPLE:sbAAy3Gh/sD1zwO7Ut75YJFCs4U="); assertEquals(filtered.getFirstHeaderOrNull("x-amz-security-token"), temporaryCredentials.getSessionToken()); } } diff --git a/apis/s3/src/test/java/org/jclouds/s3/handlers/S3RedirectionRetryHandlerExpectTest.java b/apis/s3/src/test/java/org/jclouds/s3/handlers/S3RedirectionRetryHandlerExpectTest.java index 78a6f12be6..ec2d6cc37c 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/handlers/S3RedirectionRetryHandlerExpectTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/handlers/S3RedirectionRetryHandlerExpectTest.java @@ -36,17 +36,17 @@ public class S3RedirectionRetryHandlerExpectTest extends BaseS3ClientExpectTest public void testRedirectOnHeadBucketChangesRequestToGetBucket() { HttpRequest bucketFooExists = HttpRequest.builder().method("HEAD").endpoint( - URI.create("https://foo.s3.amazonaws.com/?max-keys=0")).headers( - ImmutableMultimap. builder().put("Host", "foo.s3.amazonaws.com").put("Date", - CONSTANT_DATE).put("Authorization", "AWS identity:86P4BBb7xT+gBqq7jxM8Tc28ktY=").build()) + URI.create("https://s3.amazonaws.com/foo?max-keys=0")).headers( + ImmutableMultimap. builder().put("Date", + CONSTANT_DATE).put("Authorization", "AWS identity:p32RsBr2inawMBeCkkiA228BT2w=").build()) .build(); HttpResponse redirectResponse = HttpResponse.builder().statusCode(301).build(); HttpRequest bucketFooExistsNowUsesGET = HttpRequest.builder().method("GET").endpoint( - URI.create("https://foo.s3.amazonaws.com/?max-keys=0")).headers( - ImmutableMultimap. builder().put("Host", "foo.s3.amazonaws.com").put("Date", - CONSTANT_DATE).put("Authorization", "AWS identity:ZWVz2v/jGB+ZMmijoyfH9mFMPo0=").build()) + URI.create("https://s3.amazonaws.com/foo?max-keys=0")).headers( + ImmutableMultimap. builder().put("Date", + CONSTANT_DATE).put("Authorization", "AWS identity:p32RsBr2inawMBeCkkiA228BT2w=").build()) .build(); HttpResponse success = HttpResponse.builder().statusCode(200).build(); diff --git a/core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java b/core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java index e4a24ad6b2..76186228b0 100644 --- a/core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java +++ b/core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java @@ -103,8 +103,8 @@ import com.google.inject.TypeLiteral; * 
  * 
  * HttpRequest bucketFooExists = HttpRequest.builder().method("HEAD").endpoint(
- *          URI.create("https://foo.s3.amazonaws.com/?max-keys=0")).headers(
- *          ImmutableMultimap.<String, String> builder().put("Host", "foo.s3.amazonaws.com").put("Date", CONSTANT_DATE)
+ *          URI.create("https://s3.amazonaws.com/foo?max-keys=0")).headers(
+ *          ImmutableMultimap.<String, String> builder().put("Date", CONSTANT_DATE)
  *                   .put("Authorization", "AWS identity:86P4BBb7xT+gBqq7jxM8Tc28ktY=").build()).build();
  * 
  * S3Client clientWhenBucketExists = requestSendsResponse(bucketFooExists, HttpResponse.builder().statusCode(200).build());
diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ApiMetadata.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ApiMetadata.java
index dc616f87e6..d896a9ff8a 100644
--- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ApiMetadata.java
+++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ApiMetadata.java
@@ -17,6 +17,7 @@
 package org.jclouds.aws.s3;
 
 import static org.jclouds.reflect.Reflection2.typeToken;
+import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
 
 import java.util.Properties;
 
@@ -61,6 +62,7 @@ public class AWSS3ApiMetadata extends S3ApiMetadata {
    
    public static Properties defaultProperties() {
       Properties properties = S3ApiMetadata.defaultProperties();
+      properties.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "true");
       return properties;
    }
 
diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerExpectTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerExpectTest.java
index 0b38b0b420..aa36785ed8 100644
--- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerExpectTest.java
+++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerExpectTest.java
@@ -36,11 +36,21 @@ import com.google.inject.Module;
 @Test(groups = "unit", testName = "AWSS3BlobSignerExpectTest")
 public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest {
    private static final String DATE = "Thu, 05 Jun 2008 16:38:19 GMT";
+   private static final String HOST = "container.s3.amazonaws.com";
 
    public AWSS3BlobSignerExpectTest() {
       provider = "aws-s3";
    }
 
+   @Override
+   protected HttpRequest getBlob() {
+      return HttpRequest.builder().method("GET")
+            .endpoint("https://container.s3.amazonaws.com/name")
+            .addHeader("Host", HOST)
+            .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
+            .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build();
+   }
+
    @Override
    protected HttpRequest getBlobWithTime() {
       return HttpRequest.builder().method("GET")
@@ -50,6 +60,16 @@ public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest {
          .addHeader("Date", DATE).build();
    }
 
+   @Override
+   protected HttpRequest getBlobWithOptions() {
+      return HttpRequest.builder().method("GET")
+            .endpoint("https://container.s3.amazonaws.com/name")
+            .addHeader("Host", HOST)
+            .addHeader("Range", "bytes=0-1")
+            .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
+            .addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build();
+   }
+
    @Test
    public void testSignGetBlobWithTime() {
       BlobStore getBlobWithTime = requestsSendResponses(init());
@@ -58,6 +78,16 @@ public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest {
          compare);
    }
 
+   @Override
+   protected HttpRequest putBlob() {
+      return HttpRequest.builder().method("PUT")
+            .endpoint("https://container.s3.amazonaws.com/name")
+            .addHeader("Host", HOST)
+            .addHeader("Expect", "100-continue")
+            .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
+            .addHeader("Authorization", "AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=").build();
+   }
+
    @Override
    protected HttpRequest putBlobWithTime() {
       return HttpRequest.builder().method("PUT")
@@ -68,6 +98,15 @@ public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest {
          .addHeader("Date", DATE).build();
    }
 
+   @Override
+   protected HttpRequest removeBlob() {
+      return HttpRequest.builder().method("DELETE")
+            .endpoint("https://container.s3.amazonaws.com/name")
+            .addHeader("Host", HOST)
+            .addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
+            .addHeader("Authorization", "AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=").build();
+   }
+
    @Test
    public void testSignPutBlobWithTime() throws Exception {
       BlobStore signPutBloblWithTime = requestsSendResponses(init());