Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-305. Disable S3 virtual host buckets for generic S3 

Not all S3-compatible providers support virtual host buckets and thus
we should disable this feature by default.  Continue to enable virtual
host buckets for AWS-S3 which supports this although this feature
suffers from DNS settling issues.  Ran ran integration tests against
AWS-S3 and Scality using its S3 API.
This commit is contained in:
Andrew Gaul 2013-08-26 23:26:15 -07:00 committed by Adrian Cole
parent 73464c3877
commit 06b8cd7ac5
10 changed files with 70 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java
View File

@ -85,7 +85,7 @@ public class S3ApiMetadata extends BaseRestApiMetadata {
properties.setProperty(PROPERTY_AUTH_TAG, "AWS");
properties.setProperty(PROPERTY_HEADER_TAG, S3Headers.DEFAULT_AMAZON_HEADERTAG);
properties.setProperty(PROPERTY_S3_SERVICE_PATH, "/");
properties.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "true");
properties.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "false");
properties.setProperty(PROPERTY_RELAX_HOSTNAME, "true");
properties.setProperty(PROPERTY_BLOBSTORE_DIRECTORY_SUFFIX, "/");
properties.setProperty(PROPERTY_USER_METADATA_PREFIX, String.format("x-${%s}-meta-", PROPERTY_HEADER_TAG));

5
apis/s3/src/test/java/org/jclouds/s3/S3ClientExpectTest.java
View File

@ -36,11 +36,10 @@ public class S3ClientExpectTest extends BaseS3ClientExpectTest {
public void testBucketExistsReturnsTrueOn200AndFalseOn404() {
HttpRequest bucketFooExists = HttpRequest.builder().method("GET").endpoint(
URI.create("https://foo.s3.amazonaws.com/?max-keys=0")).headers(
URI.create("https://s3.amazonaws.com/foo?max-keys=0")).headers(
ImmutableMultimap.<String, String> builder()
.put("Host", "foo.s3.amazonaws.com")
.put("Date", CONSTANT_DATE)
.put("Authorization", "AWS identity:ZWVz2v/jGB+ZMmijoyfH9mFMPo0=")
.put("Authorization", "AWS identity:p32RsBr2inawMBeCkkiA228BT2w=")
.build()).build();
S3Client clientWhenBucketExists = requestSendsResponse(bucketFooExists, HttpResponse.builder().statusCode(200).build());

1
apis/s3/src/test/java/org/jclouds/s3/S3ClientMockTest.java
View File

@ -54,7 +54,6 @@ public class S3ClientMockTest {
static S3Client getS3Client(URL server) {
Properties overrides = new Properties();
overrides.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "false");
// prevent expect-100 bug http://code.google.com/p/mockwebserver/issues/detail?id=6
overrides.setProperty(PROPERTY_SO_TIMEOUT, "0");
overrides.setProperty(PROPERTY_MAX_RETRIES, "1");

18
apis/s3/src/test/java/org/jclouds/s3/blobstore/S3BlobSignerExpectTest.java
View File

@ -44,8 +44,7 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest {
@Override
protected HttpRequest getBlob() {
return HttpRequest.builder().method("GET")
.endpoint("https://container.s3.amazonaws.com/name")
.addHeader("Host", "container.s3.amazonaws.com")
.endpoint("https://s3.amazonaws.com/container/name")
.addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
.addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build();
}
@ -60,8 +59,7 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest {
@Override
protected HttpRequest getBlobWithTime() {
return HttpRequest.builder().method("GET")
.endpoint("https://container.s3.amazonaws.com/name")
.addHeader("Host", "container.s3.amazonaws.com")
.endpoint("https://s3.amazonaws.com/container/name")
.addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
.addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build();
}
@ -69,9 +67,8 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest {
@Override
protected HttpRequest getBlobWithOptions() {
return HttpRequest.builder().method("GET")
.endpoint("https://container.s3.amazonaws.com/name")
.endpoint("https://s3.amazonaws.com/container/name")
.addHeader("Range", "bytes=0-1")
.addHeader("Host", "container.s3.amazonaws.com")
.addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
.addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build();
}
@ -79,9 +76,8 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest {
@Override
protected HttpRequest putBlob() {
return HttpRequest.builder().method("PUT")
.endpoint("https://container.s3.amazonaws.com/name")
.endpoint("https://s3.amazonaws.com/container/name")
.addHeader("Expect", "100-continue")
.addHeader("Host", "container.s3.amazonaws.com")
.addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
.addHeader("Authorization", "AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=").build();
}
@ -96,9 +92,8 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest {
@Override
protected HttpRequest putBlobWithTime() {
return HttpRequest.builder().method("PUT")
.endpoint("https://container.s3.amazonaws.com/name")
.endpoint("https://s3.amazonaws.com/container/name")
.addHeader("Expect", "100-continue")
.addHeader("Host", "container.s3.amazonaws.com")
.addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
.addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build();
}
@ -106,8 +101,7 @@ public class S3BlobSignerExpectTest extends BaseBlobSignerExpectTest {
@Override
protected HttpRequest removeBlob() {
return HttpRequest.builder().method("DELETE")
.endpoint("https://container.s3.amazonaws.com/name")
.addHeader("Host", "container.s3.amazonaws.com")
.endpoint("https://s3.amazonaws.com/container/name")
.addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
.addHeader("Authorization", "AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=").build();
}

14
apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java
View File

@ -20,6 +20,7 @@ import static org.jclouds.Constants.PROPERTY_SESSION_INTERVAL;
import static org.jclouds.reflect.Reflection2.method;
import static org.testng.Assert.assertEquals;
import java.net.URI;
import java.util.Properties;
import javax.ws.rs.core.HttpHeaders;
@ -48,6 +49,7 @@ import com.google.common.collect.TreeMultimap;
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "RequestAuthorizeSignatureTest")
public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest<S3AsyncClient> {
String bucketName = "bucket";
@DataProvider(parallel = true)
public Object[][] dataProvider() throws NoSuchMethodException {
@ -89,7 +91,7 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest<S3Async
ImmutableList.<Object> of("bucket"));
StringBuilder builder = new StringBuilder();
filter.appendBucketName(request, builder);
assertEquals(builder.toString(), "/bucket");
assertEquals(builder.toString(), "");
}
@Test
@ -97,7 +99,7 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest<S3Async
HttpRequest request = putBucketAcl();
StringBuilder builder = new StringBuilder();
filter.appendUriPath(request, builder);
assertEquals(builder.toString(), "/?acl");
assertEquals(builder.toString(), "/" + bucketName + "?acl");
}
private GeneratedHttpRequest putBucketAcl() throws NoSuchMethodException {
@ -140,11 +142,13 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest<S3Async
}
@Test
void testAppendBucketNameURIHost() throws SecurityException, NoSuchMethodException {
void testAppendBucketNameInURIPath() throws SecurityException, NoSuchMethodException {
GeneratedHttpRequest request = processor.createRequest(
method(S3AsyncClient.class, "getBucketLocation", String.class),
ImmutableList.<Object> of("bucket"));
assertEquals(request.getEndpoint().getHost(), "bucket.s3.amazonaws.com");
ImmutableList.<Object> of(bucketName));
URI uri = request.getEndpoint();
assertEquals(uri.getHost(), "s3.amazonaws.com");
assertEquals(uri.getPath(), "/" + bucketName);
}
@Override

6
apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureWithSessionCredentialsTest.java
View File

@ -79,14 +79,14 @@ public class RequestAuthorizeSignatureWithSessionCredentialsTest {
HttpRequest bucketFooExists = GeneratedHttpRequest.builder().method("GET")
.invocation(invocation)
.endpoint("https://foo.s3.amazonaws.com/?max-keys=0")
.addHeader("Host", "foo.s3.amazonaws.com").build();
.endpoint("https://s3.amazonaws.com/foo?max-keys=0")
.build();
@Test
void testAddsSecurityToken() {
HttpRequest filtered = filter(temporaryCredentials).filter(bucketFooExists);
assertEquals(filtered.getFirstHeaderOrNull("Authorization"),
"AWS AKIAIOSFODNN7EXAMPLE:0fUhWTaRBcIvIAndg2C+5eLfE24=");
"AWS AKIAIOSFODNN7EXAMPLE:sbAAy3Gh/sD1zwO7Ut75YJFCs4U=");
assertEquals(filtered.getFirstHeaderOrNull("x-amz-security-token"), temporaryCredentials.getSessionToken());
}
}

12
apis/s3/src/test/java/org/jclouds/s3/handlers/S3RedirectionRetryHandlerExpectTest.java
View File

@ -36,17 +36,17 @@ public class S3RedirectionRetryHandlerExpectTest extends BaseS3ClientExpectTest
public void testRedirectOnHeadBucketChangesRequestToGetBucket() {
HttpRequest bucketFooExists = HttpRequest.builder().method("HEAD").endpoint(
URI.create("https://foo.s3.amazonaws.com/?max-keys=0")).headers(
ImmutableMultimap.<String, String> builder().put("Host", "foo.s3.amazonaws.com").put("Date",
CONSTANT_DATE).put("Authorization", "AWS identity:86P4BBb7xT+gBqq7jxM8Tc28ktY=").build())
URI.create("https://s3.amazonaws.com/foo?max-keys=0")).headers(
ImmutableMultimap.<String, String> builder().put("Date",
CONSTANT_DATE).put("Authorization", "AWS identity:p32RsBr2inawMBeCkkiA228BT2w=").build())
.build();
HttpResponse redirectResponse = HttpResponse.builder().statusCode(301).build();
HttpRequest bucketFooExistsNowUsesGET = HttpRequest.builder().method("GET").endpoint(
URI.create("https://foo.s3.amazonaws.com/?max-keys=0")).headers(
ImmutableMultimap.<String, String> builder().put("Host", "foo.s3.amazonaws.com").put("Date",
CONSTANT_DATE).put("Authorization", "AWS identity:ZWVz2v/jGB+ZMmijoyfH9mFMPo0=").build())
URI.create("https://s3.amazonaws.com/foo?max-keys=0")).headers(
ImmutableMultimap.<String, String> builder().put("Date",
CONSTANT_DATE).put("Authorization", "AWS identity:p32RsBr2inawMBeCkkiA228BT2w=").build())
.build();
HttpResponse success = HttpResponse.builder().statusCode(200).build();

4
core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java
View File

@ -103,8 +103,8 @@ import com.google.inject.TypeLiteral;
* <pre>
*
* HttpRequest bucketFooExists = HttpRequest.builder().method(&quot;HEAD&quot;).endpoint(
* URI.create(&quot;https://foo.s3.amazonaws.com/?max-keys=0&quot;)).headers(
* ImmutableMultimap.&lt;String, String&gt; builder().put(&quot;Host&quot;, &quot;foo.s3.amazonaws.com&quot;).put(&quot;Date&quot;, CONSTANT_DATE)
* URI.create(&quot;https://s3.amazonaws.com/foo?max-keys=0&quot;)).headers(
* ImmutableMultimap.&lt;String, String&gt; builder().put(&quot;Date&quot;, CONSTANT_DATE)
* .put(&quot;Authorization&quot;, &quot;AWS identity:86P4BBb7xT+gBqq7jxM8Tc28ktY=&quot;).build()).build();
*
* S3Client clientWhenBucketExists = requestSendsResponse(bucketFooExists, HttpResponse.builder().statusCode(200).build());

2
providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3ApiMetadata.java
View File

@ -17,6 +17,7 @@
package org.jclouds.aws.s3;
import static org.jclouds.reflect.Reflection2.typeToken;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
import java.util.Properties;
@ -61,6 +62,7 @@ public class AWSS3ApiMetadata extends S3ApiMetadata {
public static Properties defaultProperties() {
Properties properties = S3ApiMetadata.defaultProperties();
properties.setProperty(PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "true");
return properties;
}

39
providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerExpectTest.java
View File

@ -36,11 +36,21 @@ import com.google.inject.Module;
@Test(groups = "unit", testName = "AWSS3BlobSignerExpectTest")
public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest {
private static final String DATE = "Thu, 05 Jun 2008 16:38:19 GMT";
private static final String HOST = "container.s3.amazonaws.com";
public AWSS3BlobSignerExpectTest() {
provider = "aws-s3";
}
@Override
protected HttpRequest getBlob() {
return HttpRequest.builder().method("GET")
.endpoint("https://container.s3.amazonaws.com/name")
.addHeader("Host", HOST)
.addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
.addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build();
}
@Override
protected HttpRequest getBlobWithTime() {
return HttpRequest.builder().method("GET")
@ -50,6 +60,16 @@ public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest {
.addHeader("Date", DATE).build();
}
@Override
protected HttpRequest getBlobWithOptions() {
return HttpRequest.builder().method("GET")
.endpoint("https://container.s3.amazonaws.com/name")
.addHeader("Host", HOST)
.addHeader("Range", "bytes=0-1")
.addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
.addHeader("Authorization", "AWS identity:0uvBv1wEskuhFHYJF/L6kEV9A7o=").build();
}
@Test
public void testSignGetBlobWithTime() {
BlobStore getBlobWithTime = requestsSendResponses(init());
@ -58,6 +78,16 @@ public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest {
compare);
}
@Override
protected HttpRequest putBlob() {
return HttpRequest.builder().method("PUT")
.endpoint("https://container.s3.amazonaws.com/name")
.addHeader("Host", HOST)
.addHeader("Expect", "100-continue")
.addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
.addHeader("Authorization", "AWS identity:j9Dy/lmmvlCKjA4lkqZenLxMkR4=").build();
}
@Override
protected HttpRequest putBlobWithTime() {
return HttpRequest.builder().method("PUT")
@ -68,6 +98,15 @@ public class AWSS3BlobSignerExpectTest extends S3BlobSignerExpectTest {
.addHeader("Date", DATE).build();
}
@Override
protected HttpRequest removeBlob() {
return HttpRequest.builder().method("DELETE")
.endpoint("https://container.s3.amazonaws.com/name")
.addHeader("Host", HOST)
.addHeader("Date", "Thu, 05 Jun 2008 16:38:19 GMT")
.addHeader("Authorization", "AWS identity:4FnyjdX/ULdDMRbVlLNjZfEo9RQ=").build();
}
@Test
public void testSignPutBlobWithTime() throws Exception {
BlobStore signPutBloblWithTime = requestsSendResponses(init());