enforced bucket naming rules

git-svn-id: http://jclouds.googlecode.com/svn/trunk@575 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
adrian.f.cole 2009-05-10 02:06:05 +00:00
parent 151ebfb79b
commit c7dd029445
15 changed files with 165 additions and 106 deletions

View File

@ -23,7 +23,8 @@
*/
package org.jclouds.aws.s3;
import static com.google.common.base.Preconditions.*;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.File;
import java.io.FileInputStream;
@ -33,6 +34,7 @@ import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
@ -46,6 +48,24 @@ import org.jclouds.aws.s3.domain.S3Object;
public class S3Utils extends Utils {
private static final Pattern IP_PATTERN = Pattern
.compile("b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?).)"
+ "{3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)b");
public static String validateBucketName(String bucketName) {
checkNotNull(bucketName, "bucketName");
checkArgument(bucketName.matches("^[a-z0-9].*"),
"bucket name must start with a number or letter");
checkArgument(
bucketName.matches("^[-_.a-z0-9]+"),
"bucket name can only contain lowercase letters, numbers, periods (.), underscores (_), and dashes (-)");
checkArgument(bucketName.length() > 2 && bucketName.length() < 256,
"bucket name must be between 3 and 255 characters long");
checkArgument(!IP_PATTERN.matcher(bucketName).matches(),
"bucket name cannot be ip address style");
return bucketName;
}
static final byte[] HEX_CHAR_TABLE = { (byte) '0', (byte) '1', (byte) '2',
(byte) '3', (byte) '4', (byte) '5', (byte) '6', (byte) '7',
(byte) '8', (byte) '9', (byte) 'a', (byte) 'b', (byte) 'c',

View File

@ -52,7 +52,7 @@ public class CopyObject extends S3FutureCommand<S3Object.Metadata> {
getRequest().getHeaders().put(
"x-amz-copy-source",
String.format("/%1s/%2s", checkNotNull(sourceBucket,
"sourceBucket").toLowerCase(), checkNotNull(
"sourceBucket"), checkNotNull(
sourceObject, "sourceObject")));
getRequest().getHeaders().putAll(options.buildRequestHeaders());
}

View File

@ -23,11 +23,17 @@
*/
package org.jclouds.aws.s3.commands;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3ResponseException;
import org.jclouds.aws.s3.commands.options.ListBucketOptions;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.xml.ListBucketHandler;
import org.jclouds.http.commands.callables.xml.ParseSax;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.name.Named;
@ -44,4 +50,35 @@ public class ListBucket extends S3FutureCommand<S3Bucket> {
.getHandler();
handler.setBucketName(bucket);
}
@Override
public S3Bucket get() throws InterruptedException, ExecutionException {
try {
return super.get();
} catch (ExecutionException e) {
return attemptNotFound(e);
}
}
@VisibleForTesting
S3Bucket attemptNotFound(ExecutionException e) throws ExecutionException {
if (e.getCause() != null && e.getCause() instanceof S3ResponseException) {
S3ResponseException responseException = (S3ResponseException) e
.getCause();
if ("NoSuchBucket".equals(responseException.getError().getCode())) {
return S3Bucket.NOT_FOUND;
}
}
throw e;
}
@Override
public S3Bucket get(long l, TimeUnit timeUnit) throws InterruptedException,
ExecutionException, TimeoutException {
try {
return super.get(l, timeUnit);
} catch (ExecutionException e) {
return attemptNotFound(e);
}
}
}

View File

@ -23,6 +23,7 @@
*/
package org.jclouds.aws.s3.commands;
import org.jclouds.aws.s3.S3Utils;
import org.jclouds.aws.s3.commands.options.PutBucketOptions;
import org.jclouds.http.HttpHeaders;
import org.jclouds.http.commands.callables.ReturnTrueIf2xx;
@ -41,9 +42,10 @@ public class PutBucket extends S3FutureCommand<Boolean> {
@Inject
public PutBucket(@Named("jclouds.http.address") String amazonHost,
ReturnTrueIf2xx callable, @Assisted String s3Bucket,
ReturnTrueIf2xx callable, @Assisted String bucketName,
@Assisted PutBucketOptions options) {
super("PUT", "/", callable, amazonHost, s3Bucket);
super("PUT", "/", callable, amazonHost, S3Utils
.validateBucketName(bucketName));
getRequest().getHeaders().putAll(options.buildRequestHeaders());
String payload = options.buildPayload();
if (payload != null) {
@ -52,5 +54,4 @@ public class PutBucket extends S3FutureCommand<Boolean> {
payload.getBytes().length + "");
}
}
}

View File

@ -43,10 +43,9 @@ public class S3FutureCommand<T> extends HttpFutureCommand<T> {
addHostHeader(checkNotNull(amazonHost, "amazonHost"));
}
protected void addHostHeader(String amazonHost, String bucketName) {
String host = checkNotNull(bucketName, "s3Bucket.getName()") + "."
+ amazonHost;
addHostHeader(host.toLowerCase());
addHostHeader(checkNotNull(bucketName) + "." + amazonHost);
}
}

View File

@ -24,11 +24,12 @@
package org.jclouds.aws.s3.domain;
import static com.google.common.base.Preconditions.checkNotNull;
import org.joda.time.DateTime;
import java.util.HashSet;
import java.util.Set;
import org.joda.time.DateTime;
/**
* A container that provides namespace, access control and aggregation of
* {@link S3Object}s
@ -129,7 +130,7 @@ public class S3Bucket {
private S3Owner canonicalUser;
public Metadata(String name) {
this.name = checkNotNull(name, "name").toLowerCase();
this.name = checkNotNull(name, "name");
}
public String getName() {

View File

@ -32,6 +32,7 @@ public class S3Error {
private String header;
private String signatureProvided;
private String stringToSign;
private String stringSigned;
private String stringToSignBytes;
@Override
@ -47,6 +48,7 @@ public class S3Error {
sb.append(", signatureProvided='").append(signatureProvided).append(
'\'');
sb.append(", stringToSign='").append(stringToSign).append('\'');
sb.append(", stringSigned='").append(getStringSigned()).append('\'');
sb.append(", stringToSignBytes='").append(stringToSignBytes).append(
'\'');
sb.append('}');
@ -124,4 +126,12 @@ public class S3Error {
public String getHeader() {
return header;
}
public void setStringSigned(String stringSigned) {
this.stringSigned = stringSigned;
}
public String getStringSigned() {
return stringSigned;
}
}

View File

@ -64,6 +64,9 @@ public class ParseS3ErrorFromXmlContent implements HttpResponseHandler {
try {
S3Error error = parserFactory.createErrorParser().parse(
errorStream);
if ("SignatureDoesNotMatch".equals(error.getCode()))
error.setStringSigned(RequestAuthorizeSignature
.createStringToSign(command.getRequest()));
logger.trace("received the following error from s3: %1s",
error);
command.setException(new S3ResponseException(command,

View File

@ -94,27 +94,32 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
addDateHeader(request);
StringBuilder toSign = new StringBuilder();
appendMethod(request, toSign);
appendHttpHeaders(request, toSign);
appendAmzHeaders(request, toSign);
appendBucketName(request, toSign);
appendUriPath(request, toSign);
String toSign = createStringToSign(request);
addAuthHeader(request, toSign);
}
public static String createStringToSign(HttpRequest request) {
StringBuilder buffer = new StringBuilder();
appendMethod(request, buffer);
appendHttpHeaders(request, buffer);
appendAmzHeaders(request, buffer);
appendBucketName(request, buffer);
appendUriPath(request, buffer);
return buffer.toString();
}
private void removeOldHeaders(HttpRequest request) {
request.getHeaders().removeAll(S3Constants.AUTHORIZATION);
request.getHeaders().removeAll(HttpHeaders.CONTENT_TYPE);
request.getHeaders().removeAll(HttpHeaders.DATE);
}
private void addAuthHeader(HttpRequest request, StringBuilder toSign)
private void addAuthHeader(HttpRequest request, String toSign)
throws HttpException {
String signature;
try {
signature = S3Utils.hmacSha1Base64(toSign.toString(), secretKey
signature = S3Utils.hmacSha1Base64(toSign, secretKey
.getBytes());
} catch (Exception e) {
throw new HttpException("error signing request", e);
@ -123,7 +128,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
"AWS " + accessKey + ":" + signature);
}
private void appendMethod(HttpRequest request, StringBuilder toSign) {
private static void appendMethod(HttpRequest request, StringBuilder toSign) {
toSign.append(request.getMethod()).append("\n");
}
@ -131,7 +136,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
request.getHeaders().put(HttpHeaders.DATE, timestampAsHeaderString());
}
private void appendAmzHeaders(HttpRequest request, StringBuilder toSign) {
private static void appendAmzHeaders(HttpRequest request, StringBuilder toSign) {
Set<String> headers = new TreeSet<String>(request.getHeaders().keySet());
for (String header : headers) {
if (header.startsWith("x-amz-")) {
@ -144,13 +149,13 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
}
}
private void appendHttpHeaders(HttpRequest request, StringBuilder toSign) {
private static void appendHttpHeaders(HttpRequest request, StringBuilder toSign) {
for (String header : firstHeadersToSign)
toSign.append(valueOrEmpty(request.getHeaders().get(header)))
.append("\n");
}
private void appendBucketName(HttpRequest request, StringBuilder toSign) {
private static void appendBucketName(HttpRequest request, StringBuilder toSign) {
String hostHeader = request.getHeaders().get(HttpHeaders.HOST)
.iterator().next();
if (hostHeader.endsWith(".s3.amazonaws.com"))
@ -158,7 +163,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
hostHeader.substring(0, hostHeader.length() - 17));
}
private void appendUriPath(HttpRequest request, StringBuilder toSign) {
private static void appendUriPath(HttpRequest request, StringBuilder toSign) {
int queryIndex = request.getUri().indexOf('?');
if (queryIndex >= 0)
toSign.append(request.getUri().substring(0, queryIndex));
@ -166,7 +171,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
toSign.append(request.getUri());
}
private String valueOrEmpty(Collection<String> collection) {
private static String valueOrEmpty(Collection<String> collection) {
return (collection != null && collection.size() >= 1) ? collection
.iterator().next() : "";
}

View File

@ -27,46 +27,24 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import org.testng.annotations.Test;
@Test(groups = "unit", testName = "s3.AmazonS3Test")
/**
* Tests connection by listing all the buckets and their size
*
* @author Adrian Cole
*
*/
@Test(groups = "unit", testName = "s3.S3ConnectionTest")
public class S3ConnectionTest extends S3IntegrationTest {
@Test
void testListBuckets() throws Exception {
listAllMyBuckets();
}
List<S3Bucket.Metadata> listAllMyBuckets() throws Exception {
return client.getOwnedBuckets().get(10, TimeUnit.SECONDS);
}
S3Object getObject() throws Exception {
return client.getObject(bucketPrefix + "adrianjbosstest", "3366").get(
10, TimeUnit.SECONDS);
}
S3Object.Metadata headObject() throws Exception {
String bucketName = bucketPrefix + "adrianjbosstest";
return client.headObject(bucketName, "3366").get(10,
List<S3Bucket.Metadata> myBuckets = client.getOwnedBuckets().get(10,
TimeUnit.SECONDS);
}
Boolean deleteBucket() throws Exception {
String bucketName = bucketPrefix + "adrianjbosstest";
return client.deleteBucketIfEmpty(bucketName).get(10, TimeUnit.SECONDS);
}
Boolean deleteObject() throws Exception {
String bucketName = bucketPrefix + "adrianjbosstest";
return client.deleteObject(bucketName, "3366")
.get(10, TimeUnit.SECONDS);
}
S3Bucket getBucket() throws Exception {
String bucketName = bucketPrefix + "adrianjbosstest";
return client.listBucket(bucketName).get(10, TimeUnit.SECONDS);
for (S3Bucket.Metadata bucket : myBuckets) {
context.createInputStreamMap(bucket.getName()).size();
}
}
}

View File

@ -84,8 +84,9 @@ public class S3IntegrationTest {
client.putObject(sourceBucket, object).get(10, TimeUnit.SECONDS);
}
protected S3Object validateContent(String sourceBucket, String key) throws InterruptedException, ExecutionException,
TimeoutException, IOException {
protected S3Object validateContent(String sourceBucket, String key)
throws InterruptedException, ExecutionException, TimeoutException,
IOException {
assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS)
.getContents().size(), 1);
S3Object newObject = client.getObject(sourceBucket, key).get(10,
@ -126,8 +127,8 @@ public class S3IntegrationTest {
protected S3Connection client;
protected S3Context context = null;
protected String bucketPrefix = System.getProperty("user.name") + "."
+ this.getClass().getSimpleName();
protected String bucketPrefix = (System.getProperty("user.name") + "." + this
.getClass().getSimpleName()).toLowerCase();
private static final String sysAWSAccessKeyId = System
.getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
@ -177,8 +178,8 @@ public class S3IntegrationTest {
protected void deleteEverything() throws Exception {
try {
List<S3Bucket.Metadata> metaData = client
.getOwnedBuckets().get(10, TimeUnit.SECONDS);
List<S3Bucket.Metadata> metaData = client.getOwnedBuckets().get(10,
TimeUnit.SECONDS);
List<Future<Boolean>> results = new ArrayList<Future<Boolean>>();
for (S3Bucket.Metadata metaDatum : metaData) {
if (metaDatum.getName().startsWith(bucketPrefix.toLowerCase())) {

View File

@ -28,18 +28,20 @@ import org.testng.annotations.Test;
import java.util.Properties;
/**
* // TODO: Adrian: Document this!
*
* This performs the same test as {@link S3ConnectionTest}, except using SSL.
*
* @author Adrian Cole
*/
@Test(groups = "unit", sequential = true, testName = "s3.AmazonS3SSLTest")
public class AmazonS3SSLTest extends S3ConnectionTest {
@Test(groups = "unit", testName = "s3.SecureS3ConnectionTest")
public class SecureS3ConnectionTest extends S3ConnectionTest {
@Override
protected Properties buildS3Properties(String AWSAccessKeyId, String AWSSecretAccessKey) {
Properties properties = super.buildS3Properties(AWSAccessKeyId, AWSSecretAccessKey);
properties.setProperty("jclouds.http.secure", Boolean.toString(true));
protected Properties buildS3Properties(String AWSAccessKeyId,
String AWSSecretAccessKey) {
Properties properties = super.buildS3Properties(AWSAccessKeyId,
AWSSecretAccessKey);
properties.setProperty("jclouds.http.secure", Boolean.toString(true));
properties.setProperty("jclouds.http.port", "443");
return properties;
return properties;
}
}

View File

@ -57,13 +57,13 @@ import com.google.common.collect.Multimap;
*/
@Test(groups = "integration", testName = "s3.CopyObjectIntegrationTest")
public class CopyObjectIntegrationTest extends S3IntegrationTest {
String sourceKey = "apples";
String destinationKey = "pears";
@Test()
void testCopyObject() throws Exception {
String sourceBucket = bucketPrefix + "testCopyObject";
String sourceKey = "apples";
String destinationBucket = bucketPrefix + "testCopyObjectDestination";
String destinationKey = "pears";
String sourceBucket = bucketPrefix + "testcopyobject";
String destinationBucket = sourceBucket + "dest";
setupSourceBucket(sourceBucket, sourceKey);
@ -92,11 +92,8 @@ public class CopyObjectIntegrationTest extends S3IntegrationTest {
@Test
void testCopyIfModifiedSince() throws InterruptedException,
ExecutionException, TimeoutException, IOException {
String sourceBucket = bucketPrefix + "testCopyIfModifiedSince";
String sourceKey = "apples";
String destinationBucket = bucketPrefix
+ "testCopyIfModifiedSinceDestination";
String destinationKey = "pears";
String sourceBucket = bucketPrefix + "tcims";
String destinationBucket = sourceBucket + "dest";
DateTime before = new DateTime();
setupSourceBucket(sourceBucket, sourceKey);
@ -121,11 +118,8 @@ public class CopyObjectIntegrationTest extends S3IntegrationTest {
@Test
void testCopyIfUnmodifiedSince() throws InterruptedException,
ExecutionException, TimeoutException, IOException {
String sourceBucket = bucketPrefix + "testCopyIfUnmodifiedSince";
String sourceKey = "apples";
String destinationBucket = bucketPrefix
+ "testCopyIfUnmodifiedSinceDestination";
String destinationKey = "pears";
String sourceBucket = bucketPrefix + "tcius";
String destinationBucket = sourceBucket + "dest";
DateTime before = new DateTime();
setupSourceBucket(sourceBucket, sourceKey);
@ -150,13 +144,11 @@ public class CopyObjectIntegrationTest extends S3IntegrationTest {
@Test
void testCopyIfMatch() throws InterruptedException, ExecutionException,
TimeoutException, IOException {
String sourceBucket = bucketPrefix + "testCopyIfMatch";
String sourceKey = "apples";
String sourceBucket = bucketPrefix + "tcim";
byte[] realMd5 = S3Utils.md5(TEST_STRING);
byte[] badMd5 = S3Utils.md5("alf");
String destinationBucket = bucketPrefix + "testCopyIfMatchDestination";
String destinationKey = "pears";
String destinationBucket = sourceBucket + "dest";
setupSourceBucket(sourceBucket, sourceKey);
@ -179,14 +171,11 @@ public class CopyObjectIntegrationTest extends S3IntegrationTest {
@Test
void testCopyIfNoneMatch() throws IOException, InterruptedException,
ExecutionException, TimeoutException {
String sourceBucket = bucketPrefix + "testCopyIfNoneMatch";
String sourceKey = "apples";
String sourceBucket = bucketPrefix + "tcinm";
byte[] realMd5 = S3Utils.md5(TEST_STRING);
byte[] badMd5 = S3Utils.md5("alf");
String destinationBucket = bucketPrefix
+ "testCopyIfNoneMatchDestination";
String destinationKey = "pears";
String destinationBucket = sourceBucket + "dest";
setupSourceBucket(sourceBucket, sourceKey);
@ -209,11 +198,8 @@ public class CopyObjectIntegrationTest extends S3IntegrationTest {
@Test
void testCopyWithMetadata() throws InterruptedException,
ExecutionException, TimeoutException, IOException {
String sourceBucket = bucketPrefix + "testCopyWithMetadata";
String sourceKey = "apples";
String destinationBucket = bucketPrefix
+ "testCopyWithMetadataDestination";
String destinationKey = "pears";
String sourceBucket = bucketPrefix + "tcwm";
String destinationBucket = sourceBucket + "dest";
setupSourceBucket(sourceBucket, sourceKey);

View File

@ -23,6 +23,10 @@
*/
package org.jclouds.aws.s3.commands;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3IntegrationTest;
import org.testng.annotations.Test;
@ -39,8 +43,20 @@ import org.testng.annotations.Test;
public class GetObjectIntegrationTest extends S3IntegrationTest {
@Test
void testGetIfModifiedSince() {
// TODO
void testGetIfModifiedSince() throws InterruptedException, ExecutionException, TimeoutException, IOException {
String bucket = bucketPrefix + "testGetIfModifiedSince".toLowerCase();
String key = "apples";
setUpBucket(bucket, key);
}
private void setUpBucket(String sourceBucket, String sourceKey)
throws InterruptedException, ExecutionException, TimeoutException,
IOException {
createBucketAndEnsureEmpty(sourceBucket);
addObjectToBucket(sourceBucket, sourceKey);
validateContent(sourceBucket, sourceKey);
}
@Test

View File

@ -78,14 +78,14 @@ public class S3CommandFactoryTest {
@Test
void testCreateCopyObject() {
assert commandFactory.createCopyObject("sourceBucket", "sourceObject",
"destBucket", "destObject", CopyObjectOptions.NONE) != null;
assert commandFactory.createCopyObject("sourcebucket", "sourceObject",
"destbucket", "destObject", CopyObjectOptions.NONE) != null;
}
@Test
void testCreateCopyObjectOptions() {
assert commandFactory.createCopyObject("sourceBucket", "sourceObject",
"destBucket", "destObject", new CopyObjectOptions()) != null;
assert commandFactory.createCopyObject("sourcebucket", "sourceObject",
"destbucket", "destObject", new CopyObjectOptions()) != null;
}
@Test