From d91e76bdb3df332018400ecf97bd8384fba72e72 Mon Sep 17 00:00:00 2001 From: jamurty Date: Fri, 22 May 2009 05:46:22 +0000 Subject: [PATCH] Added functionality and tests for: bucket creation, retrieve object head, retrieve object data git-svn-id: http://jclouds.googlecode.com/svn/trunk@832 3d8758e0-26b5-11de-8745-db77d3ebf521 --- .../aws/s3/jets3t/JCloudsS3Service.java | 89 +++++++++----- .../java/org/jclouds/aws/s3/jets3t/Util.java | 110 ++++++++++++++++++ .../JCloudsS3ServiceIntegrationTest.java | 97 ++++++++++++--- 3 files changed, 247 insertions(+), 49 deletions(-) create mode 100644 aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/Util.java diff --git a/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java b/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java index 25f163e66d..6e569ad0c7 100644 --- a/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java +++ b/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java @@ -23,10 +23,15 @@ */ package org.jclouds.aws.s3.jets3t; -import com.google.inject.Module; +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + import org.jclouds.aws.s3.S3Connection; import org.jclouds.aws.s3.S3Context; import org.jclouds.aws.s3.S3ContextFactory; +import org.jclouds.aws.s3.commands.options.GetObjectOptions; import org.jclouds.util.Utils; import org.jets3t.service.S3ObjectsChunk; import org.jets3t.service.S3Service; @@ -38,11 +43,7 @@ import org.jets3t.service.model.S3Object; import org.jets3t.service.model.S3Owner; import org.jets3t.service.security.AWSCredentials; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; +import com.google.inject.Module; /** * A JetS3t S3Service implemented by JClouds @@ -76,7 +77,7 @@ public class JCloudsS3Service extends S3Service { } @Override - public int checkBucketStatus(String bucketName) throws S3ServiceException { + public int checkBucketStatus(final String bucketName) throws S3ServiceException { // TODO Unimplemented throw new UnsupportedOperationException(); } @@ -98,9 +99,23 @@ public class JCloudsS3Service extends S3Service { @Override protected S3Bucket createBucketImpl(String bucketName, String location, - AccessControlList acl) throws S3ServiceException { - // TODO Unimplemented - throw new UnsupportedOperationException(); + AccessControlList acl) throws S3ServiceException + { + if (location != null) + throw new UnsupportedOperationException("Bucket location is not yet supported"); + if (acl != null) + throw new UnsupportedOperationException("Bucket ACL is not yet supported"); + + try { + if (connection.putBucketIfNotExists(bucketName).get()) { + // Bucket created. + } + } catch (Exception e) { + Utils.rethrowIfRuntimeOrSameType(e); + throw new S3ServiceException( + "error creating bucket: " + bucketName, e); + } + return new S3Bucket(bucketName); } /** @@ -135,8 +150,7 @@ public class JCloudsS3Service extends S3Service { } catch (Exception e) { Utils.rethrowIfRuntimeOrSameType(e); throw new S3ServiceException(String.format( - "error deleting object: %1$s:%2$s", bucketName, objectKey), - e); + "error deleting object: %1$s:%2$s", bucketName, objectKey), e); } } @@ -172,18 +186,43 @@ public class JCloudsS3Service extends S3Service { protected S3Object getObjectDetailsImpl(String bucketName, String objectKey, Calendar ifModifiedSince, Calendar ifUnmodifiedSince, String[] ifMatchTags, - String[] ifNoneMatchTags) throws S3ServiceException { - // TODO Unimplemented - throw new UnsupportedOperationException(); + String[] ifNoneMatchTags) throws S3ServiceException + { + try { + if (ifModifiedSince != null) + throw new IllegalArgumentException("ifModifiedSince"); + if (ifUnmodifiedSince != null) + throw new IllegalArgumentException("ifUnmodifiedSince"); + if (ifMatchTags != null) + throw new IllegalArgumentException("ifMatchTags"); + if (ifNoneMatchTags != null) + throw new IllegalArgumentException("ifNoneMatchTags"); + + return Util.convertObjectHead( + connection.headObject(bucketName, objectKey).get()); + } catch (Exception e) { + Utils.rethrowIfRuntimeOrSameType(e); + throw new S3ServiceException(String.format( + "error retrieving object head: %1$s:%2$s", bucketName, objectKey), e); + } } @Override protected S3Object getObjectImpl(String bucketName, String objectKey, Calendar ifModifiedSince, Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags, - Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException { - // TODO Unimplemented - throw new UnsupportedOperationException(); + Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException + { + try { + GetObjectOptions options = Util.convertOptions( + ifModifiedSince, ifUnmodifiedSince, ifMatchTags, ifNoneMatchTags); + return Util.convertObject( + connection.getObject(bucketName, objectKey, options).get()); + } catch (Exception e) { + Utils.rethrowIfRuntimeOrSameType(e); + throw new S3ServiceException(String.format( + "error retrieving object: %1$s:%2$s", bucketName, objectKey), e); + } } @Override @@ -206,19 +245,7 @@ public class JCloudsS3Service extends S3Service { List jcBucketList = connection .listOwnedBuckets().get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS); - - ArrayList jsBucketList = new ArrayList(); - for (org.jclouds.aws.s3.domain.S3Bucket.Metadata jcBucket : jcBucketList) { - org.jets3t.service.model.S3Bucket jsBucket = new org.jets3t.service.model.S3Bucket( - jcBucket.getName()); - jsBucket.setOwner(new org.jets3t.service.model.S3Owner(jcBucket - .getOwner().getId(), jcBucket.getOwner() - .getDisplayName())); - jsBucketList.add(jsBucket); - } - return (org.jets3t.service.model.S3Bucket[]) jsBucketList - .toArray(new org.jets3t.service.model.S3Bucket[jsBucketList - .size()]); + return Util.convertBuckets(jcBucketList); } catch (Exception e) { Utils.rethrowIfRuntimeOrSameType(e); throw new S3ServiceException("error listing buckets", e); diff --git a/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/Util.java b/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/Util.java new file mode 100644 index 0000000000..c4250c3ff3 --- /dev/null +++ b/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/Util.java @@ -0,0 +1,110 @@ +/** + * + * Copyright (C) 2009 James Murty + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.aws.s3.jets3t; + +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; + +import org.jclouds.aws.s3.commands.options.GetObjectOptions; +import org.jets3t.service.model.S3Bucket; +import org.jets3t.service.model.S3Object; +import org.jets3t.service.model.S3Owner; +import org.joda.time.DateTime; + +/** + * Convert between jCloud and JetS3t objects. + * + * @author James Murty + */ +public class Util { + + public static S3Bucket convertBucket(org.jclouds.aws.s3.domain.S3Bucket.Metadata jcBucketMD) { + S3Bucket jsBucket = new S3Bucket(jcBucketMD.getName()); + if (jcBucketMD.getOwner() != null) { + jsBucket.setOwner( + new S3Owner( + jcBucketMD.getOwner().getId(), + jcBucketMD.getOwner().getDisplayName())); + } + return jsBucket; + } + + public static S3Bucket[] convertBuckets(List jcBucketMDs) { + List jsBuckets = new ArrayList(jcBucketMDs.size()); + for (org.jclouds.aws.s3.domain.S3Bucket.Metadata jcBucketMD: jcBucketMDs) { + jsBuckets.add(convertBucket(jcBucketMD)); + } + return (S3Bucket[]) jsBuckets.toArray(new S3Bucket[jsBuckets.size()]); + } + + public static S3Object convertObjectHead(org.jclouds.aws.s3.domain.S3Object.Metadata jcObjectMD) { + S3Object jsObject = new S3Object(jcObjectMD.getKey()); + if (jcObjectMD.getOwner() != null) { + jsObject.setOwner( + new S3Owner( + jcObjectMD.getOwner().getId(), + jcObjectMD.getOwner().getDisplayName())); + } + jsObject.addAllMetadata(jcObjectMD.getAllHeaders().asMap()); // TODO: Check this + return jsObject; + } + + public static S3Object convertObject(org.jclouds.aws.s3.domain.S3Object jcObject) { + S3Object jsObject = convertObjectHead(jcObject.getMetadata()); + if (jcObject.getData() != null) { + jsObject.setDataInputStream((InputStream) jcObject.getData()); + } + return jsObject; + } + + public static S3Object[] convertObjects(List jcObjects) { + List jsObjects = new ArrayList(jcObjects.size()); + for (org.jclouds.aws.s3.domain.S3Object jcObject: jcObjects) { + jsObjects.add(convertObject(jcObject)); + } + return (S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]); + } + + public static GetObjectOptions convertOptions(Calendar ifModifiedSince, + Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags) + throws UnsupportedEncodingException + { + GetObjectOptions options = new GetObjectOptions(); + if (ifModifiedSince != null) + options.ifModifiedSince(new DateTime(ifModifiedSince)); + if (ifUnmodifiedSince != null) + options.ifUnmodifiedSince(new DateTime(ifUnmodifiedSince)); + // TODO: options.ifMd5Matches should accept multiple match tags + if (ifMatchTags != null && ifMatchTags.length > 0) + options.ifMd5Matches(ifMatchTags[0].getBytes()); + // TODO: options.ifMd5DoesntMatch should accept multiple match tags + if (ifNoneMatchTags != null && ifNoneMatchTags.length > 0) + options.ifMd5DoesntMatch(ifNoneMatchTags[0].getBytes()); + return options; + } + +} diff --git a/aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java b/aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java index f1e306e769..194b37361d 100644 --- a/aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java +++ b/aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java @@ -23,17 +23,28 @@ */ package org.jclouds.aws.s3.jets3t; +import org.apache.commons.io.IOUtils; import org.jclouds.aws.s3.S3IntegrationTest; import org.jclouds.aws.s3.config.StubS3ConnectionModule; import org.jets3t.service.S3Service; import org.jets3t.service.S3ServiceException; import org.jets3t.service.model.S3Bucket; +import org.jets3t.service.model.S3Object; import org.jets3t.service.security.AWSCredentials; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; import static org.testng.Assert.fail; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.StringReader; import java.util.Iterator; import java.util.List; import java.util.concurrent.ExecutionException; @@ -88,9 +99,16 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { fail("Not yet implemented"); } - @Test(enabled = false) - public void testCreateBucketImplStringStringAccessControlList() { - fail("Not yet implemented"); + @Test + public void testCreateBucketImplStringStringAccessControlList() + throws S3ServiceException, InterruptedException, ExecutionException + { + String bucketName = bucketPrefix + ".testCreateBucketImplStringStringAccessControlList"; + S3Bucket bucket = service.createBucket(new S3Bucket(bucketName)); + assertEquals(bucket.getName(), bucketName); + assertTrue(client.bucketExists(bucketName).get()); + + client.deleteBucketIfEmpty(bucketName); } @Test @@ -106,7 +124,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { client.putBucketIfNotExists(bucketName).get(10, TimeUnit.SECONDS); } - @Test(enabled = false) + @Test public void testDeleteObjectImplStringString() throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException { String bucketName = bucketPrefix + ".testDeleteObjectImplStringString"; @@ -117,8 +135,11 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { service.deleteObject(bucketName, objectKey); - assertEquals(client.headObject(bucketName, objectKey).get(10, - TimeUnit.SECONDS), org.jclouds.aws.s3.domain.S3Object.NOT_FOUND); + assertEquals( + client.headObject(bucketName, objectKey).get(10, TimeUnit.SECONDS), + org.jclouds.aws.s3.domain.S3Object.Metadata.NOT_FOUND); + + client.deleteBucketIfEmpty(bucketName); } private void addNewObject(String name, String objectKey, String objectValue) @@ -127,7 +148,16 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { org.jclouds.aws.s3.domain.S3Object jcloudsObject = new org.jclouds.aws.s3.domain.S3Object( objectKey); jcloudsObject.setData(objectValue); + Multimap userMetadata = HashMultimap.create(); + userMetadata.put("metadata-name", "metadata-value"); + jcloudsObject.getMetadata().setUserMetadata(userMetadata); client.putObject(name, jcloudsObject).get(10, TimeUnit.SECONDS); + + org.jclouds.aws.s3.domain.S3Object createdObject = + client.getObject(name, jcloudsObject.getKey()).get(10, TimeUnit.SECONDS); + assertTrue(createdObject != org.jclouds.aws.s3.domain.S3Object.NOT_FOUND, + "object should exist but doesn't"); + assertEquals(createdObject.getMetadata().getKey(), objectKey, "object misnamed"); } @Test(enabled = false) @@ -150,14 +180,49 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { fail("Not yet implemented"); } - @Test(enabled = false) - public void testGetObjectDetailsImplStringStringCalendarCalendarStringArrayStringArray() { - fail("Not yet implemented"); + @Test + public void testGetObjectDetailsImplStringStringCalendarCalendarStringArrayStringArray() + throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException + { + String bucketName = bucketPrefix + ".testGetObjectDetailsImplStringStringCalendarCalendarStringArrayStringArray"; + String objectKey = "key"; + String objectValue = "test"; + + addNewObject(bucketName, objectKey, objectValue); + + S3Object objectDetails = service.getObjectDetails(new S3Bucket(bucketName), objectKey); + + assertEquals(objectDetails.getKey(), objectKey); + assertEquals(objectDetails.getContentLength(), 0); + assertNull(objectDetails.getDataInputStream()); + // assertEquals(objectDetails.getMetadata("metadata-name"), "metadata-value"); // TODO + + client.deleteObject(bucketName, objectKey); + client.deleteBucketIfEmpty(bucketName); } - @Test(enabled = false) - public void testGetObjectImplStringStringCalendarCalendarStringArrayStringArrayLongLong() { - fail("Not yet implemented"); + @Test + public void testGetObjectImplStringStringCalendarCalendarStringArrayStringArrayLongLong() + throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException, IOException + { + String bucketName = bucketPrefix + ".testGetObjectImplStringStringCalendarCalendarStringArrayStringArrayLongLong"; + String objectKey = "key"; + String objectValue = "test"; + + addNewObject(bucketName, objectKey, objectValue); + + S3Object object = service.getObject(new S3Bucket(bucketName), objectKey); + + assertEquals(object.getKey(), objectKey); + assertNotNull(object.getDataInputStream()); + assertEquals(IOUtils.toString(object.getDataInputStream()), objectValue); + // assertEquals(object.getContentLength(), objectKey.length()); // TODO + // assertEquals(objectDetails.getMetadata("metadata-name"), "metadata-value"); // TODO + + // TODO: Test conditional gets + + client.deleteObject(bucketName, objectKey); + client.deleteBucketIfEmpty(bucketName); } @Test(enabled = false) @@ -170,7 +235,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { fail("Not yet implemented"); } - @Test(enabled = false) + @Test public void testListAllBucketsImpl() throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException { // Ensure there is at least 1 bucket in S3 account to list and compare. @@ -183,7 +248,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { .listOwnedBuckets().get(10, TimeUnit.SECONDS); assert jsBuckets.length == jcBuckets.size(); - + Iterator jcBucketsIter = jcBuckets .iterator(); for (S3Bucket jsBucket : jsBuckets) { @@ -192,10 +257,6 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { org.jclouds.aws.s3.domain.S3Bucket.Metadata jcBucket = jcBucketsIter .next(); assert jsBucket.getName().equals(jcBucket.getName()); - assert jsBucket.getOwner().getId().equals( - jcBucket.getOwner().getId()); - assert jsBucket.getOwner().getDisplayName().equals( - jcBucket.getOwner().getDisplayName()); } client.deleteBucketIfEmpty(bucketName);