diff --git a/aws/s3/core/src/main/java/org/jclouds/aws/s3/domain/S3Bucket.java b/aws/s3/core/src/main/java/org/jclouds/aws/s3/domain/S3Bucket.java index 574cef26be..3e15f9f1d0 100644 --- a/aws/s3/core/src/main/java/org/jclouds/aws/s3/domain/S3Bucket.java +++ b/aws/s3/core/src/main/java/org/jclouds/aws/s3/domain/S3Bucket.java @@ -26,7 +26,6 @@ package org.jclouds.aws.s3.domain; import static com.google.common.base.Preconditions.checkNotNull; import org.joda.time.DateTime; -import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; diff --git a/aws/s3/core/src/test/java/org/jclouds/aws/s3/StubS3Connection.java b/aws/s3/core/src/test/java/org/jclouds/aws/s3/StubS3Connection.java index b839d52c4c..dcb93ed042 100644 --- a/aws/s3/core/src/test/java/org/jclouds/aws/s3/StubS3Connection.java +++ b/aws/s3/core/src/test/java/org/jclouds/aws/s3/StubS3Connection.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -326,8 +327,8 @@ public class StubS3Connection implements S3Connection { options.getPrefix() != null ? URLDecoder.decode(options.getPrefix(), "UTF-8") : null, URLDecoder.decode(options.getDelimiter(), "UTF-8"))); - Set commonPrefixes = iterable != null ? Sets.newTreeSet(iterable) - : new HashSet(); + SortedSet commonPrefixes = iterable != null ? Sets.newTreeSet(iterable) + : new TreeSet(); commonPrefixes.remove(CommonPrefixes.NO_PREFIX); contents = Sets.newTreeSet(Iterables.filter(contents, new DelimiterFilter(options @@ -342,9 +343,18 @@ public class StubS3Connection implements S3Connection { } if (options.getMaxKeys() != null) { - contents = firstSliceOfSize(contents, Integer.parseInt(options.getMaxKeys())); + SortedSet contentsSlice = firstSliceOfSize( + contents, Integer.parseInt(options.getMaxKeys())); returnVal.setMaxKeys(Integer.parseInt(options.getMaxKeys())); - returnVal.setTruncated(true); + if (!contentsSlice.contains(contents.last())) { + // Partial listing + returnVal.setTruncated(true); + returnVal.setMarker(contentsSlice.last().getKey()); + } else { + returnVal.setTruncated(false); + returnVal.setMarker(null); + } + contents = contentsSlice; } returnVal.setContents(contents); diff --git a/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java b/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java index 8699829f32..f30524cb63 100644 --- a/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java +++ b/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java @@ -23,6 +23,8 @@ */ package org.jclouds.aws.s3.jets3t; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; import java.util.List; import java.util.Map; @@ -32,6 +34,7 @@ import org.jclouds.aws.s3.S3Connection; import org.jclouds.aws.s3.S3Context; import org.jclouds.aws.s3.S3ContextFactory; import org.jclouds.aws.s3.commands.options.GetObjectOptions; +import org.jclouds.aws.s3.commands.options.ListBucketOptions; import org.jclouds.util.Utils; import org.jets3t.service.S3ObjectsChunk; import org.jets3t.service.S3Service; @@ -201,8 +204,8 @@ public class JCloudsS3Service extends S3Service { Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags, Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException { try { - GetObjectOptions options = Util.convertOptions(ifModifiedSince, ifUnmodifiedSince, - ifMatchTags, ifNoneMatchTags); + GetObjectOptions options = Util.convertGetObjectOptions(ifModifiedSince, + ifUnmodifiedSince, ifMatchTags, ifNoneMatchTags); return Util.convertObject(connection.getObject(bucketName, objectKey, options).get()); } catch (Exception e) { Utils. rethrowIfRuntimeOrSameType(e); @@ -239,15 +242,49 @@ public class JCloudsS3Service extends S3Service { protected S3ObjectsChunk listObjectsChunkedImpl(String bucketName, String prefix, String delimiter, long maxListingLength, String priorLastKey, boolean completeListing) throws S3ServiceException { - // TODO Unimplemented - throw new UnsupportedOperationException(); + try { + List jsObjects = new ArrayList(); + List commonPrefixes = new ArrayList(); + org.jclouds.aws.s3.domain.S3Bucket jcBucket = null; + do { + ListBucketOptions options = Util.convertListObjectOptions(prefix, priorLastKey, + delimiter, maxListingLength); + + jcBucket = connection.listBucket(bucketName, options) + .get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS); + + jsObjects.addAll(Arrays.asList(Util.convertObjectHeads(jcBucket.getContents()))); + commonPrefixes.addAll(jcBucket.getCommonPrefixes()); + if (jcBucket.isTruncated()) { + priorLastKey = jsObjects.get(jsObjects.size() - 1).getKey(); + } else { + priorLastKey = null; + } + } while (completeListing && jcBucket.isTruncated()); // Build entire listing if requested + + return new S3ObjectsChunk( + prefix, // Return the supplied prefix, not the one in the S3 response. + jcBucket.getDelimiter(), + (S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]), + (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]), + priorLastKey); + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3ServiceException("error listing objects in bucket " + bucketName, e); + } } @Override protected S3Object[] listObjectsImpl(String bucketName, String prefix, String delimiter, - long maxListingLength) throws S3ServiceException { - // TODO Unimplemented - throw new UnsupportedOperationException(); + long maxListingLength) throws S3ServiceException + { + try { + return listObjectsChunked(bucketName, prefix, delimiter, maxListingLength, null, true) + .getObjects(); + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3ServiceException("error listing objects in bucket " + bucketName, e); + } } @Override diff --git a/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/Util.java b/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/Util.java index 95aa94af56..f325e67edf 100644 --- a/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/Util.java +++ b/aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/Util.java @@ -28,8 +28,10 @@ import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Calendar; import java.util.List; +import java.util.Set; import org.jclouds.aws.s3.commands.options.GetObjectOptions; +import org.jclouds.aws.s3.commands.options.ListBucketOptions; import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Object; import org.jets3t.service.model.S3Owner; @@ -61,6 +63,15 @@ public class Util { return (S3Bucket[]) jsBuckets.toArray(new S3Bucket[jsBuckets.size()]); } + public static S3Object[] convertObjectHeads(Set + jcObjectMDs) { + List jsObjects = new ArrayList(jcObjectMDs.size()); + for (org.jclouds.aws.s3.domain.S3Object.Metadata jcObjectMD: jcObjectMDs) { + jsObjects.add(convertObjectHead(jcObjectMD)); + } + return (S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]); + } + public static S3Object convertObjectHead(org.jclouds.aws.s3.domain.S3Object.Metadata jcObjectMD) { S3Object jsObject = new S3Object(jcObjectMD.getKey()); if (jcObjectMD.getOwner() != null) { @@ -89,22 +100,45 @@ public class Util { return (S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]); } - public static GetObjectOptions convertOptions(Calendar ifModifiedSince, + public static GetObjectOptions convertGetObjectOptions(Calendar ifModifiedSince, Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags) throws UnsupportedEncodingException { GetObjectOptions options = new GetObjectOptions(); - if (ifModifiedSince != null) + if (ifModifiedSince != null) { options.ifModifiedSince(new DateTime(ifModifiedSince)); - if (ifUnmodifiedSince != null) + } + if (ifUnmodifiedSince != null) { options.ifUnmodifiedSince(new DateTime(ifUnmodifiedSince)); + } // TODO: options.ifMd5Matches should accept multiple match tags - if (ifMatchTags != null && ifMatchTags.length > 0) + if (ifMatchTags != null && ifMatchTags.length > 0) { options.ifMd5Matches(ifMatchTags[0].getBytes()); + } // TODO: options.ifMd5DoesntMatch should accept multiple match tags - if (ifNoneMatchTags != null && ifNoneMatchTags.length > 0) + if (ifNoneMatchTags != null && ifNoneMatchTags.length > 0) { options.ifMd5DoesntMatch(ifNoneMatchTags[0].getBytes()); + } return options; } + + public static ListBucketOptions convertListObjectOptions(String prefix, String marker, + String delimiter, Long maxKeys) throws UnsupportedEncodingException + { + ListBucketOptions options = new ListBucketOptions(); + if (prefix != null) { + options.withPrefix(prefix); + } + if (marker != null) { + options.afterMarker(marker); + } + if (maxKeys != null) { + options.maxResults(maxKeys); + } + if (delimiter != null) { + options.delimiter(delimiter); + } + return options; + } } diff --git a/aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java b/aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java index dc2dd5073e..552618d995 100644 --- a/aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java +++ b/aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java @@ -39,10 +39,12 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.io.IOUtils; import org.jclouds.aws.s3.S3IntegrationTest; import org.jclouds.aws.s3.config.StubS3ConnectionModule; +import org.jets3t.service.S3ObjectsChunk; import org.jets3t.service.S3Service; import org.jets3t.service.S3ServiceException; import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Object; +import org.jets3t.service.multithread.CreateObjectsEvent; import org.jets3t.service.security.AWSCredentials; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -164,6 +166,9 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { assertTrue(createdObject != org.jclouds.aws.s3.domain.S3Object.NOT_FOUND, "object should exist but doesn't"); assertEquals(createdObject.getMetadata().getKey(), objectKey, "object misnamed"); + + client.deleteObject(name, objectKey); + client.deleteBucketIfEmpty(name); } @Test(enabled = false) @@ -187,7 +192,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { } @Test - public void testGetObjectDetailsImplStringStringCalendarCalendarStringArrayStringArray() + public void testGetObjectDetailsImpl() throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException { String bucketName = bucketPrefix + ".testGetObjectDetailsImplStringStringCalendarCalendarStringArrayStringArray"; @@ -210,7 +215,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { } @Test - public void testGetObjectImplStringStringCalendarCalendarStringArrayStringArrayLongLong() + public void testGetObjectImpl() throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException, IOException { String bucketName = bucketPrefix + ".testGetObjectImplStringStringCalendarCalendarStringArrayStringArrayLongLong"; @@ -274,16 +279,137 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest { client.deleteBucketIfEmpty(bucketName); } - @Test(enabled = false) - public void testListObjectsChunkedImplStringStringStringLongStringBoolean() { - fail("Not yet implemented"); + @Test + public void testListObjectsChunkedImpl() throws InterruptedException, ExecutionException, + TimeoutException, IOException, S3ServiceException + { + String bucketName = bucketPrefix + ".testListAllBucketsImplString".toLowerCase(); + createBucket(bucketName); + + addObjectToBucket(bucketName, "item1/subobject2"); + addObjectToBucket(bucketName, "item2"); + addObjectToBucket(bucketName, "object1"); + addObjectToBucket(bucketName, "object2/subobject1"); + + S3ObjectsChunk chunk; + + // Normal complete listing + chunk = service.listObjectsChunked(bucketName, null, null, 1000, null, true); + assertEquals(chunk.getObjects().length, 4); + assertEquals(chunk.getCommonPrefixes().length, 0); + assertNull(chunk.getDelimiter()); + assertNull(chunk.getPrefix()); + assertNull(chunk.getPriorLastKey()); + + // Partial listing + chunk = service.listObjectsChunked(bucketName, null, null, 2, null, false); + assertEquals(chunk.getObjects().length, 2); + assertEquals(chunk.getCommonPrefixes().length, 0); + assertNull(chunk.getDelimiter()); + assertNull(chunk.getPrefix()); + assertEquals(chunk.getPriorLastKey(), "item2"); + + // Complete listing, in two chunks + chunk = service.listObjectsChunked(bucketName, null, null, 2, null, true); + assertEquals(chunk.getObjects().length, 4); + assertEquals(chunk.getCommonPrefixes().length, 0); + assertNull(chunk.getDelimiter()); + assertNull(chunk.getPrefix()); + assertNull(chunk.getPriorLastKey()); + + // Partial listing with marker + chunk = service.listObjectsChunked(bucketName, null, null, 1000, "item1/subobject2", true); + assertEquals(chunk.getObjects().length, 3); + assertEquals(chunk.getCommonPrefixes().length, 0); + assertNull(chunk.getDelimiter()); + assertNull(chunk.getPrefix()); + assertNull(chunk.getPriorLastKey()); + + // Partial listing with marker + chunk = service.listObjectsChunked(bucketName, null, null, 1000, "object1", true); + assertEquals(chunk.getObjects().length, 1); + assertEquals(chunk.getCommonPrefixes().length, 0); + assertNull(chunk.getDelimiter()); + assertNull(chunk.getPrefix()); + assertNull(chunk.getPriorLastKey()); + + // Prefix test + chunk = service.listObjectsChunked(bucketName, "item", null, 1000, null, true); + assertEquals(chunk.getObjects().length, 2); + assertEquals(chunk.getCommonPrefixes().length, 0); + assertNull(chunk.getDelimiter()); + assertEquals(chunk.getPrefix(), "item"); + assertNull(chunk.getPriorLastKey()); + + // Delimiter test + chunk = service.listObjectsChunked(bucketName, null, "/", 1000, null, true); + assertEquals(chunk.getObjects().length, 2); + assertEquals(chunk.getCommonPrefixes().length, 2); + assertEquals(chunk.getDelimiter(), "/"); + assertNull(chunk.getPrefix()); + assertNull(chunk.getPriorLastKey()); + + // Prefix & delimiter test + chunk = service.listObjectsChunked(bucketName, "item", "/", 1000, null, true); + assertEquals(chunk.getObjects().length, 1); + assertEquals(chunk.getCommonPrefixes().length, 1); + assertEquals(chunk.getDelimiter(), "/"); + assertEquals(chunk.getPrefix(), "item"); + assertNull(chunk.getPriorLastKey()); + + client.deleteObject(bucketName, "item1/subobject2"); + client.deleteObject(bucketName, "item2"); + client.deleteObject(bucketName, "object1"); + client.deleteObject(bucketName, "object2/subobject1"); + client.deleteBucketIfEmpty(bucketName); } - @Test(enabled = false) - public void testListObjectsImplStringStringStringLong() { - fail("Not yet implemented"); - } + @Test + public void testListObjectsImpl() throws InterruptedException, ExecutionException, + TimeoutException, IOException, S3ServiceException + { + String bucketName = bucketPrefix + ".testListAllBucketsImplString".toLowerCase(); + createBucket(bucketName); + + addObjectToBucket(bucketName, "item1/subobject2"); + addObjectToBucket(bucketName, "item2"); + addObjectToBucket(bucketName, "object1"); + addObjectToBucket(bucketName, "object2/subobject1"); + + S3Object[] objects; + + // Normal complete listing + objects = service.listObjects(bucketName, null, null, 1000); + assertEquals(objects.length, 4); + + // Complete listing, in two chunks + objects = service.listObjects(bucketName, null, null, 2); + assertEquals(objects.length, 4); + assertEquals(objects[0].getKey(), "item1/subobject2"); + assertEquals(objects[3].getKey(), "object2/subobject1"); + // Prefix test + objects = service.listObjects(bucketName, "item", null, 1000); + assertEquals(objects.length, 2); + + // Delimiter test + objects = service.listObjects(bucketName, null, "/", 1000); + assertEquals(objects.length, 2); + assertEquals(objects[0].getKey(), "item2"); + assertEquals(objects[1].getKey(), "object1"); + + // Prefix & delimiter test + objects = service.listObjects(bucketName, "item", "/", 1000); + assertEquals(objects.length, 1); + assertEquals(objects[0].getKey(), "item2"); + + client.deleteObject(bucketName, "item1/subobject2"); + client.deleteObject(bucketName, "item2"); + client.deleteObject(bucketName, "object1"); + client.deleteObject(bucketName, "object2/subobject1"); + client.deleteBucketIfEmpty(bucketName); + } + @Test(enabled = false) public void testPutBucketAclImplStringAccessControlList() { fail("Not yet implemented");