Added object listing capabilities to JetS3t service wrapper. Also fixed S3 stub's handling of MaxKeys constraint when listing objects

git-svn-id: http://jclouds.googlecode.com/svn/trunk@873 3d8758e0-26b5-11de-8745-db77d3ebf521

aws/s3/core/src/main/java/org/jclouds/aws/s3/domain/S3Bucket.java

@@ -26,7 +26,6 @@ package org.jclouds.aws.s3.domain;
 import static com.google.common.base.Preconditions.checkNotNull;
 import org.joda.time.DateTime;
 
-import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
 

aws/s3/core/src/test/java/org/jclouds/aws/s3/StubS3Connection.java

@@ -39,6 +39,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedSet;
+import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
@@ -326,8 +327,8 @@ public class StubS3Connection implements S3Connection {
                            options.getPrefix() != null ? URLDecoder.decode(options.getPrefix(),
                                     "UTF-8") : null, URLDecoder.decode(options.getDelimiter(),
                                     "UTF-8")));
-                  Set<String> commonPrefixes = iterable != null ? Sets.newTreeSet(iterable)
+                  SortedSet<String> commonPrefixes = iterable != null ? Sets.newTreeSet(iterable)
-                           : new HashSet<String>();
+                           : new TreeSet<String>();
                   commonPrefixes.remove(CommonPrefixes.NO_PREFIX);
 
                   contents = Sets.newTreeSet(Iterables.filter(contents, new DelimiterFilter(options
@@ -342,9 +343,18 @@ public class StubS3Connection implements S3Connection {
             }
 
             if (options.getMaxKeys() != null) {
-               contents = firstSliceOfSize(contents, Integer.parseInt(options.getMaxKeys()));
+               SortedSet<S3Object.Metadata> contentsSlice = firstSliceOfSize(
+                     contents, Integer.parseInt(options.getMaxKeys()));
                returnVal.setMaxKeys(Integer.parseInt(options.getMaxKeys()));
-               returnVal.setTruncated(true);
+               if (!contentsSlice.contains(contents.last())) {
+                  // Partial listing
+                  returnVal.setTruncated(true);
+                  returnVal.setMarker(contentsSlice.last().getKey());
+               } else {
+                  returnVal.setTruncated(false);                  
+                  returnVal.setMarker(null);
+               }
+               contents = contentsSlice;
             }
 
             returnVal.setContents(contents);

aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java

@@ -23,6 +23,8 @@
  */
 package org.jclouds.aws.s3.jets3t;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Calendar;
 import java.util.List;
 import java.util.Map;
@@ -32,6 +34,7 @@ import org.jclouds.aws.s3.S3Connection;
 import org.jclouds.aws.s3.S3Context;
 import org.jclouds.aws.s3.S3ContextFactory;
 import org.jclouds.aws.s3.commands.options.GetObjectOptions;
+import org.jclouds.aws.s3.commands.options.ListBucketOptions;
 import org.jclouds.util.Utils;
 import org.jets3t.service.S3ObjectsChunk;
 import org.jets3t.service.S3Service;
@@ -201,8 +204,8 @@ public class JCloudsS3Service extends S3Service {
             Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags,
             Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException {
       try {
-         GetObjectOptions options = Util.convertOptions(ifModifiedSince, ifUnmodifiedSince,
+         GetObjectOptions options = Util.convertGetObjectOptions(ifModifiedSince, 
-                  ifMatchTags, ifNoneMatchTags);
+               ifUnmodifiedSince, ifMatchTags, ifNoneMatchTags);
          return Util.convertObject(connection.getObject(bucketName, objectKey, options).get());
       } catch (Exception e) {
          Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
@@ -239,15 +242,49 @@ public class JCloudsS3Service extends S3Service {
    protected S3ObjectsChunk listObjectsChunkedImpl(String bucketName, String prefix,
             String delimiter, long maxListingLength, String priorLastKey, boolean completeListing)
             throws S3ServiceException {
-      // TODO Unimplemented
+      try {         
-      throw new UnsupportedOperationException();
+         List<S3Object> jsObjects = new ArrayList<S3Object>();
+         List<String> commonPrefixes = new ArrayList<String>();
+         org.jclouds.aws.s3.domain.S3Bucket jcBucket = null;
+         do {            
+            ListBucketOptions options = Util.convertListObjectOptions(prefix, priorLastKey, 
+                  delimiter, maxListingLength);
+            
+            jcBucket = connection.listBucket(bucketName, options)
+                  .get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
+
+            jsObjects.addAll(Arrays.asList(Util.convertObjectHeads(jcBucket.getContents())));
+            commonPrefixes.addAll(jcBucket.getCommonPrefixes());
+            if (jcBucket.isTruncated()) {
+               priorLastKey = jsObjects.get(jsObjects.size() - 1).getKey();
+            } else {
+               priorLastKey = null;
+            }
+         } while (completeListing && jcBucket.isTruncated());  // Build entire listing if requested
+
+         return new S3ObjectsChunk(
+               prefix,  // Return the supplied prefix, not the one in the S3 response.
+               jcBucket.getDelimiter(),
+               (S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]),
+               (String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
+               priorLastKey);
+      } catch (Exception e) {
+         Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
+         throw new S3ServiceException("error listing objects in bucket " + bucketName, e);         
+      }
    }
 
    @Override
    protected S3Object[] listObjectsImpl(String bucketName, String prefix, String delimiter,
-            long maxListingLength) throws S3ServiceException {
+            long maxListingLength) throws S3ServiceException 
-      // TODO Unimplemented
+   {
-      throw new UnsupportedOperationException();
+      try {
+         return listObjectsChunked(bucketName, prefix, delimiter, maxListingLength, null, true)
+            .getObjects();
+      } catch (Exception e) {
+         Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
+         throw new S3ServiceException("error listing objects in bucket " + bucketName, e);         
+      }
    }
 
    @Override

aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/Util.java

@@ -28,8 +28,10 @@ import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.List;
+import java.util.Set;
 
 import org.jclouds.aws.s3.commands.options.GetObjectOptions;
+import org.jclouds.aws.s3.commands.options.ListBucketOptions;
 import org.jets3t.service.model.S3Bucket;
 import org.jets3t.service.model.S3Object;
 import org.jets3t.service.model.S3Owner;
@@ -61,6 +63,15 @@ public class Util {
         return (S3Bucket[]) jsBuckets.toArray(new S3Bucket[jsBuckets.size()]);
     }
     
+    public static S3Object[] convertObjectHeads(Set<org.jclouds.aws.s3.domain.S3Object.Metadata> 
+          jcObjectMDs) {
+       List<S3Object> jsObjects = new ArrayList<S3Object>(jcObjectMDs.size());
+       for (org.jclouds.aws.s3.domain.S3Object.Metadata jcObjectMD: jcObjectMDs) {
+          jsObjects.add(convertObjectHead(jcObjectMD));
+       }
+       return (S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]);       
+    }
+    
     public static S3Object convertObjectHead(org.jclouds.aws.s3.domain.S3Object.Metadata jcObjectMD) {
         S3Object jsObject = new S3Object(jcObjectMD.getKey());
         if (jcObjectMD.getOwner() != null) {
@@ -89,22 +100,45 @@ public class Util {
         return (S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]);
     }
     
-    public static GetObjectOptions convertOptions(Calendar ifModifiedSince,
+    public static GetObjectOptions convertGetObjectOptions(Calendar ifModifiedSince,
         Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags) 
     	throws UnsupportedEncodingException 
     {
     	GetObjectOptions options = new GetObjectOptions();
-    	if (ifModifiedSince != null) 
+    	if (ifModifiedSince != null) { 
     		options.ifModifiedSince(new DateTime(ifModifiedSince));
-    	if (ifUnmodifiedSince != null)
+    	}
+    	if (ifUnmodifiedSince != null) {
     		options.ifUnmodifiedSince(new DateTime(ifUnmodifiedSince));
+    	}
     	// TODO: options.ifMd5Matches should accept multiple match tags
-    	if (ifMatchTags != null && ifMatchTags.length > 0)
+    	if (ifMatchTags != null && ifMatchTags.length > 0) {
     		options.ifMd5Matches(ifMatchTags[0].getBytes());
+    	}
     	// TODO: options.ifMd5DoesntMatch should accept multiple match tags
-    	if (ifNoneMatchTags != null && ifNoneMatchTags.length > 0)
+    	if (ifNoneMatchTags != null && ifNoneMatchTags.length > 0) {
     		options.ifMd5DoesntMatch(ifNoneMatchTags[0].getBytes());
+    	}
     	return options;
     }
+    
+    public static ListBucketOptions convertListObjectOptions(String prefix, String marker, 
+          String delimiter, Long maxKeys) throws UnsupportedEncodingException 
+    {
+       ListBucketOptions options = new ListBucketOptions();
+       if (prefix != null) {
+          options.withPrefix(prefix);
+       }
+       if (marker != null) {
+          options.afterMarker(marker);
+       }
+       if (maxKeys != null) {
+          options.maxResults(maxKeys);
+       }
+       if (delimiter != null) {
+          options.delimiter(delimiter);
+       }
+       return options;
+    }
 
 }

aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java

@@ -39,10 +39,12 @@ import java.util.concurrent.TimeoutException;
 import org.apache.commons.io.IOUtils;
 import org.jclouds.aws.s3.S3IntegrationTest;
 import org.jclouds.aws.s3.config.StubS3ConnectionModule;
+import org.jets3t.service.S3ObjectsChunk;
 import org.jets3t.service.S3Service;
 import org.jets3t.service.S3ServiceException;
 import org.jets3t.service.model.S3Bucket;
 import org.jets3t.service.model.S3Object;
+import org.jets3t.service.multithread.CreateObjectsEvent;
 import org.jets3t.service.security.AWSCredentials;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -164,6 +166,9 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
 		assertTrue(createdObject != org.jclouds.aws.s3.domain.S3Object.NOT_FOUND,
 			"object should exist but doesn't");
 		assertEquals(createdObject.getMetadata().getKey(), objectKey, "object misnamed");
+
+      client.deleteObject(name, objectKey);
+      client.deleteBucketIfEmpty(name);
     }
 
     @Test(enabled = false)
@@ -187,7 +192,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
     }
 
     @Test
-    public void testGetObjectDetailsImplStringStringCalendarCalendarStringArrayStringArray() 
+    public void testGetObjectDetailsImpl() 
     	throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException 
 	{
         String bucketName = bucketPrefix + ".testGetObjectDetailsImplStringStringCalendarCalendarStringArrayStringArray";
@@ -210,7 +215,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
     }
 
     @Test
-    public void testGetObjectImplStringStringCalendarCalendarStringArrayStringArrayLongLong() 
+    public void testGetObjectImpl() 
     	throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException, IOException 
 	{
         String bucketName = bucketPrefix + ".testGetObjectImplStringStringCalendarCalendarStringArrayStringArrayLongLong";
@@ -274,16 +279,137 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
         client.deleteBucketIfEmpty(bucketName);
     }
 
-    @Test(enabled = false)
+    @Test
-    public void testListObjectsChunkedImplStringStringStringLongStringBoolean() {
+    public void testListObjectsChunkedImpl() throws InterruptedException, ExecutionException, 
-        fail("Not yet implemented");
+          TimeoutException, IOException, S3ServiceException 
+    {
+       String bucketName = bucketPrefix + ".testListAllBucketsImplString".toLowerCase();
+       createBucket(bucketName);
+       
+       addObjectToBucket(bucketName, "item1/subobject2");
+       addObjectToBucket(bucketName, "item2");
+       addObjectToBucket(bucketName, "object1");
+       addObjectToBucket(bucketName, "object2/subobject1");
+       
+       S3ObjectsChunk chunk;
+       
+       // Normal complete listing
+       chunk = service.listObjectsChunked(bucketName, null, null, 1000, null, true);
+       assertEquals(chunk.getObjects().length, 4);
+       assertEquals(chunk.getCommonPrefixes().length, 0);
+       assertNull(chunk.getDelimiter());
+       assertNull(chunk.getPrefix());
+       assertNull(chunk.getPriorLastKey());
+       
+       // Partial listing
+       chunk = service.listObjectsChunked(bucketName, null, null, 2, null, false);
+       assertEquals(chunk.getObjects().length, 2);
+       assertEquals(chunk.getCommonPrefixes().length, 0);
+       assertNull(chunk.getDelimiter());
+       assertNull(chunk.getPrefix());
+       assertEquals(chunk.getPriorLastKey(), "item2");
+
+       // Complete listing, in two chunks
+       chunk = service.listObjectsChunked(bucketName, null, null, 2, null, true);
+       assertEquals(chunk.getObjects().length, 4);
+       assertEquals(chunk.getCommonPrefixes().length, 0);
+       assertNull(chunk.getDelimiter());
+       assertNull(chunk.getPrefix());
+       assertNull(chunk.getPriorLastKey());
+
+       // Partial listing with marker
+       chunk = service.listObjectsChunked(bucketName, null, null, 1000, "item1/subobject2", true);
+       assertEquals(chunk.getObjects().length, 3);
+       assertEquals(chunk.getCommonPrefixes().length, 0);
+       assertNull(chunk.getDelimiter());
+       assertNull(chunk.getPrefix());
+       assertNull(chunk.getPriorLastKey());
+
+       // Partial listing with marker
+       chunk = service.listObjectsChunked(bucketName, null, null, 1000, "object1", true);
+       assertEquals(chunk.getObjects().length, 1);
+       assertEquals(chunk.getCommonPrefixes().length, 0);
+       assertNull(chunk.getDelimiter());
+       assertNull(chunk.getPrefix());
+       assertNull(chunk.getPriorLastKey());
+
+       // Prefix test
+       chunk = service.listObjectsChunked(bucketName, "item", null, 1000, null, true);
+       assertEquals(chunk.getObjects().length, 2);
+       assertEquals(chunk.getCommonPrefixes().length, 0);
+       assertNull(chunk.getDelimiter());
+       assertEquals(chunk.getPrefix(), "item");
+       assertNull(chunk.getPriorLastKey());
+
+       // Delimiter test
+       chunk = service.listObjectsChunked(bucketName, null, "/", 1000, null, true);
+       assertEquals(chunk.getObjects().length, 2);
+       assertEquals(chunk.getCommonPrefixes().length, 2);
+       assertEquals(chunk.getDelimiter(), "/");
+       assertNull(chunk.getPrefix());
+       assertNull(chunk.getPriorLastKey());
+
+       // Prefix & delimiter test
+       chunk = service.listObjectsChunked(bucketName, "item", "/", 1000, null, true);
+       assertEquals(chunk.getObjects().length, 1);
+       assertEquals(chunk.getCommonPrefixes().length, 1);
+       assertEquals(chunk.getDelimiter(), "/");
+       assertEquals(chunk.getPrefix(), "item");
+       assertNull(chunk.getPriorLastKey());
+
+       client.deleteObject(bucketName, "item1/subobject2");
+       client.deleteObject(bucketName, "item2");
+       client.deleteObject(bucketName, "object1");
+       client.deleteObject(bucketName, "object2/subobject1");
+       client.deleteBucketIfEmpty(bucketName);
     }
 
-    @Test(enabled = false)
+    @Test
-    public void testListObjectsImplStringStringStringLong() {
+    public void testListObjectsImpl() throws InterruptedException, ExecutionException, 
-        fail("Not yet implemented");
+          TimeoutException, IOException, S3ServiceException 
-    }
+    {
+       String bucketName = bucketPrefix + ".testListAllBucketsImplString".toLowerCase();
+       createBucket(bucketName);
+       
+       addObjectToBucket(bucketName, "item1/subobject2");
+       addObjectToBucket(bucketName, "item2");
+       addObjectToBucket(bucketName, "object1");
+       addObjectToBucket(bucketName, "object2/subobject1");
+       
+       S3Object[] objects;
+       
+       // Normal complete listing
+       objects = service.listObjects(bucketName, null, null, 1000);
+       assertEquals(objects.length, 4);
+       
+       // Complete listing, in two chunks
+       objects = service.listObjects(bucketName, null, null, 2);
+       assertEquals(objects.length, 4);
+       assertEquals(objects[0].getKey(), "item1/subobject2");
+       assertEquals(objects[3].getKey(), "object2/subobject1");
 
+       // Prefix test
+       objects = service.listObjects(bucketName, "item", null, 1000);
+       assertEquals(objects.length, 2);
+
+       // Delimiter test
+       objects = service.listObjects(bucketName, null, "/", 1000);
+       assertEquals(objects.length, 2);
+       assertEquals(objects[0].getKey(), "item2");
+       assertEquals(objects[1].getKey(), "object1");
+
+       // Prefix & delimiter test
+       objects = service.listObjects(bucketName, "item", "/", 1000);
+       assertEquals(objects.length, 1);
+       assertEquals(objects[0].getKey(), "item2");
+
+       client.deleteObject(bucketName, "item1/subobject2");
+       client.deleteObject(bucketName, "item2");
+       client.deleteObject(bucketName, "object1");
+       client.deleteObject(bucketName, "object2/subobject1");
+       client.deleteBucketIfEmpty(bucketName);
+    }
+    
     @Test(enabled = false)
     public void testPutBucketAclImplStringAccessControlList() {
         fail("Not yet implemented");