Issue 572: added BlobStores.listAll for a lazy iterable

blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java

@@ -0,0 +1,51 @@
+package org.jclouds.blobstore;
+
+import java.util.Iterator;
+
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.blobstore.options.ListContainerOptions;
+
+import com.google.common.annotations.Beta;
+import com.google.common.collect.AbstractIterator;
+
+public class BlobStores {
+
+   /**
+    * A variant of BlobStore.list(String, ListContainerOptions) that
+    * produces an Iterable over the entire set of results, not just one
+    * page, making multiple calls to BlobStore.list as needed.
+    */
+   @Beta
+   public static Iterable<StorageMetadata> listAll(final BlobStore blobStore, final String container,
+            final ListContainerOptions options) {
+
+      return new Iterable<StorageMetadata>() {
+         public Iterator<StorageMetadata> iterator() {
+            return new AbstractIterator<StorageMetadata>() {
+               private Iterator<? extends StorageMetadata> iterator;
+               private String marker;
+
+               public StorageMetadata computeNext() {
+                  while (true) {
+                     if (iterator == null) {
+                        ListContainerOptions nextOptions = marker == null ? options : options.clone().afterMarker(marker);
+                        PageSet<? extends StorageMetadata> list = blobStore.list(container, nextOptions);
+                        iterator = list.iterator();
+                        marker = list.getNextMarker();
+                     }
+                     if (iterator.hasNext()) {
+                        return iterator.next();
+                     }
+                     if (marker == null) {
+                        return endOfData();
+                     }
+                     iterator = null;
+                  }
+               }
+            };
+         }
+      };
+   }
+
+}

blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java

@@ -0,0 +1,105 @@
+package org.jclouds.blobstore;
+
+import static org.easymock.classextension.EasyMock.createMock;
+import static org.testng.Assert.assertEquals;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.easymock.classextension.EasyMock;
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.blobstore.domain.internal.PageSetImpl;
+import org.jclouds.blobstore.options.ListContainerOptions;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+
+public class BlobStoresTest {
+
+   private final String containerName = "mycontainer";
+
+   @Test(expectedExceptions={ContainerNotFoundException.class})
+   public void testListAllForUnknownContainerFromTransientBlobStore() throws Exception {
+      ListContainerOptions options = ListContainerOptions.NONE;
+      BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
+      try {
+         BlobStore blobStore = context.getBlobStore();
+   
+         // Arguably it would be best to throw the exception as soon as listAll is called; but because
+         // the iterator is lazy we don't the exception until we first call iterator().next() or hasNext().
+         Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, "wrongcontainer", options);
+         iterable.iterator().hasNext();
+      } finally {
+         context.close();
+      }
+   }
+   
+   @Test
+   public void testListAllFromTransientBlobStore() throws Exception {
+      final int NUM_BLOBS = 31;
+      ListContainerOptions options = ListContainerOptions.Builder.maxResults(10);
+      BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
+      BlobStore blobStore = null;
+      try {
+         blobStore = context.getBlobStore();
+         blobStore.createContainerInLocation(null, containerName);   
+         Set<String> expectedNames = new HashSet<String>();
+         for (int i = 0; i < NUM_BLOBS; i++) {
+            String blobName = "myname"+i;
+            blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload("payload"+i).build());
+            expectedNames.add(blobName);
+         }
+         
+         Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
+         Iterable<String> iterableNames = Iterables.transform(iterable, new Function<StorageMetadata,String>() {
+            @Override public String apply(StorageMetadata input) {
+               return input.getName();
+            }});
+         
+         // Note that blob.getMetadata being put does not equal blob metadata being retrieved 
+         // because uri is null in one and populated in the other.
+         // Therefore we just compare names to ensure the iterator worked.
+         assertEquals(ImmutableSet.copyOf(iterableNames), expectedNames);
+      } finally {
+         if (blobStore != null) blobStore.deleteContainer(containerName);
+         context.close();
+      }
+   }
+   
+   @Test
+   public void testListAllWhenOnePage() throws Exception {
+      BlobStore blobStore = createMock(BlobStore.class);
+      ListContainerOptions options = ListContainerOptions.NONE;
+      StorageMetadata v1 = createMock(StorageMetadata.class);
+      PageSet<StorageMetadata> pageSet = new PageSetImpl<StorageMetadata>(Collections.singletonList(v1), null);
+      
+      EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options)).andReturn(pageSet).once();
+      EasyMock.replay(blobStore);
+      
+      Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
+      assertEquals(ImmutableList.copyOf(iterable), ImmutableList.of(v1));
+   }
+   
+   @Test
+   public void testListAllWhenTwoPages() throws Exception {
+      BlobStore blobStore = createMock(BlobStore.class);
+      ListContainerOptions options = ListContainerOptions.NONE;
+      ListContainerOptions options2 = ListContainerOptions.Builder.afterMarker("marker1");
+      StorageMetadata v1 = createMock(StorageMetadata.class);
+      StorageMetadata v2 = createMock(StorageMetadata.class);
+      PageSet<StorageMetadata> pageSet = new PageSetImpl<StorageMetadata>(Collections.singletonList(v1), "marker1");
+      PageSet<StorageMetadata> pageSet2 = new PageSetImpl<StorageMetadata>(Collections.singletonList(v2), null);
+      
+      EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options)).andReturn(pageSet).once();
+      EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options2)).andReturn(pageSet2).once();
+      EasyMock.replay(blobStore);
+      
+      Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
+      assertEquals(ImmutableList.copyOf(iterable), ImmutableList.of(v1, v2));
+   }
+}