Merge pull request #247 from aledsage/Issue-572-BlobStore-listAll

Issue 572: added BlobStores.listAll for a lazy iterable

blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java

@@ -0,0 +1,51 @@
+package org.jclouds.blobstore;
+
+import java.util.Iterator;
+
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.blobstore.options.ListContainerOptions;
+
+import com.google.common.annotations.Beta;
+import com.google.common.collect.AbstractIterator;
+
+public class BlobStores {
+
+   /**
+    * A variant of BlobStore.list(String, ListContainerOptions) that
+    * produces an Iterable over the entire set of results, not just one
+    * page, making multiple calls to BlobStore.list as needed.
+    */
+   @Beta
+   public static Iterable<StorageMetadata> listAll(final BlobStore blobStore, final String container,
+            final ListContainerOptions options) {
+
+      return new Iterable<StorageMetadata>() {
+         public Iterator<StorageMetadata> iterator() {
+            return new AbstractIterator<StorageMetadata>() {
+               private Iterator<? extends StorageMetadata> iterator;
+               private String marker;
+
+               public StorageMetadata computeNext() {
+                  while (true) {
+                     if (iterator == null) {
+                        ListContainerOptions nextOptions = marker == null ? options : options.clone().afterMarker(marker);
+                        PageSet<? extends StorageMetadata> list = blobStore.list(container, nextOptions);
+                        iterator = list.iterator();
+                        marker = list.getNextMarker();
+                     }
+                     if (iterator.hasNext()) {
+                        return iterator.next();
+                     }
+                     if (marker == null) {
+                        return endOfData();
+                     }
+                     iterator = null;
+                  }
+               }
+            };
+         }
+      };
+   }
+
+}

blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java

@@ -21,6 +21,8 @@ package org.jclouds.blobstore.options;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.google.common.base.Objects;
+
 /**
  * Contains options supported in the list container operation. <h2>
  * Usage</h2> The recommended way to instantiate a ListOptions object is to statically import
@@ -228,4 +230,27 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
       return "[dir=" + dir + ", recursive=" + recursive + ", detailed=" + detailed
                + ", maxResults=" + getMaxResults() + "]";
    }
+
+   @Override
+   public int hashCode() {
+      return Objects.hashCode(detailed, recursive, dir, getMarker(), getMaxResults());
+   }
+
+   @Override
+   public boolean equals(Object obj) {
+      if (this == obj)
+         return true;
+      if (obj == null)
+         return false;
+      if (getClass() != obj.getClass())
+         return false;
+      ListContainerOptions other = (ListContainerOptions) obj;
+      return (detailed == other.detailed) &&
+               recursive == other.recursive &&
+               Objects.equal(dir, other.dir) &&
+               Objects.equal(getMarker(), other.getMarker()) &&
+               Objects.equal(getMaxResults(), other.getMaxResults());
+   }
+   
+   
 }

blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java

@@ -0,0 +1,105 @@
+package org.jclouds.blobstore;
+
+import static org.easymock.classextension.EasyMock.createMock;
+import static org.testng.Assert.assertEquals;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.easymock.classextension.EasyMock;
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.blobstore.domain.internal.PageSetImpl;
+import org.jclouds.blobstore.options.ListContainerOptions;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+
+public class BlobStoresTest {
+
+   private final String containerName = "mycontainer";
+
+   @Test(expectedExceptions={ContainerNotFoundException.class})
+   public void testListAllForUnknownContainerFromTransientBlobStore() throws Exception {
+      ListContainerOptions options = ListContainerOptions.NONE;
+      BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
+      try {
+         BlobStore blobStore = context.getBlobStore();
+   
+         // Arguably it would be best to throw the exception as soon as listAll is called; but because
+         // the iterator is lazy we don't the exception until we first call iterator().next() or hasNext().
+         Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, "wrongcontainer", options);
+         iterable.iterator().hasNext();
+      } finally {
+         context.close();
+      }
+   }
+   
+   @Test
+   public void testListAllFromTransientBlobStore() throws Exception {
+      final int NUM_BLOBS = 31;
+      ListContainerOptions options = ListContainerOptions.Builder.maxResults(10);
+      BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
+      BlobStore blobStore = null;
+      try {
+         blobStore = context.getBlobStore();
+         blobStore.createContainerInLocation(null, containerName);   
+         Set<String> expectedNames = new HashSet<String>();
+         for (int i = 0; i < NUM_BLOBS; i++) {
+            String blobName = "myname"+i;
+            blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload("payload"+i).build());
+            expectedNames.add(blobName);
+         }
+         
+         Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
+         Iterable<String> iterableNames = Iterables.transform(iterable, new Function<StorageMetadata,String>() {
+            @Override public String apply(StorageMetadata input) {
+               return input.getName();
+            }});
+         
+         // Note that blob.getMetadata being put does not equal blob metadata being retrieved 
+         // because uri is null in one and populated in the other.
+         // Therefore we just compare names to ensure the iterator worked.
+         assertEquals(ImmutableSet.copyOf(iterableNames), expectedNames);
+      } finally {
+         if (blobStore != null) blobStore.deleteContainer(containerName);
+         context.close();
+      }
+   }
+   
+   @Test
+   public void testListAllWhenOnePage() throws Exception {
+      BlobStore blobStore = createMock(BlobStore.class);
+      ListContainerOptions options = ListContainerOptions.NONE;
+      StorageMetadata v1 = createMock(StorageMetadata.class);
+      PageSet<StorageMetadata> pageSet = new PageSetImpl<StorageMetadata>(Collections.singletonList(v1), null);
+      
+      EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options)).andReturn(pageSet).once();
+      EasyMock.replay(blobStore);
+      
+      Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
+      assertEquals(ImmutableList.copyOf(iterable), ImmutableList.of(v1));
+   }
+   
+   @Test
+   public void testListAllWhenTwoPages() throws Exception {
+      BlobStore blobStore = createMock(BlobStore.class);
+      ListContainerOptions options = ListContainerOptions.NONE;
+      ListContainerOptions options2 = ListContainerOptions.Builder.afterMarker("marker1");
+      StorageMetadata v1 = createMock(StorageMetadata.class);
+      StorageMetadata v2 = createMock(StorageMetadata.class);
+      PageSet<StorageMetadata> pageSet = new PageSetImpl<StorageMetadata>(Collections.singletonList(v1), "marker1");
+      PageSet<StorageMetadata> pageSet2 = new PageSetImpl<StorageMetadata>(Collections.singletonList(v2), null);
+      
+      EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options)).andReturn(pageSet).once();
+      EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options2)).andReturn(pageSet2).once();
+      EasyMock.replay(blobStore);
+      
+      Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
+      assertEquals(ImmutableList.copyOf(iterable), ImmutableList.of(v1, v2));
+   }
+}