Issue-572: added ListAllOptions for configuring BlobStores.listAll as eager

blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java

@@ -22,21 +22,54 @@ import java.util.Iterator;
 
 import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.blobstore.options.ListAllOptions;
 import org.jclouds.blobstore.options.ListContainerOptions;
 
 import com.google.common.annotations.Beta;
 import com.google.common.collect.AbstractIterator;
 
+/**
+ * Utilities for using Blob Stores.
+ * 
+ * @author Aled Sage
+ * @since 1.3
+ */
 public class BlobStores {
 
+   /**
+    * @see listAll(BlobStore, String, ListContainerOptions, ListAllOptions)
+    */
+   @Beta
+   public static Iterable<StorageMetadata> listAll(BlobStore blobStore, String container,
+            ListContainerOptions containerOptions) {
+      return listAll(blobStore, container, containerOptions, ListAllOptions.NONE);
+   }
+   
    /**
     * A variant of BlobStore.list(String, ListContainerOptions) that
     * produces an Iterable over the entire set of results, not just one
     * page, making multiple calls to BlobStore.list as needed.
+    * 
+    * Note that if listAllOptions.isEager, then the first page will be fetched
+    * immediately and cached. Repeatedly iterating will not re-fetch (and thus
+    * will not refresh) the first page.
+    *  
+    * @throws ContainerNotFoundException If listAllOptions.isEager and container cannot be found
     */
    @Beta
    public static Iterable<StorageMetadata> listAll(final BlobStore blobStore, final String container,
-            final ListContainerOptions options) {
+            final ListContainerOptions containerOptions, final ListAllOptions listAllOptions) {
+      final boolean eager = listAllOptions.isEager();
+      final PageSet<? extends StorageMetadata> firstList;
+      final String firstMarker;
+
+      if (eager) {
+         firstList = blobStore.list(container, containerOptions);
+         firstMarker = firstList.getNextMarker();
+      } else {
+         firstList = null;
+         firstMarker = null;
+      }
       
       return new Iterable<StorageMetadata>() {
          public Iterator<StorageMetadata> iterator() {
@@ -47,11 +80,17 @@ public class BlobStores {
                public StorageMetadata computeNext() {
                   while (true) {
                      if (iterator == null) {
-                        ListContainerOptions nextOptions = marker == null ? options : options.clone().afterMarker(marker);
+                        PageSet<? extends StorageMetadata> list;
-                        PageSet<? extends StorageMetadata> list = blobStore.list(container, nextOptions);
+                        if (eager && marker == null) {
-                        iterator = list.iterator();
+                           list = firstList;
+                           marker = firstMarker;
+                        } else {
+                           ListContainerOptions nextOptions = marker == null ? containerOptions : containerOptions.clone().afterMarker(marker);
+                           list = blobStore.list(container, nextOptions);
                            marker = list.getNextMarker();
                         }
+                        iterator = list.iterator();
+                     }
                      if (iterator.hasNext()) {
                         return iterator.next();
                      }

blobstore/src/main/java/org/jclouds/blobstore/options/ListAllOptions.java

@@ -0,0 +1,93 @@
+package org.jclouds.blobstore.options;
+
+import com.google.common.base.Objects;
+
+/**
+ * Contains options supported by BlobStores.listAll.
+ *  
+ * @see ListOptions for recommended usage patterns
+ * 
+ * @author Aled Sage
+ * @since 1.3
+ */
+public class ListAllOptions implements Cloneable {
+
+   public static final ImmutableListAllOptions NONE = new ImmutableListAllOptions(new ListAllOptions());
+
+   private boolean eager = false;
+
+   public ListAllOptions() {
+   }
+
+   ListAllOptions(boolean eagerness) {
+      this.eager = eagerness;
+   }
+
+   public static class ImmutableListAllOptions extends ListAllOptions {
+      private final ListAllOptions delegate;
+
+      public ImmutableListAllOptions(ListAllOptions delegate) {
+         this.delegate = delegate;
+      }
+
+      @Override
+      public boolean isEager() {
+         return delegate.isEager();
+      }
+      
+      @Override
+      public ListAllOptions eager(boolean val) {
+         throw new UnsupportedOperationException();
+      }
+   }
+
+   public boolean isEager() {
+      return eager;
+   }
+
+   /**
+    * If eager, will connect to container immediately and fail-fast, rather than failing when 
+    * first iterating over the list.
+    */
+   public ListAllOptions eager(boolean val) {
+      this.eager = val;
+      return this;
+   }
+
+   public static class Builder {
+      /**
+       * @see ListAllOptions#eager(boolean)
+       */
+      public static ListAllOptions eager(boolean eager) {
+         ListAllOptions options = new ListAllOptions();
+         return options.eager(eager);
+      }
+   }
+
+   @Override
+   public ListAllOptions clone() {
+      return new ListAllOptions(isEager());
+   }
+
+   @Override
+   public String toString() {
+      return "[eager=" + eager + "]";
+   }
+
+   @Override
+   public int hashCode() {
+      return Objects.hashCode(eager);
+   }
+
+   @Override
+   public boolean equals(Object obj) {
+      if (this == obj)
+         return true;
+      if (obj == null)
+         return false;
+      if (getClass() != obj.getClass())
+         return false;
+      ListAllOptions other = (ListAllOptions) obj;
+      return (eager == other.eager);
+   }
+}

blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java

@@ -29,6 +29,7 @@ import org.easymock.classextension.EasyMock;
 import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.domain.internal.PageSetImpl;
+import org.jclouds.blobstore.options.ListAllOptions;
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.testng.annotations.Test;
 
@@ -41,15 +42,28 @@ public class BlobStoresTest {
 
    private final String containerName = "mycontainer";
 
+   @Test(expectedExceptions={ContainerNotFoundException.class})
+   public void testListAllForUnknownContainerFromTransientBlobStoreEagerly() throws Exception {
+      ListContainerOptions containerOptions = ListContainerOptions.NONE;
+      ListAllOptions listAllOptions = ListAllOptions.Builder.eager(true);
+      BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
+      try {
+         BlobStore blobStore = context.getBlobStore();
+         BlobStores.listAll(blobStore, "wrongcontainer", containerOptions, listAllOptions);
+      } finally {
+         context.close();
+      }
+   }
+
+   /**
+    * Default listAll is not eager, so test that exception is thrown when first attempt to iterate.
+    */
    @Test(expectedExceptions={ContainerNotFoundException.class})
    public void testListAllForUnknownContainerFromTransientBlobStore() throws Exception {
       ListContainerOptions options = ListContainerOptions.NONE;
       BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
       try {
          BlobStore blobStore = context.getBlobStore();
-   
-         // Arguably it would be best to throw the exception as soon as listAll is called; but because
-         // the iterator is lazy we don't the exception until we first call iterator().next() or hasNext().
          Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, "wrongcontainer", options);
          iterable.iterator().hasNext();
       } finally {
@@ -59,8 +73,18 @@ public class BlobStoresTest {
    
    @Test
    public void testListAllFromTransientBlobStore() throws Exception {
+      runListAllFromTransientBlobStore(false);
+   }
+
+   @Test
+   public void testListAllFromTransientBlobStoreEagerly() throws Exception {
+      runListAllFromTransientBlobStore(true);
+   }
+
+   private void runListAllFromTransientBlobStore(boolean eager) throws Exception {
+      final int numTimesToIterate = 2;
       final int NUM_BLOBS = 31;
-      ListContainerOptions options = ListContainerOptions.Builder.maxResults(10);
+      ListContainerOptions containerOptions = ListContainerOptions.Builder.maxResults(10);
       BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
       BlobStore blobStore = null;
       try {
@@ -73,7 +97,10 @@ public class BlobStoresTest {
             expectedNames.add(blobName);
          }
          
-         Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
+         ListAllOptions listAllOptions = ListAllOptions.Builder.eager(eager);
+         Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, containerOptions, listAllOptions);
+         
+         for (int i = 0; i < numTimesToIterate; i++) {
             Iterable<String> iterableNames = Iterables.transform(iterable, new Function<StorageMetadata,String>() {
                @Override public String apply(StorageMetadata input) {
                   return input.getName();
@@ -83,6 +110,7 @@ public class BlobStoresTest {
             // because uri is null in one and populated in the other.
             // Therefore we just compare names to ensure the iterator worked.
             assertEquals(ImmutableSet.copyOf(iterableNames), expectedNames);
+         }
       } finally {
          if (blobStore != null) blobStore.deleteContainer(containerName);
          context.close();