From 657c268db7eb4a2269965f25f2c3422762fa7e5e Mon Sep 17 00:00:00 2001 From: Aled Sage Date: Wed, 11 Jan 2012 20:42:25 +0000 Subject: [PATCH] Issue-572: added ListAllOptions for configuring BlobStores.listAll as eager --- .../org/jclouds/blobstore/BlobStores.java | 47 +++++++++- .../blobstore/options/ListAllOptions.java | 93 +++++++++++++++++++ .../org/jclouds/blobstore/BlobStoresTest.java | 54 ++++++++--- 3 files changed, 177 insertions(+), 17 deletions(-) create mode 100644 blobstore/src/main/java/org/jclouds/blobstore/options/ListAllOptions.java diff --git a/blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java b/blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java index fb3b41840a..b59d60f44b 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java @@ -22,22 +22,55 @@ import java.util.Iterator; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.options.ListAllOptions; import org.jclouds.blobstore.options.ListContainerOptions; import com.google.common.annotations.Beta; import com.google.common.collect.AbstractIterator; +/** + * Utilities for using Blob Stores. + * + * @author Aled Sage + * @since 1.3 + */ public class BlobStores { + /** + * @see listAll(BlobStore, String, ListContainerOptions, ListAllOptions) + */ + @Beta + public static Iterable listAll(BlobStore blobStore, String container, + ListContainerOptions containerOptions) { + return listAll(blobStore, container, containerOptions, ListAllOptions.NONE); + } + /** * A variant of BlobStore.list(String, ListContainerOptions) that * produces an Iterable over the entire set of results, not just one * page, making multiple calls to BlobStore.list as needed. + * + * Note that if listAllOptions.isEager, then the first page will be fetched + * immediately and cached. Repeatedly iterating will not re-fetch (and thus + * will not refresh) the first page. + * + * @throws ContainerNotFoundException If listAllOptions.isEager and container cannot be found */ @Beta public static Iterable listAll(final BlobStore blobStore, final String container, - final ListContainerOptions options) { + final ListContainerOptions containerOptions, final ListAllOptions listAllOptions) { + final boolean eager = listAllOptions.isEager(); + final PageSet firstList; + final String firstMarker; + if (eager) { + firstList = blobStore.list(container, containerOptions); + firstMarker = firstList.getNextMarker(); + } else { + firstList = null; + firstMarker = null; + } + return new Iterable() { public Iterator iterator() { return new AbstractIterator() { @@ -47,10 +80,16 @@ public class BlobStores { public StorageMetadata computeNext() { while (true) { if (iterator == null) { - ListContainerOptions nextOptions = marker == null ? options : options.clone().afterMarker(marker); - PageSet list = blobStore.list(container, nextOptions); + PageSet list; + if (eager && marker == null) { + list = firstList; + marker = firstMarker; + } else { + ListContainerOptions nextOptions = marker == null ? containerOptions : containerOptions.clone().afterMarker(marker); + list = blobStore.list(container, nextOptions); + marker = list.getNextMarker(); + } iterator = list.iterator(); - marker = list.getNextMarker(); } if (iterator.hasNext()) { return iterator.next(); diff --git a/blobstore/src/main/java/org/jclouds/blobstore/options/ListAllOptions.java b/blobstore/src/main/java/org/jclouds/blobstore/options/ListAllOptions.java new file mode 100644 index 0000000000..4deb3d61cd --- /dev/null +++ b/blobstore/src/main/java/org/jclouds/blobstore/options/ListAllOptions.java @@ -0,0 +1,93 @@ +package org.jclouds.blobstore.options; + +import com.google.common.base.Objects; + +/** + * Contains options supported by BlobStores.listAll. + * + * @see ListOptions for recommended usage patterns + * + * @author Aled Sage + * @since 1.3 + */ +public class ListAllOptions implements Cloneable { + + public static final ImmutableListAllOptions NONE = new ImmutableListAllOptions(new ListAllOptions()); + + private boolean eager = false; + + public ListAllOptions() { + } + + ListAllOptions(boolean eagerness) { + this.eager = eagerness; + } + + public static class ImmutableListAllOptions extends ListAllOptions { + private final ListAllOptions delegate; + + public ImmutableListAllOptions(ListAllOptions delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEager() { + return delegate.isEager(); + } + + @Override + public ListAllOptions eager(boolean val) { + throw new UnsupportedOperationException(); + } + } + + public boolean isEager() { + return eager; + } + + /** + * If eager, will connect to container immediately and fail-fast, rather than failing when + * first iterating over the list. + */ + public ListAllOptions eager(boolean val) { + this.eager = val; + return this; + } + + public static class Builder { + /** + * @see ListAllOptions#eager(boolean) + */ + public static ListAllOptions eager(boolean eager) { + ListAllOptions options = new ListAllOptions(); + return options.eager(eager); + } + } + + @Override + public ListAllOptions clone() { + return new ListAllOptions(isEager()); + } + + @Override + public String toString() { + return "[eager=" + eager + "]"; + } + + @Override + public int hashCode() { + return Objects.hashCode(eager); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ListAllOptions other = (ListAllOptions) obj; + return (eager == other.eager); + } +} diff --git a/blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java b/blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java index 9ea6d131b1..2e2c11f42d 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java @@ -29,6 +29,7 @@ import org.easymock.classextension.EasyMock; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.domain.internal.PageSetImpl; +import org.jclouds.blobstore.options.ListAllOptions; import org.jclouds.blobstore.options.ListContainerOptions; import org.testng.annotations.Test; @@ -41,15 +42,28 @@ public class BlobStoresTest { private final String containerName = "mycontainer"; + @Test(expectedExceptions={ContainerNotFoundException.class}) + public void testListAllForUnknownContainerFromTransientBlobStoreEagerly() throws Exception { + ListContainerOptions containerOptions = ListContainerOptions.NONE; + ListAllOptions listAllOptions = ListAllOptions.Builder.eager(true); + BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey"); + try { + BlobStore blobStore = context.getBlobStore(); + BlobStores.listAll(blobStore, "wrongcontainer", containerOptions, listAllOptions); + } finally { + context.close(); + } + } + + /** + * Default listAll is not eager, so test that exception is thrown when first attempt to iterate. + */ @Test(expectedExceptions={ContainerNotFoundException.class}) public void testListAllForUnknownContainerFromTransientBlobStore() throws Exception { ListContainerOptions options = ListContainerOptions.NONE; BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey"); try { BlobStore blobStore = context.getBlobStore(); - - // Arguably it would be best to throw the exception as soon as listAll is called; but because - // the iterator is lazy we don't the exception until we first call iterator().next() or hasNext(). Iterable iterable = BlobStores.listAll(blobStore, "wrongcontainer", options); iterable.iterator().hasNext(); } finally { @@ -59,8 +73,18 @@ public class BlobStoresTest { @Test public void testListAllFromTransientBlobStore() throws Exception { + runListAllFromTransientBlobStore(false); + } + + @Test + public void testListAllFromTransientBlobStoreEagerly() throws Exception { + runListAllFromTransientBlobStore(true); + } + + private void runListAllFromTransientBlobStore(boolean eager) throws Exception { + final int numTimesToIterate = 2; final int NUM_BLOBS = 31; - ListContainerOptions options = ListContainerOptions.Builder.maxResults(10); + ListContainerOptions containerOptions = ListContainerOptions.Builder.maxResults(10); BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey"); BlobStore blobStore = null; try { @@ -73,16 +97,20 @@ public class BlobStoresTest { expectedNames.add(blobName); } - Iterable iterable = BlobStores.listAll(blobStore, containerName, options); - Iterable iterableNames = Iterables.transform(iterable, new Function() { - @Override public String apply(StorageMetadata input) { - return input.getName(); - }}); + ListAllOptions listAllOptions = ListAllOptions.Builder.eager(eager); + Iterable iterable = BlobStores.listAll(blobStore, containerName, containerOptions, listAllOptions); - // Note that blob.getMetadata being put does not equal blob metadata being retrieved - // because uri is null in one and populated in the other. - // Therefore we just compare names to ensure the iterator worked. - assertEquals(ImmutableSet.copyOf(iterableNames), expectedNames); + for (int i = 0; i < numTimesToIterate; i++) { + Iterable iterableNames = Iterables.transform(iterable, new Function() { + @Override public String apply(StorageMetadata input) { + return input.getName(); + }}); + + // Note that blob.getMetadata being put does not equal blob metadata being retrieved + // because uri is null in one and populated in the other. + // Therefore we just compare names to ensure the iterator worked. + assertEquals(ImmutableSet.copyOf(iterableNames), expectedNames); + } } finally { if (blobStore != null) blobStore.deleteContainer(containerName); context.close();