From 82bfcbfbfaa2a4a2ac9b263d3b518e8235df520c Mon Sep 17 00:00:00 2001 From: Aled Sage Date: Wed, 14 Dec 2011 23:50:51 +0000 Subject: [PATCH] Issue 572: added BlobStores.listAll for a lazy iterable --- .../org/jclouds/blobstore/BlobStores.java | 51 +++++++++ .../org/jclouds/blobstore/BlobStoresTest.java | 105 ++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java create mode 100644 blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java diff --git a/blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java b/blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java new file mode 100644 index 0000000000..1aac1a09db --- /dev/null +++ b/blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java @@ -0,0 +1,51 @@ +package org.jclouds.blobstore; + +import java.util.Iterator; + +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.options.ListContainerOptions; + +import com.google.common.annotations.Beta; +import com.google.common.collect.AbstractIterator; + +public class BlobStores { + + /** + * A variant of BlobStore.list(String, ListContainerOptions) that + * produces an Iterable over the entire set of results, not just one + * page, making multiple calls to BlobStore.list as needed. + */ + @Beta + public static Iterable listAll(final BlobStore blobStore, final String container, + final ListContainerOptions options) { + + return new Iterable() { + public Iterator iterator() { + return new AbstractIterator() { + private Iterator iterator; + private String marker; + + public StorageMetadata computeNext() { + while (true) { + if (iterator == null) { + ListContainerOptions nextOptions = marker == null ? options : options.clone().afterMarker(marker); + PageSet list = blobStore.list(container, nextOptions); + iterator = list.iterator(); + marker = list.getNextMarker(); + } + if (iterator.hasNext()) { + return iterator.next(); + } + if (marker == null) { + return endOfData(); + } + iterator = null; + } + } + }; + } + }; + } + +} diff --git a/blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java b/blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java new file mode 100644 index 0000000000..42e35b99dc --- /dev/null +++ b/blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java @@ -0,0 +1,105 @@ +package org.jclouds.blobstore; + +import static org.easymock.classextension.EasyMock.createMock; +import static org.testng.Assert.assertEquals; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import org.easymock.classextension.EasyMock; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.domain.internal.PageSetImpl; +import org.jclouds.blobstore.options.ListContainerOptions; +import org.testng.annotations.Test; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; + +public class BlobStoresTest { + + private final String containerName = "mycontainer"; + + @Test(expectedExceptions={ContainerNotFoundException.class}) + public void testListAllForUnknownContainerFromTransientBlobStore() throws Exception { + ListContainerOptions options = ListContainerOptions.NONE; + BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey"); + try { + BlobStore blobStore = context.getBlobStore(); + + // Arguably it would be best to throw the exception as soon as listAll is called; but because + // the iterator is lazy we don't the exception until we first call iterator().next() or hasNext(). + Iterable iterable = BlobStores.listAll(blobStore, "wrongcontainer", options); + iterable.iterator().hasNext(); + } finally { + context.close(); + } + } + + @Test + public void testListAllFromTransientBlobStore() throws Exception { + final int NUM_BLOBS = 31; + ListContainerOptions options = ListContainerOptions.Builder.maxResults(10); + BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey"); + BlobStore blobStore = null; + try { + blobStore = context.getBlobStore(); + blobStore.createContainerInLocation(null, containerName); + Set expectedNames = new HashSet(); + for (int i = 0; i < NUM_BLOBS; i++) { + String blobName = "myname"+i; + blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload("payload"+i).build()); + expectedNames.add(blobName); + } + + Iterable iterable = BlobStores.listAll(blobStore, containerName, options); + Iterable iterableNames = Iterables.transform(iterable, new Function() { + @Override public String apply(StorageMetadata input) { + return input.getName(); + }}); + + // Note that blob.getMetadata being put does not equal blob metadata being retrieved + // because uri is null in one and populated in the other. + // Therefore we just compare names to ensure the iterator worked. + assertEquals(ImmutableSet.copyOf(iterableNames), expectedNames); + } finally { + if (blobStore != null) blobStore.deleteContainer(containerName); + context.close(); + } + } + + @Test + public void testListAllWhenOnePage() throws Exception { + BlobStore blobStore = createMock(BlobStore.class); + ListContainerOptions options = ListContainerOptions.NONE; + StorageMetadata v1 = createMock(StorageMetadata.class); + PageSet pageSet = new PageSetImpl(Collections.singletonList(v1), null); + + EasyMock.>expect(blobStore.list(containerName, options)).andReturn(pageSet).once(); + EasyMock.replay(blobStore); + + Iterable iterable = BlobStores.listAll(blobStore, containerName, options); + assertEquals(ImmutableList.copyOf(iterable), ImmutableList.of(v1)); + } + + @Test + public void testListAllWhenTwoPages() throws Exception { + BlobStore blobStore = createMock(BlobStore.class); + ListContainerOptions options = ListContainerOptions.NONE; + ListContainerOptions options2 = ListContainerOptions.Builder.afterMarker("marker1"); + StorageMetadata v1 = createMock(StorageMetadata.class); + StorageMetadata v2 = createMock(StorageMetadata.class); + PageSet pageSet = new PageSetImpl(Collections.singletonList(v1), "marker1"); + PageSet pageSet2 = new PageSetImpl(Collections.singletonList(v2), null); + + EasyMock.>expect(blobStore.list(containerName, options)).andReturn(pageSet).once(); + EasyMock.>expect(blobStore.list(containerName, options2)).andReturn(pageSet2).once(); + EasyMock.replay(blobStore); + + Iterable iterable = BlobStores.listAll(blobStore, containerName, options); + assertEquals(ImmutableList.copyOf(iterable), ImmutableList.of(v1, v2)); + } +}