mirror of https://github.com/apache/jclouds.git
Issue 572: added BlobStores.listAll for a lazy iterable
This commit is contained in:
parent
80ed183ca4
commit
82bfcbfbfa
|
@ -0,0 +1,51 @@
|
|||
package org.jclouds.blobstore;
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.jclouds.blobstore.domain.PageSet;
|
||||
import org.jclouds.blobstore.domain.StorageMetadata;
|
||||
import org.jclouds.blobstore.options.ListContainerOptions;
|
||||
|
||||
import com.google.common.annotations.Beta;
|
||||
import com.google.common.collect.AbstractIterator;
|
||||
|
||||
public class BlobStores {
|
||||
|
||||
/**
|
||||
* A variant of BlobStore.list(String, ListContainerOptions) that
|
||||
* produces an Iterable over the entire set of results, not just one
|
||||
* page, making multiple calls to BlobStore.list as needed.
|
||||
*/
|
||||
@Beta
|
||||
public static Iterable<StorageMetadata> listAll(final BlobStore blobStore, final String container,
|
||||
final ListContainerOptions options) {
|
||||
|
||||
return new Iterable<StorageMetadata>() {
|
||||
public Iterator<StorageMetadata> iterator() {
|
||||
return new AbstractIterator<StorageMetadata>() {
|
||||
private Iterator<? extends StorageMetadata> iterator;
|
||||
private String marker;
|
||||
|
||||
public StorageMetadata computeNext() {
|
||||
while (true) {
|
||||
if (iterator == null) {
|
||||
ListContainerOptions nextOptions = marker == null ? options : options.clone().afterMarker(marker);
|
||||
PageSet<? extends StorageMetadata> list = blobStore.list(container, nextOptions);
|
||||
iterator = list.iterator();
|
||||
marker = list.getNextMarker();
|
||||
}
|
||||
if (iterator.hasNext()) {
|
||||
return iterator.next();
|
||||
}
|
||||
if (marker == null) {
|
||||
return endOfData();
|
||||
}
|
||||
iterator = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,105 @@
|
|||
package org.jclouds.blobstore;
|
||||
|
||||
import static org.easymock.classextension.EasyMock.createMock;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import org.easymock.classextension.EasyMock;
|
||||
import org.jclouds.blobstore.domain.PageSet;
|
||||
import org.jclouds.blobstore.domain.StorageMetadata;
|
||||
import org.jclouds.blobstore.domain.internal.PageSetImpl;
|
||||
import org.jclouds.blobstore.options.ListContainerOptions;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Iterables;
|
||||
|
||||
public class BlobStoresTest {
|
||||
|
||||
private final String containerName = "mycontainer";
|
||||
|
||||
@Test(expectedExceptions={ContainerNotFoundException.class})
|
||||
public void testListAllForUnknownContainerFromTransientBlobStore() throws Exception {
|
||||
ListContainerOptions options = ListContainerOptions.NONE;
|
||||
BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
|
||||
try {
|
||||
BlobStore blobStore = context.getBlobStore();
|
||||
|
||||
// Arguably it would be best to throw the exception as soon as listAll is called; but because
|
||||
// the iterator is lazy we don't the exception until we first call iterator().next() or hasNext().
|
||||
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, "wrongcontainer", options);
|
||||
iterable.iterator().hasNext();
|
||||
} finally {
|
||||
context.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testListAllFromTransientBlobStore() throws Exception {
|
||||
final int NUM_BLOBS = 31;
|
||||
ListContainerOptions options = ListContainerOptions.Builder.maxResults(10);
|
||||
BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
|
||||
BlobStore blobStore = null;
|
||||
try {
|
||||
blobStore = context.getBlobStore();
|
||||
blobStore.createContainerInLocation(null, containerName);
|
||||
Set<String> expectedNames = new HashSet<String>();
|
||||
for (int i = 0; i < NUM_BLOBS; i++) {
|
||||
String blobName = "myname"+i;
|
||||
blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload("payload"+i).build());
|
||||
expectedNames.add(blobName);
|
||||
}
|
||||
|
||||
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
|
||||
Iterable<String> iterableNames = Iterables.transform(iterable, new Function<StorageMetadata,String>() {
|
||||
@Override public String apply(StorageMetadata input) {
|
||||
return input.getName();
|
||||
}});
|
||||
|
||||
// Note that blob.getMetadata being put does not equal blob metadata being retrieved
|
||||
// because uri is null in one and populated in the other.
|
||||
// Therefore we just compare names to ensure the iterator worked.
|
||||
assertEquals(ImmutableSet.copyOf(iterableNames), expectedNames);
|
||||
} finally {
|
||||
if (blobStore != null) blobStore.deleteContainer(containerName);
|
||||
context.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testListAllWhenOnePage() throws Exception {
|
||||
BlobStore blobStore = createMock(BlobStore.class);
|
||||
ListContainerOptions options = ListContainerOptions.NONE;
|
||||
StorageMetadata v1 = createMock(StorageMetadata.class);
|
||||
PageSet<StorageMetadata> pageSet = new PageSetImpl<StorageMetadata>(Collections.singletonList(v1), null);
|
||||
|
||||
EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options)).andReturn(pageSet).once();
|
||||
EasyMock.replay(blobStore);
|
||||
|
||||
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
|
||||
assertEquals(ImmutableList.copyOf(iterable), ImmutableList.of(v1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testListAllWhenTwoPages() throws Exception {
|
||||
BlobStore blobStore = createMock(BlobStore.class);
|
||||
ListContainerOptions options = ListContainerOptions.NONE;
|
||||
ListContainerOptions options2 = ListContainerOptions.Builder.afterMarker("marker1");
|
||||
StorageMetadata v1 = createMock(StorageMetadata.class);
|
||||
StorageMetadata v2 = createMock(StorageMetadata.class);
|
||||
PageSet<StorageMetadata> pageSet = new PageSetImpl<StorageMetadata>(Collections.singletonList(v1), "marker1");
|
||||
PageSet<StorageMetadata> pageSet2 = new PageSetImpl<StorageMetadata>(Collections.singletonList(v2), null);
|
||||
|
||||
EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options)).andReturn(pageSet).once();
|
||||
EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options2)).andReturn(pageSet2).once();
|
||||
EasyMock.replay(blobStore);
|
||||
|
||||
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
|
||||
assertEquals(ImmutableList.copyOf(iterable), ImmutableList.of(v1, v2));
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue