Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue-572: added ListAllOptions for configuring BlobStores.listAll as eager

This commit is contained in:
Aled Sage 2012-01-11 20:42:25 +00:00
parent bf1311be9b
commit 657c268db7
3 changed files with 177 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java
View File

@ -22,21 +22,54 @@ import java.util.Iterator;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.options.ListAllOptions;
import org.jclouds.blobstore.options.ListContainerOptions;
import com.google.common.annotations.Beta;
import com.google.common.collect.AbstractIterator;
/**
* Utilities for using Blob Stores.
*
* @author Aled Sage
* @since 1.3
*/
public class BlobStores {
/**
* @see listAll(BlobStore, String, ListContainerOptions, ListAllOptions)
*/
@Beta
public static Iterable<StorageMetadata> listAll(BlobStore blobStore, String container,
ListContainerOptions containerOptions) {
return listAll(blobStore, container, containerOptions, ListAllOptions.NONE);
}
/**
* A variant of BlobStore.list(String, ListContainerOptions) that
* produces an Iterable over the entire set of results, not just one
* page, making multiple calls to BlobStore.list as needed.
*
* Note that if listAllOptions.isEager, then the first page will be fetched
* immediately and cached. Repeatedly iterating will not re-fetch (and thus
* will not refresh) the first page.
*
* @throws ContainerNotFoundException If listAllOptions.isEager and container cannot be found
*/
@Beta
public static Iterable<StorageMetadata> listAll(final BlobStore blobStore, final String container,
final ListContainerOptions options) {
final ListContainerOptions containerOptions, final ListAllOptions listAllOptions) {
final boolean eager = listAllOptions.isEager();
final PageSet<? extends StorageMetadata> firstList;
final String firstMarker;
if (eager) {
firstList = blobStore.list(container, containerOptions);
firstMarker = firstList.getNextMarker();
} else {
firstList = null;
firstMarker = null;
}
return new Iterable<StorageMetadata>() {
public Iterator<StorageMetadata> iterator() {
@ -47,11 +80,17 @@ public class BlobStores {
public StorageMetadata computeNext() {
while (true) {
if (iterator == null) {
ListContainerOptions nextOptions = marker == null ? options : options.clone().afterMarker(marker);
PageSet<? extends StorageMetadata> list = blobStore.list(container, nextOptions);
iterator = list.iterator();
PageSet<? extends StorageMetadata> list;
if (eager && marker == null) {
list = firstList;
marker = firstMarker;
} else {
ListContainerOptions nextOptions = marker == null ? containerOptions : containerOptions.clone().afterMarker(marker);
list = blobStore.list(container, nextOptions);
marker = list.getNextMarker();
}
iterator = list.iterator();
}
if (iterator.hasNext()) {
return iterator.next();
}

93
blobstore/src/main/java/org/jclouds/blobstore/options/ListAllOptions.java Normal file
View File

@ -0,0 +1,93 @@
package org.jclouds.blobstore.options;
import com.google.common.base.Objects;
/**
* Contains options supported by BlobStores.listAll.
*
* @see ListOptions for recommended usage patterns
*
* @author Aled Sage
* @since 1.3
*/
public class ListAllOptions implements Cloneable {
public static final ImmutableListAllOptions NONE = new ImmutableListAllOptions(new ListAllOptions());
private boolean eager = false;
public ListAllOptions() {
}
ListAllOptions(boolean eagerness) {
this.eager = eagerness;
}
public static class ImmutableListAllOptions extends ListAllOptions {
private final ListAllOptions delegate;
public ImmutableListAllOptions(ListAllOptions delegate) {
this.delegate = delegate;
}
@Override
public boolean isEager() {
return delegate.isEager();
}
@Override
public ListAllOptions eager(boolean val) {
throw new UnsupportedOperationException();
}
}
public boolean isEager() {
return eager;
}
/**
* If eager, will connect to container immediately and fail-fast, rather than failing when
* first iterating over the list.
*/
public ListAllOptions eager(boolean val) {
this.eager = val;
return this;
}
public static class Builder {
/**
* @see ListAllOptions#eager(boolean)
*/
public static ListAllOptions eager(boolean eager) {
ListAllOptions options = new ListAllOptions();
return options.eager(eager);
}
}
@Override
public ListAllOptions clone() {
return new ListAllOptions(isEager());
}
@Override
public String toString() {
return "[eager=" + eager + "]";
}
@Override
public int hashCode() {
return Objects.hashCode(eager);
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
ListAllOptions other = (ListAllOptions) obj;
return (eager == other.eager);
}
}

38
blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java
View File

@ -29,6 +29,7 @@ import org.easymock.classextension.EasyMock;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.options.ListAllOptions;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.testng.annotations.Test;
@ -41,15 +42,28 @@ public class BlobStoresTest {
private final String containerName = "mycontainer";
@Test(expectedExceptions={ContainerNotFoundException.class})
public void testListAllForUnknownContainerFromTransientBlobStoreEagerly() throws Exception {
ListContainerOptions containerOptions = ListContainerOptions.NONE;
ListAllOptions listAllOptions = ListAllOptions.Builder.eager(true);
BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
try {
BlobStore blobStore = context.getBlobStore();
BlobStores.listAll(blobStore, "wrongcontainer", containerOptions, listAllOptions);
} finally {
context.close();
}
}
/**
* Default listAll is not eager, so test that exception is thrown when first attempt to iterate.
*/
@Test(expectedExceptions={ContainerNotFoundException.class})
public void testListAllForUnknownContainerFromTransientBlobStore() throws Exception {
ListContainerOptions options = ListContainerOptions.NONE;
BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
try {
BlobStore blobStore = context.getBlobStore();
// Arguably it would be best to throw the exception as soon as listAll is called; but because
// the iterator is lazy we don't the exception until we first call iterator().next() or hasNext().
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, "wrongcontainer", options);
iterable.iterator().hasNext();
} finally {
@ -59,8 +73,18 @@ public class BlobStoresTest {
@Test
public void testListAllFromTransientBlobStore() throws Exception {
runListAllFromTransientBlobStore(false);
}
@Test
public void testListAllFromTransientBlobStoreEagerly() throws Exception {
runListAllFromTransientBlobStore(true);
}
private void runListAllFromTransientBlobStore(boolean eager) throws Exception {
final int numTimesToIterate = 2;
final int NUM_BLOBS = 31;
ListContainerOptions options = ListContainerOptions.Builder.maxResults(10);
ListContainerOptions containerOptions = ListContainerOptions.Builder.maxResults(10);
BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
BlobStore blobStore = null;
try {
@ -73,7 +97,10 @@ public class BlobStoresTest {
expectedNames.add(blobName);
}
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
ListAllOptions listAllOptions = ListAllOptions.Builder.eager(eager);
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, containerOptions, listAllOptions);
for (int i = 0; i < numTimesToIterate; i++) {
Iterable<String> iterableNames = Iterables.transform(iterable, new Function<StorageMetadata,String>() {
@Override public String apply(StorageMetadata input) {
return input.getName();
@ -83,6 +110,7 @@ public class BlobStoresTest {
// because uri is null in one and populated in the other.
// Therefore we just compare names to ensure the iterator worked.
assertEquals(ImmutableSet.copyOf(iterableNames), expectedNames);
}
} finally {
if (blobStore != null) blobStore.deleteContainer(containerName);
context.close();