mirror of https://github.com/apache/jclouds.git
Merge pull request #310 from aledsage/Issue-572-BlobStore-listAll-Options
Issue-572: added ListAllOptions for configuring BlobStores.listAll as eager
This commit is contained in:
commit
8f5b46d50a
|
@ -22,21 +22,54 @@ import java.util.Iterator;
|
|||
|
||||
import org.jclouds.blobstore.domain.PageSet;
|
||||
import org.jclouds.blobstore.domain.StorageMetadata;
|
||||
import org.jclouds.blobstore.options.ListAllOptions;
|
||||
import org.jclouds.blobstore.options.ListContainerOptions;
|
||||
|
||||
import com.google.common.annotations.Beta;
|
||||
import com.google.common.collect.AbstractIterator;
|
||||
|
||||
/**
|
||||
* Utilities for using Blob Stores.
|
||||
*
|
||||
* @author Aled Sage
|
||||
* @since 1.3
|
||||
*/
|
||||
public class BlobStores {
|
||||
|
||||
/**
|
||||
* @see listAll(BlobStore, String, ListContainerOptions, ListAllOptions)
|
||||
*/
|
||||
@Beta
|
||||
public static Iterable<StorageMetadata> listAll(BlobStore blobStore, String container,
|
||||
ListContainerOptions containerOptions) {
|
||||
return listAll(blobStore, container, containerOptions, ListAllOptions.NONE);
|
||||
}
|
||||
|
||||
/**
|
||||
* A variant of BlobStore.list(String, ListContainerOptions) that
|
||||
* produces an Iterable over the entire set of results, not just one
|
||||
* page, making multiple calls to BlobStore.list as needed.
|
||||
*
|
||||
* Note that if listAllOptions.isEager, then the first page will be fetched
|
||||
* immediately and cached. Repeatedly iterating will not re-fetch (and thus
|
||||
* will not refresh) the first page.
|
||||
*
|
||||
* @throws ContainerNotFoundException If listAllOptions.isEager and container cannot be found
|
||||
*/
|
||||
@Beta
|
||||
public static Iterable<StorageMetadata> listAll(final BlobStore blobStore, final String container,
|
||||
final ListContainerOptions options) {
|
||||
final ListContainerOptions containerOptions, final ListAllOptions listAllOptions) {
|
||||
final boolean eager = listAllOptions.isEager();
|
||||
final PageSet<? extends StorageMetadata> firstList;
|
||||
final String firstMarker;
|
||||
|
||||
if (eager) {
|
||||
firstList = blobStore.list(container, containerOptions);
|
||||
firstMarker = firstList.getNextMarker();
|
||||
} else {
|
||||
firstList = null;
|
||||
firstMarker = null;
|
||||
}
|
||||
|
||||
return new Iterable<StorageMetadata>() {
|
||||
public Iterator<StorageMetadata> iterator() {
|
||||
|
@ -47,11 +80,17 @@ public class BlobStores {
|
|||
public StorageMetadata computeNext() {
|
||||
while (true) {
|
||||
if (iterator == null) {
|
||||
ListContainerOptions nextOptions = marker == null ? options : options.clone().afterMarker(marker);
|
||||
PageSet<? extends StorageMetadata> list = blobStore.list(container, nextOptions);
|
||||
iterator = list.iterator();
|
||||
PageSet<? extends StorageMetadata> list;
|
||||
if (eager && marker == null) {
|
||||
list = firstList;
|
||||
marker = firstMarker;
|
||||
} else {
|
||||
ListContainerOptions nextOptions = marker == null ? containerOptions : containerOptions.clone().afterMarker(marker);
|
||||
list = blobStore.list(container, nextOptions);
|
||||
marker = list.getNextMarker();
|
||||
}
|
||||
iterator = list.iterator();
|
||||
}
|
||||
if (iterator.hasNext()) {
|
||||
return iterator.next();
|
||||
}
|
||||
|
|
|
@ -0,0 +1,93 @@
|
|||
package org.jclouds.blobstore.options;
|
||||
|
||||
import com.google.common.base.Objects;
|
||||
|
||||
/**
|
||||
* Contains options supported by BlobStores.listAll.
|
||||
*
|
||||
* @see ListOptions for recommended usage patterns
|
||||
*
|
||||
* @author Aled Sage
|
||||
* @since 1.3
|
||||
*/
|
||||
public class ListAllOptions implements Cloneable {
|
||||
|
||||
public static final ImmutableListAllOptions NONE = new ImmutableListAllOptions(new ListAllOptions());
|
||||
|
||||
private boolean eager = false;
|
||||
|
||||
public ListAllOptions() {
|
||||
}
|
||||
|
||||
ListAllOptions(boolean eagerness) {
|
||||
this.eager = eagerness;
|
||||
}
|
||||
|
||||
public static class ImmutableListAllOptions extends ListAllOptions {
|
||||
private final ListAllOptions delegate;
|
||||
|
||||
public ImmutableListAllOptions(ListAllOptions delegate) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEager() {
|
||||
return delegate.isEager();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ListAllOptions eager(boolean val) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isEager() {
|
||||
return eager;
|
||||
}
|
||||
|
||||
/**
|
||||
* If eager, will connect to container immediately and fail-fast, rather than failing when
|
||||
* first iterating over the list.
|
||||
*/
|
||||
public ListAllOptions eager(boolean val) {
|
||||
this.eager = val;
|
||||
return this;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
/**
|
||||
* @see ListAllOptions#eager(boolean)
|
||||
*/
|
||||
public static ListAllOptions eager(boolean eager) {
|
||||
ListAllOptions options = new ListAllOptions();
|
||||
return options.eager(eager);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ListAllOptions clone() {
|
||||
return new ListAllOptions(isEager());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "[eager=" + eager + "]";
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hashCode(eager);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
ListAllOptions other = (ListAllOptions) obj;
|
||||
return (eager == other.eager);
|
||||
}
|
||||
}
|
|
@ -29,6 +29,7 @@ import org.easymock.classextension.EasyMock;
|
|||
import org.jclouds.blobstore.domain.PageSet;
|
||||
import org.jclouds.blobstore.domain.StorageMetadata;
|
||||
import org.jclouds.blobstore.domain.internal.PageSetImpl;
|
||||
import org.jclouds.blobstore.options.ListAllOptions;
|
||||
import org.jclouds.blobstore.options.ListContainerOptions;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
|
@ -41,15 +42,28 @@ public class BlobStoresTest {
|
|||
|
||||
private final String containerName = "mycontainer";
|
||||
|
||||
@Test(expectedExceptions={ContainerNotFoundException.class})
|
||||
public void testListAllForUnknownContainerFromTransientBlobStoreEagerly() throws Exception {
|
||||
ListContainerOptions containerOptions = ListContainerOptions.NONE;
|
||||
ListAllOptions listAllOptions = ListAllOptions.Builder.eager(true);
|
||||
BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
|
||||
try {
|
||||
BlobStore blobStore = context.getBlobStore();
|
||||
BlobStores.listAll(blobStore, "wrongcontainer", containerOptions, listAllOptions);
|
||||
} finally {
|
||||
context.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Default listAll is not eager, so test that exception is thrown when first attempt to iterate.
|
||||
*/
|
||||
@Test(expectedExceptions={ContainerNotFoundException.class})
|
||||
public void testListAllForUnknownContainerFromTransientBlobStore() throws Exception {
|
||||
ListContainerOptions options = ListContainerOptions.NONE;
|
||||
BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
|
||||
try {
|
||||
BlobStore blobStore = context.getBlobStore();
|
||||
|
||||
// Arguably it would be best to throw the exception as soon as listAll is called; but because
|
||||
// the iterator is lazy we don't the exception until we first call iterator().next() or hasNext().
|
||||
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, "wrongcontainer", options);
|
||||
iterable.iterator().hasNext();
|
||||
} finally {
|
||||
|
@ -59,8 +73,18 @@ public class BlobStoresTest {
|
|||
|
||||
@Test
|
||||
public void testListAllFromTransientBlobStore() throws Exception {
|
||||
runListAllFromTransientBlobStore(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testListAllFromTransientBlobStoreEagerly() throws Exception {
|
||||
runListAllFromTransientBlobStore(true);
|
||||
}
|
||||
|
||||
private void runListAllFromTransientBlobStore(boolean eager) throws Exception {
|
||||
final int numTimesToIterate = 2;
|
||||
final int NUM_BLOBS = 31;
|
||||
ListContainerOptions options = ListContainerOptions.Builder.maxResults(10);
|
||||
ListContainerOptions containerOptions = ListContainerOptions.Builder.maxResults(10);
|
||||
BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
|
||||
BlobStore blobStore = null;
|
||||
try {
|
||||
|
@ -73,7 +97,10 @@ public class BlobStoresTest {
|
|||
expectedNames.add(blobName);
|
||||
}
|
||||
|
||||
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
|
||||
ListAllOptions listAllOptions = ListAllOptions.Builder.eager(eager);
|
||||
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, containerOptions, listAllOptions);
|
||||
|
||||
for (int i = 0; i < numTimesToIterate; i++) {
|
||||
Iterable<String> iterableNames = Iterables.transform(iterable, new Function<StorageMetadata,String>() {
|
||||
@Override public String apply(StorageMetadata input) {
|
||||
return input.getName();
|
||||
|
@ -83,6 +110,7 @@ public class BlobStoresTest {
|
|||
// because uri is null in one and populated in the other.
|
||||
// Therefore we just compare names to ensure the iterator worked.
|
||||
assertEquals(ImmutableSet.copyOf(iterableNames), expectedNames);
|
||||
}
|
||||
} finally {
|
||||
if (blobStore != null) blobStore.deleteContainer(containerName);
|
||||
context.close();
|
||||
|
|
Loading…
Reference in New Issue