Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #247 from aledsage/Issue-572-BlobStore-listAll 

Issue 572: added BlobStores.listAll for a lazy iterable
This commit is contained in:
Adrian Cole 2011-12-15 11:38:17 -08:00
parent beb87a2131 82bfcbfbfa
commit 9899372737
3 changed files with 181 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
blobstore/src/main/java/org/jclouds/blobstore/BlobStores.java Normal file
View File

@ -0,0 +1,51 @@
package org.jclouds.blobstore;
import java.util.Iterator;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.options.ListContainerOptions;
import com.google.common.annotations.Beta;
import com.google.common.collect.AbstractIterator;
public class BlobStores {
/**
* A variant of BlobStore.list(String, ListContainerOptions) that
* produces an Iterable over the entire set of results, not just one
* page, making multiple calls to BlobStore.list as needed.
*/
@Beta
public static Iterable<StorageMetadata> listAll(final BlobStore blobStore, final String container,
final ListContainerOptions options) {
return new Iterable<StorageMetadata>() {
public Iterator<StorageMetadata> iterator() {
return new AbstractIterator<StorageMetadata>() {
private Iterator<? extends StorageMetadata> iterator;
private String marker;
public StorageMetadata computeNext() {
while (true) {
if (iterator == null) {
ListContainerOptions nextOptions = marker == null ? options : options.clone().afterMarker(marker);
PageSet<? extends StorageMetadata> list = blobStore.list(container, nextOptions);
iterator = list.iterator();
marker = list.getNextMarker();
}
if (iterator.hasNext()) {
return iterator.next();
}
if (marker == null) {
return endOfData();
}
iterator = null;
}
}
};
}
};
}
}

25
blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java
View File

@ -21,6 +21,8 @@ package org.jclouds.blobstore.options;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.base.Objects;
/**
* Contains options supported in the list container operation. <h2>
* Usage</h2> The recommended way to instantiate a ListOptions object is to statically import
@ -228,4 +230,27 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
return "[dir=" + dir + ", recursive=" + recursive + ", detailed=" + detailed
+ ", maxResults=" + getMaxResults() + "]";
}
@Override
public int hashCode() {
return Objects.hashCode(detailed, recursive, dir, getMarker(), getMaxResults());
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
ListContainerOptions other = (ListContainerOptions) obj;
return (detailed == other.detailed) &&
recursive == other.recursive &&
Objects.equal(dir, other.dir) &&
Objects.equal(getMarker(), other.getMarker()) &&
Objects.equal(getMaxResults(), other.getMaxResults());
}
}

105
blobstore/src/test/java/org/jclouds/blobstore/BlobStoresTest.java Normal file
View File

@ -0,0 +1,105 @@
package org.jclouds.blobstore;
import static org.easymock.classextension.EasyMock.createMock;
import static org.testng.Assert.assertEquals;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.easymock.classextension.EasyMock;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.testng.annotations.Test;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
public class BlobStoresTest {
private final String containerName = "mycontainer";
@Test(expectedExceptions={ContainerNotFoundException.class})
public void testListAllForUnknownContainerFromTransientBlobStore() throws Exception {
ListContainerOptions options = ListContainerOptions.NONE;
BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
try {
BlobStore blobStore = context.getBlobStore();
// Arguably it would be best to throw the exception as soon as listAll is called; but because
// the iterator is lazy we don't the exception until we first call iterator().next() or hasNext().
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, "wrongcontainer", options);
iterable.iterator().hasNext();
} finally {
context.close();
}
}
@Test
public void testListAllFromTransientBlobStore() throws Exception {
final int NUM_BLOBS = 31;
ListContainerOptions options = ListContainerOptions.Builder.maxResults(10);
BlobStoreContext context = new BlobStoreContextFactory().createContext("transient", "dummyid", "dummykey");
BlobStore blobStore = null;
try {
blobStore = context.getBlobStore();
blobStore.createContainerInLocation(null, containerName);
Set<String> expectedNames = new HashSet<String>();
for (int i = 0; i < NUM_BLOBS; i++) {
String blobName = "myname"+i;
blobStore.putBlob(containerName, blobStore.blobBuilder(blobName).payload("payload"+i).build());
expectedNames.add(blobName);
}
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
Iterable<String> iterableNames = Iterables.transform(iterable, new Function<StorageMetadata,String>() {
@Override public String apply(StorageMetadata input) {
return input.getName();
}});
// Note that blob.getMetadata being put does not equal blob metadata being retrieved
// because uri is null in one and populated in the other.
// Therefore we just compare names to ensure the iterator worked.
assertEquals(ImmutableSet.copyOf(iterableNames), expectedNames);
} finally {
if (blobStore != null) blobStore.deleteContainer(containerName);
context.close();
}
}
@Test
public void testListAllWhenOnePage() throws Exception {
BlobStore blobStore = createMock(BlobStore.class);
ListContainerOptions options = ListContainerOptions.NONE;
StorageMetadata v1 = createMock(StorageMetadata.class);
PageSet<StorageMetadata> pageSet = new PageSetImpl<StorageMetadata>(Collections.singletonList(v1), null);
EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options)).andReturn(pageSet).once();
EasyMock.replay(blobStore);
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
assertEquals(ImmutableList.copyOf(iterable), ImmutableList.of(v1));
}
@Test
public void testListAllWhenTwoPages() throws Exception {
BlobStore blobStore = createMock(BlobStore.class);
ListContainerOptions options = ListContainerOptions.NONE;
ListContainerOptions options2 = ListContainerOptions.Builder.afterMarker("marker1");
StorageMetadata v1 = createMock(StorageMetadata.class);
StorageMetadata v2 = createMock(StorageMetadata.class);
PageSet<StorageMetadata> pageSet = new PageSetImpl<StorageMetadata>(Collections.singletonList(v1), "marker1");
PageSet<StorageMetadata> pageSet2 = new PageSetImpl<StorageMetadata>(Collections.singletonList(v2), null);
EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options)).andReturn(pageSet).once();
EasyMock.<PageSet<? extends StorageMetadata>>expect(blobStore.list(containerName, options2)).andReturn(pageSet2).once();
EasyMock.replay(blobStore);
Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options);
assertEquals(ImmutableList.copyOf(iterable), ImmutableList.of(v1, v2));
}
}