Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 1011: remove PagedIterator

This commit is contained in:
Adrian Cole 2012-07-16 12:11:59 -04:00
parent addeb0d031
commit 3154371433
8 changed files with 135 additions and 360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apis/cloudwatch/src/main/java/org/jclouds/cloudwatch/CloudWatch.java
View File

@ -26,7 +26,6 @@ import org.jclouds.cloudwatch.features.MetricApi;
import org.jclouds.cloudwatch.options.ListMetricsOptions;
import org.jclouds.collect.IterableWithMarker;
import org.jclouds.collect.PagedIterables;
import org.jclouds.collect.PagedIterators;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
@ -47,7 +46,7 @@ public class CloudWatch {
* @return iterable of metrics fitting the criteria
*/
public static Iterable<Metric> listMetrics(final MetricApi metricApi, final ListMetricsOptions options) {
return Iterables.concat(PagedIterables.create(PagedIterators.advancing(metricApi.list(options),
return Iterables.concat(PagedIterables.advance(metricApi.list(options),
new Function<Object, IterableWithMarker<Metric>>() {
@Override
@ -59,7 +58,7 @@ public class CloudWatch {
public String toString() {
return "listMetrics(" + options + ")";
}
})));
}));
}
/**

31
core/src/main/java/org/jclouds/collect/PagedIterable.java
View File

@ -26,27 +26,31 @@ import com.google.common.collect.Iterators;
import com.google.common.collect.UnmodifiableIterator;
/**
* Allows you to advance through sequence of pages in a resultset. Typically
* used in apis that return only a certain number of records at a time.
* Extends {@link FluentIterable} allowing you to lazily advance through
* sequence of pages in a resultset. Typically used in apis that return only a
* certain number of records at a time.
*
* Simplest usage is to employ the {@link #concat} convenience function.
* Simplest usage is to employ the {@link #concat} convenience function, and one
* of the methods from {@link FluentIterable}.
*
* <pre>
* FluentIterable<StorageMetadata> blobs = blobstore.list(...).concat();
* for (StorageMetadata blob : blobs) {
* process(blob);
* }
* // pull in new pages until it we see something interesting.
* Optional<StorageMetadata> firstInterestingBlob = blobstore
* .list(// options //)
* .concat()
* .firstMatch(isInterestingBlob());
* </pre>
*
* Some may be interested in each page, for example to
* For those seeking manual control of page advances, don't use concat, and
* instead look at the value of {@link IterableWithMarker#nextToken}.
*
* <pre>
* PagedIterator<StorageMetadata> blobs = blobstore.list(...).iterator();
* while (blobs.hasNext()) {
* FluentIterable<StorageMetadata> page = blobs.next();
* IterableWithMarker<StorageMetadata> page = blobs.next();
* ProcessedResults results = process(page);
* if (results.shouldBeBookmarked() && blobs.nextMarker().isPresent()) {
* saveBookmark(blobs.nextMarker().get());
* if (results.shouldBeBookmarked() && page.nextMarker().isPresent()) {
* saveBookmark(page.nextMarker().get());
* }
* }
* </pre>
@ -56,9 +60,6 @@ import com.google.common.collect.UnmodifiableIterator;
@Beta
public abstract class PagedIterable<E> extends FluentIterable<IterableWithMarker<E>> {
@Override
public abstract PagedIterator<E> iterator();
/**
* Combines all the pages into a single unmodifiable iterable. ex.
*
@ -72,7 +73,7 @@ public abstract class PagedIterable<E> extends FluentIterable<IterableWithMarker
* @see Iterators#concat
*/
public FluentIterable<E> concat() {
final PagedIterator<E> iterator = iterator();
final Iterator<IterableWithMarker<E>> iterator = iterator();
final UnmodifiableIterator<Iterator<E>> unmodifiable = new UnmodifiableIterator<Iterator<E>>() {
@Override
public boolean hasNext() {

121
core/src/main/java/org/jclouds/collect/PagedIterables.java
View File

@ -18,7 +18,15 @@
*/
package org.jclouds.collect;
import static com.google.common.base.Preconditions.checkNotNull;
import java.util.Iterator;
import com.google.common.annotations.Beta;
import com.google.common.base.Function;
import com.google.common.base.Objects;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.ImmutableSet;
/**
* Utilities for using {@link PagedIterable}s.
@ -27,23 +35,120 @@ import com.google.common.annotations.Beta;
*/
@Beta
public class PagedIterables {
/**
* @param only
* the only page of data
*
* @param iterator
* how to advance pages
*
* @return iterable current data which continues if the user iterates beyond the first page
* @return iterable with only the one page
*/
public static <T> PagedIterable<T> create(final PagedIterator<T> iterator) {
public static <T> PagedIterable<T> of(final IterableWithMarker<T> only) {
return new PagedIterable<T>() {
@Override
public PagedIterator<T> iterator() {
return iterator;
public Iterator<IterableWithMarker<T>> iterator() {
return ImmutableSet.of(only).iterator();
}
};
}
/**
*
*
* @param initial
* the initial set current data
* @param markerToNext
* produces the next set based on the marker
*
* @return iterable current data which continues if the user iterates beyond
* the first page
*/
public static <T> PagedIterable<T> advance(final IterableWithMarker<T> initial,
final Function<Object, IterableWithMarker<T>> markerToNext) {
return new PagedIterable<T>() {
@Override
public Iterator<IterableWithMarker<T>> iterator() {
return advancingIterator(initial, markerToNext);
}
};
}
private static class AdvancingIterator<T> extends AbstractIterator<IterableWithMarker<T>> {
private final Function<Object, IterableWithMarker<T>> markerToNext;
private transient IterableWithMarker<T> current;
private transient boolean unread = true;
private AdvancingIterator(IterableWithMarker<T> initial, Function<Object, IterableWithMarker<T>> markerToNext) {
this.current = checkNotNull(initial, "initial iterable");
this.markerToNext = checkNotNull(markerToNext, "marker to next iterable");
}
/**
* {@inheritDoc}
*/
@Override
protected IterableWithMarker<T> computeNext() {
if (unread)
try {
return current;
} finally {
unread = false;
}
else if (current.nextMarker().isPresent())
return current = markerToNext.apply(current.nextMarker().get());
else
return endOfData();
}
/**
* {@inheritDoc}
*/
@Override
public int hashCode() {
return Objects.hashCode(current, unread);
}
/**
* {@inheritDoc}
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null || getClass() != obj.getClass())
return false;
AdvancingIterator<?> other = AdvancingIterator.class.cast(obj);
return Objects.equal(this.current, other.current) && Objects.equal(this.unread, other.unread);
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return Objects.toStringHelper("").add("current", current).add("unread", unread).toString();
}
}
/**
*
* @param initial
* the initial set current data
* @param markerToNext
* produces the next set based on the marker
*
* @return iterable current data which continues if the user iterates beyond
* the first page
*/
public static <T> Iterator<IterableWithMarker<T>> advancingIterator(IterableWithMarker<T> initial,
Function<Object, IterableWithMarker<T>> markerToNext) {
if (!initial.nextMarker().isPresent()) {
return ImmutableSet.of(initial).iterator();
}
return new AdvancingIterator<T>(initial, markerToNext);
}
}

40
core/src/main/java/org/jclouds/collect/PagedIterator.java
View File

@ -1,40 +0,0 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.collect;
import com.google.common.annotations.Beta;
import com.google.common.base.Optional;
import com.google.common.collect.AbstractIterator;
/**
*
* @author Adrian Cole
*/
@Beta
public abstract class PagedIterator<T> extends AbstractIterator<IterableWithMarker<T>> {
/**
* If there is a next marker, then the set is incomplete and you should issue another command to
* retrieve the rest, setting the option {@code marker} to this value
*
* @return next marker, or absent if list is complete
*/
public abstract Optional<Object> nextMarker();
}

184
core/src/main/java/org/jclouds/collect/PagedIterators.java
View File

@ -1,184 +0,0 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy current the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.collect;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.annotations.Beta;
import com.google.common.base.Function;
import com.google.common.base.Objects;
import com.google.common.base.Optional;
/**
* Utilities for using {@link PagedIterator}s.
*
* @author Adrian Cole
*/
@Beta
public class PagedIterators {
private static class AdvancingPagedIterator<T> extends PagedIterator<T> {
private final Function<Object, IterableWithMarker<T>> markerToNext;
private transient IterableWithMarker<T> current;
private transient boolean unread = true;
private AdvancingPagedIterator(IterableWithMarker<T> initial, Function<Object, IterableWithMarker<T>> markerToNext) {
this.current = checkNotNull(initial, "initial iterable");
this.markerToNext = checkNotNull(markerToNext, "marker to next iterable");
}
/**
* {@inheritDoc}
*/
@Override
protected IterableWithMarker<T> computeNext() {
if (unread)
try {
return current;
} finally {
unread = false;
}
else if (nextMarker().isPresent())
return current = markerToNext.apply(nextMarker().get());
else
return endOfData();
}
@Override
public Optional<Object> nextMarker() {
return current.nextMarker();
}
/**
* {@inheritDoc}
*/
@Override
public int hashCode() {
return Objects.hashCode(current, unread);
}
/**
* {@inheritDoc}
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null || getClass() != obj.getClass())
return false;
AdvancingPagedIterator<?> other = AdvancingPagedIterator.class.cast(obj);
return Objects.equal(this.current, other.current) && Objects.equal(this.unread, other.unread);
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return Objects.toStringHelper("").add("current", current).add("unread", unread).toString();
}
}
/**
*
* @param initial
* the initial set current data
* @param markerToNext
* produces the next set based on the marker
*
* @return iterable current data which continues if the user iterates beyond the first page
*/
public static <T> PagedIterator<T> advancing(IterableWithMarker<T> initial,
Function<Object, IterableWithMarker<T>> markerToNext) {
if (!initial.nextMarker().isPresent()) {
return of(initial);
}
return new AdvancingPagedIterator<T>(initial, markerToNext);
}
/**
*
* @param initial
* the initial set current data
* @return iterable current data which only contains the single element
*/
public static <T> PagedIterator<T> of(IterableWithMarker<T> initial) {
return new OnlyElementIterator<T>(initial);
}
private static class OnlyElementIterator<T> extends PagedIterator<T> {
private transient IterableWithMarker<T> onlyElement;
private transient boolean unread = true;
private OnlyElementIterator(IterableWithMarker<T> onlyElement) {
this.onlyElement = checkNotNull(onlyElement, "onlyElement");
}
/**
* {@inheritDoc}
*/
@Override
protected IterableWithMarker<T> computeNext() {
if (unread)
try {
return onlyElement;
} finally {
unread = false;
}
else
return endOfData();
}
@Override
public Optional<Object> nextMarker() {
return onlyElement.nextMarker();
}
/**
* {@inheritDoc}
*/
@Override
public int hashCode() {
return Objects.hashCode(onlyElement);
}
/**
* {@inheritDoc}
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null || getClass() != obj.getClass())
return false;
OnlyElementIterator<?> other = OnlyElementIterator.class.cast(obj);
return Objects.equal(this.onlyElement, other.onlyElement) && Objects.equal(this.unread, other.unread);
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return Objects.toStringHelper("").add("onlyElement", onlyElement).add("unread", unread).toString();
}
}
}

5
core/src/main/java/org/jclouds/collect/internal/CallerArg0ToPagedIterable.java
View File

@ -40,7 +40,6 @@ package org.jclouds.collect.internal;
import org.jclouds.collect.IterableWithMarker;
import org.jclouds.collect.PagedIterable;
import org.jclouds.collect.PagedIterables;
import org.jclouds.collect.PagedIterators;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.InvocationContext;
import org.jclouds.rest.internal.GeneratedHttpRequest;
@ -61,7 +60,7 @@ public abstract class CallerArg0ToPagedIterable<T, I extends CallerArg0ToPagedIt
@Override
public PagedIterable<T> apply(IterableWithMarker<T> input) {
if (input.nextMarker() == null)
return PagedIterables.create(PagedIterators.of(input));
return PagedIterables.of(input);
Optional<String> arg0Option = Optional.absent();
if (request.getCaller().get().getArgs() != null && request.getCaller().get().getArgs().length > 0) {
@ -70,7 +69,7 @@ public abstract class CallerArg0ToPagedIterable<T, I extends CallerArg0ToPagedIt
arg0Option = Optional.of(arg0.toString());
}
final String arg0 = arg0Option.orNull();
return PagedIterables.create(PagedIterators.advancing(input, markerToNextForCallingArg0(arg0)));
return PagedIterables.advance(input, markerToNextForCallingArg0(arg0));
}
protected abstract Function<Object, IterableWithMarker<T>> markerToNextForCallingArg0(String arg0);

4
core/src/test/java/org/jclouds/collect/PagedIterablesTest.java
View File

@ -27,7 +27,7 @@ public class PagedIterablesTest {
EasyMock.replay(markerToNext);
PagedIterable<String> iterable = PagedIterables.create(PagedIterators.advancing(initial, markerToNext));
PagedIterable<String> iterable = PagedIterables.advance(initial, markerToNext);
Assert.assertSame(iterable.get(0), initial);
@ -48,7 +48,7 @@ public class PagedIterablesTest {
EasyMock.replay(markerToNext);
PagedIterable<String> iterable = PagedIterables.create(PagedIterators.advancing(initial, markerToNext));
PagedIterable<String> iterable = PagedIterables.advance(initial, markerToNext);
Assert.assertEquals(iterable.concat().toImmutableSet(),
ImmutableSet.of("foo", "bar", "boo", "baz", "ham", "cheeze"));

105
core/src/test/java/org/jclouds/collect/PagedIteratorsTest.java
View File

@ -1,105 +0,0 @@
package org.jclouds.collect;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import org.easymock.EasyMock;
import org.testng.Assert;
import org.testng.annotations.Test;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
/**
* Tests behavior of {@code IterableWithMarkers}.
*
* @author Adrian Cole
*/
@Test(testName = "PagedIteratorsTest")
public class PagedIteratorsTest {
@SuppressWarnings("unchecked")
@Test
public void testSinglePageResultReturnsSame() {
IterableWithMarker<String> initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar"));
Function<Object, IterableWithMarker<String>> markerToNext = createMock(Function.class);
EasyMock.replay(markerToNext);
Assert.assertSame(PagedIterators.advancing(initial, markerToNext).next(), initial);
EasyMock.verify(markerToNext);
}
@SuppressWarnings("unchecked")
@Test
public void testMultiPage2Pages() {
IterableWithMarker<String> initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar"), "MARKER1");
Function<Object, IterableWithMarker<String>> markerToNext = createMock(Function.class);
expect(markerToNext.apply("MARKER1")).andReturn(IterableWithMarkers.from(ImmutableSet.of("boo", "baz"), null));
EasyMock.replay(markerToNext);
Assert.assertEquals(ImmutableSet.copyOf(Iterables.concat(ImmutableSet.copyOf(PagedIterators.advancing(initial,
markerToNext)))), ImmutableSet.of("foo", "bar", "boo", "baz"));
EasyMock.verify(markerToNext);
}
@SuppressWarnings("unchecked")
@Test
public void testMultiPage3Pages() {
IterableWithMarker<String> initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar"), "MARKER1");
Function<Object, IterableWithMarker<String>> markerToNext = createMock(Function.class);
expect(markerToNext.apply("MARKER1")).andReturn(
IterableWithMarkers.from(ImmutableSet.of("boo", "baz"), "MARKER2"));
expect(markerToNext.apply("MARKER2")).andReturn(IterableWithMarkers.from(ImmutableSet.of("ham", "cheeze"), null));
EasyMock.replay(markerToNext);
Assert.assertEquals(ImmutableSet.copyOf(Iterables.concat(ImmutableSet.copyOf(PagedIterators.advancing(initial,
markerToNext)))), ImmutableSet.of("foo", "bar", "boo", "baz", "ham", "cheeze"));
EasyMock.verify(markerToNext);
}
@SuppressWarnings("unchecked")
@Test
public void testMultiPage3PagesNextMarkerSetCorrectly() {
IterableWithMarker<String> initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar"), "MARKER1");
Function<Object, IterableWithMarker<String>> markerToNext = createMock(Function.class);
IterableWithMarker<String> second = IterableWithMarkers.from(ImmutableSet.of("boo", "baz"), "MARKER2");
expect(markerToNext.apply("MARKER1")).andReturn(second);
IterableWithMarker<String> third = IterableWithMarkers.from(ImmutableSet.of("ham", "cheeze"), null);
expect(markerToNext.apply("MARKER2")).andReturn(third);
EasyMock.replay(markerToNext);
PagedIterator<String> iterator = PagedIterators.advancing(initial, markerToNext);
Assert.assertEquals(iterator.hasNext(), true);
Assert.assertEquals(iterator.nextMarker(), Optional.of("MARKER1"));
Assert.assertEquals(iterator.next(), initial);
Assert.assertEquals(iterator.hasNext(), true);
Assert.assertEquals(iterator.nextMarker(), Optional.of("MARKER2"));
Assert.assertEquals(iterator.next(), second);
Assert.assertEquals(iterator.hasNext(), true);
Assert.assertEquals(iterator.nextMarker(), Optional.absent());
Assert.assertEquals(iterator.next(), third);
Assert.assertEquals(iterator.hasNext(), false);
Assert.assertEquals(iterator.nextMarker(), Optional.absent());
EasyMock.verify(markerToNext);
}
}