diff --git a/apis/cloudwatch/src/main/java/org/jclouds/cloudwatch/CloudWatch.java b/apis/cloudwatch/src/main/java/org/jclouds/cloudwatch/CloudWatch.java index a289a77783..d433926d68 100644 --- a/apis/cloudwatch/src/main/java/org/jclouds/cloudwatch/CloudWatch.java +++ b/apis/cloudwatch/src/main/java/org/jclouds/cloudwatch/CloudWatch.java @@ -26,7 +26,6 @@ import org.jclouds.cloudwatch.features.MetricApi; import org.jclouds.cloudwatch.options.ListMetricsOptions; import org.jclouds.collect.IterableWithMarker; import org.jclouds.collect.PagedIterables; -import org.jclouds.collect.PagedIterators; import com.google.common.base.Function; import com.google.common.collect.Iterables; @@ -47,7 +46,7 @@ public class CloudWatch { * @return iterable of metrics fitting the criteria */ public static Iterable listMetrics(final MetricApi metricApi, final ListMetricsOptions options) { - return Iterables.concat(PagedIterables.create(PagedIterators.advancing(metricApi.list(options), + return Iterables.concat(PagedIterables.advance(metricApi.list(options), new Function>() { @Override @@ -59,7 +58,7 @@ public class CloudWatch { public String toString() { return "listMetrics(" + options + ")"; } - }))); + })); } /** diff --git a/core/src/main/java/org/jclouds/collect/PagedIterable.java b/core/src/main/java/org/jclouds/collect/PagedIterable.java index 09175469a6..d5cf683a64 100644 --- a/core/src/main/java/org/jclouds/collect/PagedIterable.java +++ b/core/src/main/java/org/jclouds/collect/PagedIterable.java @@ -26,27 +26,31 @@ import com.google.common.collect.Iterators; import com.google.common.collect.UnmodifiableIterator; /** - * Allows you to advance through sequence of pages in a resultset. Typically - * used in apis that return only a certain number of records at a time. + * Extends {@link FluentIterable} allowing you to lazily advance through + * sequence of pages in a resultset. Typically used in apis that return only a + * certain number of records at a time. * - * Simplest usage is to employ the {@link #concat} convenience function. + * Simplest usage is to employ the {@link #concat} convenience function, and one + * of the methods from {@link FluentIterable}. * * 
- * FluentIterable blobs = blobstore.list(...).concat();
- * for (StorageMetadata blob : blobs) {
- *     process(blob);
- * }
+ *    // pull in new pages until it we see something interesting.
+ *     Optional firstInterestingBlob = blobstore
+ *         .list(// options //)
+ *         .concat()
+ *         .firstMatch(isInterestingBlob());
  *
* - * Some may be interested in each page, for example to + * For those seeking manual control of page advances, don't use concat, and + * instead look at the value of {@link IterableWithMarker#nextToken}. * * 
  * PagedIterator blobs = blobstore.list(...).iterator();
  * while (blobs.hasNext()) {
- *     FluentIterable page = blobs.next();
+ *     IterableWithMarker page = blobs.next();
  *     ProcessedResults results = process(page);
- *     if (results.shouldBeBookmarked() && blobs.nextMarker().isPresent()) {
- *         saveBookmark(blobs.nextMarker().get());
+ *     if (results.shouldBeBookmarked() && page.nextMarker().isPresent()) {
+ *         saveBookmark(page.nextMarker().get());
  *     }
  * }
  *
@@ -56,9 +60,6 @@ import com.google.common.collect.UnmodifiableIterator; @Beta public abstract class PagedIterable extends FluentIterable> { - @Override - public abstract PagedIterator iterator(); - /** * Combines all the pages into a single unmodifiable iterable. ex. * @@ -72,7 +73,7 @@ public abstract class PagedIterable extends FluentIterable concat() { - final PagedIterator iterator = iterator(); + final Iterator> iterator = iterator(); final UnmodifiableIterator> unmodifiable = new UnmodifiableIterator>() { @Override public boolean hasNext() { diff --git a/core/src/main/java/org/jclouds/collect/PagedIterables.java b/core/src/main/java/org/jclouds/collect/PagedIterables.java index 5267f57416..b98c03865d 100644 --- a/core/src/main/java/org/jclouds/collect/PagedIterables.java +++ b/core/src/main/java/org/jclouds/collect/PagedIterables.java @@ -18,7 +18,15 @@ */ package org.jclouds.collect; +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.Iterator; + import com.google.common.annotations.Beta; +import com.google.common.base.Function; +import com.google.common.base.Objects; +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableSet; /** * Utilities for using {@link PagedIterable}s. @@ -27,23 +35,120 @@ import com.google.common.annotations.Beta; */ @Beta public class PagedIterables { - /** + * @param only + * the only page of data * - * @param iterator - * how to advance pages - * - * @return iterable current data which continues if the user iterates beyond the first page + * @return iterable with only the one page */ - public static PagedIterable create(final PagedIterator iterator) { + public static PagedIterable of(final IterableWithMarker only) { return new PagedIterable() { @Override - public PagedIterator iterator() { - return iterator; + public Iterator> iterator() { + return ImmutableSet.of(only).iterator(); } }; } + /** + * + * + * @param initial + * the initial set current data + * @param markerToNext + * produces the next set based on the marker + * + * @return iterable current data which continues if the user iterates beyond + * the first page + */ + public static PagedIterable advance(final IterableWithMarker initial, + final Function> markerToNext) { + return new PagedIterable() { + + @Override + public Iterator> iterator() { + return advancingIterator(initial, markerToNext); + } + + }; + } + + private static class AdvancingIterator extends AbstractIterator> { + + private final Function> markerToNext; + private transient IterableWithMarker current; + private transient boolean unread = true; + + private AdvancingIterator(IterableWithMarker initial, Function> markerToNext) { + this.current = checkNotNull(initial, "initial iterable"); + this.markerToNext = checkNotNull(markerToNext, "marker to next iterable"); + } + + /** + * {@inheritDoc} + */ + @Override + protected IterableWithMarker computeNext() { + if (unread) + try { + return current; + } finally { + unread = false; + } + else if (current.nextMarker().isPresent()) + return current = markerToNext.apply(current.nextMarker().get()); + else + return endOfData(); + } + + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + return Objects.hashCode(current, unread); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; + AdvancingIterator other = AdvancingIterator.class.cast(obj); + return Objects.equal(this.current, other.current) && Objects.equal(this.unread, other.unread); + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return Objects.toStringHelper("").add("current", current).add("unread", unread).toString(); + } + } + + /** + * + * @param initial + * the initial set current data + * @param markerToNext + * produces the next set based on the marker + * + * @return iterable current data which continues if the user iterates beyond + * the first page + */ + public static Iterator> advancingIterator(IterableWithMarker initial, + Function> markerToNext) { + if (!initial.nextMarker().isPresent()) { + return ImmutableSet.of(initial).iterator(); + } + return new AdvancingIterator(initial, markerToNext); + } + } diff --git a/core/src/main/java/org/jclouds/collect/PagedIterator.java b/core/src/main/java/org/jclouds/collect/PagedIterator.java deleted file mode 100644 index 9d47c6d1fb..0000000000 --- a/core/src/main/java/org/jclouds/collect/PagedIterator.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed to jclouds, Inc. (jclouds) under one or more - * contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. jclouds licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.jclouds.collect; - -import com.google.common.annotations.Beta; -import com.google.common.base.Optional; -import com.google.common.collect.AbstractIterator; - -/** - * - * @author Adrian Cole - */ -@Beta -public abstract class PagedIterator extends AbstractIterator> { - - /** - * If there is a next marker, then the set is incomplete and you should issue another command to - * retrieve the rest, setting the option {@code marker} to this value - * - * @return next marker, or absent if list is complete - */ - public abstract Optional nextMarker(); - -} \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/collect/PagedIterators.java b/core/src/main/java/org/jclouds/collect/PagedIterators.java deleted file mode 100644 index 4ff50e9bd5..0000000000 --- a/core/src/main/java/org/jclouds/collect/PagedIterators.java +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Licensed to jclouds, Inc. (jclouds) under one or more - * contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. jclouds licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy current the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.jclouds.collect; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.common.annotations.Beta; -import com.google.common.base.Function; -import com.google.common.base.Objects; -import com.google.common.base.Optional; - -/** - * Utilities for using {@link PagedIterator}s. - * - * @author Adrian Cole - */ -@Beta -public class PagedIterators { - - private static class AdvancingPagedIterator extends PagedIterator { - - private final Function> markerToNext; - private transient IterableWithMarker current; - private transient boolean unread = true; - - private AdvancingPagedIterator(IterableWithMarker initial, Function> markerToNext) { - this.current = checkNotNull(initial, "initial iterable"); - this.markerToNext = checkNotNull(markerToNext, "marker to next iterable"); - } - - /** - * {@inheritDoc} - */ - @Override - protected IterableWithMarker computeNext() { - if (unread) - try { - return current; - } finally { - unread = false; - } - else if (nextMarker().isPresent()) - return current = markerToNext.apply(nextMarker().get()); - else - return endOfData(); - } - - @Override - public Optional nextMarker() { - return current.nextMarker(); - } - - /** - * {@inheritDoc} - */ - @Override - public int hashCode() { - return Objects.hashCode(current, unread); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null || getClass() != obj.getClass()) - return false; - AdvancingPagedIterator other = AdvancingPagedIterator.class.cast(obj); - return Objects.equal(this.current, other.current) && Objects.equal(this.unread, other.unread); - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() { - return Objects.toStringHelper("").add("current", current).add("unread", unread).toString(); - } - } - - /** - * - * @param initial - * the initial set current data - * @param markerToNext - * produces the next set based on the marker - * - * @return iterable current data which continues if the user iterates beyond the first page - */ - public static PagedIterator advancing(IterableWithMarker initial, - Function> markerToNext) { - if (!initial.nextMarker().isPresent()) { - return of(initial); - } - return new AdvancingPagedIterator(initial, markerToNext); - } - - /** - * - * @param initial - * the initial set current data - * @return iterable current data which only contains the single element - */ - public static PagedIterator of(IterableWithMarker initial) { - return new OnlyElementIterator(initial); - } - - private static class OnlyElementIterator extends PagedIterator { - - private transient IterableWithMarker onlyElement; - private transient boolean unread = true; - - private OnlyElementIterator(IterableWithMarker onlyElement) { - this.onlyElement = checkNotNull(onlyElement, "onlyElement"); - } - - /** - * {@inheritDoc} - */ - @Override - protected IterableWithMarker computeNext() { - if (unread) - try { - return onlyElement; - } finally { - unread = false; - } - else - return endOfData(); - } - - @Override - public Optional nextMarker() { - return onlyElement.nextMarker(); - } - - /** - * {@inheritDoc} - */ - @Override - public int hashCode() { - return Objects.hashCode(onlyElement); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null || getClass() != obj.getClass()) - return false; - OnlyElementIterator other = OnlyElementIterator.class.cast(obj); - return Objects.equal(this.onlyElement, other.onlyElement) && Objects.equal(this.unread, other.unread); - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() { - return Objects.toStringHelper("").add("onlyElement", onlyElement).add("unread", unread).toString(); - } - } -} diff --git a/core/src/main/java/org/jclouds/collect/internal/CallerArg0ToPagedIterable.java b/core/src/main/java/org/jclouds/collect/internal/CallerArg0ToPagedIterable.java index c93738fc2b..a1ddedb14b 100644 --- a/core/src/main/java/org/jclouds/collect/internal/CallerArg0ToPagedIterable.java +++ b/core/src/main/java/org/jclouds/collect/internal/CallerArg0ToPagedIterable.java @@ -40,7 +40,6 @@ package org.jclouds.collect.internal; import org.jclouds.collect.IterableWithMarker; import org.jclouds.collect.PagedIterable; import org.jclouds.collect.PagedIterables; -import org.jclouds.collect.PagedIterators; import org.jclouds.http.HttpRequest; import org.jclouds.rest.InvocationContext; import org.jclouds.rest.internal.GeneratedHttpRequest; @@ -61,7 +60,7 @@ public abstract class CallerArg0ToPagedIterable apply(IterableWithMarker input) { if (input.nextMarker() == null) - return PagedIterables.create(PagedIterators.of(input)); + return PagedIterables.of(input); Optional arg0Option = Optional.absent(); if (request.getCaller().get().getArgs() != null && request.getCaller().get().getArgs().length > 0) { @@ -70,7 +69,7 @@ public abstract class CallerArg0ToPagedIterable> markerToNextForCallingArg0(String arg0); diff --git a/core/src/test/java/org/jclouds/collect/PagedIterablesTest.java b/core/src/test/java/org/jclouds/collect/PagedIterablesTest.java index e8a0cf933f..9ce207dc6b 100644 --- a/core/src/test/java/org/jclouds/collect/PagedIterablesTest.java +++ b/core/src/test/java/org/jclouds/collect/PagedIterablesTest.java @@ -27,7 +27,7 @@ public class PagedIterablesTest { EasyMock.replay(markerToNext); - PagedIterable iterable = PagedIterables.create(PagedIterators.advancing(initial, markerToNext)); + PagedIterable iterable = PagedIterables.advance(initial, markerToNext); Assert.assertSame(iterable.get(0), initial); @@ -48,7 +48,7 @@ public class PagedIterablesTest { EasyMock.replay(markerToNext); - PagedIterable iterable = PagedIterables.create(PagedIterators.advancing(initial, markerToNext)); + PagedIterable iterable = PagedIterables.advance(initial, markerToNext); Assert.assertEquals(iterable.concat().toImmutableSet(), ImmutableSet.of("foo", "bar", "boo", "baz", "ham", "cheeze")); diff --git a/core/src/test/java/org/jclouds/collect/PagedIteratorsTest.java b/core/src/test/java/org/jclouds/collect/PagedIteratorsTest.java deleted file mode 100644 index c1348c460f..0000000000 --- a/core/src/test/java/org/jclouds/collect/PagedIteratorsTest.java +++ /dev/null @@ -1,105 +0,0 @@ -package org.jclouds.collect; - -import static org.easymock.EasyMock.createMock; -import static org.easymock.EasyMock.expect; - -import org.easymock.EasyMock; -import org.testng.Assert; -import org.testng.annotations.Test; - -import com.google.common.base.Function; -import com.google.common.base.Optional; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; - -/** - * Tests behavior of {@code IterableWithMarkers}. - * - * @author Adrian Cole - */ -@Test(testName = "PagedIteratorsTest") -public class PagedIteratorsTest { - - @SuppressWarnings("unchecked") - @Test - public void testSinglePageResultReturnsSame() { - - IterableWithMarker initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar")); - Function> markerToNext = createMock(Function.class); - - EasyMock.replay(markerToNext); - - Assert.assertSame(PagedIterators.advancing(initial, markerToNext).next(), initial); - - EasyMock.verify(markerToNext); - } - - @SuppressWarnings("unchecked") - @Test - public void testMultiPage2Pages() { - - IterableWithMarker initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar"), "MARKER1"); - Function> markerToNext = createMock(Function.class); - - expect(markerToNext.apply("MARKER1")).andReturn(IterableWithMarkers.from(ImmutableSet.of("boo", "baz"), null)); - - EasyMock.replay(markerToNext); - - Assert.assertEquals(ImmutableSet.copyOf(Iterables.concat(ImmutableSet.copyOf(PagedIterators.advancing(initial, - markerToNext)))), ImmutableSet.of("foo", "bar", "boo", "baz")); - - EasyMock.verify(markerToNext); - - } - - @SuppressWarnings("unchecked") - @Test - public void testMultiPage3Pages() { - - IterableWithMarker initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar"), "MARKER1"); - Function> markerToNext = createMock(Function.class); - - expect(markerToNext.apply("MARKER1")).andReturn( - IterableWithMarkers.from(ImmutableSet.of("boo", "baz"), "MARKER2")); - - expect(markerToNext.apply("MARKER2")).andReturn(IterableWithMarkers.from(ImmutableSet.of("ham", "cheeze"), null)); - - EasyMock.replay(markerToNext); - - Assert.assertEquals(ImmutableSet.copyOf(Iterables.concat(ImmutableSet.copyOf(PagedIterators.advancing(initial, - markerToNext)))), ImmutableSet.of("foo", "bar", "boo", "baz", "ham", "cheeze")); - - EasyMock.verify(markerToNext); - - } - - @SuppressWarnings("unchecked") - @Test - public void testMultiPage3PagesNextMarkerSetCorrectly() { - - IterableWithMarker initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar"), "MARKER1"); - Function> markerToNext = createMock(Function.class); - IterableWithMarker second = IterableWithMarkers.from(ImmutableSet.of("boo", "baz"), "MARKER2"); - expect(markerToNext.apply("MARKER1")).andReturn(second); - IterableWithMarker third = IterableWithMarkers.from(ImmutableSet.of("ham", "cheeze"), null); - expect(markerToNext.apply("MARKER2")).andReturn(third); - - EasyMock.replay(markerToNext); - PagedIterator iterator = PagedIterators.advancing(initial, markerToNext); - - Assert.assertEquals(iterator.hasNext(), true); - Assert.assertEquals(iterator.nextMarker(), Optional.of("MARKER1")); - Assert.assertEquals(iterator.next(), initial); - Assert.assertEquals(iterator.hasNext(), true); - Assert.assertEquals(iterator.nextMarker(), Optional.of("MARKER2")); - Assert.assertEquals(iterator.next(), second); - Assert.assertEquals(iterator.hasNext(), true); - Assert.assertEquals(iterator.nextMarker(), Optional.absent()); - Assert.assertEquals(iterator.next(), third); - Assert.assertEquals(iterator.hasNext(), false); - Assert.assertEquals(iterator.nextMarker(), Optional.absent()); - EasyMock.verify(markerToNext); - - } - -}