Issue 1011: remove PagedIterator

apis/cloudwatch/src/main/java/org/jclouds/cloudwatch/CloudWatch.java

@@ -26,7 +26,6 @@ import org.jclouds.cloudwatch.features.MetricApi;
 import org.jclouds.cloudwatch.options.ListMetricsOptions;
 import org.jclouds.collect.IterableWithMarker;
 import org.jclouds.collect.PagedIterables;
-import org.jclouds.collect.PagedIterators;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
@@ -47,7 +46,7 @@ public class CloudWatch {
     * @return iterable of metrics fitting the criteria
     */
    public static Iterable<Metric> listMetrics(final MetricApi metricApi, final ListMetricsOptions options) {
-      return Iterables.concat(PagedIterables.create(PagedIterators.advancing(metricApi.list(options),
+      return Iterables.concat(PagedIterables.advance(metricApi.list(options),
                new Function<Object, IterableWithMarker<Metric>>() {
 
                   @Override
@@ -59,7 +58,7 @@ public class CloudWatch {
                   public String toString() {
                      return "listMetrics(" + options + ")";
                   }
-               })));
+               }));
    }
 
    /**

core/src/main/java/org/jclouds/collect/PagedIterable.java

@@ -26,27 +26,31 @@ import com.google.common.collect.Iterators;
 import com.google.common.collect.UnmodifiableIterator;
 
 /**
- * Allows you to advance through sequence of pages in a resultset. Typically
+ * Extends {@link FluentIterable} allowing you to lazily advance through
- * used in apis that return only a certain number of records at a time.
+ * sequence of pages in a resultset. Typically used in apis that return only a
+ * certain number of records at a time.
  * 
- * Simplest usage is to employ the {@link #concat} convenience function.
+ * Simplest usage is to employ the {@link #concat} convenience function, and one
+ * of the methods from {@link FluentIterable}.
  * 
  * <pre>
- * FluentIterable<StorageMetadata> blobs = blobstore.list(...).concat();
+ *    // pull in new pages until it we see something interesting.
- * for (StorageMetadata blob : blobs) {
+ *     Optional<StorageMetadata> firstInterestingBlob = blobstore
- *     process(blob);
+ *         .list(// options //)
- * }
+ *         .concat()
+ *         .firstMatch(isInterestingBlob());
  * </pre>
  * 
- * Some may be interested in each page, for example to
+ * For those seeking manual control of page advances, don't use concat, and
+ * instead look at the value of {@link IterableWithMarker#nextToken}.
  * 
  * <pre>
  * PagedIterator<StorageMetadata> blobs = blobstore.list(...).iterator();
  * while (blobs.hasNext()) {
- *     FluentIterable<StorageMetadata> page = blobs.next();
+ *     IterableWithMarker<StorageMetadata> page = blobs.next();
  *     ProcessedResults results = process(page);
- *     if (results.shouldBeBookmarked() && blobs.nextMarker().isPresent()) {
+ *     if (results.shouldBeBookmarked() && page.nextMarker().isPresent()) {
- *         saveBookmark(blobs.nextMarker().get());
+ *         saveBookmark(page.nextMarker().get());
  *     }
  * }
  * </pre>
@@ -56,9 +60,6 @@ import com.google.common.collect.UnmodifiableIterator;
 @Beta
 public abstract class PagedIterable<E> extends FluentIterable<IterableWithMarker<E>> {
 
-   @Override
-   public abstract PagedIterator<E> iterator();
-
    /**
     * Combines all the pages into a single unmodifiable iterable. ex.
     * 
@@ -72,7 +73,7 @@ public abstract class PagedIterable<E> extends FluentIterable<IterableWithMarker
     * @see Iterators#concat
     */
    public FluentIterable<E> concat() {
-      final PagedIterator<E> iterator = iterator();
+      final Iterator<IterableWithMarker<E>> iterator = iterator();
       final UnmodifiableIterator<Iterator<E>> unmodifiable = new UnmodifiableIterator<Iterator<E>>() {
          @Override
          public boolean hasNext() {

core/src/main/java/org/jclouds/collect/PagedIterables.java

@@ -18,7 +18,15 @@
  */
 package org.jclouds.collect;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Iterator;
+
 import com.google.common.annotations.Beta;
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.ImmutableSet;
 
 /**
  * Utilities for using {@link PagedIterable}s.
@@ -27,23 +35,120 @@ import com.google.common.annotations.Beta;
  */
 @Beta
 public class PagedIterables {
-
    /**
+    * @param only
+    *           the only page of data
     * 
-    * @param iterator
+    * @return iterable with only the one page
-    *           how to advance pages
-    * 
-    * @return iterable current data which continues if the user iterates beyond the first page
     */
-   public static <T> PagedIterable<T> create(final PagedIterator<T> iterator) {
+   public static <T> PagedIterable<T> of(final IterableWithMarker<T> only) {
       return new PagedIterable<T>() {
 
          @Override
-         public PagedIterator<T> iterator() {
+         public Iterator<IterableWithMarker<T>> iterator() {
-            return iterator;
+            return ImmutableSet.of(only).iterator();
          }
 
       };
    }
 
+   /**
+    * 
+    * 
+    * @param initial
+    *           the initial set current data
+    * @param markerToNext
+    *           produces the next set based on the marker
+    * 
+    * @return iterable current data which continues if the user iterates beyond
+    *         the first page
+    */
+   public static <T> PagedIterable<T> advance(final IterableWithMarker<T> initial,
+         final Function<Object, IterableWithMarker<T>> markerToNext) {
+      return new PagedIterable<T>() {
+
+         @Override
+         public Iterator<IterableWithMarker<T>> iterator() {
+            return advancingIterator(initial, markerToNext);
+         }
+
+      };
+   }
+
+   private static class AdvancingIterator<T> extends AbstractIterator<IterableWithMarker<T>> {
+
+      private final Function<Object, IterableWithMarker<T>> markerToNext;
+      private transient IterableWithMarker<T> current;
+      private transient boolean unread = true;
+
+      private AdvancingIterator(IterableWithMarker<T> initial, Function<Object, IterableWithMarker<T>> markerToNext) {
+         this.current = checkNotNull(initial, "initial iterable");
+         this.markerToNext = checkNotNull(markerToNext, "marker to next iterable");
+      }
+
+      /**
+       * {@inheritDoc}
+       */
+      @Override
+      protected IterableWithMarker<T> computeNext() {
+         if (unread)
+            try {
+               return current;
+            } finally {
+               unread = false;
+            }
+         else if (current.nextMarker().isPresent())
+            return current = markerToNext.apply(current.nextMarker().get());
+         else
+            return endOfData();
+      }
+
+      /**
+       * {@inheritDoc}
+       */
+      @Override
+      public int hashCode() {
+         return Objects.hashCode(current, unread);
+      }
+
+      /**
+       * {@inheritDoc}
+       */
+      @Override
+      public boolean equals(Object obj) {
+         if (this == obj)
+            return true;
+         if (obj == null || getClass() != obj.getClass())
+            return false;
+         AdvancingIterator<?> other = AdvancingIterator.class.cast(obj);
+         return Objects.equal(this.current, other.current) && Objects.equal(this.unread, other.unread);
+      }
+
+      /**
+       * {@inheritDoc}
+       */
+      @Override
+      public String toString() {
+         return Objects.toStringHelper("").add("current", current).add("unread", unread).toString();
+      }
+   }
+
+   /**
+    * 
+    * @param initial
+    *           the initial set current data
+    * @param markerToNext
+    *           produces the next set based on the marker
+    * 
+    * @return iterable current data which continues if the user iterates beyond
+    *         the first page
+    */
+   public static <T> Iterator<IterableWithMarker<T>> advancingIterator(IterableWithMarker<T> initial,
+         Function<Object, IterableWithMarker<T>> markerToNext) {
+      if (!initial.nextMarker().isPresent()) {
+         return ImmutableSet.of(initial).iterator();
+      }
+      return new AdvancingIterator<T>(initial, markerToNext);
+   }
+
 }

core/src/main/java/org/jclouds/collect/PagedIterator.java

@@ -1,40 +0,0 @@
-/**
- * Licensed to jclouds, Inc. (jclouds) under one or more
- * contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  jclouds licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.jclouds.collect;
-
-import com.google.common.annotations.Beta;
-import com.google.common.base.Optional;
-import com.google.common.collect.AbstractIterator;
-
-/**
- * 
- * @author Adrian Cole
- */
-@Beta
-public abstract class PagedIterator<T> extends AbstractIterator<IterableWithMarker<T>> {
-
-   /**
-    * If there is a next marker, then the set is incomplete and you should issue another command to
-    * retrieve the rest, setting the option {@code marker} to this value 
-    * 
-    * @return next marker, or absent if list is complete
-    */
-   public abstract Optional<Object> nextMarker();
-
-}

core/src/main/java/org/jclouds/collect/PagedIterators.java

@@ -1,184 +0,0 @@
-/**
- * Licensed to jclouds, Inc. (jclouds) under one or more
- * contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  jclouds licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy current the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.jclouds.collect;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import com.google.common.annotations.Beta;
-import com.google.common.base.Function;
-import com.google.common.base.Objects;
-import com.google.common.base.Optional;
-
-/**
- * Utilities for using {@link PagedIterator}s.
- * 
- * @author Adrian Cole
- */
-@Beta
-public class PagedIterators {
-
-   private static class AdvancingPagedIterator<T> extends PagedIterator<T> {
-
-      private final Function<Object, IterableWithMarker<T>> markerToNext;
-      private transient IterableWithMarker<T> current;
-      private transient boolean unread = true;
-
-      private AdvancingPagedIterator(IterableWithMarker<T> initial, Function<Object, IterableWithMarker<T>> markerToNext) {
-         this.current = checkNotNull(initial, "initial iterable");
-         this.markerToNext = checkNotNull(markerToNext, "marker to next iterable");
-      }
-
-      /**
-       * {@inheritDoc}
-       */
-      @Override
-      protected IterableWithMarker<T> computeNext() {
-         if (unread)
-            try {
-               return current;
-            } finally {
-               unread = false;
-            }
-         else if (nextMarker().isPresent())
-            return current = markerToNext.apply(nextMarker().get());
-         else
-            return endOfData();
-      }
-
-      @Override
-      public Optional<Object> nextMarker() {
-         return current.nextMarker();
-      }
-
-      /**
-       * {@inheritDoc}
-       */
-      @Override
-      public int hashCode() {
-         return Objects.hashCode(current, unread);
-      }
-
-      /**
-       * {@inheritDoc}
-       */
-      @Override
-      public boolean equals(Object obj) {
-         if (this == obj)
-            return true;
-         if (obj == null || getClass() != obj.getClass())
-            return false;
-         AdvancingPagedIterator<?> other = AdvancingPagedIterator.class.cast(obj);
-         return Objects.equal(this.current, other.current) && Objects.equal(this.unread, other.unread);
-      }
-
-      /**
-       * {@inheritDoc}
-       */
-      @Override
-      public String toString() {
-         return Objects.toStringHelper("").add("current", current).add("unread", unread).toString();
-      }
-   }
-
-   /**
-    * 
-    * @param initial
-    *           the initial set current data
-    * @param markerToNext
-    *           produces the next set based on the marker
-    * 
-    * @return iterable current data which continues if the user iterates beyond the first page
-    */
-   public static <T> PagedIterator<T> advancing(IterableWithMarker<T> initial,
-            Function<Object, IterableWithMarker<T>> markerToNext) {
-      if (!initial.nextMarker().isPresent()) {
-         return of(initial);
-      }
-      return new AdvancingPagedIterator<T>(initial, markerToNext);
-   }
-
-   /**
-    * 
-    * @param initial
-    *           the initial set current data
-    * @return iterable current data which only contains the single element
-    */
-   public static <T> PagedIterator<T> of(IterableWithMarker<T> initial) {
-      return new OnlyElementIterator<T>(initial);
-   }
-
-   private static class OnlyElementIterator<T> extends PagedIterator<T> {
-
-      private transient IterableWithMarker<T> onlyElement;
-      private transient boolean unread = true;
-
-      private OnlyElementIterator(IterableWithMarker<T> onlyElement) {
-         this.onlyElement = checkNotNull(onlyElement, "onlyElement");
-      }
-
-      /**
-       * {@inheritDoc}
-       */
-      @Override
-      protected IterableWithMarker<T> computeNext() {
-         if (unread)
-            try {
-               return onlyElement;
-            } finally {
-               unread = false;
-            }
-         else
-            return endOfData();
-      }
-
-      @Override
-      public Optional<Object> nextMarker() {
-         return onlyElement.nextMarker();
-      }
-
-      /**
-       * {@inheritDoc}
-       */
-      @Override
-      public int hashCode() {
-         return Objects.hashCode(onlyElement);
-      }
-
-      /**
-       * {@inheritDoc}
-       */
-      @Override
-      public boolean equals(Object obj) {
-         if (this == obj)
-            return true;
-         if (obj == null || getClass() != obj.getClass())
-            return false;
-         OnlyElementIterator<?> other = OnlyElementIterator.class.cast(obj);
-         return Objects.equal(this.onlyElement, other.onlyElement) && Objects.equal(this.unread, other.unread);
-      }
-
-      /**
-       * {@inheritDoc}
-       */
-      @Override
-      public String toString() {
-         return Objects.toStringHelper("").add("onlyElement", onlyElement).add("unread", unread).toString();
-      }
-   }
-}

core/src/main/java/org/jclouds/collect/internal/CallerArg0ToPagedIterable.java

@@ -40,7 +40,6 @@ package org.jclouds.collect.internal;
 import org.jclouds.collect.IterableWithMarker;
 import org.jclouds.collect.PagedIterable;
 import org.jclouds.collect.PagedIterables;
-import org.jclouds.collect.PagedIterators;
 import org.jclouds.http.HttpRequest;
 import org.jclouds.rest.InvocationContext;
 import org.jclouds.rest.internal.GeneratedHttpRequest;
@@ -61,7 +60,7 @@ public abstract class CallerArg0ToPagedIterable<T, I extends CallerArg0ToPagedIt
    @Override
    public PagedIterable<T> apply(IterableWithMarker<T> input) {
       if (input.nextMarker() == null)
-         return PagedIterables.create(PagedIterators.of(input));
+         return PagedIterables.of(input);
 
       Optional<String> arg0Option = Optional.absent();
       if (request.getCaller().get().getArgs() != null && request.getCaller().get().getArgs().length > 0) {
@@ -70,7 +69,7 @@ public abstract class CallerArg0ToPagedIterable<T, I extends CallerArg0ToPagedIt
             arg0Option = Optional.of(arg0.toString());
       }
       final String arg0 = arg0Option.orNull();
-      return PagedIterables.create(PagedIterators.advancing(input, markerToNextForCallingArg0(arg0)));
+      return PagedIterables.advance(input, markerToNextForCallingArg0(arg0));
    }
 
    protected abstract Function<Object, IterableWithMarker<T>> markerToNextForCallingArg0(String arg0);

core/src/test/java/org/jclouds/collect/PagedIterablesTest.java

@@ -27,7 +27,7 @@ public class PagedIterablesTest {
 
       EasyMock.replay(markerToNext);
 
-      PagedIterable<String> iterable = PagedIterables.create(PagedIterators.advancing(initial, markerToNext));
+      PagedIterable<String> iterable = PagedIterables.advance(initial, markerToNext);
 
       Assert.assertSame(iterable.get(0), initial);
 
@@ -48,7 +48,7 @@ public class PagedIterablesTest {
 
       EasyMock.replay(markerToNext);
 
-      PagedIterable<String> iterable = PagedIterables.create(PagedIterators.advancing(initial, markerToNext));
+      PagedIterable<String> iterable = PagedIterables.advance(initial, markerToNext);
 
       Assert.assertEquals(iterable.concat().toImmutableSet(),
                ImmutableSet.of("foo", "bar", "boo", "baz", "ham", "cheeze"));

core/src/test/java/org/jclouds/collect/PagedIteratorsTest.java

@@ -1,105 +0,0 @@
-package org.jclouds.collect;
-
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.expect;
-
-import org.easymock.EasyMock;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-
-/**
- * Tests behavior of {@code IterableWithMarkers}.
- * 
- * @author Adrian Cole
- */
-@Test(testName = "PagedIteratorsTest")
-public class PagedIteratorsTest {
-
-   @SuppressWarnings("unchecked")
-   @Test
-   public void testSinglePageResultReturnsSame() {
-
-      IterableWithMarker<String> initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar"));
-      Function<Object, IterableWithMarker<String>> markerToNext = createMock(Function.class);
-
-      EasyMock.replay(markerToNext);
-
-      Assert.assertSame(PagedIterators.advancing(initial, markerToNext).next(), initial);
-
-      EasyMock.verify(markerToNext);
-   }
-
-   @SuppressWarnings("unchecked")
-   @Test
-   public void testMultiPage2Pages() {
-
-      IterableWithMarker<String> initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar"), "MARKER1");
-      Function<Object, IterableWithMarker<String>> markerToNext = createMock(Function.class);
-
-      expect(markerToNext.apply("MARKER1")).andReturn(IterableWithMarkers.from(ImmutableSet.of("boo", "baz"), null));
-
-      EasyMock.replay(markerToNext);
-
-      Assert.assertEquals(ImmutableSet.copyOf(Iterables.concat(ImmutableSet.copyOf(PagedIterators.advancing(initial,
-               markerToNext)))), ImmutableSet.of("foo", "bar", "boo", "baz"));
-
-      EasyMock.verify(markerToNext);
-
-   }
-
-   @SuppressWarnings("unchecked")
-   @Test
-   public void testMultiPage3Pages() {
-
-      IterableWithMarker<String> initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar"), "MARKER1");
-      Function<Object, IterableWithMarker<String>> markerToNext = createMock(Function.class);
-
-      expect(markerToNext.apply("MARKER1")).andReturn(
-               IterableWithMarkers.from(ImmutableSet.of("boo", "baz"), "MARKER2"));
-
-      expect(markerToNext.apply("MARKER2")).andReturn(IterableWithMarkers.from(ImmutableSet.of("ham", "cheeze"), null));
-
-      EasyMock.replay(markerToNext);
-
-      Assert.assertEquals(ImmutableSet.copyOf(Iterables.concat(ImmutableSet.copyOf(PagedIterators.advancing(initial,
-               markerToNext)))), ImmutableSet.of("foo", "bar", "boo", "baz", "ham", "cheeze"));
-
-      EasyMock.verify(markerToNext);
-
-   }
-
-   @SuppressWarnings("unchecked")
-   @Test
-   public void testMultiPage3PagesNextMarkerSetCorrectly() {
-
-      IterableWithMarker<String> initial = IterableWithMarkers.from(ImmutableSet.of("foo", "bar"), "MARKER1");
-      Function<Object, IterableWithMarker<String>> markerToNext = createMock(Function.class);
-      IterableWithMarker<String> second = IterableWithMarkers.from(ImmutableSet.of("boo", "baz"), "MARKER2");
-      expect(markerToNext.apply("MARKER1")).andReturn(second);
-      IterableWithMarker<String> third = IterableWithMarkers.from(ImmutableSet.of("ham", "cheeze"), null);
-      expect(markerToNext.apply("MARKER2")).andReturn(third);
-
-      EasyMock.replay(markerToNext);
-      PagedIterator<String> iterator = PagedIterators.advancing(initial, markerToNext);
-
-      Assert.assertEquals(iterator.hasNext(), true);
-      Assert.assertEquals(iterator.nextMarker(), Optional.of("MARKER1"));
-      Assert.assertEquals(iterator.next(), initial);
-      Assert.assertEquals(iterator.hasNext(), true);
-      Assert.assertEquals(iterator.nextMarker(), Optional.of("MARKER2"));
-      Assert.assertEquals(iterator.next(), second);
-      Assert.assertEquals(iterator.hasNext(), true);
-      Assert.assertEquals(iterator.nextMarker(), Optional.absent());
-      Assert.assertEquals(iterator.next(), third);
-      Assert.assertEquals(iterator.hasNext(), false);
-      Assert.assertEquals(iterator.nextMarker(), Optional.absent());
-      EasyMock.verify(markerToNext);
-
-   }
-
-}