Merge pull request #904 from jclouds/s3-pagediterable

added S3 PagedIterable adapter

apis/s3/src/main/java/org/jclouds/s3/S3.java

@@ -0,0 +1,85 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.s3;
+
+import org.jclouds.collect.IterableWithMarker;
+import org.jclouds.collect.IterableWithMarkers;
+import org.jclouds.collect.PagedIterable;
+import org.jclouds.collect.PagedIterables;
+import org.jclouds.s3.domain.ListBucketResponse;
+import org.jclouds.s3.domain.ObjectMetadata;
+import org.jclouds.s3.options.ListBucketOptions;
+
+import com.google.common.base.Function;
+
+/**
+ * Utilities for using S3.
+ *
+ * @author Adrian Cole
+ */
+public class S3 {
+
+   /**
+    * List all objects in a bucket, in a way that manages pagination, based on
+    * the criteria in the {@link ListBucketOptions} passed in.
+    *
+    * ex.
+    *
+    * <pre>
+    * continueAfterEachPage = listBucket(s3Client, bucket, options).concat();
+    *
+    * </pre>
+    *
+    * @param s3Client
+    *           the {@link S3Client} to use for the requests
+    * @param bucket
+    *           the bucket to list
+    * @param options
+    *           the {@link ListBucketOptions} describing the listBucket requests
+    *
+    * @return iterable of objects fitting the criteria
+    * @see PagedIterable
+    */
+   public static PagedIterable<ObjectMetadata> listBucket(final S3Client s3Client, final String bucket,
+         final ListBucketOptions options) {
+      return PagedIterables.advance(ToIterableWithMarker.INSTANCE.apply(s3Client.listBucket(bucket, options)),
+            new Function<Object, IterableWithMarker<ObjectMetadata>>() {
+
+               @Override
+               public IterableWithMarker<ObjectMetadata> apply(Object input) {
+                  return ToIterableWithMarker.INSTANCE.apply(s3Client.listBucket(bucket,
+                        options.clone().afterMarker(input.toString())));
+               }
+
+               @Override
+               public String toString() {
+                  return "listBucket(" + options + ")";
+               }
+            });
+   }
+
+   private enum ToIterableWithMarker implements Function<ListBucketResponse, IterableWithMarker<ObjectMetadata>> {
+      INSTANCE;
+      @Override
+      public IterableWithMarker<ObjectMetadata> apply(ListBucketResponse in) {
+         return IterableWithMarkers.from(in, in.getNextMarker());
+      }
+   }
+
+}

apis/s3/src/main/java/org/jclouds/s3/options/ListBucketOptions.java

@@ -41,7 +41,7 @@ import org.jclouds.http.options.BaseHttpRequestOptions;
  *      href="http://docs.amazonwebservices.com/AmazonS3/2006-03-01/index.html?RESTBucketGET.html?"
  *      />
  */
-public class ListBucketOptions extends BaseHttpRequestOptions {
+public class ListBucketOptions extends BaseHttpRequestOptions implements Cloneable {
    public static final ListBucketOptions NONE = new ListBucketOptions();
 
    /**
@@ -138,4 +138,11 @@ public class ListBucketOptions extends BaseHttpRequestOptions {
       }
 
    }
+
+   @Override
+   public ListBucketOptions clone() {
+      ListBucketOptions newOptions = new ListBucketOptions();
+      newOptions.queryParameters.putAll(queryParameters);
+      return newOptions;
+   }
 }

apis/s3/src/test/java/org/jclouds/s3/S3Test.java

@@ -0,0 +1,93 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.s3;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.testng.Assert.assertEquals;
+
+import org.easymock.EasyMock;
+import org.jclouds.collect.PagedIterable;
+import org.jclouds.s3.domain.ListBucketResponse;
+import org.jclouds.s3.domain.ObjectMetadata;
+import org.jclouds.s3.domain.internal.ListBucketResponseImpl;
+import org.jclouds.s3.options.ListBucketOptions;
+import org.jclouds.s3.xml.ListBucketHandlerTest;
+import org.testng.annotations.Test;
+
+/**
+ * Tests behavior of {@code S3}.
+ *
+ * @author Adrian Cole
+ */
+public class S3Test {
+
+   /**
+    * Tests {@link S3#listBucket(S3Client, String, ListBucketOptions)} where a
+    * single response returns all results.
+    *
+    * @throws Exception
+    *            if anything goes wrong
+    */
+   @Test
+   public void testSinglePageResult() throws Exception {
+      S3Client api = createMock(S3Client.class);
+      ListBucketOptions options = new ListBucketOptions();
+      ListBucketResponse response = new ListBucketHandlerTest().expected();
+
+      expect(api.listBucket("bucket", options)).andReturn(response).once();
+
+      EasyMock.replay(api);
+
+      PagedIterable<ObjectMetadata> result = S3.listBucket(api, "bucket", options);
+
+      // number of pages
+      assertEquals(result.size(), 1);
+      // number of objects
+      assertEquals(result.get(0).size(), 10);
+   }
+
+   /**
+    * Tests {@link S3#listBucket(S3Client, String, ListBucketOptions)} where
+    * retrieving all results requires multiple requests.
+    *
+    * @throws Exception
+    *            if anything goes wrong
+    */
+   @Test
+   public void testMultiPageResult() throws Exception {
+      String nextMarker = "FOO";
+      S3Client api = createMock(S3Client.class);
+      ListBucketOptions options = new ListBucketOptions();
+      ListBucketResponse response2 = new ListBucketHandlerTest().expected();
+      ListBucketResponse response1 = new ListBucketResponseImpl(response2.getName(), response2, response2.getPrefix(),
+            null, nextMarker, response2.getMaxKeys(), response2.getDelimiter(), false, response2.getCommonPrefixes());
+
+      expect(api.listBucket("bucket", options)).andReturn(response1).once();
+      expect(api.listBucket("bucket", options.afterMarker(nextMarker))).andReturn(response2).once();
+
+      EasyMock.replay(api);
+
+      PagedIterable<ObjectMetadata> result = S3.listBucket(api, "bucket", options);
+
+      // number of objects
+      assertEquals(result.concat().size(), 20);
+   }
+
+}

apis/s3/src/test/java/org/jclouds/s3/xml/ListBucketHandlerTest.java

@@ -26,6 +26,7 @@ import java.util.TreeSet;
 
 import org.jclouds.crypto.CryptoStreams;
 import org.jclouds.date.DateService;
+import org.jclouds.date.internal.SimpleDateFormatDateService;
 import org.jclouds.http.HttpException;
 import org.jclouds.http.HttpRequest;
 import org.jclouds.http.functions.BaseHandlerTest;
@@ -36,7 +37,6 @@ import org.jclouds.s3.domain.ObjectMetadata;
 import org.jclouds.s3.domain.ObjectMetadataBuilder;
 import org.jclouds.s3.domain.internal.ListBucketResponseImpl;
 import org.jclouds.util.Strings2;
-import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
 import com.google.common.collect.ImmutableList;
@@ -51,21 +51,19 @@ import com.google.common.collect.ImmutableList;
 public class ListBucketHandlerTest extends BaseHandlerTest {
    public static final String listBucketWithPrefixAppsSlash = "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>adriancole.org.jclouds.s3.amazons3testdelimiter</Name><Prefix>apps/</Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>apps/0</Key><LastModified>2009-05-07T18:27:08.000Z</LastModified><ETag>&quot;c82e6a0025c31c5de5947fda62ac51ab&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/1</Key><LastModified>2009-05-07T18:27:09.000Z</LastModified><ETag>&quot;944fab2c5a9a6bacf07db5e688310d7a&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/2</Key><LastModified>2009-05-07T18:27:09.000Z</LastModified><ETag>&quot;a227b8888045c8fd159fb495214000f0&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/3</Key><LastModified>2009-05-07T18:27:09.000Z</LastModified><ETag>&quot;c9caa76c3dec53e2a192608ce73eef03&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/4</Key><LastModified>2009-05-07T18:27:09.000Z</LastModified><ETag>&quot;1ce5d0dcc6154a647ea90c7bdf82a224&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/5</Key><LastModified>2009-05-07T18:27:09.000Z</LastModified><ETag>&quot;79433524d87462ee05708a8ef894ed55&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/6</Key><LastModified>2009-05-07T18:27:10.000Z</LastModified><ETag>&quot;dd00a060b28ddca8bc5a21a49e306f67&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/7</Key><LastModified>2009-05-07T18:27:10.000Z</LastModified><ETag>&quot;8cd06eca6e819a927b07a285d750b100&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/8</Key><LastModified>2009-05-07T18:27:10.000Z</LastModified><ETag>&quot;174495094d0633b92cbe46603eee6bad&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/9</Key><LastModified>2009-05-07T18:27:10.000Z</LastModified><ETag>&quot;cd8a19b26fea8a827276df0ad11c580d&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents></ListBucketResult>";
    public static final String listBucketWithSlashDelimiterAndCommonPrefixApps = "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"> <Delimiter>/</Delimiter> <CommonPrefixes><Prefix>apps/</Prefix></CommonPrefixes></ListBucketResult>";
-   private DateService dateService;
+   private DateService dateService = new SimpleDateFormatDateService();
-
-   @BeforeTest
-   @Override
-   protected void setUpInjector() {
-      super.setUpInjector();
-      dateService = injector.getInstance(DateService.class);
-      assert dateService != null;
-   }
 
    public void testApplyInputStream() {
       InputStream is = getClass().getResourceAsStream("/list_bucket.xml");
 
       ListBucketResponse result = createParser().parse(is);
 
+      ListBucketResponse expected = expected();
+
+      assertEquals(result.toString(), expected.toString());
+   }
+
+   public ListBucketResponse expected() {
       CanonicalUser owner = new CanonicalUser("e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0",
                "ferncam");
       String bucket = "adriancole.org.jclouds.aws.s3.amazons3testdelimiter";
@@ -111,8 +109,7 @@ public class ListBucketHandlerTest extends BaseHandlerTest {
                                  "\"cd8a19b26fea8a827276df0ad11c580d\"").owner(owner).contentMD5(
                                  CryptoStreams.hex("cd8a19b26fea8a827276df0ad11c580d")).contentLength(8l).build()),
                "apps/", null, null, 1000, null, false, new TreeSet<String>());
-
+      return expected;
-      assertEquals(result.toString(), expected.toString());
    }
 
    ParseSax<ListBucketResponse> createParser() {