From 081046d4d6a79477d79d1a5e5f6498d7f1e5a33b Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Thu, 18 Oct 2012 17:27:32 -0700 Subject: [PATCH] added S3 PagedIterable adapter --- apis/s3/src/main/java/org/jclouds/s3/S3.java | 85 +++++++++++++++++ .../jclouds/s3/options/ListBucketOptions.java | 9 +- .../src/test/java/org/jclouds/s3/S3Test.java | 93 +++++++++++++++++++ .../jclouds/s3/xml/ListBucketHandlerTest.java | 21 ++--- 4 files changed, 195 insertions(+), 13 deletions(-) create mode 100644 apis/s3/src/main/java/org/jclouds/s3/S3.java create mode 100644 apis/s3/src/test/java/org/jclouds/s3/S3Test.java diff --git a/apis/s3/src/main/java/org/jclouds/s3/S3.java b/apis/s3/src/main/java/org/jclouds/s3/S3.java new file mode 100644 index 0000000000..f3090dd868 --- /dev/null +++ b/apis/s3/src/main/java/org/jclouds/s3/S3.java @@ -0,0 +1,85 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3; + +import org.jclouds.collect.IterableWithMarker; +import org.jclouds.collect.IterableWithMarkers; +import org.jclouds.collect.PagedIterable; +import org.jclouds.collect.PagedIterables; +import org.jclouds.s3.domain.ListBucketResponse; +import org.jclouds.s3.domain.ObjectMetadata; +import org.jclouds.s3.options.ListBucketOptions; + +import com.google.common.base.Function; + +/** + * Utilities for using S3. + * + * @author Adrian Cole + */ +public class S3 { + + /** + * List all objects in a bucket, in a way that manages pagination, based on + * the criteria in the {@link ListBucketOptions} passed in. + * + * ex. + * + * 
+    * continueAfterEachPage = listBucket(s3Client, bucket, options).concat();
+    *
+    *
+ * + * @param s3Client + * the {@link S3Client} to use for the requests + * @param bucket + * the bucket to list + * @param options + * the {@link ListBucketOptions} describing the listBucket requests + * + * @return iterable of objects fitting the criteria + * @see PagedIterable + */ + public static PagedIterable listBucket(final S3Client s3Client, final String bucket, + final ListBucketOptions options) { + return PagedIterables.advance(ToIterableWithMarker.INSTANCE.apply(s3Client.listBucket(bucket, options)), + new Function>() { + + @Override + public IterableWithMarker apply(Object input) { + return ToIterableWithMarker.INSTANCE.apply(s3Client.listBucket(bucket, + options.clone().afterMarker(input.toString()))); + } + + @Override + public String toString() { + return "listBucket(" + options + ")"; + } + }); + } + + private enum ToIterableWithMarker implements Function> { + INSTANCE; + @Override + public IterableWithMarker apply(ListBucketResponse in) { + return IterableWithMarkers.from(in, in.getNextMarker()); + } + } + +} diff --git a/apis/s3/src/main/java/org/jclouds/s3/options/ListBucketOptions.java b/apis/s3/src/main/java/org/jclouds/s3/options/ListBucketOptions.java index 53f48c7b62..345087e9a3 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/options/ListBucketOptions.java +++ b/apis/s3/src/main/java/org/jclouds/s3/options/ListBucketOptions.java @@ -41,7 +41,7 @@ import org.jclouds.http.options.BaseHttpRequestOptions; * href="http://docs.amazonwebservices.com/AmazonS3/2006-03-01/index.html?RESTBucketGET.html?" * /> */ -public class ListBucketOptions extends BaseHttpRequestOptions { +public class ListBucketOptions extends BaseHttpRequestOptions implements Cloneable { public static final ListBucketOptions NONE = new ListBucketOptions(); /** @@ -138,4 +138,11 @@ public class ListBucketOptions extends BaseHttpRequestOptions { } } + + @Override + public ListBucketOptions clone() { + ListBucketOptions newOptions = new ListBucketOptions(); + newOptions.queryParameters.putAll(queryParameters); + return newOptions; + } } diff --git a/apis/s3/src/test/java/org/jclouds/s3/S3Test.java b/apis/s3/src/test/java/org/jclouds/s3/S3Test.java new file mode 100644 index 0000000000..0a1f11d8e6 --- /dev/null +++ b/apis/s3/src/test/java/org/jclouds/s3/S3Test.java @@ -0,0 +1,93 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.s3; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.testng.Assert.assertEquals; + +import org.easymock.EasyMock; +import org.jclouds.collect.PagedIterable; +import org.jclouds.s3.domain.ListBucketResponse; +import org.jclouds.s3.domain.ObjectMetadata; +import org.jclouds.s3.domain.internal.ListBucketResponseImpl; +import org.jclouds.s3.options.ListBucketOptions; +import org.jclouds.s3.xml.ListBucketHandlerTest; +import org.testng.annotations.Test; + +/** + * Tests behavior of {@code S3}. + * + * @author Adrian Cole + */ +public class S3Test { + + /** + * Tests {@link S3#listBucket(S3Client, String, ListBucketOptions)} where a + * single response returns all results. + * + * @throws Exception + * if anything goes wrong + */ + @Test + public void testSinglePageResult() throws Exception { + S3Client api = createMock(S3Client.class); + ListBucketOptions options = new ListBucketOptions(); + ListBucketResponse response = new ListBucketHandlerTest().expected(); + + expect(api.listBucket("bucket", options)).andReturn(response).once(); + + EasyMock.replay(api); + + PagedIterable result = S3.listBucket(api, "bucket", options); + + // number of pages + assertEquals(result.size(), 1); + // number of objects + assertEquals(result.get(0).size(), 10); + } + + /** + * Tests {@link S3#listBucket(S3Client, String, ListBucketOptions)} where + * retrieving all results requires multiple requests. + * + * @throws Exception + * if anything goes wrong + */ + @Test + public void testMultiPageResult() throws Exception { + String nextMarker = "FOO"; + S3Client api = createMock(S3Client.class); + ListBucketOptions options = new ListBucketOptions(); + ListBucketResponse response2 = new ListBucketHandlerTest().expected(); + ListBucketResponse response1 = new ListBucketResponseImpl(response2.getName(), response2, response2.getPrefix(), + null, nextMarker, response2.getMaxKeys(), response2.getDelimiter(), false, response2.getCommonPrefixes()); + + expect(api.listBucket("bucket", options)).andReturn(response1).once(); + expect(api.listBucket("bucket", options.afterMarker(nextMarker))).andReturn(response2).once(); + + EasyMock.replay(api); + + PagedIterable result = S3.listBucket(api, "bucket", options); + + // number of objects + assertEquals(result.concat().size(), 20); + } + +} diff --git a/apis/s3/src/test/java/org/jclouds/s3/xml/ListBucketHandlerTest.java b/apis/s3/src/test/java/org/jclouds/s3/xml/ListBucketHandlerTest.java index 69f6301eb2..da2a282876 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/xml/ListBucketHandlerTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/xml/ListBucketHandlerTest.java @@ -26,6 +26,7 @@ import java.util.TreeSet; import org.jclouds.crypto.CryptoStreams; import org.jclouds.date.DateService; +import org.jclouds.date.internal.SimpleDateFormatDateService; import org.jclouds.http.HttpException; import org.jclouds.http.HttpRequest; import org.jclouds.http.functions.BaseHandlerTest; @@ -36,7 +37,6 @@ import org.jclouds.s3.domain.ObjectMetadata; import org.jclouds.s3.domain.ObjectMetadataBuilder; import org.jclouds.s3.domain.internal.ListBucketResponseImpl; import org.jclouds.util.Strings2; -import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; import com.google.common.collect.ImmutableList; @@ -51,21 +51,19 @@ import com.google.common.collect.ImmutableList; public class ListBucketHandlerTest extends BaseHandlerTest { public static final String listBucketWithPrefixAppsSlash = "adriancole.org.jclouds.s3.amazons3testdelimiterapps/1000falseapps/02009-05-07T18:27:08.000Z"c82e6a0025c31c5de5947fda62ac51ab"8e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0ferncamSTANDARDapps/12009-05-07T18:27:09.000Z"944fab2c5a9a6bacf07db5e688310d7a"8e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0ferncamSTANDARDapps/22009-05-07T18:27:09.000Z"a227b8888045c8fd159fb495214000f0"8e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0ferncamSTANDARDapps/32009-05-07T18:27:09.000Z"c9caa76c3dec53e2a192608ce73eef03"8e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0ferncamSTANDARDapps/42009-05-07T18:27:09.000Z"1ce5d0dcc6154a647ea90c7bdf82a224"8e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0ferncamSTANDARDapps/52009-05-07T18:27:09.000Z"79433524d87462ee05708a8ef894ed55"8e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0ferncamSTANDARDapps/62009-05-07T18:27:10.000Z"dd00a060b28ddca8bc5a21a49e306f67"8e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0ferncamSTANDARDapps/72009-05-07T18:27:10.000Z"8cd06eca6e819a927b07a285d750b100"8e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0ferncamSTANDARDapps/82009-05-07T18:27:10.000Z"174495094d0633b92cbe46603eee6bad"8e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0ferncamSTANDARDapps/92009-05-07T18:27:10.000Z"cd8a19b26fea8a827276df0ad11c580d"8e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0ferncamSTANDARD"; public static final String listBucketWithSlashDelimiterAndCommonPrefixApps = " / apps/"; - private DateService dateService; - - @BeforeTest - @Override - protected void setUpInjector() { - super.setUpInjector(); - dateService = injector.getInstance(DateService.class); - assert dateService != null; - } + private DateService dateService = new SimpleDateFormatDateService(); public void testApplyInputStream() { InputStream is = getClass().getResourceAsStream("/list_bucket.xml"); ListBucketResponse result = createParser().parse(is); + ListBucketResponse expected = expected(); + + assertEquals(result.toString(), expected.toString()); + } + + public ListBucketResponse expected() { CanonicalUser owner = new CanonicalUser("e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0", "ferncam"); String bucket = "adriancole.org.jclouds.aws.s3.amazons3testdelimiter"; @@ -111,8 +109,7 @@ public class ListBucketHandlerTest extends BaseHandlerTest { "\"cd8a19b26fea8a827276df0ad11c580d\"").owner(owner).contentMD5( CryptoStreams.hex("cd8a19b26fea8a827276df0ad11c580d")).contentLength(8l).build()), "apps/", null, null, 1000, null, false, new TreeSet()); - - assertEquals(result.toString(), expected.toString()); + return expected; } ParseSax createParser() {