Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #904 from jclouds/s3-pagediterable 

added S3 PagedIterable adapter
This commit is contained in:
Adrian Cole 2012-10-18 22:24:50 -07:00
parent 53fcdbb790 081046d4d6
commit 3cd57073e0
4 changed files with 195 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
apis/s3/src/main/java/org/jclouds/s3/S3.java Normal file
View File

@ -0,0 +1,85 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3;
import org.jclouds.collect.IterableWithMarker;
import org.jclouds.collect.IterableWithMarkers;
import org.jclouds.collect.PagedIterable;
import org.jclouds.collect.PagedIterables;
import org.jclouds.s3.domain.ListBucketResponse;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.options.ListBucketOptions;
import com.google.common.base.Function;
/**
* Utilities for using S3.
*
* @author Adrian Cole
*/
public class S3 {
/**
* List all objects in a bucket, in a way that manages pagination, based on
* the criteria in the {@link ListBucketOptions} passed in.
*
* ex.
*
* <pre>
* continueAfterEachPage = listBucket(s3Client, bucket, options).concat();
*
* </pre>
*
* @param s3Client
* the {@link S3Client} to use for the requests
* @param bucket
* the bucket to list
* @param options
* the {@link ListBucketOptions} describing the listBucket requests
*
* @return iterable of objects fitting the criteria
* @see PagedIterable
*/
public static PagedIterable<ObjectMetadata> listBucket(final S3Client s3Client, final String bucket,
final ListBucketOptions options) {
return PagedIterables.advance(ToIterableWithMarker.INSTANCE.apply(s3Client.listBucket(bucket, options)),
new Function<Object, IterableWithMarker<ObjectMetadata>>() {
@Override
public IterableWithMarker<ObjectMetadata> apply(Object input) {
return ToIterableWithMarker.INSTANCE.apply(s3Client.listBucket(bucket,
options.clone().afterMarker(input.toString())));
}
@Override
public String toString() {
return "listBucket(" + options + ")";
}
});
}
private enum ToIterableWithMarker implements Function<ListBucketResponse, IterableWithMarker<ObjectMetadata>> {
INSTANCE;
@Override
public IterableWithMarker<ObjectMetadata> apply(ListBucketResponse in) {
return IterableWithMarkers.from(in, in.getNextMarker());
}
}
}

9
apis/s3/src/main/java/org/jclouds/s3/options/ListBucketOptions.java
View File

@ -41,7 +41,7 @@ import org.jclouds.http.options.BaseHttpRequestOptions;
* href="http://docs.amazonwebservices.com/AmazonS3/2006-03-01/index.html?RESTBucketGET.html?"
* />
*/
public class ListBucketOptions extends BaseHttpRequestOptions {
public class ListBucketOptions extends BaseHttpRequestOptions implements Cloneable {
public static final ListBucketOptions NONE = new ListBucketOptions();
/**
@ -138,4 +138,11 @@ public class ListBucketOptions extends BaseHttpRequestOptions {
}
}
@Override
public ListBucketOptions clone() {
ListBucketOptions newOptions = new ListBucketOptions();
newOptions.queryParameters.putAll(queryParameters);
return newOptions;
}
}

93
apis/s3/src/test/java/org/jclouds/s3/S3Test.java Normal file
View File

@ -0,0 +1,93 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.s3;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.testng.Assert.assertEquals;
import org.easymock.EasyMock;
import org.jclouds.collect.PagedIterable;
import org.jclouds.s3.domain.ListBucketResponse;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.internal.ListBucketResponseImpl;
import org.jclouds.s3.options.ListBucketOptions;
import org.jclouds.s3.xml.ListBucketHandlerTest;
import org.testng.annotations.Test;
/**
* Tests behavior of {@code S3}.
*
* @author Adrian Cole
*/
public class S3Test {
/**
* Tests {@link S3#listBucket(S3Client, String, ListBucketOptions)} where a
* single response returns all results.
*
* @throws Exception
* if anything goes wrong
*/
@Test
public void testSinglePageResult() throws Exception {
S3Client api = createMock(S3Client.class);
ListBucketOptions options = new ListBucketOptions();
ListBucketResponse response = new ListBucketHandlerTest().expected();
expect(api.listBucket("bucket", options)).andReturn(response).once();
EasyMock.replay(api);
PagedIterable<ObjectMetadata> result = S3.listBucket(api, "bucket", options);
// number of pages
assertEquals(result.size(), 1);
// number of objects
assertEquals(result.get(0).size(), 10);
}
/**
* Tests {@link S3#listBucket(S3Client, String, ListBucketOptions)} where
* retrieving all results requires multiple requests.
*
* @throws Exception
* if anything goes wrong
*/
@Test
public void testMultiPageResult() throws Exception {
String nextMarker = "FOO";
S3Client api = createMock(S3Client.class);
ListBucketOptions options = new ListBucketOptions();
ListBucketResponse response2 = new ListBucketHandlerTest().expected();
ListBucketResponse response1 = new ListBucketResponseImpl(response2.getName(), response2, response2.getPrefix(),
null, nextMarker, response2.getMaxKeys(), response2.getDelimiter(), false, response2.getCommonPrefixes());
expect(api.listBucket("bucket", options)).andReturn(response1).once();
expect(api.listBucket("bucket", options.afterMarker(nextMarker))).andReturn(response2).once();
EasyMock.replay(api);
PagedIterable<ObjectMetadata> result = S3.listBucket(api, "bucket", options);
// number of objects
assertEquals(result.concat().size(), 20);
}
}

21
apis/s3/src/test/java/org/jclouds/s3/xml/ListBucketHandlerTest.java
View File

@ -26,6 +26,7 @@ import java.util.TreeSet;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.DateService;
import org.jclouds.date.internal.SimpleDateFormatDateService;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.functions.BaseHandlerTest;
@ -36,7 +37,6 @@ import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.ObjectMetadataBuilder;
import org.jclouds.s3.domain.internal.ListBucketResponseImpl;
import org.jclouds.util.Strings2;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList;
@ -51,21 +51,19 @@ import com.google.common.collect.ImmutableList;
public class ListBucketHandlerTest extends BaseHandlerTest {
public static final String listBucketWithPrefixAppsSlash = "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>adriancole.org.jclouds.s3.amazons3testdelimiter</Name><Prefix>apps/</Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>apps/0</Key><LastModified>2009-05-07T18:27:08.000Z</LastModified><ETag>&quot;c82e6a0025c31c5de5947fda62ac51ab&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/1</Key><LastModified>2009-05-07T18:27:09.000Z</LastModified><ETag>&quot;944fab2c5a9a6bacf07db5e688310d7a&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/2</Key><LastModified>2009-05-07T18:27:09.000Z</LastModified><ETag>&quot;a227b8888045c8fd159fb495214000f0&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/3</Key><LastModified>2009-05-07T18:27:09.000Z</LastModified><ETag>&quot;c9caa76c3dec53e2a192608ce73eef03&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/4</Key><LastModified>2009-05-07T18:27:09.000Z</LastModified><ETag>&quot;1ce5d0dcc6154a647ea90c7bdf82a224&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/5</Key><LastModified>2009-05-07T18:27:09.000Z</LastModified><ETag>&quot;79433524d87462ee05708a8ef894ed55&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/6</Key><LastModified>2009-05-07T18:27:10.000Z</LastModified><ETag>&quot;dd00a060b28ddca8bc5a21a49e306f67&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/7</Key><LastModified>2009-05-07T18:27:10.000Z</LastModified><ETag>&quot;8cd06eca6e819a927b07a285d750b100&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/8</Key><LastModified>2009-05-07T18:27:10.000Z</LastModified><ETag>&quot;174495094d0633b92cbe46603eee6bad&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>apps/9</Key><LastModified>2009-05-07T18:27:10.000Z</LastModified><ETag>&quot;cd8a19b26fea8a827276df0ad11c580d&quot;</ETag><Size>8</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents></ListBucketResult>";
public static final String listBucketWithSlashDelimiterAndCommonPrefixApps = "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"> <Delimiter>/</Delimiter> <CommonPrefixes><Prefix>apps/</Prefix></CommonPrefixes></ListBucketResult>";
private DateService dateService;
@BeforeTest
@Override
protected void setUpInjector() {
super.setUpInjector();
dateService = injector.getInstance(DateService.class);
assert dateService != null;
}
private DateService dateService = new SimpleDateFormatDateService();
public void testApplyInputStream() {
InputStream is = getClass().getResourceAsStream("/list_bucket.xml");
ListBucketResponse result = createParser().parse(is);
ListBucketResponse expected = expected();
assertEquals(result.toString(), expected.toString());
}
public ListBucketResponse expected() {
CanonicalUser owner = new CanonicalUser("e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0",
"ferncam");
String bucket = "adriancole.org.jclouds.aws.s3.amazons3testdelimiter";
@ -111,8 +109,7 @@ public class ListBucketHandlerTest extends BaseHandlerTest {
"\"cd8a19b26fea8a827276df0ad11c580d\"").owner(owner).contentMD5(
CryptoStreams.hex("cd8a19b26fea8a827276df0ad11c580d")).contentLength(8l).build()),
"apps/", null, null, 1000, null, false, new TreeSet<String>());
assertEquals(result.toString(), expected.toString());
return expected;
}
ParseSax<ListBucketResponse> createParser() {