Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added object listing capabilities to JetS3t service wrapper. Also fixed S3 stub's handling of MaxKeys constraint when listing objects 

git-svn-id: http://jclouds.googlecode.com/svn/trunk@873 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
jamurty 2009-05-29 16:54:45 +00:00
parent 345e56695d
commit b34e7180b9
5 changed files with 232 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
aws/s3/core/src/main/java/org/jclouds/aws/s3/domain/S3Bucket.java
View File

@ -26,7 +26,6 @@ package org.jclouds.aws.s3.domain;
import static com.google.common.base.Preconditions.checkNotNull;
import org.joda.time.DateTime;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

18
aws/s3/core/src/test/java/org/jclouds/aws/s3/StubS3Connection.java
View File

@ -39,6 +39,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@ -326,8 +327,8 @@ public class StubS3Connection implements S3Connection {
options.getPrefix() != null ? URLDecoder.decode(options.getPrefix(),
"UTF-8") : null, URLDecoder.decode(options.getDelimiter(),
"UTF-8")));
Set<String> commonPrefixes = iterable != null ? Sets.newTreeSet(iterable)
: new HashSet<String>();
SortedSet<String> commonPrefixes = iterable != null ? Sets.newTreeSet(iterable)
: new TreeSet<String>();
commonPrefixes.remove(CommonPrefixes.NO_PREFIX);
contents = Sets.newTreeSet(Iterables.filter(contents, new DelimiterFilter(options
@ -342,9 +343,18 @@ public class StubS3Connection implements S3Connection {
}
if (options.getMaxKeys() != null) {
contents = firstSliceOfSize(contents, Integer.parseInt(options.getMaxKeys()));
SortedSet<S3Object.Metadata> contentsSlice = firstSliceOfSize(
contents, Integer.parseInt(options.getMaxKeys()));
returnVal.setMaxKeys(Integer.parseInt(options.getMaxKeys()));
returnVal.setTruncated(true);
if (!contentsSlice.contains(contents.last())) {
// Partial listing
returnVal.setTruncated(true);
returnVal.setMarker(contentsSlice.last().getKey());
} else {
returnVal.setTruncated(false);
returnVal.setMarker(null);
}
contents = contentsSlice;
}
returnVal.setContents(contents);

51
aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java
View File

@ -23,6 +23,8 @@
*/
package org.jclouds.aws.s3.jets3t;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.List;
import java.util.Map;
@ -32,6 +34,7 @@ import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3Context;
import org.jclouds.aws.s3.S3ContextFactory;
import org.jclouds.aws.s3.commands.options.GetObjectOptions;
import org.jclouds.aws.s3.commands.options.ListBucketOptions;
import org.jclouds.util.Utils;
import org.jets3t.service.S3ObjectsChunk;
import org.jets3t.service.S3Service;
@ -201,8 +204,8 @@ public class JCloudsS3Service extends S3Service {
Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags,
Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException {
try {
GetObjectOptions options = Util.convertOptions(ifModifiedSince, ifUnmodifiedSince,
ifMatchTags, ifNoneMatchTags);
GetObjectOptions options = Util.convertGetObjectOptions(ifModifiedSince,
ifUnmodifiedSince, ifMatchTags, ifNoneMatchTags);
return Util.convertObject(connection.getObject(bucketName, objectKey, options).get());
} catch (Exception e) {
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
@ -239,15 +242,49 @@ public class JCloudsS3Service extends S3Service {
protected S3ObjectsChunk listObjectsChunkedImpl(String bucketName, String prefix,
String delimiter, long maxListingLength, String priorLastKey, boolean completeListing)
throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
try {
List<S3Object> jsObjects = new ArrayList<S3Object>();
List<String> commonPrefixes = new ArrayList<String>();
org.jclouds.aws.s3.domain.S3Bucket jcBucket = null;
do {
ListBucketOptions options = Util.convertListObjectOptions(prefix, priorLastKey,
delimiter, maxListingLength);
jcBucket = connection.listBucket(bucketName, options)
.get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
jsObjects.addAll(Arrays.asList(Util.convertObjectHeads(jcBucket.getContents())));
commonPrefixes.addAll(jcBucket.getCommonPrefixes());
if (jcBucket.isTruncated()) {
priorLastKey = jsObjects.get(jsObjects.size() - 1).getKey();
} else {
priorLastKey = null;
}
} while (completeListing && jcBucket.isTruncated()); // Build entire listing if requested
return new S3ObjectsChunk(
prefix, // Return the supplied prefix, not the one in the S3 response.
jcBucket.getDelimiter(),
(S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]),
(String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
priorLastKey);
} catch (Exception e) {
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException("error listing objects in bucket " + bucketName, e);
}
}
@Override
protected S3Object[] listObjectsImpl(String bucketName, String prefix, String delimiter,
long maxListingLength) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
long maxListingLength) throws S3ServiceException
{
try {
return listObjectsChunked(bucketName, prefix, delimiter, maxListingLength, null, true)
.getObjects();
} catch (Exception e) {
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException("error listing objects in bucket " + bucketName, e);
}
}
@Override

44
aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/Util.java
View File

@ -28,8 +28,10 @@ import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.Set;
import org.jclouds.aws.s3.commands.options.GetObjectOptions;
import org.jclouds.aws.s3.commands.options.ListBucketOptions;
import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3Object;
import org.jets3t.service.model.S3Owner;
@ -61,6 +63,15 @@ public class Util {
return (S3Bucket[]) jsBuckets.toArray(new S3Bucket[jsBuckets.size()]);
}
public static S3Object[] convertObjectHeads(Set<org.jclouds.aws.s3.domain.S3Object.Metadata>
jcObjectMDs) {
List<S3Object> jsObjects = new ArrayList<S3Object>(jcObjectMDs.size());
for (org.jclouds.aws.s3.domain.S3Object.Metadata jcObjectMD: jcObjectMDs) {
jsObjects.add(convertObjectHead(jcObjectMD));
}
return (S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]);
}
public static S3Object convertObjectHead(org.jclouds.aws.s3.domain.S3Object.Metadata jcObjectMD) {
S3Object jsObject = new S3Object(jcObjectMD.getKey());
if (jcObjectMD.getOwner() != null) {
@ -89,22 +100,45 @@ public class Util {
return (S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]);
}
public static GetObjectOptions convertOptions(Calendar ifModifiedSince,
public static GetObjectOptions convertGetObjectOptions(Calendar ifModifiedSince,
Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags)
throws UnsupportedEncodingException
{
GetObjectOptions options = new GetObjectOptions();
if (ifModifiedSince != null)
if (ifModifiedSince != null) {
options.ifModifiedSince(new DateTime(ifModifiedSince));
if (ifUnmodifiedSince != null)
}
if (ifUnmodifiedSince != null) {
options.ifUnmodifiedSince(new DateTime(ifUnmodifiedSince));
}
// TODO: options.ifMd5Matches should accept multiple match tags
if (ifMatchTags != null && ifMatchTags.length > 0)
if (ifMatchTags != null && ifMatchTags.length > 0) {
options.ifMd5Matches(ifMatchTags[0].getBytes());
}
// TODO: options.ifMd5DoesntMatch should accept multiple match tags
if (ifNoneMatchTags != null && ifNoneMatchTags.length > 0)
if (ifNoneMatchTags != null && ifNoneMatchTags.length > 0) {
options.ifMd5DoesntMatch(ifNoneMatchTags[0].getBytes());
}
return options;
}
public static ListBucketOptions convertListObjectOptions(String prefix, String marker,
String delimiter, Long maxKeys) throws UnsupportedEncodingException
{
ListBucketOptions options = new ListBucketOptions();
if (prefix != null) {
options.withPrefix(prefix);
}
if (marker != null) {
options.afterMarker(marker);
}
if (maxKeys != null) {
options.maxResults(maxKeys);
}
if (delimiter != null) {
options.delimiter(delimiter);
}
return options;
}
}

142
aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java
View File

@ -39,10 +39,12 @@ import java.util.concurrent.TimeoutException;
import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.S3IntegrationTest;
import org.jclouds.aws.s3.config.StubS3ConnectionModule;
import org.jets3t.service.S3ObjectsChunk;
import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3Object;
import org.jets3t.service.multithread.CreateObjectsEvent;
import org.jets3t.service.security.AWSCredentials;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@ -164,6 +166,9 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
assertTrue(createdObject != org.jclouds.aws.s3.domain.S3Object.NOT_FOUND,
"object should exist but doesn't");
assertEquals(createdObject.getMetadata().getKey(), objectKey, "object misnamed");
client.deleteObject(name, objectKey);
client.deleteBucketIfEmpty(name);
}
@Test(enabled = false)
@ -187,7 +192,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
}
@Test
public void testGetObjectDetailsImplStringStringCalendarCalendarStringArrayStringArray()
public void testGetObjectDetailsImpl()
throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException
{
String bucketName = bucketPrefix + ".testGetObjectDetailsImplStringStringCalendarCalendarStringArrayStringArray";
@ -210,7 +215,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
}
@Test
public void testGetObjectImplStringStringCalendarCalendarStringArrayStringArrayLongLong()
public void testGetObjectImpl()
throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException, IOException
{
String bucketName = bucketPrefix + ".testGetObjectImplStringStringCalendarCalendarStringArrayStringArrayLongLong";
@ -274,14 +279,135 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
client.deleteBucketIfEmpty(bucketName);
}
@Test(enabled = false)
public void testListObjectsChunkedImplStringStringStringLongStringBoolean() {
fail("Not yet implemented");
@Test
public void testListObjectsChunkedImpl() throws InterruptedException, ExecutionException,
TimeoutException, IOException, S3ServiceException
{
String bucketName = bucketPrefix + ".testListAllBucketsImplString".toLowerCase();
createBucket(bucketName);
addObjectToBucket(bucketName, "item1/subobject2");
addObjectToBucket(bucketName, "item2");
addObjectToBucket(bucketName, "object1");
addObjectToBucket(bucketName, "object2/subobject1");
S3ObjectsChunk chunk;
// Normal complete listing
chunk = service.listObjectsChunked(bucketName, null, null, 1000, null, true);
assertEquals(chunk.getObjects().length, 4);
assertEquals(chunk.getCommonPrefixes().length, 0);
assertNull(chunk.getDelimiter());
assertNull(chunk.getPrefix());
assertNull(chunk.getPriorLastKey());
// Partial listing
chunk = service.listObjectsChunked(bucketName, null, null, 2, null, false);
assertEquals(chunk.getObjects().length, 2);
assertEquals(chunk.getCommonPrefixes().length, 0);
assertNull(chunk.getDelimiter());
assertNull(chunk.getPrefix());
assertEquals(chunk.getPriorLastKey(), "item2");
// Complete listing, in two chunks
chunk = service.listObjectsChunked(bucketName, null, null, 2, null, true);
assertEquals(chunk.getObjects().length, 4);
assertEquals(chunk.getCommonPrefixes().length, 0);
assertNull(chunk.getDelimiter());
assertNull(chunk.getPrefix());
assertNull(chunk.getPriorLastKey());
// Partial listing with marker
chunk = service.listObjectsChunked(bucketName, null, null, 1000, "item1/subobject2", true);
assertEquals(chunk.getObjects().length, 3);
assertEquals(chunk.getCommonPrefixes().length, 0);
assertNull(chunk.getDelimiter());
assertNull(chunk.getPrefix());
assertNull(chunk.getPriorLastKey());
// Partial listing with marker
chunk = service.listObjectsChunked(bucketName, null, null, 1000, "object1", true);
assertEquals(chunk.getObjects().length, 1);
assertEquals(chunk.getCommonPrefixes().length, 0);
assertNull(chunk.getDelimiter());
assertNull(chunk.getPrefix());
assertNull(chunk.getPriorLastKey());
// Prefix test
chunk = service.listObjectsChunked(bucketName, "item", null, 1000, null, true);
assertEquals(chunk.getObjects().length, 2);
assertEquals(chunk.getCommonPrefixes().length, 0);
assertNull(chunk.getDelimiter());
assertEquals(chunk.getPrefix(), "item");
assertNull(chunk.getPriorLastKey());
// Delimiter test
chunk = service.listObjectsChunked(bucketName, null, "/", 1000, null, true);
assertEquals(chunk.getObjects().length, 2);
assertEquals(chunk.getCommonPrefixes().length, 2);
assertEquals(chunk.getDelimiter(), "/");
assertNull(chunk.getPrefix());
assertNull(chunk.getPriorLastKey());
// Prefix & delimiter test
chunk = service.listObjectsChunked(bucketName, "item", "/", 1000, null, true);
assertEquals(chunk.getObjects().length, 1);
assertEquals(chunk.getCommonPrefixes().length, 1);
assertEquals(chunk.getDelimiter(), "/");
assertEquals(chunk.getPrefix(), "item");
assertNull(chunk.getPriorLastKey());
client.deleteObject(bucketName, "item1/subobject2");
client.deleteObject(bucketName, "item2");
client.deleteObject(bucketName, "object1");
client.deleteObject(bucketName, "object2/subobject1");
client.deleteBucketIfEmpty(bucketName);
}
@Test(enabled = false)
public void testListObjectsImplStringStringStringLong() {
fail("Not yet implemented");
@Test
public void testListObjectsImpl() throws InterruptedException, ExecutionException,
TimeoutException, IOException, S3ServiceException
{
String bucketName = bucketPrefix + ".testListAllBucketsImplString".toLowerCase();
createBucket(bucketName);
addObjectToBucket(bucketName, "item1/subobject2");
addObjectToBucket(bucketName, "item2");
addObjectToBucket(bucketName, "object1");
addObjectToBucket(bucketName, "object2/subobject1");
S3Object[] objects;
// Normal complete listing
objects = service.listObjects(bucketName, null, null, 1000);
assertEquals(objects.length, 4);
// Complete listing, in two chunks
objects = service.listObjects(bucketName, null, null, 2);
assertEquals(objects.length, 4);
assertEquals(objects[0].getKey(), "item1/subobject2");
assertEquals(objects[3].getKey(), "object2/subobject1");
// Prefix test
objects = service.listObjects(bucketName, "item", null, 1000);
assertEquals(objects.length, 2);
// Delimiter test
objects = service.listObjects(bucketName, null, "/", 1000);
assertEquals(objects.length, 2);
assertEquals(objects[0].getKey(), "item2");
assertEquals(objects[1].getKey(), "object1");
// Prefix & delimiter test
objects = service.listObjects(bucketName, "item", "/", 1000);
assertEquals(objects.length, 1);
assertEquals(objects[0].getKey(), "item2");
client.deleteObject(bucketName, "item1/subobject2");
client.deleteObject(bucketName, "item2");
client.deleteObject(bucketName, "object1");
client.deleteObject(bucketName, "object2/subobject1");
client.deleteBucketIfEmpty(bucketName);
}
@Test(enabled = false)