mirror of https://github.com/apache/jclouds.git
Added object listing capabilities to JetS3t service wrapper. Also fixed S3 stub's handling of MaxKeys constraint when listing objects
git-svn-id: http://jclouds.googlecode.com/svn/trunk@873 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
jamurty
parent
345e56695d
commit
b34e7180b9
|
|
@ -26,7 +26,6 @@ package org.jclouds.aws.s3.domain;
|
|||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
import java.util.Set;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
|
|
@ -326,8 +327,8 @@ public class StubS3Connection implements S3Connection {
|
|||
options.getPrefix() != null ? URLDecoder.decode(options.getPrefix(),
|
||||
"UTF-8") : null, URLDecoder.decode(options.getDelimiter(),
|
||||
"UTF-8")));
|
||||
Set<String> commonPrefixes = iterable != null ? Sets.newTreeSet(iterable)
|
||||
: new HashSet<String>();
|
||||
SortedSet<String> commonPrefixes = iterable != null ? Sets.newTreeSet(iterable)
|
||||
: new TreeSet<String>();
|
||||
commonPrefixes.remove(CommonPrefixes.NO_PREFIX);
|
||||
|
||||
contents = Sets.newTreeSet(Iterables.filter(contents, new DelimiterFilter(options
|
||||
|
|
@ -342,9 +343,18 @@ public class StubS3Connection implements S3Connection {
|
|||
}
|
||||
|
||||
if (options.getMaxKeys() != null) {
|
||||
contents = firstSliceOfSize(contents, Integer.parseInt(options.getMaxKeys()));
|
||||
SortedSet<S3Object.Metadata> contentsSlice = firstSliceOfSize(
|
||||
contents, Integer.parseInt(options.getMaxKeys()));
|
||||
returnVal.setMaxKeys(Integer.parseInt(options.getMaxKeys()));
|
||||
returnVal.setTruncated(true);
|
||||
if (!contentsSlice.contains(contents.last())) {
|
||||
// Partial listing
|
||||
returnVal.setTruncated(true);
|
||||
returnVal.setMarker(contentsSlice.last().getKey());
|
||||
} else {
|
||||
returnVal.setTruncated(false);
|
||||
returnVal.setMarker(null);
|
||||
}
|
||||
contents = contentsSlice;
|
||||
}
|
||||
|
||||
returnVal.setContents(contents);
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@
|
|||
*/
|
||||
package org.jclouds.aws.s3.jets3t;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Calendar;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -32,6 +34,7 @@ import org.jclouds.aws.s3.S3Connection;
|
|||
import org.jclouds.aws.s3.S3Context;
|
||||
import org.jclouds.aws.s3.S3ContextFactory;
|
||||
import org.jclouds.aws.s3.commands.options.GetObjectOptions;
|
||||
import org.jclouds.aws.s3.commands.options.ListBucketOptions;
|
||||
import org.jclouds.util.Utils;
|
||||
import org.jets3t.service.S3ObjectsChunk;
|
||||
import org.jets3t.service.S3Service;
|
||||
|
|
@ -201,8 +204,8 @@ public class JCloudsS3Service extends S3Service {
|
|||
Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags,
|
||||
Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException {
|
||||
try {
|
||||
GetObjectOptions options = Util.convertOptions(ifModifiedSince, ifUnmodifiedSince,
|
||||
ifMatchTags, ifNoneMatchTags);
|
||||
GetObjectOptions options = Util.convertGetObjectOptions(ifModifiedSince,
|
||||
ifUnmodifiedSince, ifMatchTags, ifNoneMatchTags);
|
||||
return Util.convertObject(connection.getObject(bucketName, objectKey, options).get());
|
||||
} catch (Exception e) {
|
||||
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
|
||||
|
|
@ -239,15 +242,49 @@ public class JCloudsS3Service extends S3Service {
|
|||
protected S3ObjectsChunk listObjectsChunkedImpl(String bucketName, String prefix,
|
||||
String delimiter, long maxListingLength, String priorLastKey, boolean completeListing)
|
||||
throws S3ServiceException {
|
||||
// TODO Unimplemented
|
||||
throw new UnsupportedOperationException();
|
||||
try {
|
||||
List<S3Object> jsObjects = new ArrayList<S3Object>();
|
||||
List<String> commonPrefixes = new ArrayList<String>();
|
||||
org.jclouds.aws.s3.domain.S3Bucket jcBucket = null;
|
||||
do {
|
||||
ListBucketOptions options = Util.convertListObjectOptions(prefix, priorLastKey,
|
||||
delimiter, maxListingLength);
|
||||
|
||||
jcBucket = connection.listBucket(bucketName, options)
|
||||
.get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
|
||||
|
||||
jsObjects.addAll(Arrays.asList(Util.convertObjectHeads(jcBucket.getContents())));
|
||||
commonPrefixes.addAll(jcBucket.getCommonPrefixes());
|
||||
if (jcBucket.isTruncated()) {
|
||||
priorLastKey = jsObjects.get(jsObjects.size() - 1).getKey();
|
||||
} else {
|
||||
priorLastKey = null;
|
||||
}
|
||||
} while (completeListing && jcBucket.isTruncated()); // Build entire listing if requested
|
||||
|
||||
return new S3ObjectsChunk(
|
||||
prefix, // Return the supplied prefix, not the one in the S3 response.
|
||||
jcBucket.getDelimiter(),
|
||||
(S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]),
|
||||
(String[]) commonPrefixes.toArray(new String[commonPrefixes.size()]),
|
||||
priorLastKey);
|
||||
} catch (Exception e) {
|
||||
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
|
||||
throw new S3ServiceException("error listing objects in bucket " + bucketName, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected S3Object[] listObjectsImpl(String bucketName, String prefix, String delimiter,
|
||||
long maxListingLength) throws S3ServiceException {
|
||||
// TODO Unimplemented
|
||||
throw new UnsupportedOperationException();
|
||||
long maxListingLength) throws S3ServiceException
|
||||
{
|
||||
try {
|
||||
return listObjectsChunked(bucketName, prefix, delimiter, maxListingLength, null, true)
|
||||
.getObjects();
|
||||
} catch (Exception e) {
|
||||
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
|
||||
throw new S3ServiceException("error listing objects in bucket " + bucketName, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -28,8 +28,10 @@ import java.io.UnsupportedEncodingException;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.jclouds.aws.s3.commands.options.GetObjectOptions;
|
||||
import org.jclouds.aws.s3.commands.options.ListBucketOptions;
|
||||
import org.jets3t.service.model.S3Bucket;
|
||||
import org.jets3t.service.model.S3Object;
|
||||
import org.jets3t.service.model.S3Owner;
|
||||
|
|
@ -61,6 +63,15 @@ public class Util {
|
|||
return (S3Bucket[]) jsBuckets.toArray(new S3Bucket[jsBuckets.size()]);
|
||||
}
|
||||
|
||||
public static S3Object[] convertObjectHeads(Set<org.jclouds.aws.s3.domain.S3Object.Metadata>
|
||||
jcObjectMDs) {
|
||||
List<S3Object> jsObjects = new ArrayList<S3Object>(jcObjectMDs.size());
|
||||
for (org.jclouds.aws.s3.domain.S3Object.Metadata jcObjectMD: jcObjectMDs) {
|
||||
jsObjects.add(convertObjectHead(jcObjectMD));
|
||||
}
|
||||
return (S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]);
|
||||
}
|
||||
|
||||
public static S3Object convertObjectHead(org.jclouds.aws.s3.domain.S3Object.Metadata jcObjectMD) {
|
||||
S3Object jsObject = new S3Object(jcObjectMD.getKey());
|
||||
if (jcObjectMD.getOwner() != null) {
|
||||
|
|
@ -89,22 +100,45 @@ public class Util {
|
|||
return (S3Object[]) jsObjects.toArray(new S3Object[jsObjects.size()]);
|
||||
}
|
||||
|
||||
public static GetObjectOptions convertOptions(Calendar ifModifiedSince,
|
||||
public static GetObjectOptions convertGetObjectOptions(Calendar ifModifiedSince,
|
||||
Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags)
|
||||
throws UnsupportedEncodingException
|
||||
{
|
||||
GetObjectOptions options = new GetObjectOptions();
|
||||
if (ifModifiedSince != null)
|
||||
if (ifModifiedSince != null) {
|
||||
options.ifModifiedSince(new DateTime(ifModifiedSince));
|
||||
if (ifUnmodifiedSince != null)
|
||||
}
|
||||
if (ifUnmodifiedSince != null) {
|
||||
options.ifUnmodifiedSince(new DateTime(ifUnmodifiedSince));
|
||||
}
|
||||
// TODO: options.ifMd5Matches should accept multiple match tags
|
||||
if (ifMatchTags != null && ifMatchTags.length > 0)
|
||||
if (ifMatchTags != null && ifMatchTags.length > 0) {
|
||||
options.ifMd5Matches(ifMatchTags[0].getBytes());
|
||||
}
|
||||
// TODO: options.ifMd5DoesntMatch should accept multiple match tags
|
||||
if (ifNoneMatchTags != null && ifNoneMatchTags.length > 0)
|
||||
if (ifNoneMatchTags != null && ifNoneMatchTags.length > 0) {
|
||||
options.ifMd5DoesntMatch(ifNoneMatchTags[0].getBytes());
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
public static ListBucketOptions convertListObjectOptions(String prefix, String marker,
|
||||
String delimiter, Long maxKeys) throws UnsupportedEncodingException
|
||||
{
|
||||
ListBucketOptions options = new ListBucketOptions();
|
||||
if (prefix != null) {
|
||||
options.withPrefix(prefix);
|
||||
}
|
||||
if (marker != null) {
|
||||
options.afterMarker(marker);
|
||||
}
|
||||
if (maxKeys != null) {
|
||||
options.maxResults(maxKeys);
|
||||
}
|
||||
if (delimiter != null) {
|
||||
options.delimiter(delimiter);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,10 +39,12 @@ import java.util.concurrent.TimeoutException;
|
|||
import org.apache.commons.io.IOUtils;
|
||||
import org.jclouds.aws.s3.S3IntegrationTest;
|
||||
import org.jclouds.aws.s3.config.StubS3ConnectionModule;
|
||||
import org.jets3t.service.S3ObjectsChunk;
|
||||
import org.jets3t.service.S3Service;
|
||||
import org.jets3t.service.S3ServiceException;
|
||||
import org.jets3t.service.model.S3Bucket;
|
||||
import org.jets3t.service.model.S3Object;
|
||||
import org.jets3t.service.multithread.CreateObjectsEvent;
|
||||
import org.jets3t.service.security.AWSCredentials;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.Test;
|
||||
|
|
@ -164,6 +166,9 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
|
|||
assertTrue(createdObject != org.jclouds.aws.s3.domain.S3Object.NOT_FOUND,
|
||||
"object should exist but doesn't");
|
||||
assertEquals(createdObject.getMetadata().getKey(), objectKey, "object misnamed");
|
||||
|
||||
client.deleteObject(name, objectKey);
|
||||
client.deleteBucketIfEmpty(name);
|
||||
}
|
||||
|
||||
@Test(enabled = false)
|
||||
|
|
@ -187,7 +192,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testGetObjectDetailsImplStringStringCalendarCalendarStringArrayStringArray()
|
||||
public void testGetObjectDetailsImpl()
|
||||
throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException
|
||||
{
|
||||
String bucketName = bucketPrefix + ".testGetObjectDetailsImplStringStringCalendarCalendarStringArrayStringArray";
|
||||
|
|
@ -210,7 +215,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testGetObjectImplStringStringCalendarCalendarStringArrayStringArrayLongLong()
|
||||
public void testGetObjectImpl()
|
||||
throws InterruptedException, ExecutionException, TimeoutException, S3ServiceException, IOException
|
||||
{
|
||||
String bucketName = bucketPrefix + ".testGetObjectImplStringStringCalendarCalendarStringArrayStringArrayLongLong";
|
||||
|
|
@ -274,16 +279,137 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
|
|||
client.deleteBucketIfEmpty(bucketName);
|
||||
}
|
||||
|
||||
@Test(enabled = false)
|
||||
public void testListObjectsChunkedImplStringStringStringLongStringBoolean() {
|
||||
fail("Not yet implemented");
|
||||
@Test
|
||||
public void testListObjectsChunkedImpl() throws InterruptedException, ExecutionException,
|
||||
TimeoutException, IOException, S3ServiceException
|
||||
{
|
||||
String bucketName = bucketPrefix + ".testListAllBucketsImplString".toLowerCase();
|
||||
createBucket(bucketName);
|
||||
|
||||
addObjectToBucket(bucketName, "item1/subobject2");
|
||||
addObjectToBucket(bucketName, "item2");
|
||||
addObjectToBucket(bucketName, "object1");
|
||||
addObjectToBucket(bucketName, "object2/subobject1");
|
||||
|
||||
S3ObjectsChunk chunk;
|
||||
|
||||
// Normal complete listing
|
||||
chunk = service.listObjectsChunked(bucketName, null, null, 1000, null, true);
|
||||
assertEquals(chunk.getObjects().length, 4);
|
||||
assertEquals(chunk.getCommonPrefixes().length, 0);
|
||||
assertNull(chunk.getDelimiter());
|
||||
assertNull(chunk.getPrefix());
|
||||
assertNull(chunk.getPriorLastKey());
|
||||
|
||||
// Partial listing
|
||||
chunk = service.listObjectsChunked(bucketName, null, null, 2, null, false);
|
||||
assertEquals(chunk.getObjects().length, 2);
|
||||
assertEquals(chunk.getCommonPrefixes().length, 0);
|
||||
assertNull(chunk.getDelimiter());
|
||||
assertNull(chunk.getPrefix());
|
||||
assertEquals(chunk.getPriorLastKey(), "item2");
|
||||
|
||||
// Complete listing, in two chunks
|
||||
chunk = service.listObjectsChunked(bucketName, null, null, 2, null, true);
|
||||
assertEquals(chunk.getObjects().length, 4);
|
||||
assertEquals(chunk.getCommonPrefixes().length, 0);
|
||||
assertNull(chunk.getDelimiter());
|
||||
assertNull(chunk.getPrefix());
|
||||
assertNull(chunk.getPriorLastKey());
|
||||
|
||||
// Partial listing with marker
|
||||
chunk = service.listObjectsChunked(bucketName, null, null, 1000, "item1/subobject2", true);
|
||||
assertEquals(chunk.getObjects().length, 3);
|
||||
assertEquals(chunk.getCommonPrefixes().length, 0);
|
||||
assertNull(chunk.getDelimiter());
|
||||
assertNull(chunk.getPrefix());
|
||||
assertNull(chunk.getPriorLastKey());
|
||||
|
||||
// Partial listing with marker
|
||||
chunk = service.listObjectsChunked(bucketName, null, null, 1000, "object1", true);
|
||||
assertEquals(chunk.getObjects().length, 1);
|
||||
assertEquals(chunk.getCommonPrefixes().length, 0);
|
||||
assertNull(chunk.getDelimiter());
|
||||
assertNull(chunk.getPrefix());
|
||||
assertNull(chunk.getPriorLastKey());
|
||||
|
||||
// Prefix test
|
||||
chunk = service.listObjectsChunked(bucketName, "item", null, 1000, null, true);
|
||||
assertEquals(chunk.getObjects().length, 2);
|
||||
assertEquals(chunk.getCommonPrefixes().length, 0);
|
||||
assertNull(chunk.getDelimiter());
|
||||
assertEquals(chunk.getPrefix(), "item");
|
||||
assertNull(chunk.getPriorLastKey());
|
||||
|
||||
// Delimiter test
|
||||
chunk = service.listObjectsChunked(bucketName, null, "/", 1000, null, true);
|
||||
assertEquals(chunk.getObjects().length, 2);
|
||||
assertEquals(chunk.getCommonPrefixes().length, 2);
|
||||
assertEquals(chunk.getDelimiter(), "/");
|
||||
assertNull(chunk.getPrefix());
|
||||
assertNull(chunk.getPriorLastKey());
|
||||
|
||||
// Prefix & delimiter test
|
||||
chunk = service.listObjectsChunked(bucketName, "item", "/", 1000, null, true);
|
||||
assertEquals(chunk.getObjects().length, 1);
|
||||
assertEquals(chunk.getCommonPrefixes().length, 1);
|
||||
assertEquals(chunk.getDelimiter(), "/");
|
||||
assertEquals(chunk.getPrefix(), "item");
|
||||
assertNull(chunk.getPriorLastKey());
|
||||
|
||||
client.deleteObject(bucketName, "item1/subobject2");
|
||||
client.deleteObject(bucketName, "item2");
|
||||
client.deleteObject(bucketName, "object1");
|
||||
client.deleteObject(bucketName, "object2/subobject1");
|
||||
client.deleteBucketIfEmpty(bucketName);
|
||||
}
|
||||
|
||||
@Test(enabled = false)
|
||||
public void testListObjectsImplStringStringStringLong() {
|
||||
fail("Not yet implemented");
|
||||
}
|
||||
@Test
|
||||
public void testListObjectsImpl() throws InterruptedException, ExecutionException,
|
||||
TimeoutException, IOException, S3ServiceException
|
||||
{
|
||||
String bucketName = bucketPrefix + ".testListAllBucketsImplString".toLowerCase();
|
||||
createBucket(bucketName);
|
||||
|
||||
addObjectToBucket(bucketName, "item1/subobject2");
|
||||
addObjectToBucket(bucketName, "item2");
|
||||
addObjectToBucket(bucketName, "object1");
|
||||
addObjectToBucket(bucketName, "object2/subobject1");
|
||||
|
||||
S3Object[] objects;
|
||||
|
||||
// Normal complete listing
|
||||
objects = service.listObjects(bucketName, null, null, 1000);
|
||||
assertEquals(objects.length, 4);
|
||||
|
||||
// Complete listing, in two chunks
|
||||
objects = service.listObjects(bucketName, null, null, 2);
|
||||
assertEquals(objects.length, 4);
|
||||
assertEquals(objects[0].getKey(), "item1/subobject2");
|
||||
assertEquals(objects[3].getKey(), "object2/subobject1");
|
||||
|
||||
// Prefix test
|
||||
objects = service.listObjects(bucketName, "item", null, 1000);
|
||||
assertEquals(objects.length, 2);
|
||||
|
||||
// Delimiter test
|
||||
objects = service.listObjects(bucketName, null, "/", 1000);
|
||||
assertEquals(objects.length, 2);
|
||||
assertEquals(objects[0].getKey(), "item2");
|
||||
assertEquals(objects[1].getKey(), "object1");
|
||||
|
||||
// Prefix & delimiter test
|
||||
objects = service.listObjects(bucketName, "item", "/", 1000);
|
||||
assertEquals(objects.length, 1);
|
||||
assertEquals(objects[0].getKey(), "item2");
|
||||
|
||||
client.deleteObject(bucketName, "item1/subobject2");
|
||||
client.deleteObject(bucketName, "item2");
|
||||
client.deleteObject(bucketName, "object1");
|
||||
client.deleteObject(bucketName, "object2/subobject1");
|
||||
client.deleteBucketIfEmpty(bucketName);
|
||||
}
|
||||
|
||||
@Test(enabled = false)
|
||||
public void testPutBucketAclImplStringAccessControlList() {
|
||||
fail("Not yet implemented");
|
||||
|
|
|
|||
Loading…
Reference in New Issue