JCLOUDS-820: Support multi-delete for generic S3

Tested against AWS and DreamObjects.  This commit only moves and
renames code.
This commit is contained in:
Andrew Gaul 2015-02-13 05:43:16 -08:00
parent b46ec7eb29
commit 4bb319a0cc
13 changed files with 138 additions and 152 deletions

View File

@ -62,6 +62,7 @@ import org.jclouds.rest.annotations.XMLResponseParser;
import org.jclouds.s3.binders.BindACLToXMLPayload;
import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured;
import org.jclouds.s3.binders.BindBucketLoggingToXmlPayload;
import org.jclouds.s3.binders.BindIterableAsPayloadToDeleteRequest;
import org.jclouds.s3.binders.BindNoBucketLoggingToXmlPayload;
import org.jclouds.s3.binders.BindObjectMetadataToRequest;
import org.jclouds.s3.binders.BindPartIdsAndETagsToRequest;
@ -70,6 +71,7 @@ import org.jclouds.s3.binders.BindS3ObjectMetadataToRequest;
import org.jclouds.s3.domain.AccessControlList;
import org.jclouds.s3.domain.BucketLogging;
import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.domain.DeleteResult;
import org.jclouds.s3.domain.ListBucketResponse;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.Payer;
@ -93,6 +95,7 @@ import org.jclouds.s3.predicates.validators.BucketNameValidator;
import org.jclouds.s3.xml.AccessControlListHandler;
import org.jclouds.s3.xml.BucketLoggingHandler;
import org.jclouds.s3.xml.CopyObjectHandler;
import org.jclouds.s3.xml.DeleteResultHandler;
import org.jclouds.s3.xml.ListAllMyBucketsHandler;
import org.jclouds.s3.xml.ListBucketHandler;
import org.jclouds.s3.xml.LocationConstraintHandler;
@ -203,6 +206,34 @@ public interface S3Client extends Closeable {
BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@PathParam("key") String key);
/**
* The Multi-Object Delete operation enables you to delete multiple objects from a bucket using a
* single HTTP request. If you know the object keys that you want to delete, then this operation
* provides a suitable alternative to sending individual delete requests (see DELETE Object),
* reducing per-request overhead.
*
* The Multi-Object Delete request contains a set of up to 1000 keys that you want to delete.
*
* If a key does not exist is considered to be deleted.
*
* The Multi-Object Delete operation supports two modes for the response; verbose and quiet.
* By default, the operation uses verbose mode in which the response includes the result of
* deletion of each key in your request.
*
* @param bucketName
* namespace of the objects you are deleting
* @param keys
* set of unique keys identifying objects
*/
@Named("DeleteObject")
@POST
@Path("/")
@QueryParams(keys = "delete")
@XMLResponseParser(DeleteResultHandler.class)
DeleteResult deleteObjects(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(
BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@BinderParam(BindIterableAsPayloadToDeleteRequest.class) Iterable<String> keys);
/**
* Store data by creating or overwriting an object.
* <p/>

View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.aws.s3.binders;
package org.jclouds.s3.binders;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.base.Preconditions.checkArgument;

View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.aws.s3.domain;
package org.jclouds.s3.domain;
import static com.google.common.base.Preconditions.checkNotNull;

View File

@ -14,11 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.aws.s3.xml;
package org.jclouds.s3.xml;
import static org.jclouds.util.SaxUtils.equalsOrSuffix;
import org.jclouds.aws.s3.domain.DeleteResult;
import org.jclouds.s3.domain.DeleteResult;
import org.jclouds.http.functions.ParseSax;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;

View File

@ -14,13 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.aws.s3.xml;
package org.jclouds.s3.xml;
import static org.jclouds.util.SaxUtils.equalsOrSuffix;
import java.util.Map;
import org.jclouds.aws.s3.domain.DeleteResult;
import org.jclouds.s3.domain.DeleteResult;
import org.jclouds.http.functions.ParseSax;
import org.xml.sax.SAXException;

View File

@ -16,14 +16,23 @@
*/
package org.jclouds.s3;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.hash.Hashing.md5;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import java.net.URI;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpResponse;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.s3.domain.DeleteResult;
import org.jclouds.s3.internal.BaseS3ClientExpectTest;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
@Test(groups = "unit", testName = "S3ClientExpectTest")
public class S3ClientExpectTest extends BaseS3ClientExpectTest {
@ -46,4 +55,60 @@ public class S3ClientExpectTest extends BaseS3ClientExpectTest {
}
@Test
public void testDeleteMultipleObjects() {
final String request = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
"<Delete>" +
"<Object><Key>key1</Key></Object>" +
"<Object><Key>key2</Key></Object>" +
"</Delete>";
final Payload requestPayload = Payloads.newStringPayload(request);
requestPayload.getContentMetadata().setContentType("text/xml");
requestPayload.getContentMetadata().setContentMD5(md5().hashString(request, UTF_8));
final String response = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
"<DeleteResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n" +
" <Deleted>\n" +
" <Key>key1</Key>\n" +
" </Deleted>\n" +
" <Deleted>\n" +
" <Key>key1.1</Key>\n" +
" </Deleted>\n" +
" <Error>\n" +
" <Key>key2</Key>\n" +
" <Code>AccessDenied</Code>\n" +
" <Message>Access Denied</Message>\n" +
" </Error>\n" +
"</DeleteResult>";
final Payload responsePayload = Payloads.newStringPayload(response);
responsePayload.getContentMetadata().setContentType("text/xml");
S3Client client = requestSendsResponse(
HttpRequest.builder()
.method("POST")
.endpoint("http://localhost/test?delete")
.addHeader("Date", CONSTANT_DATE)
.addHeader("Authorization", "AWS identity:XptAJrBvfz68TEfPkhXj4R58uvE=")
.payload(requestPayload)
.build(),
HttpResponse.builder()
.statusCode(200)
.addHeader("x-amz-request-id", "7A84C3CD4437A4C0")
.addHeader("Date", CONSTANT_DATE)
.addHeader("ETag", "437b930db84b8079c2dd804a71936b5f")
.addHeader("Server", "AmazonS3")
.payload(responsePayload)
.build()
);
DeleteResult result = client.deleteObjects("test", ImmutableSet.of("key1", "key2"));
assertNotNull(result, "result is null");
assertEquals(result.getDeleted(), ImmutableSet.of("key1", "key1.1"));
assertEquals(result.getErrors().size(), 1);
assertEquals(result.getErrors().get("key2"), new DeleteResult.Error("AccessDenied", "Access Denied"));
}
}

View File

@ -36,10 +36,13 @@ import java.net.URI;
import java.net.URL;
import java.util.Date;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import org.jclouds.blobstore.KeyNotFoundException;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
import org.jclouds.http.HttpResponseException;
import org.jclouds.io.ByteStreams2;
@ -50,6 +53,7 @@ import org.jclouds.s3.domain.AccessControlList.EmailAddressGrantee;
import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI;
import org.jclouds.s3.domain.AccessControlList.Permission;
import org.jclouds.s3.domain.CannedAccessPolicy;
import org.jclouds.s3.domain.DeleteResult;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.ObjectMetadataBuilder;
import org.jclouds.s3.domain.S3Object;
@ -60,6 +64,7 @@ import org.testng.annotations.Test;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.hash.HashCode;
import com.google.common.io.ByteSource;
@ -538,6 +543,34 @@ public class S3ClientLiveTest extends BaseBlobStoreIntegrationTest {
}
}
public void testDeleteMultipleObjects() throws InterruptedException {
String container = getContainerName();
try {
ImmutableSet.Builder<String> builder = ImmutableSet.builder();
for (int i = 0; i < 5; i++) {
String key = UUID.randomUUID().toString();
Blob blob = view.getBlobStore().blobBuilder(key).payload("").build();
view.getBlobStore().putBlob(container, blob);
builder.add(key);
}
Set<String> keys = builder.build();
DeleteResult result = getApi().deleteObjects(container, keys);
assertTrue(result.getDeleted().containsAll(keys));
assertEquals(result.getErrors().size(), 0);
for (String key : keys) {
assertConsistencyAwareBlobDoesntExist(container, key);
}
} finally {
returnContainer(container);
}
}
private void checkGrants(AccessControlList acl) {
String ownerId = acl.getOwner().getId();

View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.aws.s3.binders;
package org.jclouds.s3.binders;
import static org.testng.Assert.assertEquals;

View File

@ -14,14 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.aws.s3.xml;
package org.jclouds.s3.xml;
import static org.testng.Assert.assertEquals;
import java.io.InputStream;
import org.jclouds.aws.s3.domain.DeleteResult;
import org.jclouds.http.functions.BaseHandlerTest;
import org.jclouds.s3.domain.DeleteResult;
import org.testng.annotations.Test;
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire

View File

@ -18,26 +18,10 @@ package org.jclouds.aws.s3;
import static org.jclouds.blobstore.attr.BlobScopes.CONTAINER;
import javax.inject.Named;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import org.jclouds.aws.s3.binders.BindIterableAsPayloadToDeleteRequest;
import org.jclouds.aws.s3.domain.DeleteResult;
import org.jclouds.aws.s3.xml.DeleteResultHandler;
import org.jclouds.blobstore.attr.BlobScope;
import org.jclouds.rest.annotations.BinderParam;
import org.jclouds.rest.annotations.EndpointParam;
import org.jclouds.rest.annotations.ParamValidators;
import org.jclouds.rest.annotations.QueryParams;
import org.jclouds.rest.annotations.RequestFilters;
import org.jclouds.rest.annotations.XMLResponseParser;
import org.jclouds.s3.Bucket;
import org.jclouds.s3.S3Client;
import org.jclouds.s3.binders.BindAsHostPrefixIfConfigured;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
import org.jclouds.s3.functions.AssignCorrectHostnameForBucket;
import org.jclouds.s3.predicates.validators.BucketNameValidator;
/**
* Provides access to amazon-specific S3 features
@ -45,32 +29,4 @@ import org.jclouds.s3.predicates.validators.BucketNameValidator;
@RequestFilters(RequestAuthorizeSignature.class)
@BlobScope(CONTAINER)
public interface AWSS3Client extends S3Client {
/**
* The Multi-Object Delete operation enables you to delete multiple objects from a bucket using a
* single HTTP request. If you know the object keys that you want to delete, then this operation
* provides a suitable alternative to sending individual delete requests (see DELETE Object),
* reducing per-request overhead.
*
* The Multi-Object Delete request contains a set of up to 1000 keys that you want to delete.
*
* If a key does not exist is considered to be deleted.
*
* The Multi-Object Delete operation supports two modes for the response; verbose and quiet.
* By default, the operation uses verbose mode in which the response includes the result of
* deletion of each key in your request.
*
* @param bucketName
* namespace of the objects you are deleting
* @param keys
* set of unique keys identifying objects
*/
@Named("DeleteObject")
@POST
@Path("/")
@QueryParams(keys = "delete")
@XMLResponseParser(DeleteResultHandler.class)
DeleteResult deleteObjects(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(
BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@BinderParam(BindIterableAsPayloadToDeleteRequest.class) Iterable<String> keys);
}

View File

@ -16,27 +16,19 @@
*/
package org.jclouds.aws.s3;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.hash.Hashing.md5;
import static org.jclouds.aws.s3.blobstore.options.AWSS3PutObjectOptions.Builder.storageClass;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import org.jclouds.aws.s3.domain.DeleteResult;
import org.jclouds.aws.s3.internal.BaseAWSS3ClientExpectTest;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobBuilder;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpResponse;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.s3.blobstore.functions.BlobToObject;
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
import org.testng.annotations.Test;
import com.google.common.base.Functions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Injector;
@Test
@ -86,62 +78,4 @@ public class AWSS3ClientExpectTest extends BaseAWSS3ClientExpectTest {
client.putObject("test", blobToObject.apply(blob),
storageClass(StorageClass.REDUCED_REDUNDANCY));
}
@Test
public void testDeleteMultipleObjects() {
final String request = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
"<Delete>" +
"<Object><Key>key1</Key></Object>" +
"<Object><Key>key2</Key></Object>" +
"</Delete>";
final Payload requestPayload = Payloads.newStringPayload(request);
requestPayload.getContentMetadata().setContentType("text/xml");
requestPayload.getContentMetadata().setContentMD5(md5().hashString(request, UTF_8));
final String response = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
"<DeleteResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n" +
" <Deleted>\n" +
" <Key>key1</Key>\n" +
" </Deleted>\n" +
" <Deleted>\n" +
" <Key>key1.1</Key>\n" +
" </Deleted>\n" +
" <Error>\n" +
" <Key>key2</Key>\n" +
" <Code>AccessDenied</Code>\n" +
" <Message>Access Denied</Message>\n" +
" </Error>\n" +
"</DeleteResult>";
final Payload responsePayload = Payloads.newStringPayload(response);
responsePayload.getContentMetadata().setContentType("text/xml");
AWSS3Client client = requestsSendResponses(bucketLocationRequest, bucketLocationResponse,
HttpRequest.builder()
.method("POST")
.endpoint("https://test.s3-eu-west-1.amazonaws.com/?delete")
.addHeader("Host", "test.s3-eu-west-1.amazonaws.com")
.addHeader("Date", CONSTANT_DATE)
.addHeader("Authorization", "AWS identity:/k3HQNVVyAQMsr9qhx6hajocVu4=")
.payload(requestPayload)
.build(),
HttpResponse.builder()
.statusCode(200)
.addHeader("x-amz-request-id", "7A84C3CD4437A4C0")
.addHeader("Date", CONSTANT_DATE)
.addHeader("ETag", "437b930db84b8079c2dd804a71936b5f")
.addHeader("Server", "AmazonS3")
.payload(responsePayload)
.build()
);
DeleteResult result = client.deleteObjects("test", ImmutableSet.of("key1", "key2"));
assertNotNull(result, "result is null");
assertEquals(result.getDeleted(), ImmutableSet.of("key1", "key1.1"));
assertEquals(result.getErrors().size(), 1);
assertEquals(result.getErrors().get("key2"), new DeleteResult.Error("AccessDenied", "Access Denied"));
}
}

View File

@ -20,15 +20,10 @@ import static org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions.Builder.stora
import static org.jclouds.s3.options.ListBucketOptions.Builder.withPrefix;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import java.util.Set;
import java.util.UUID;
import org.jclouds.aws.AWSResponseException;
import org.jclouds.aws.domain.Region;
import org.jclouds.aws.s3.domain.DeleteResult;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.StorageMetadata;
@ -139,34 +134,6 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
assertEquals("InvalidBucketName", e.getError().getCode());
}
}
public void testDeleteMultipleObjects() throws InterruptedException {
String container = getContainerName();
try {
ImmutableSet.Builder<String> builder = ImmutableSet.builder();
for (int i = 0; i < 5; i++) {
String key = UUID.randomUUID().toString();
Blob blob = view.getBlobStore().blobBuilder(key).payload("").build();
view.getBlobStore().putBlob(container, blob);
builder.add(key);
}
Set<String> keys = builder.build();
DeleteResult result = getApi().deleteObjects(container, keys);
assertTrue(result.getDeleted().containsAll(keys));
assertEquals(result.getErrors().size(), 0);
for (String key : keys) {
assertConsistencyAwareBlobDoesntExist(container, key);
}
} finally {
returnContainer(container);
}
}
public void testDirectoryEndingWithSlash() throws InterruptedException {
String containerName = getContainerName();