From 4aa493657644547ae50a7488350369643191bec4 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Fri, 19 Oct 2018 10:13:08 -0700 Subject: [PATCH] HDDS-701. Support key multi-delete. Contributed by Elek, Marton. (cherry picked from commit 9aebafd2da44fd048d201b6ea5a043d7dda3dad9) --- .../main/smoketest/s3/objectmultidelete.robot | 44 +++++ .../hadoop/ozone/s3/HeaderPreprocessor.java | 47 ++++++ .../ozone/s3/endpoint/BucketEndpoint.java | 51 ++++++ .../ozone/s3/endpoint/MultiDeleteRequest.java | 96 +++++++++++ .../s3/endpoint/MultiDeleteResponse.java | 154 ++++++++++++++++++ .../s3/endpoint/TestObjectMultiDelete.java | 86 ++++++++++ 6 files changed, 478 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteRequest.java create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot new file mode 100644 index 00000000000..83c967b1ba5 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation S3 gateway test with aws cli +Library OperatingSystem +Library String +Resource ../commonlib.robot +Resource commonawslib.robot +Test Setup Setup s3 tests + +*** Variables *** +${ENDPOINT_URL} http://s3g:9878 +${BUCKET} generated + +*** Test Cases *** + +Delete file with multi delete + Execute date > /tmp/testfile + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f1 --body /tmp/testfile + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f2 --body /tmp/testfile + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f3 --body /tmp/testfile + ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix multidelete/ + Should contain ${result} multidelete/f1 + Should contain ${result} multidelete/f2 + Should contain ${result} multidelete/f3 + ${result} = Execute AWSS3APICli delete-objects --bucket ${BUCKET} --delete 'Objects=[{Key=multidelete/f1},{Key=multidelete/f2},{Key=multidelete/f4}]' + Should not contain ${result} Error + ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix multidelete/ + Should not contain ${result} multidelete/f1 + Should not contain ${result} multidelete/f2 + Should contain ${result} multidelete/f3 diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java new file mode 100644 index 00000000000..f51142dd4f4 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3; + +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.container.PreMatching; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.ext.Provider; +import java.io.IOException; + +/** + * Filter to adjust request headers for compatible reasons. + */ + +@Provider +@PreMatching +public class HeaderPreprocessor implements ContainerRequestFilter { + + @Override + public void filter(ContainerRequestContext requestContext) throws + IOException { + if (requestContext.getUriInfo().getQueryParameters() + .containsKey("delete")) { + //aws cli doesn't send proper Content-Type and by default POST requests + //processed as form-url-encoded. Here we can fix this. + requestContext.getHeaders() + .putSingle("Content-Type", MediaType.APPLICATION_XML); + } + } + +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 1fa19c422ff..1c31dd41f36 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -21,9 +21,11 @@ import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.HEAD; +import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.HttpHeaders; @@ -34,10 +36,15 @@ import java.io.IOException; import java.io.InputStream; import java.time.Instant; import java.util.Iterator; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response.ResponseBuilder; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata; +import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteRequest.DeleteObject; +import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteResponse.DeletedObject; +import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteResponse.Error; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; @@ -208,4 +215,48 @@ public class BucketEndpoint extends EndpointBase { .build(); } + + /** + * Implement multi delete. + *

+ * see: https://docs.aws.amazon + * .com/AmazonS3/latest/API/multiobjectdeleteapi.html + */ + @POST + @Produces(MediaType.APPLICATION_XML) + public Response multiDelete(@PathParam("bucket") String bucketName, + @QueryParam("delete") String delete, + MultiDeleteRequest request) throws OS3Exception, IOException { + OzoneBucket bucket = getBucket(bucketName); + MultiDeleteResponse result = new MultiDeleteResponse(); + if (request.getObjects() != null) { + for (DeleteObject keyToDelete : request.getObjects()) { + try { + bucket.deleteKey(keyToDelete.getKey()); + + if (!request.isQuiet()) { + result.addDeleted(new DeletedObject(keyToDelete.getKey())); + } + } catch (IOException ex) { + if (!ex.getMessage().contains("KEY_NOT_FOUND")) { + result.addError( + new Error(keyToDelete.getKey(), "InternalError", + ex.getMessage())); + } else if (!request.isQuiet()) { + result.addDeleted(new DeletedObject(keyToDelete.getKey())); + } + } catch (Exception ex) { + result.addError( + new Error(keyToDelete.getKey(), "InternalError", + ex.getMessage())); + } + } + } + ResponseBuilder response = Response.ok(); + if (!request.isQuiet() || result.getErrors().size() > 0) { + response = response.entity(result); + } + return response.build(); + + } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteRequest.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteRequest.java new file mode 100644 index 00000000000..d9dd043bd6c --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteRequest.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.endpoint; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + +/** + * Request for multi object delete request. + */ + +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "Delete", namespace = "http://s3.amazonaws" + + ".com/doc/2006-03-01/") +public class MultiDeleteRequest { + + @XmlElement(name = "Quiet") + private boolean quiet; + + @XmlElement(name = "Object") + private List objects = new ArrayList<>(); + + public boolean isQuiet() { + return quiet; + } + + public void setQuiet(boolean quiet) { + this.quiet = quiet; + } + + public List getObjects() { + return objects; + } + + public void setObjects( + List objects) { + this.objects = objects; + } + + /** + * JAXB entity for child element. + */ + @XmlAccessorType(XmlAccessType.FIELD) + @XmlRootElement(name = "Object", namespace = "http://s3.amazonaws" + + ".com/doc/2006-03-01/") + public static class DeleteObject { + + @XmlElement(name = "Key") + private String key; + + @XmlElement(name = "VersionId") + private String versionId; + + public DeleteObject() { + } + + public DeleteObject(String key) { + this.key = key; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getVersionId() { + return versionId; + } + + public void setVersionId(String versionId) { + this.versionId = versionId; + } + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java new file mode 100644 index 00000000000..f2e21e6cb54 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.endpoint; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + +/** + * Response for multi object delete request. + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "DeleteResult", namespace = "http://s3.amazonaws" + + ".com/doc/2006-03-01/") +public class MultiDeleteResponse { + + @XmlElement(name = "Deleted") + private List deletedObjects = new ArrayList<>(); + + @XmlElement(name = "Error") + private List errors = new ArrayList<>(); + + public void addDeleted(DeletedObject deletedObject) { + deletedObjects.add(deletedObject); + } + + public void addError(Error error) { + errors.add(error); + } + + public List getDeletedObjects() { + return deletedObjects; + } + + public void setDeletedObjects( + List deletedObjects) { + this.deletedObjects = deletedObjects; + } + + public List getErrors() { + return errors; + } + + public void setErrors( + List errors) { + this.errors = errors; + } + + /** + * JAXB entity for child element. + */ + @XmlAccessorType(XmlAccessType.FIELD) + @XmlRootElement(name = "Deleted", namespace = "http://s3.amazonaws" + + ".com/doc/2006-03-01/") + public static class DeletedObject { + + @XmlElement(name = "Key") + private String key; + + private String versionId; + + public DeletedObject() { + } + + public DeletedObject(String key) { + this.key = key; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getVersionId() { + return versionId; + } + + public void setVersionId(String versionId) { + this.versionId = versionId; + } + } + + /** + * JAXB entity for child element. + */ + @XmlAccessorType(XmlAccessType.FIELD) + @XmlRootElement(name = "Error", namespace = "http://s3.amazonaws" + + ".com/doc/2006-03-01/") + public static class Error { + + @XmlElement(name = "Key") + private String key; + + @XmlElement(name = "Code") + private String code; + + @XmlElement(name = "Message") + private String message; + + public Error() { + } + + public Error(String key, String code, String message) { + this.key = key; + this.code = code; + this.message = message; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + } +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java new file mode 100644 index 00000000000..12a14a0989c --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.hadoop.ozone.s3.endpoint; + +import javax.ws.rs.core.Response; +import javax.xml.bind.JAXBException; +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteRequest.DeleteObject; +import org.apache.hadoop.ozone.s3.exception.OS3Exception; + +import com.google.common.collect.Sets; +import org.junit.Assert; +import org.junit.Test; + +/** + * Test object multi delete. + */ +public class TestObjectMultiDelete { + + @Test + public void delete() throws IOException, OS3Exception, JAXBException { + //GIVEN + OzoneClient client = new OzoneClientStub(); + client.getObjectStore().createS3Bucket("bilbo", "b1"); + + String volumeName = client.getObjectStore().getOzoneVolumeName("b1"); + + OzoneBucket bucket = + client.getObjectStore().getVolume(volumeName).getBucket("b1"); + + bucket.createKey("key1", 0).close(); + bucket.createKey("key2", 0).close(); + bucket.createKey("key3", 0).close(); + + BucketEndpoint rest = new BucketEndpoint(); + rest.setClient(client); + + MultiDeleteRequest mdr = new MultiDeleteRequest(); + mdr.getObjects().add(new DeleteObject("key1")); + mdr.getObjects().add(new DeleteObject("key2")); + mdr.getObjects().add(new DeleteObject("key4")); + + //WHEN + Response response = rest.multiDelete("b1", "", mdr); + + //THEN + MultiDeleteResponse mdresponse = (MultiDeleteResponse) response.getEntity(); + + Set keysAtTheEnd = Sets.newHashSet(bucket.listKeys("")).stream() + .map(OzoneKey::getName) + .collect(Collectors.toSet()); + + Set expectedResult = new HashSet<>(); + expectedResult.add("key3"); + + //THEN + Assert.assertEquals(expectedResult, keysAtTheEnd); + Assert.assertEquals(3, mdresponse.getDeletedObjects().size()); + Assert.assertEquals(0, mdresponse.getErrors().size()); + } +} \ No newline at end of file