HADOOP-18438: AliyunOSSFileSystemStore deleteObjects interface should return the objects that failed to delete (#4857)

Merged to trunk, thank @chenshuang778  for your contribution

hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java

@@ -72,6 +72,7 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.NoSuchElementException;
+import java.util.stream.Collectors;
 
 import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
 
@@ -203,31 +204,29 @@ public class AliyunOSSFileSystemStore {
 
     int retry = 10;
     int tries = 0;
-    List<String> deleteFailed = keysToDelete;
-    while(CollectionUtils.isNotEmpty(deleteFailed)) {
+    while (CollectionUtils.isNotEmpty(keysToDelete)) {
       DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucketName);
-      deleteRequest.setKeys(deleteFailed);
+      deleteRequest.setKeys(keysToDelete);
       // There are two modes to do batch delete:
-      // 1. detail mode: DeleteObjectsResult.getDeletedObjects returns objects
-      // which were deleted successfully.
-      // 2. simple mode: DeleteObjectsResult.getDeletedObjects returns objects
-      // which were deleted unsuccessfully.
-      // Here, we choose the simple mode to do batch delete.
-      deleteRequest.setQuiet(true);
+      // 1. verbose mode: A list of all deleted objects is returned.
+      // 2. quiet mode: No message body is returned.
+      // Here, we choose the verbose mode to do batch delete.
+      deleteRequest.setQuiet(false);
       DeleteObjectsResult result = ossClient.deleteObjects(deleteRequest);
       statistics.incrementWriteOps(1);
-      deleteFailed = result.getDeletedObjects();
+      final List<String> deletedObjects = result.getDeletedObjects();
+      keysToDelete = keysToDelete.stream().filter(item -> !deletedObjects.contains(item))
+          .collect(Collectors.toList());
       tries++;
       if (tries == retry) {
         break;
       }
     }
 
-    if (tries == retry && CollectionUtils.isNotEmpty(deleteFailed)) {
+    if (tries == retry && CollectionUtils.isNotEmpty(keysToDelete)) {
       // Most of time, it is impossible to try 10 times, expect the
       // Aliyun OSS service problems.
-      throw new IOException("Failed to delete Aliyun OSS objects for " +
-          tries + " times.");
+      throw new IOException("Failed to delete Aliyun OSS objects for " + tries + " times.");
     }
   }
 

hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java

@@ -18,9 +18,12 @@
 
 package org.apache.hadoop.fs.aliyun.oss;
 
+import com.aliyun.oss.model.OSSObjectSummary;
 import com.aliyun.oss.model.ObjectMetadata;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -36,7 +39,10 @@ import java.security.DigestInputStream;
 import java.security.DigestOutputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
 
+import static org.apache.hadoop.fs.aliyun.oss.Constants.MAX_PAGING_KEYS_DEFAULT;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -128,4 +134,29 @@ public class TestAliyunOSSFileSystemStore {
     writeRenameReadCompare(new Path("/test/xlarge"),
         Constants.MULTIPART_UPLOAD_PART_SIZE_DEFAULT + 1);
   }
+
+  @Test
+  public void testDeleteObjects() throws IOException, NoSuchAlgorithmException {
+    // generate test files
+    final int files = 10;
+    final long size = 5 * 1024 * 1024;
+    final String prefix = "dir";
+    for (int i = 0; i < files; i++) {
+      Path path = new Path(String.format("/%s/testFile-%d.txt", prefix, i));
+      ContractTestUtils.generateTestFile(this.fs, path, size, 256, 255);
+    }
+    OSSListRequest listRequest =
+        store.createListObjectsRequest(prefix, MAX_PAGING_KEYS_DEFAULT, null, null, true);
+    List<String> keysToDelete = new ArrayList<>();
+    OSSListResult objects = store.listObjects(listRequest);
+    assertEquals(files, objects.getObjectSummaries().size());
+
+    // test delete files
+    for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
+      keysToDelete.add(objectSummary.getKey());
+    }
+    store.deleteObjects(keysToDelete);
+    objects = store.listObjects(listRequest);
+    assertEquals(0, objects.getObjectSummaries().size());
+  }
 }