HADOOP-12169 ListStatus on empty dir in S3A lists itself instead of returning an empty list. author: Pieter Reuse.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java

@@ -19,10 +19,14 @@
 package org.apache.hadoop.fs.contract;
 
 import java.io.FileNotFoundException;
+import java.io.IOException;
 
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -58,4 +62,23 @@ public abstract class AbstractContractGetFileStatusTest extends
       handleExpectedException(e);
     }
   }
+
+  @Test
+  public void testListStatusEmptyDirectory() throws IOException {
+    // remove the test directory
+    FileSystem fs = getFileSystem();
+    assertTrue(fs.delete(getContract().getTestPath(), true));
+
+    // create a - non-qualified - Path for a subdir
+    Path subfolder = getContract().getTestPath().suffix("/"+testPath.getName());
+    assertTrue(fs.mkdirs(subfolder));
+
+    // assert empty ls on the empty dir
+    assertEquals("ls on an empty directory not of length 0", 0,
+        fs.listStatus(subfolder).length);
+
+    // assert non-empty ls on parent dir
+    assertTrue("ls on a non-empty directory of length 0",
+        fs.listStatus(getContract().getTestPath()).length > 0);
+  }
 }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -846,11 +846,14 @@ public class S3AFileSystem extends FileSystem {
       ObjectListing objects = s3.listObjects(request);
       statistics.incrementReadOps(1);
 
+      Path fQualified = f.makeQualified(uri, workingDir);
+
       while (true) {
         for (S3ObjectSummary summary : objects.getObjectSummaries()) {
           Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir);
           // Skip over keys that are ourselves and old S3N _$folder$ files
-          if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
+          if (keyPath.equals(fQualified) ||
+              summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
             if (LOG.isDebugEnabled()) {
               LOG.debug("Ignoring: " + keyPath);
             }
@@ -865,7 +868,7 @@ public class S3AFileSystem extends FileSystem {
           } else {
             result.add(new S3AFileStatus(summary.getSize(),
                 dateToLong(summary.getLastModified()), keyPath,
-                getDefaultBlockSize(f.makeQualified(uri, workingDir))));
+                getDefaultBlockSize(fQualified)));
             if (LOG.isDebugEnabled()) {
               LOG.debug("Adding: fi: " + keyPath);
             }

hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml

@@ -77,6 +77,11 @@
     <value>false</value>
   </property>
 
+  <property>
+    <name>fs.contract.supports-getfilestatus</name>
+    <value>true</value>
+  </property>
+
   <property>
     <name>fs.contract.supports-seek</name>
     <value>true</value>