HADOOP-12169 ListStatus on empty dir in S3A lists itself instead of returning an empty list. author: Pieter Reuse.

This commit is contained in:
Steve Loughran 2016-04-03 16:39:14 +01:00
parent aa38748997
commit 20e808d83e
3 changed files with 33 additions and 2 deletions

View File

@ -19,10 +19,14 @@
package org.apache.hadoop.fs.contract;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -58,4 +62,23 @@ public abstract class AbstractContractGetFileStatusTest extends
handleExpectedException(e);
}
}
@Test
public void testListStatusEmptyDirectory() throws IOException {
// remove the test directory
FileSystem fs = getFileSystem();
assertTrue(fs.delete(getContract().getTestPath(), true));
// create a - non-qualified - Path for a subdir
Path subfolder = getContract().getTestPath().suffix("/"+testPath.getName());
assertTrue(fs.mkdirs(subfolder));
// assert empty ls on the empty dir
assertEquals("ls on an empty directory not of length 0", 0,
fs.listStatus(subfolder).length);
// assert non-empty ls on parent dir
assertTrue("ls on a non-empty directory of length 0",
fs.listStatus(getContract().getTestPath()).length > 0);
}
}

View File

@ -846,11 +846,14 @@ public class S3AFileSystem extends FileSystem {
ObjectListing objects = s3.listObjects(request);
statistics.incrementReadOps(1);
Path fQualified = f.makeQualified(uri, workingDir);
while (true) {
for (S3ObjectSummary summary : objects.getObjectSummaries()) {
Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir);
// Skip over keys that are ourselves and old S3N _$folder$ files
if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
if (keyPath.equals(fQualified) ||
summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Ignoring: " + keyPath);
}
@ -865,7 +868,7 @@ public class S3AFileSystem extends FileSystem {
} else {
result.add(new S3AFileStatus(summary.getSize(),
dateToLong(summary.getLastModified()), keyPath,
getDefaultBlockSize(f.makeQualified(uri, workingDir))));
getDefaultBlockSize(fQualified)));
if (LOG.isDebugEnabled()) {
LOG.debug("Adding: fi: " + keyPath);
}

View File

@ -77,6 +77,11 @@
<value>false</value>
</property>
<property>
<name>fs.contract.supports-getfilestatus</name>
<value>true</value>
</property>
<property>
<name>fs.contract.supports-seek</name>
<value>true</value>