Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-18662. ListFiles with recursive fails with FNF. (#5477). Contributed by Ayush Saxena. 

Reviewed-by: Steve Loughran <stevel@apache.org>
This commit is contained in:
Ayush Saxena 2023-03-23 08:30:08 +05:30
parent 6b6bd82bf0
commit a226016c52
No known key found for this signature in database
GPG Key ID: D09AE71061AB564D
2 changed files with 62 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
View File

@ -2415,8 +2415,14 @@ public abstract class FileSystem extends Configured
if (stat.isFile()) { // file
curFile = stat;
} else if (recursive) { // directory
itors.push(curItor);
curItor = listLocatedStatus(stat.getPath());
try {
RemoteIterator<LocatedFileStatus> newDirItor = listLocatedStatus(stat.getPath());
itors.push(curItor);
curItor = newDirItor;
} catch (FileNotFoundException ignored) {
LOGGER.debug("Directory {} deleted while attempting for recursive listing",
stat.getPath());
}
}
}

54
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
View File

@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CONTEXT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@ -28,6 +29,7 @@ import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import java.io.File;
import java.io.FileNotFoundException;
@ -118,9 +120,11 @@ import org.apache.hadoop.test.Whitebox;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.util.functional.RemoteIterators;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.InOrder;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;
@ -1552,6 +1556,56 @@ public class TestDistributedFileSystem {
}
}
@Test
public void testListFilesRecursive() throws IOException {
Configuration conf = getTestConfiguration();
try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();) {
DistributedFileSystem fs = cluster.getFileSystem();
// Create some directories and files.
Path dir = new Path("/dir");
Path subDir1 = fs.makeQualified(new Path(dir, "subDir1"));
Path subDir2 = fs.makeQualified(new Path(dir, "subDir2"));
fs.create(new Path(dir, "foo1")).close();
fs.create(new Path(dir, "foo2")).close();
fs.create(new Path(subDir1, "foo3")).close();
fs.create(new Path(subDir2, "foo4")).close();
// Mock the filesystem, and throw FNF when listing is triggered for the subdirectory.
FileSystem mockFs = spy(fs);
Mockito.doThrow(new FileNotFoundException("")).when(mockFs).listLocatedStatus(eq(subDir1));
List<LocatedFileStatus> str = RemoteIterators.toList(mockFs.listFiles(dir, true));
assertThat(str).hasSize(3);
// Mock the filesystem to depict a scenario where the directory got deleted and a file
// got created with the same name.
Mockito.doReturn(getMockedIterator(subDir1)).when(mockFs).listLocatedStatus(eq(subDir1));
str = RemoteIterators.toList(mockFs.listFiles(dir, true));
assertThat(str).hasSize(4);
}
}
private static RemoteIterator<LocatedFileStatus> getMockedIterator(Path subDir1) {
return new RemoteIterator<LocatedFileStatus>() {
private int remainingEntries = 1;
@Override
public boolean hasNext() throws IOException {
return remainingEntries > 0;
}
@Override
public LocatedFileStatus next() throws IOException {
remainingEntries--;
return new LocatedFileStatus(0, false, 1, 1024, 0L, 0, null, null, null, null, subDir1,
false, false, false, null);
}
};
}
@Test
public void testListStatusOfSnapshotDirs() throws IOException {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(getTestConfiguration())