diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 049dcf906bf..bd0f5b5e46f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -427,6 +427,8 @@ Release 2.7.0 - UNRELEASED HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics collection. (Shanyu Zhao via cnauroth) + HADOOP-11421. Add IOUtils#listDirectory (cmccabe) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java index 6be7446c994..e6c00c940bb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java @@ -23,6 +23,12 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.WritableByteChannel; +import java.nio.file.DirectoryStream; +import java.nio.file.DirectoryIteratorException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -30,6 +36,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ChunkedArrayList; /** * An utility class for I/O related functionality. @@ -313,4 +320,33 @@ public static void writeFully(FileChannel fc, ByteBuffer buf, offset += fc.write(buf, offset); } while (buf.remaining() > 0); } + + /** + * Return the complete list of files in a directory as strings.

+ * + * This is better than File#listDir because it does not ignore IOExceptions. + * + * @param dir The directory to list. + * @param filter If non-null, the filter to use when listing + * this directory. + * @return The list of files in the directory. + * + * @throws IOException On I/O error + */ + public static List listDirectory(File dir, FilenameFilter filter) + throws IOException { + ArrayList list = new ArrayList (); + try (DirectoryStream stream = + Files.newDirectoryStream(dir.toPath())) { + for (Path entry: stream) { + String fileName = entry.getFileName().toString(); + if ((filter == null) || filter.accept(dir, fileName)) { + list.add(fileName); + } + } + } catch (DirectoryIteratorException e) { + throw e.getCause(); + } + return list; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java index 5db3c91a482..ac4fb483f67 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java @@ -24,14 +24,21 @@ import java.io.ByteArrayInputStream; import java.io.EOFException; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -245,4 +252,41 @@ public void testSkipFully() throws IOException { in.close(); } } + + private static enum NoEntry3Filter implements FilenameFilter { + INSTANCE; + + @Override + public boolean accept(File dir, String name) { + return !name.equals("entry3"); + } + } + + @Test + public void testListDirectory() throws IOException { + File dir = new File("testListDirectory"); + Files.createDirectory(dir.toPath()); + try { + Set entries = new HashSet(); + entries.add("entry1"); + entries.add("entry2"); + entries.add("entry3"); + for (String entry : entries) { + Files.createDirectory(new File(dir, entry).toPath()); + } + List list = IOUtils.listDirectory(dir, + NoEntry3Filter.INSTANCE); + for (String entry : list) { + Assert.assertTrue(entries.remove(entry)); + } + Assert.assertTrue(entries.contains("entry3")); + list = IOUtils.listDirectory(dir, null); + for (String entry : list) { + entries.remove(entry); + } + Assert.assertTrue(entries.isEmpty()); + } finally { + FileUtils.deleteDirectory(dir); + } + } }