HADOOP-11421. Add IOUtils#listDirectory (cmccabe)

This commit is contained in:
Colin Patrick Mccabe 2014-12-17 10:31:19 -08:00
parent 6ba8fd7e8e
commit 9937eef7f7
3 changed files with 82 additions and 0 deletions

View File

@ -427,6 +427,8 @@ Release 2.7.0 - UNRELEASED
HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics
collection. (Shanyu Zhao via cnauroth) collection. (Shanyu Zhao via cnauroth)
HADOOP-11421. Add IOUtils#listDirectory (cmccabe)
OPTIMIZATIONS OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array. HADOOP-11323. WritableComparator#compare keeps reference to byte array.

View File

@ -23,6 +23,12 @@
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.channels.FileChannel; import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel; import java.nio.channels.WritableByteChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.DirectoryIteratorException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
@ -30,6 +36,7 @@
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ChunkedArrayList;
/** /**
* An utility class for I/O related functionality. * An utility class for I/O related functionality.
@ -313,4 +320,33 @@ public static void writeFully(FileChannel fc, ByteBuffer buf,
offset += fc.write(buf, offset); offset += fc.write(buf, offset);
} while (buf.remaining() > 0); } while (buf.remaining() > 0);
} }
/**
* Return the complete list of files in a directory as strings.<p/>
*
* This is better than File#listDir because it does not ignore IOExceptions.
*
* @param dir The directory to list.
* @param filter If non-null, the filter to use when listing
* this directory.
* @return The list of files in the directory.
*
* @throws IOException On I/O error
*/
public static List<String> listDirectory(File dir, FilenameFilter filter)
throws IOException {
ArrayList<String> list = new ArrayList<String> ();
try (DirectoryStream<Path> stream =
Files.newDirectoryStream(dir.toPath())) {
for (Path entry: stream) {
String fileName = entry.getFileName().toString();
if ((filter == null) || filter.accept(dir, fileName)) {
list.add(fileName);
}
}
} catch (DirectoryIteratorException e) {
throw e.getCause();
}
return list;
}
} }

View File

@ -24,14 +24,21 @@
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.EOFException; import java.io.EOFException;
import java.io.File; import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.channels.FileChannel; import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
@ -245,4 +252,41 @@ public void testSkipFully() throws IOException {
in.close(); in.close();
} }
} }
private static enum NoEntry3Filter implements FilenameFilter {
INSTANCE;
@Override
public boolean accept(File dir, String name) {
return !name.equals("entry3");
}
}
@Test
public void testListDirectory() throws IOException {
File dir = new File("testListDirectory");
Files.createDirectory(dir.toPath());
try {
Set<String> entries = new HashSet<String>();
entries.add("entry1");
entries.add("entry2");
entries.add("entry3");
for (String entry : entries) {
Files.createDirectory(new File(dir, entry).toPath());
}
List<String> list = IOUtils.listDirectory(dir,
NoEntry3Filter.INSTANCE);
for (String entry : list) {
Assert.assertTrue(entries.remove(entry));
}
Assert.assertTrue(entries.contains("entry3"));
list = IOUtils.listDirectory(dir, null);
for (String entry : list) {
entries.remove(entry);
}
Assert.assertTrue(entries.isEmpty());
} finally {
FileUtils.deleteDirectory(dir);
}
}
} }