diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 93304dc3677..4057c4e7eb2 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -22,6 +22,9 @@ Release 2.8.0 - UNRELEASED HADOOP-8934. Shell command ls should include sort options (Jonathan Allen via aw) + HADOOP-10971. Add -C flag to make `hadoop fs -ls` print filenames only. + (Kengo Seki via aajisaka) + IMPROVEMENTS HADOOP-6842. "hadoop fs -text" does not give a useful text representation diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java index 0e467008d4b..171d221569d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java @@ -43,6 +43,7 @@ class Ls extends FsCommand { factory.addClass(Lsr.class, "-lsr"); } + private static final String OPTION_PATHONLY = "C"; private static final String OPTION_DIRECTORY = "d"; private static final String OPTION_HUMAN = "h"; private static final String OPTION_RECURSIVE = "R"; @@ -52,10 +53,10 @@ class Ls extends FsCommand { private static final String OPTION_SIZE = "S"; public static final String NAME = "ls"; - public static final String USAGE = "[-" + OPTION_DIRECTORY + "] [-" - + OPTION_HUMAN + "] " + "[-" + OPTION_RECURSIVE + "] [-" + OPTION_MTIME - + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] " + "[-" - + OPTION_ATIME + "] [ ...]"; + public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-" + + OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" + OPTION_RECURSIVE + + "] [-" + OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + + "] [-" + OPTION_ATIME + "] [ ...]"; public static final String DESCRIPTION = "List the contents that match the specified file pattern. If " + @@ -67,6 +68,8 @@ class Ls extends FsCommand { "\tpermissions - userId groupId sizeOfDirectory(in bytes) modificationDate(yyyy-MM-dd HH:mm) directoryName\n\n" + "and file entries are of the form:\n" + "\tpermissions numberOfReplicas userId groupId sizeOfFile(in bytes) modificationDate(yyyy-MM-dd HH:mm) fileName\n\n" + + " -" + OPTION_PATHONLY + + " Display the paths of files and directories only.\n" + " -" + OPTION_DIRECTORY + " Directories are listed as plain files.\n" + " -" + OPTION_HUMAN + @@ -89,6 +92,7 @@ class Ls extends FsCommand { protected int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0; protected String lineFormat; + private boolean pathOnly; protected boolean dirRecurse; private boolean orderReverse; private boolean orderTime; @@ -107,10 +111,11 @@ class Ls extends FsCommand { @Override protected void processOptions(LinkedList args) throws IOException { - CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, + CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, OPTION_PATHONLY, OPTION_DIRECTORY, OPTION_HUMAN, OPTION_RECURSIVE, OPTION_REVERSE, OPTION_MTIME, OPTION_SIZE, OPTION_ATIME); cf.parse(args); + pathOnly = cf.getOpt(OPTION_PATHONLY); dirRecurse = !cf.getOpt(OPTION_DIRECTORY); setRecursive(cf.getOpt(OPTION_RECURSIVE) && dirRecurse); humanReadable = cf.getOpt(OPTION_HUMAN); @@ -123,6 +128,15 @@ class Ls extends FsCommand { initialiseOrderComparator(); } + /** + * Should display only paths of files and directories. + * @return true display paths only, false display all fields + */ + @InterfaceAudience.Private + boolean isPathOnly() { + return this.pathOnly; + } + /** * Should the contents of the directory be shown or just the directory? * @return true if directory contents, false if just directory @@ -191,15 +205,23 @@ class Ls extends FsCommand { protected void processPaths(PathData parent, PathData ... items) throws IOException { if (parent != null && !isRecursive() && items.length != 0) { - out.println("Found " + items.length + " items"); + if (!pathOnly) { + out.println("Found " + items.length + " items"); + } Arrays.sort(items, getOrderComparator()); } - adjustColumnWidths(items); + if (!pathOnly) { + adjustColumnWidths(items); + } super.processPaths(parent, items); } @Override protected void processPath(PathData item) throws IOException { + if (pathOnly) { + out.println(item.toString()); + return; + } FileStatus stat = item.stat; String line = String.format(lineFormat, (stat.isDirectory() ? "d" : "-"), diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index aaf7ef4326a..ae2b0ef340e 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -382,10 +382,11 @@ Return usage output. ls ---- -Usage: `hadoop fs -ls [-d] [-h] [-R] [-t] [-S] [-r] [-u] ` +Usage: `hadoop fs -ls [-C] [-d] [-h] [-R] [-t] [-S] [-r] [-u] ` Options: +* -C: Display the paths of files and directories only. * -d: Directories are listed as plain files. * -h: Format file sizes in a human-readable fashion (eg 64.0m instead of 67108864). * -R: Recursively list subdirectories encountered. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java index 66403db3377..4a9103ffc18 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java @@ -74,6 +74,24 @@ public class TestLs { LinkedList options = new LinkedList(); Ls ls = new Ls(); ls.processOptions(options); + assertFalse(ls.isPathOnly()); + assertTrue(ls.isDirRecurse()); + assertFalse(ls.isHumanReadable()); + assertFalse(ls.isRecursive()); + assertFalse(ls.isOrderReverse()); + assertFalse(ls.isOrderSize()); + assertFalse(ls.isOrderTime()); + assertFalse(ls.isUseAtime()); + } + + // check the -C option is recognised + @Test + public void processOptionsPathOnly() throws IOException { + LinkedList options = new LinkedList(); + options.add("-C"); + Ls ls = new Ls(); + ls.processOptions(options); + assertTrue(ls.isPathOnly()); assertTrue(ls.isDirRecurse()); assertFalse(ls.isHumanReadable()); assertFalse(ls.isRecursive()); @@ -90,6 +108,7 @@ public class TestLs { options.add("-d"); Ls ls = new Ls(); ls.processOptions(options); + assertFalse(ls.isPathOnly()); assertFalse(ls.isDirRecurse()); assertFalse(ls.isHumanReadable()); assertFalse(ls.isRecursive()); @@ -106,6 +125,7 @@ public class TestLs { options.add("-h"); Ls ls = new Ls(); ls.processOptions(options); + assertFalse(ls.isPathOnly()); assertTrue(ls.isDirRecurse()); assertTrue(ls.isHumanReadable()); assertFalse(ls.isRecursive()); @@ -122,6 +142,7 @@ public class TestLs { options.add("-R"); Ls ls = new Ls(); ls.processOptions(options); + assertFalse(ls.isPathOnly()); assertTrue(ls.isDirRecurse()); assertFalse(ls.isHumanReadable()); assertTrue(ls.isRecursive()); @@ -138,6 +159,7 @@ public class TestLs { options.add("-r"); Ls ls = new Ls(); ls.processOptions(options); + assertFalse(ls.isPathOnly()); assertTrue(ls.isDirRecurse()); assertFalse(ls.isHumanReadable()); assertFalse(ls.isRecursive()); @@ -154,6 +176,7 @@ public class TestLs { options.add("-S"); Ls ls = new Ls(); ls.processOptions(options); + assertFalse(ls.isPathOnly()); assertTrue(ls.isDirRecurse()); assertFalse(ls.isHumanReadable()); assertFalse(ls.isRecursive()); @@ -170,6 +193,7 @@ public class TestLs { options.add("-t"); Ls ls = new Ls(); ls.processOptions(options); + assertFalse(ls.isPathOnly()); assertTrue(ls.isDirRecurse()); assertFalse(ls.isHumanReadable()); assertFalse(ls.isRecursive()); @@ -187,6 +211,7 @@ public class TestLs { options.add("-S"); Ls ls = new Ls(); ls.processOptions(options); + assertFalse(ls.isPathOnly()); assertTrue(ls.isDirRecurse()); assertFalse(ls.isHumanReadable()); assertFalse(ls.isRecursive()); @@ -205,6 +230,7 @@ public class TestLs { options.add("-r"); Ls ls = new Ls(); ls.processOptions(options); + assertFalse(ls.isPathOnly()); assertTrue(ls.isDirRecurse()); assertFalse(ls.isHumanReadable()); assertFalse(ls.isRecursive()); @@ -221,6 +247,7 @@ public class TestLs { options.add("-u"); Ls ls = new Ls(); ls.processOptions(options); + assertFalse(ls.isPathOnly()); assertTrue(ls.isDirRecurse()); assertFalse(ls.isHumanReadable()); assertFalse(ls.isRecursive()); @@ -234,6 +261,7 @@ public class TestLs { @Test public void processOptionsAll() throws IOException { LinkedList options = new LinkedList(); + options.add("-C"); // show file path only options.add("-d"); // directory options.add("-h"); // human readable options.add("-R"); // recursive @@ -243,6 +271,7 @@ public class TestLs { options.add("-u"); // show atime Ls ls = new Ls(); ls.processOptions(options); + assertTrue(ls.isPathOnly()); assertFalse(ls.isDirRecurse()); assertTrue(ls.isHumanReadable()); assertFalse(ls.isRecursive()); // -d overrules -R @@ -981,6 +1010,44 @@ public class TestLs { verifyNoMoreInteractions(out); } + // check path only display (-C option) + @Test + public void processPathDirectoryPathOnly() throws IOException { + TestFile testfile01 = new TestFile("testDirectory", "testFile01"); + TestFile testfile02 = new TestFile("testDirectory", "testFile02"); + TestFile testfile03 = new TestFile("testDirectory", "testFile03"); + TestFile testfile04 = new TestFile("testDirectory", "testFile04"); + TestFile testfile05 = new TestFile("testDirectory", "testFile05"); + TestFile testfile06 = new TestFile("testDirectory", "testFile06"); + + TestFile testDir = new TestFile("", "testDirectory"); + testDir.setIsDir(true); + testDir.addContents(testfile01, testfile02, testfile03, testfile04, + testfile05, testfile06); + + LinkedList pathData = new LinkedList(); + pathData.add(testDir.getPathData()); + + PrintStream out = mock(PrintStream.class); + + Ls ls = new Ls(); + ls.out = out; + + LinkedList options = new LinkedList(); + options.add("-C"); + ls.processOptions(options); + + ls.processArguments(pathData); + InOrder inOrder = inOrder(out); + inOrder.verify(out).println(testfile01.getPath().toString()); + inOrder.verify(out).println(testfile02.getPath().toString()); + inOrder.verify(out).println(testfile03.getPath().toString()); + inOrder.verify(out).println(testfile04.getPath().toString()); + inOrder.verify(out).println(testfile05.getPath().toString()); + inOrder.verify(out).println(testfile06.getPath().toString()); + verifyNoMoreInteractions(out); + } + // check the deprecated flag isn't set @Test public void isDeprecated() { diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index ae5f44c34df..8f48ccc1e2f 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -54,7 +54,7 @@ RegexpComparator - ^-ls \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)* + ^-ls \[-C\] \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)* RegexpComparator @@ -92,6 +92,10 @@ RegexpComparator ^\s*modificationDate\(yyyy-MM-dd HH:mm\) fileName( )* + + RegexpComparator + ^\s*-C\s+Display the paths of files and directories only\.( )* + RegexpComparator ^\s*-d\s+Directories are listed as plain files\.( )*