HADOOP-10971. Add -C flag to make `hadoop fs -ls` print filenames only. Contributed by Kengo Seki.
(cherry picked from commit 3b50dcdce4
)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
This commit is contained in:
parent
df6d242e28
commit
fb49967e97
|
@ -22,6 +22,9 @@ Release 2.8.0 - UNRELEASED
|
|||
HADOOP-8934. Shell command ls should include sort options (Jonathan Allen
|
||||
via aw)
|
||||
|
||||
HADOOP-10971. Add -C flag to make `hadoop fs -ls` print filenames only.
|
||||
(Kengo Seki via aajisaka)
|
||||
|
||||
IMPROVEMENTS
|
||||
|
||||
HADOOP-6842. "hadoop fs -text" does not give a useful text representation
|
||||
|
|
|
@ -43,6 +43,7 @@ class Ls extends FsCommand {
|
|||
factory.addClass(Lsr.class, "-lsr");
|
||||
}
|
||||
|
||||
private static final String OPTION_PATHONLY = "C";
|
||||
private static final String OPTION_DIRECTORY = "d";
|
||||
private static final String OPTION_HUMAN = "h";
|
||||
private static final String OPTION_RECURSIVE = "R";
|
||||
|
@ -52,10 +53,10 @@ class Ls extends FsCommand {
|
|||
private static final String OPTION_SIZE = "S";
|
||||
|
||||
public static final String NAME = "ls";
|
||||
public static final String USAGE = "[-" + OPTION_DIRECTORY + "] [-"
|
||||
+ OPTION_HUMAN + "] " + "[-" + OPTION_RECURSIVE + "] [-" + OPTION_MTIME
|
||||
+ "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] " + "[-"
|
||||
+ OPTION_ATIME + "] [<path> ...]";
|
||||
public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-"
|
||||
+ OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" + OPTION_RECURSIVE
|
||||
+ "] [-" + OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE
|
||||
+ "] [-" + OPTION_ATIME + "] [<path> ...]";
|
||||
|
||||
public static final String DESCRIPTION =
|
||||
"List the contents that match the specified file pattern. If " +
|
||||
|
@ -67,6 +68,8 @@ class Ls extends FsCommand {
|
|||
"\tpermissions - userId groupId sizeOfDirectory(in bytes) modificationDate(yyyy-MM-dd HH:mm) directoryName\n\n" +
|
||||
"and file entries are of the form:\n" +
|
||||
"\tpermissions numberOfReplicas userId groupId sizeOfFile(in bytes) modificationDate(yyyy-MM-dd HH:mm) fileName\n\n" +
|
||||
" -" + OPTION_PATHONLY +
|
||||
" Display the paths of files and directories only.\n" +
|
||||
" -" + OPTION_DIRECTORY +
|
||||
" Directories are listed as plain files.\n" +
|
||||
" -" + OPTION_HUMAN +
|
||||
|
@ -89,6 +92,7 @@ class Ls extends FsCommand {
|
|||
|
||||
protected int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0;
|
||||
protected String lineFormat;
|
||||
private boolean pathOnly;
|
||||
protected boolean dirRecurse;
|
||||
private boolean orderReverse;
|
||||
private boolean orderTime;
|
||||
|
@ -107,10 +111,11 @@ class Ls extends FsCommand {
|
|||
@Override
|
||||
protected void processOptions(LinkedList<String> args)
|
||||
throws IOException {
|
||||
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE,
|
||||
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, OPTION_PATHONLY,
|
||||
OPTION_DIRECTORY, OPTION_HUMAN, OPTION_RECURSIVE, OPTION_REVERSE,
|
||||
OPTION_MTIME, OPTION_SIZE, OPTION_ATIME);
|
||||
cf.parse(args);
|
||||
pathOnly = cf.getOpt(OPTION_PATHONLY);
|
||||
dirRecurse = !cf.getOpt(OPTION_DIRECTORY);
|
||||
setRecursive(cf.getOpt(OPTION_RECURSIVE) && dirRecurse);
|
||||
humanReadable = cf.getOpt(OPTION_HUMAN);
|
||||
|
@ -123,6 +128,15 @@ class Ls extends FsCommand {
|
|||
initialiseOrderComparator();
|
||||
}
|
||||
|
||||
/**
|
||||
* Should display only paths of files and directories.
|
||||
* @return true display paths only, false display all fields
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
boolean isPathOnly() {
|
||||
return this.pathOnly;
|
||||
}
|
||||
|
||||
/**
|
||||
* Should the contents of the directory be shown or just the directory?
|
||||
* @return true if directory contents, false if just directory
|
||||
|
@ -191,15 +205,23 @@ class Ls extends FsCommand {
|
|||
protected void processPaths(PathData parent, PathData ... items)
|
||||
throws IOException {
|
||||
if (parent != null && !isRecursive() && items.length != 0) {
|
||||
out.println("Found " + items.length + " items");
|
||||
if (!pathOnly) {
|
||||
out.println("Found " + items.length + " items");
|
||||
}
|
||||
Arrays.sort(items, getOrderComparator());
|
||||
}
|
||||
adjustColumnWidths(items);
|
||||
if (!pathOnly) {
|
||||
adjustColumnWidths(items);
|
||||
}
|
||||
super.processPaths(parent, items);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void processPath(PathData item) throws IOException {
|
||||
if (pathOnly) {
|
||||
out.println(item.toString());
|
||||
return;
|
||||
}
|
||||
FileStatus stat = item.stat;
|
||||
String line = String.format(lineFormat,
|
||||
(stat.isDirectory() ? "d" : "-"),
|
||||
|
|
|
@ -382,10 +382,11 @@ Return usage output.
|
|||
ls
|
||||
----
|
||||
|
||||
Usage: `hadoop fs -ls [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
|
||||
Usage: `hadoop fs -ls [-C] [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
|
||||
|
||||
Options:
|
||||
|
||||
* -C: Display the paths of files and directories only.
|
||||
* -d: Directories are listed as plain files.
|
||||
* -h: Format file sizes in a human-readable fashion (eg 64.0m instead of 67108864).
|
||||
* -R: Recursively list subdirectories encountered.
|
||||
|
|
|
@ -74,6 +74,24 @@ public class TestLs {
|
|||
LinkedList<String> options = new LinkedList<String>();
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertFalse(ls.isPathOnly());
|
||||
assertTrue(ls.isDirRecurse());
|
||||
assertFalse(ls.isHumanReadable());
|
||||
assertFalse(ls.isRecursive());
|
||||
assertFalse(ls.isOrderReverse());
|
||||
assertFalse(ls.isOrderSize());
|
||||
assertFalse(ls.isOrderTime());
|
||||
assertFalse(ls.isUseAtime());
|
||||
}
|
||||
|
||||
// check the -C option is recognised
|
||||
@Test
|
||||
public void processOptionsPathOnly() throws IOException {
|
||||
LinkedList<String> options = new LinkedList<String>();
|
||||
options.add("-C");
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertTrue(ls.isPathOnly());
|
||||
assertTrue(ls.isDirRecurse());
|
||||
assertFalse(ls.isHumanReadable());
|
||||
assertFalse(ls.isRecursive());
|
||||
|
@ -90,6 +108,7 @@ public class TestLs {
|
|||
options.add("-d");
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertFalse(ls.isPathOnly());
|
||||
assertFalse(ls.isDirRecurse());
|
||||
assertFalse(ls.isHumanReadable());
|
||||
assertFalse(ls.isRecursive());
|
||||
|
@ -106,6 +125,7 @@ public class TestLs {
|
|||
options.add("-h");
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertFalse(ls.isPathOnly());
|
||||
assertTrue(ls.isDirRecurse());
|
||||
assertTrue(ls.isHumanReadable());
|
||||
assertFalse(ls.isRecursive());
|
||||
|
@ -122,6 +142,7 @@ public class TestLs {
|
|||
options.add("-R");
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertFalse(ls.isPathOnly());
|
||||
assertTrue(ls.isDirRecurse());
|
||||
assertFalse(ls.isHumanReadable());
|
||||
assertTrue(ls.isRecursive());
|
||||
|
@ -138,6 +159,7 @@ public class TestLs {
|
|||
options.add("-r");
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertFalse(ls.isPathOnly());
|
||||
assertTrue(ls.isDirRecurse());
|
||||
assertFalse(ls.isHumanReadable());
|
||||
assertFalse(ls.isRecursive());
|
||||
|
@ -154,6 +176,7 @@ public class TestLs {
|
|||
options.add("-S");
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertFalse(ls.isPathOnly());
|
||||
assertTrue(ls.isDirRecurse());
|
||||
assertFalse(ls.isHumanReadable());
|
||||
assertFalse(ls.isRecursive());
|
||||
|
@ -170,6 +193,7 @@ public class TestLs {
|
|||
options.add("-t");
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertFalse(ls.isPathOnly());
|
||||
assertTrue(ls.isDirRecurse());
|
||||
assertFalse(ls.isHumanReadable());
|
||||
assertFalse(ls.isRecursive());
|
||||
|
@ -187,6 +211,7 @@ public class TestLs {
|
|||
options.add("-S");
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertFalse(ls.isPathOnly());
|
||||
assertTrue(ls.isDirRecurse());
|
||||
assertFalse(ls.isHumanReadable());
|
||||
assertFalse(ls.isRecursive());
|
||||
|
@ -205,6 +230,7 @@ public class TestLs {
|
|||
options.add("-r");
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertFalse(ls.isPathOnly());
|
||||
assertTrue(ls.isDirRecurse());
|
||||
assertFalse(ls.isHumanReadable());
|
||||
assertFalse(ls.isRecursive());
|
||||
|
@ -221,6 +247,7 @@ public class TestLs {
|
|||
options.add("-u");
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertFalse(ls.isPathOnly());
|
||||
assertTrue(ls.isDirRecurse());
|
||||
assertFalse(ls.isHumanReadable());
|
||||
assertFalse(ls.isRecursive());
|
||||
|
@ -234,6 +261,7 @@ public class TestLs {
|
|||
@Test
|
||||
public void processOptionsAll() throws IOException {
|
||||
LinkedList<String> options = new LinkedList<String>();
|
||||
options.add("-C"); // show file path only
|
||||
options.add("-d"); // directory
|
||||
options.add("-h"); // human readable
|
||||
options.add("-R"); // recursive
|
||||
|
@ -243,6 +271,7 @@ public class TestLs {
|
|||
options.add("-u"); // show atime
|
||||
Ls ls = new Ls();
|
||||
ls.processOptions(options);
|
||||
assertTrue(ls.isPathOnly());
|
||||
assertFalse(ls.isDirRecurse());
|
||||
assertTrue(ls.isHumanReadable());
|
||||
assertFalse(ls.isRecursive()); // -d overrules -R
|
||||
|
@ -981,6 +1010,44 @@ public class TestLs {
|
|||
verifyNoMoreInteractions(out);
|
||||
}
|
||||
|
||||
// check path only display (-C option)
|
||||
@Test
|
||||
public void processPathDirectoryPathOnly() throws IOException {
|
||||
TestFile testfile01 = new TestFile("testDirectory", "testFile01");
|
||||
TestFile testfile02 = new TestFile("testDirectory", "testFile02");
|
||||
TestFile testfile03 = new TestFile("testDirectory", "testFile03");
|
||||
TestFile testfile04 = new TestFile("testDirectory", "testFile04");
|
||||
TestFile testfile05 = new TestFile("testDirectory", "testFile05");
|
||||
TestFile testfile06 = new TestFile("testDirectory", "testFile06");
|
||||
|
||||
TestFile testDir = new TestFile("", "testDirectory");
|
||||
testDir.setIsDir(true);
|
||||
testDir.addContents(testfile01, testfile02, testfile03, testfile04,
|
||||
testfile05, testfile06);
|
||||
|
||||
LinkedList<PathData> pathData = new LinkedList<PathData>();
|
||||
pathData.add(testDir.getPathData());
|
||||
|
||||
PrintStream out = mock(PrintStream.class);
|
||||
|
||||
Ls ls = new Ls();
|
||||
ls.out = out;
|
||||
|
||||
LinkedList<String> options = new LinkedList<String>();
|
||||
options.add("-C");
|
||||
ls.processOptions(options);
|
||||
|
||||
ls.processArguments(pathData);
|
||||
InOrder inOrder = inOrder(out);
|
||||
inOrder.verify(out).println(testfile01.getPath().toString());
|
||||
inOrder.verify(out).println(testfile02.getPath().toString());
|
||||
inOrder.verify(out).println(testfile03.getPath().toString());
|
||||
inOrder.verify(out).println(testfile04.getPath().toString());
|
||||
inOrder.verify(out).println(testfile05.getPath().toString());
|
||||
inOrder.verify(out).println(testfile06.getPath().toString());
|
||||
verifyNoMoreInteractions(out);
|
||||
}
|
||||
|
||||
// check the deprecated flag isn't set
|
||||
@Test
|
||||
public void isDeprecated() {
|
||||
|
|
|
@ -54,7 +54,7 @@
|
|||
<comparators>
|
||||
<comparator>
|
||||
<type>RegexpComparator</type>
|
||||
<expected-output>^-ls \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)*</expected-output>
|
||||
<expected-output>^-ls \[-C\] \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)*</expected-output>
|
||||
</comparator>
|
||||
<comparator>
|
||||
<type>RegexpComparator</type>
|
||||
|
@ -92,6 +92,10 @@
|
|||
<type>RegexpComparator</type>
|
||||
<expected-output>^\s*modificationDate\(yyyy-MM-dd HH:mm\) fileName( )*</expected-output>
|
||||
</comparator>
|
||||
<comparator>
|
||||
<type>RegexpComparator</type>
|
||||
<expected-output>^\s*-C\s+Display the paths of files and directories only\.( )*</expected-output>
|
||||
</comparator>
|
||||
<comparator>
|
||||
<type>RegexpComparator</type>
|
||||
<expected-output>^\s*-d\s+Directories are listed as plain files\.( )*</expected-output>
|
||||
|
|
Loading…
Reference in New Issue