Add -E option in 'ls' to list erasure coding policy of each file and directory if applicable. Contributed by luhuichun via lei.
This commit is contained in:
parent
25d891a784
commit
d6df0fdbbd
|
@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.fs.FileStatus;
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.fs.ContentSummary;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a listing of all files in that match the file patterns.
|
* Get a listing of all files in that match the file patterns.
|
||||||
|
@ -54,13 +55,14 @@ class Ls extends FsCommand {
|
||||||
private static final String OPTION_MTIME = "t";
|
private static final String OPTION_MTIME = "t";
|
||||||
private static final String OPTION_ATIME = "u";
|
private static final String OPTION_ATIME = "u";
|
||||||
private static final String OPTION_SIZE = "S";
|
private static final String OPTION_SIZE = "S";
|
||||||
|
private static final String OPTION_ECPOLICY = "e";
|
||||||
|
|
||||||
public static final String NAME = "ls";
|
public static final String NAME = "ls";
|
||||||
public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-" +
|
public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-" +
|
||||||
OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" +
|
OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" +
|
||||||
OPTION_HIDENONPRINTABLE + "] [-" + OPTION_RECURSIVE + "] [-" +
|
OPTION_HIDENONPRINTABLE + "] [-" + OPTION_RECURSIVE + "] [-" +
|
||||||
OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] [-" +
|
OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] [-" +
|
||||||
OPTION_ATIME + "] [<path> ...]";
|
OPTION_ATIME + "] [-" + OPTION_ECPOLICY +"] [<path> ...]";
|
||||||
|
|
||||||
public static final String DESCRIPTION =
|
public static final String DESCRIPTION =
|
||||||
"List the contents that match the specified file pattern. If " +
|
"List the contents that match the specified file pattern. If " +
|
||||||
|
@ -91,7 +93,9 @@ class Ls extends FsCommand {
|
||||||
" Reverse the order of the sort.\n" +
|
" Reverse the order of the sort.\n" +
|
||||||
" -" + OPTION_ATIME +
|
" -" + OPTION_ATIME +
|
||||||
" Use time of last access instead of modification for\n" +
|
" Use time of last access instead of modification for\n" +
|
||||||
" display and sorting.";
|
" display and sorting.\n"+
|
||||||
|
" -" + OPTION_ECPOLICY +
|
||||||
|
" Display the erasure coding policy of files and directories.\n";
|
||||||
|
|
||||||
protected final SimpleDateFormat dateFormat =
|
protected final SimpleDateFormat dateFormat =
|
||||||
new SimpleDateFormat("yyyy-MM-dd HH:mm");
|
new SimpleDateFormat("yyyy-MM-dd HH:mm");
|
||||||
|
@ -104,6 +108,7 @@ class Ls extends FsCommand {
|
||||||
private boolean orderTime;
|
private boolean orderTime;
|
||||||
private boolean orderSize;
|
private boolean orderSize;
|
||||||
private boolean useAtime;
|
private boolean useAtime;
|
||||||
|
private boolean displayECPolicy;
|
||||||
private Comparator<PathData> orderComparator;
|
private Comparator<PathData> orderComparator;
|
||||||
|
|
||||||
protected boolean humanReadable = false;
|
protected boolean humanReadable = false;
|
||||||
|
@ -129,7 +134,7 @@ class Ls extends FsCommand {
|
||||||
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE,
|
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE,
|
||||||
OPTION_PATHONLY, OPTION_DIRECTORY, OPTION_HUMAN,
|
OPTION_PATHONLY, OPTION_DIRECTORY, OPTION_HUMAN,
|
||||||
OPTION_HIDENONPRINTABLE, OPTION_RECURSIVE, OPTION_REVERSE,
|
OPTION_HIDENONPRINTABLE, OPTION_RECURSIVE, OPTION_REVERSE,
|
||||||
OPTION_MTIME, OPTION_SIZE, OPTION_ATIME);
|
OPTION_MTIME, OPTION_SIZE, OPTION_ATIME, OPTION_ECPOLICY);
|
||||||
cf.parse(args);
|
cf.parse(args);
|
||||||
pathOnly = cf.getOpt(OPTION_PATHONLY);
|
pathOnly = cf.getOpt(OPTION_PATHONLY);
|
||||||
dirRecurse = !cf.getOpt(OPTION_DIRECTORY);
|
dirRecurse = !cf.getOpt(OPTION_DIRECTORY);
|
||||||
|
@ -140,6 +145,7 @@ class Ls extends FsCommand {
|
||||||
orderTime = cf.getOpt(OPTION_MTIME);
|
orderTime = cf.getOpt(OPTION_MTIME);
|
||||||
orderSize = !orderTime && cf.getOpt(OPTION_SIZE);
|
orderSize = !orderTime && cf.getOpt(OPTION_SIZE);
|
||||||
useAtime = cf.getOpt(OPTION_ATIME);
|
useAtime = cf.getOpt(OPTION_ATIME);
|
||||||
|
displayECPolicy = cf.getOpt(OPTION_ECPOLICY);
|
||||||
if (args.isEmpty()) args.add(Path.CUR_DIR);
|
if (args.isEmpty()) args.add(Path.CUR_DIR);
|
||||||
|
|
||||||
initialiseOrderComparator();
|
initialiseOrderComparator();
|
||||||
|
@ -245,25 +251,42 @@ class Ls extends FsCommand {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
FileStatus stat = item.stat;
|
FileStatus stat = item.stat;
|
||||||
String line = String.format(lineFormat,
|
if (displayECPolicy) {
|
||||||
(stat.isDirectory() ? "d" : "-"),
|
ContentSummary contentSummary = item.fs.getContentSummary(item.path);
|
||||||
stat.getPermission() + (stat.getPermission().getAclBit() ? "+" : " "),
|
String line = String.format(lineFormat,
|
||||||
(stat.isFile() ? stat.getReplication() : "-"),
|
(stat.isDirectory() ? "d" : "-"),
|
||||||
stat.getOwner(),
|
stat.getPermission() + (stat.getPermission().getAclBit() ? "+" : " "),
|
||||||
stat.getGroup(),
|
(stat.isFile() ? stat.getReplication() : "-"),
|
||||||
formatSize(stat.getLen()),
|
stat.getOwner(),
|
||||||
dateFormat.format(new Date(isUseAtime()
|
stat.getGroup(),
|
||||||
? stat.getAccessTime()
|
contentSummary.getErasureCodingPolicy(),
|
||||||
: stat.getModificationTime())),
|
formatSize(stat.getLen()),
|
||||||
isHideNonPrintable() ? new PrintableString(item.toString()) : item);
|
dateFormat.format(new Date(isUseAtime()
|
||||||
out.println(line);
|
? stat.getAccessTime()
|
||||||
|
: stat.getModificationTime())),
|
||||||
|
isHideNonPrintable() ? new PrintableString(item.toString()) : item);
|
||||||
|
out.println(line);
|
||||||
|
} else {
|
||||||
|
String line = String.format(lineFormat,
|
||||||
|
(stat.isDirectory() ? "d" : "-"),
|
||||||
|
stat.getPermission() + (stat.getPermission().getAclBit() ? "+" : " "),
|
||||||
|
(stat.isFile() ? stat.getReplication() : "-"),
|
||||||
|
stat.getOwner(),
|
||||||
|
stat.getGroup(),
|
||||||
|
formatSize(stat.getLen()),
|
||||||
|
dateFormat.format(new Date(isUseAtime()
|
||||||
|
? stat.getAccessTime()
|
||||||
|
: stat.getModificationTime())),
|
||||||
|
isHideNonPrintable() ? new PrintableString(item.toString()) : item);
|
||||||
|
out.println(line);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compute column widths and rebuild the format string
|
* Compute column widths and rebuild the format string
|
||||||
* @param items to find the max field width for each column
|
* @param items to find the max field width for each column
|
||||||
*/
|
*/
|
||||||
private void adjustColumnWidths(PathData items[]) {
|
private void adjustColumnWidths(PathData items[]) throws IOException {
|
||||||
for (PathData item : items) {
|
for (PathData item : items) {
|
||||||
FileStatus stat = item.stat;
|
FileStatus stat = item.stat;
|
||||||
maxRepl = maxLength(maxRepl, stat.getReplication());
|
maxRepl = maxLength(maxRepl, stat.getReplication());
|
||||||
|
@ -278,6 +301,14 @@ class Ls extends FsCommand {
|
||||||
// Do not use '%-0s' as a formatting conversion, since it will throw a
|
// Do not use '%-0s' as a formatting conversion, since it will throw a
|
||||||
// a MissingFormatWidthException if it is used in String.format().
|
// a MissingFormatWidthException if it is used in String.format().
|
||||||
// http://docs.oracle.com/javase/1.5.0/docs/api/java/util/Formatter.html#intFlags
|
// http://docs.oracle.com/javase/1.5.0/docs/api/java/util/Formatter.html#intFlags
|
||||||
|
if(displayECPolicy){
|
||||||
|
int maxEC=0;
|
||||||
|
for (PathData item : items) {
|
||||||
|
ContentSummary contentSummary = item.fs.getContentSummary(item.path);
|
||||||
|
maxEC=maxLength(maxEC,contentSummary.getErasureCodingPolicy().length());
|
||||||
|
}
|
||||||
|
fmt.append(" %"+maxEC+"s ");
|
||||||
|
}
|
||||||
fmt.append((maxOwner > 0) ? "%-" + maxOwner + "s " : "%s");
|
fmt.append((maxOwner > 0) ? "%-" + maxOwner + "s " : "%s");
|
||||||
fmt.append((maxGroup > 0) ? "%-" + maxGroup + "s " : "%s");
|
fmt.append((maxGroup > 0) ? "%-" + maxGroup + "s " : "%s");
|
||||||
fmt.append("%" + maxLen + "s ");
|
fmt.append("%" + maxLen + "s ");
|
||||||
|
|
|
@ -410,7 +410,7 @@ Return usage output.
|
||||||
ls
|
ls
|
||||||
----
|
----
|
||||||
|
|
||||||
Usage: `hadoop fs -ls [-C] [-d] [-h] [-q] [-R] [-t] [-S] [-r] [-u] <args> `
|
Usage: `hadoop fs -ls [-C] [-d] [-h] [-q] [-R] [-t] [-S] [-r] [-u] [-e] <args> `
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
|
|
||||||
|
@ -423,6 +423,7 @@ Options:
|
||||||
* -S: Sort output by file size.
|
* -S: Sort output by file size.
|
||||||
* -r: Reverse the sort order.
|
* -r: Reverse the sort order.
|
||||||
* -u: Use access time rather than modification time for display and sorting.
|
* -u: Use access time rather than modification time for display and sorting.
|
||||||
|
* -e: Display the erasure coding policy of files and directories only.
|
||||||
|
|
||||||
For a file ls returns stat on the file with the following format:
|
For a file ls returns stat on the file with the following format:
|
||||||
|
|
||||||
|
@ -437,6 +438,7 @@ Files within a directory are order by filename by default.
|
||||||
Example:
|
Example:
|
||||||
|
|
||||||
* `hadoop fs -ls /user/hadoop/file1`
|
* `hadoop fs -ls /user/hadoop/file1`
|
||||||
|
* `hadoop fs -ls -e /ecdir`
|
||||||
|
|
||||||
Exit Code:
|
Exit Code:
|
||||||
|
|
||||||
|
|
|
@ -54,7 +54,7 @@
|
||||||
<comparators>
|
<comparators>
|
||||||
<comparator>
|
<comparator>
|
||||||
<type>RegexpComparator</type>
|
<type>RegexpComparator</type>
|
||||||
<expected-output>^-ls \[-C\] \[-d\] \[-h\] \[-q\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)*</expected-output>
|
<expected-output>^-ls \[-C\] \[-d\] \[-h\] \[-q\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[-e\] \[<path> \.\.\.\] :( |\t)*</expected-output>
|
||||||
</comparator>
|
</comparator>
|
||||||
<comparator>
|
<comparator>
|
||||||
<type>RegexpComparator</type>
|
<type>RegexpComparator</type>
|
||||||
|
@ -136,6 +136,10 @@
|
||||||
<type>RegexpComparator</type>
|
<type>RegexpComparator</type>
|
||||||
<expected-output>^( |\t)*display and sorting\.</expected-output>
|
<expected-output>^( |\t)*display and sorting\.</expected-output>
|
||||||
</comparator>
|
</comparator>
|
||||||
|
<comparator>
|
||||||
|
<type>RegexpComparator</type>
|
||||||
|
<expected-output>^( |\t)*-e\s+Display the erasure coding policy of files and directories\.</expected-output>
|
||||||
|
</comparator>
|
||||||
</comparators>
|
</comparators>
|
||||||
</test>
|
</test>
|
||||||
|
|
||||||
|
|
|
@ -773,5 +773,39 @@
|
||||||
</comparator>
|
</comparator>
|
||||||
</comparators>
|
</comparators>
|
||||||
</test>
|
</test>
|
||||||
|
|
||||||
|
<test> <!-- TESTED -->
|
||||||
|
<description>ls: file using absolute path and option -e to show erasure coding policy of a directory</description>
|
||||||
|
<test-commands>
|
||||||
|
<command>-fs NAMENODE -mkdir -p /ecdir</command>
|
||||||
|
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-6-3-64k</ec-admin-command>
|
||||||
|
<command>-fs NAMENODE -touchz /ecdir/file1</command>
|
||||||
|
<command>-fs NAMENODE -touchz /ecdir/file2</command>
|
||||||
|
<command>-fs NAMENODE -touchz /ecdir/file3</command>
|
||||||
|
<command>-fs NAMENODE -ls -e /ecdir</command>
|
||||||
|
</test-commands>
|
||||||
|
<cleanup-commands>
|
||||||
|
<command>-fs NAMENODE -rmdir /ecdir</command>
|
||||||
|
</cleanup-commands>
|
||||||
|
<comparators>
|
||||||
|
<comparator>
|
||||||
|
<type>RegexpComparator</type>
|
||||||
|
<expected-output>Found [0-9] items</expected-output>
|
||||||
|
</comparator>
|
||||||
|
<comparator>
|
||||||
|
<type>RegexpComparator</type>
|
||||||
|
<expected-output>^-rw-r--r--( )*1( )*USERNAME( )*supergroup( )*[A-Za-z0-9-]{1,}( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/ecdir/file1</expected-output>
|
||||||
|
</comparator>
|
||||||
|
<comparator>
|
||||||
|
<type>RegexpComparator</type>
|
||||||
|
<expected-output>^-rw-r--r--( )*1( )*USERNAME( )*supergroup( )*[A-Za-z0-9-]{1,}( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/ecdir/file2</expected-output>
|
||||||
|
</comparator>
|
||||||
|
<comparator>
|
||||||
|
<type>RegexpComparator</type>
|
||||||
|
<expected-output>^-rw-r--r--( )*1( )*USERNAME( )*supergroup( )*[A-Za-z0-9-]{1,}( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/ecdir/file3</expected-output>
|
||||||
|
</comparator>
|
||||||
|
</comparators>
|
||||||
|
</test>
|
||||||
|
|
||||||
</tests>
|
</tests>
|
||||||
</configuration>
|
</configuration>
|
||||||
|
|
Loading…
Reference in New Issue