HADOOP-8944. Shell command fs -count should include human readable option (Jonathan Allen via aw)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617775 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
5197f8c3c5
commit
a34dafe325
|
@ -202,6 +202,9 @@ Trunk (Unreleased)
|
|||
HADOOP-10224. JavaKeyStoreProvider has to protect against corrupting
|
||||
underlying store. (asuresh via tucu)
|
||||
|
||||
HADOOP-8944. Shell command fs -count should include human readable option
|
||||
(Jonathan Allen via aw)
|
||||
|
||||
BUG FIXES
|
||||
|
||||
HADOOP-9451. Fault single-layer config if node group topology is enabled.
|
||||
|
|
|
@ -24,6 +24,7 @@ import java.io.IOException;
|
|||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.io.Writable;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
|
||||
/** Store the summary of a content (a directory or a file). */
|
||||
@InterfaceAudience.Public
|
||||
|
@ -102,7 +103,7 @@ public class ContentSummary implements Writable{
|
|||
* <----12----> <----12----> <-------18------->
|
||||
* DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME
|
||||
*/
|
||||
private static final String STRING_FORMAT = "%12d %12d %18d ";
|
||||
private static final String STRING_FORMAT = "%12s %12s %18s ";
|
||||
/**
|
||||
* Output format:
|
||||
* <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18------->
|
||||
|
@ -117,7 +118,7 @@ public class ContentSummary implements Writable{
|
|||
|
||||
private static final String QUOTA_HEADER = String.format(
|
||||
QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
|
||||
"quota", "remaining quota", "space quota", "reamaining quota") +
|
||||
"name quota", "rem name quota", "space quota", "rem space quota") +
|
||||
HEADER;
|
||||
|
||||
/** Return the header of the output.
|
||||
|
@ -142,8 +143,22 @@ public class ContentSummary implements Writable{
|
|||
*
|
||||
* @param qOption a flag indicating if quota needs to be printed or not
|
||||
* @return the string representation of the object
|
||||
*/
|
||||
*/
|
||||
public String toString(boolean qOption) {
|
||||
return toString(qOption, false);
|
||||
}
|
||||
|
||||
/** Return the string representation of the object in the output format.
|
||||
* if qOption is false, output directory count, file count, and content size;
|
||||
* if qOption is true, output quota and remaining quota as well.
|
||||
* if hOption is false file sizes are returned in bytes
|
||||
* if hOption is true file sizes are returned in human readable
|
||||
*
|
||||
* @param qOption a flag indicating if quota needs to be printed or not
|
||||
* @param hOption a flag indicating if human readable output if to be used
|
||||
* @return the string representation of the object
|
||||
*/
|
||||
public String toString(boolean qOption, boolean hOption) {
|
||||
String prefix = "";
|
||||
if (qOption) {
|
||||
String quotaStr = "none";
|
||||
|
@ -152,19 +167,32 @@ public class ContentSummary implements Writable{
|
|||
String spaceQuotaRem = "inf";
|
||||
|
||||
if (quota>0) {
|
||||
quotaStr = Long.toString(quota);
|
||||
quotaRem = Long.toString(quota-(directoryCount+fileCount));
|
||||
quotaStr = formatSize(quota, hOption);
|
||||
quotaRem = formatSize(quota-(directoryCount+fileCount), hOption);
|
||||
}
|
||||
if (spaceQuota>0) {
|
||||
spaceQuotaStr = Long.toString(spaceQuota);
|
||||
spaceQuotaRem = Long.toString(spaceQuota - spaceConsumed);
|
||||
spaceQuotaStr = formatSize(spaceQuota, hOption);
|
||||
spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
|
||||
}
|
||||
|
||||
prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
|
||||
quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem);
|
||||
}
|
||||
|
||||
return prefix + String.format(STRING_FORMAT, directoryCount,
|
||||
fileCount, length);
|
||||
return prefix + String.format(STRING_FORMAT,
|
||||
formatSize(directoryCount, hOption),
|
||||
formatSize(fileCount, hOption),
|
||||
formatSize(length, hOption));
|
||||
}
|
||||
/**
|
||||
* Formats a size to be human readable or in bytes
|
||||
* @param size value to be formatted
|
||||
* @param humanReadable flag indicating human readable or not
|
||||
* @return String representation of the size
|
||||
*/
|
||||
private String formatSize(long size, boolean humanReadable) {
|
||||
return humanReadable
|
||||
? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
|
||||
: String.valueOf(size);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -42,16 +42,22 @@ public class Count extends FsCommand {
|
|||
factory.addClass(Count.class, "-count");
|
||||
}
|
||||
|
||||
private static final String OPTION_QUOTA = "q";
|
||||
private static final String OPTION_HUMAN = "h";
|
||||
|
||||
public static final String NAME = "count";
|
||||
public static final String USAGE = "[-q] <path> ...";
|
||||
public static final String USAGE =
|
||||
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] <path> ...";
|
||||
public static final String DESCRIPTION =
|
||||
"Count the number of directories, files and bytes under the paths\n" +
|
||||
"that match the specified file pattern. The output columns are:\n" +
|
||||
"DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" +
|
||||
"QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" +
|
||||
" DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME";
|
||||
" DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" +
|
||||
"The -h option shows file sizes in human readable format.";
|
||||
|
||||
private boolean showQuotas;
|
||||
private boolean humanReadable;
|
||||
|
||||
/** Constructor */
|
||||
public Count() {}
|
||||
|
@ -70,17 +76,37 @@ public class Count extends FsCommand {
|
|||
|
||||
@Override
|
||||
protected void processOptions(LinkedList<String> args) {
|
||||
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "q");
|
||||
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
|
||||
OPTION_QUOTA, OPTION_HUMAN);
|
||||
cf.parse(args);
|
||||
if (args.isEmpty()) { // default path is the current working directory
|
||||
args.add(".");
|
||||
}
|
||||
showQuotas = cf.getOpt("q");
|
||||
showQuotas = cf.getOpt(OPTION_QUOTA);
|
||||
humanReadable = cf.getOpt(OPTION_HUMAN);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void processPath(PathData src) throws IOException {
|
||||
ContentSummary summary = src.fs.getContentSummary(src.path);
|
||||
out.println(summary.toString(showQuotas) + src);
|
||||
out.println(summary.toString(showQuotas, isHumanReadable()) + src);
|
||||
}
|
||||
|
||||
/**
|
||||
* Should quotas get shown as part of the report?
|
||||
* @return if quotas should be shown then true otherwise false
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
boolean isShowQuotas() {
|
||||
return showQuotas;
|
||||
}
|
||||
|
||||
/**
|
||||
* Should sizes be shown in human readable format rather than bytes?
|
||||
* @return true if human readable format
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
boolean isHumanReadable() {
|
||||
return humanReadable;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -138,7 +138,7 @@ copyToLocal
|
|||
|
||||
count
|
||||
|
||||
Usage: <<<hdfs dfs -count [-q] <paths> >>>
|
||||
Usage: <<<hdfs dfs -count [-q] [-h] <paths> >>>
|
||||
|
||||
Count the number of directories, files and bytes under the paths that match
|
||||
the specified file pattern. The output columns with -count are: DIR_COUNT,
|
||||
|
@ -147,12 +147,16 @@ count
|
|||
The output columns with -count -q are: QUOTA, REMAINING_QUATA, SPACE_QUOTA,
|
||||
REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, FILE_NAME
|
||||
|
||||
The -h option shows sizes in human readable format.
|
||||
|
||||
Example:
|
||||
|
||||
* <<<hdfs dfs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>>
|
||||
|
||||
* <<<hdfs dfs -count -q hdfs://nn1.example.com/file1>>>
|
||||
|
||||
* <<<hdfs dfs -count -q -h hdfs://nn1.example.com/file1>>>
|
||||
|
||||
Exit Code:
|
||||
|
||||
Returns 0 on success and -1 on error.
|
||||
|
|
|
@ -238,7 +238,7 @@
|
|||
<comparators>
|
||||
<comparator>
|
||||
<type>RegexpComparator</type>
|
||||
<expected-output>^-count \[-q\] <path> \.\.\. :\s*</expected-output>
|
||||
<expected-output>^-count \[-q\] \[-h\] <path> \.\.\. :( )*</expected-output>
|
||||
</comparator>
|
||||
<comparator>
|
||||
<type>RegexpComparator</type>
|
||||
|
@ -260,6 +260,10 @@
|
|||
<type>RegexpComparator</type>
|
||||
<expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )*</expected-output>
|
||||
</comparator>
|
||||
<comparator>
|
||||
<type>RegexpComparator</type>
|
||||
<expected-output>^( |\t)*The -h option shows file sizes in human readable format.( )*</expected-output>
|
||||
</comparator>
|
||||
</comparators>
|
||||
</test>
|
||||
|
||||
|
|
|
@ -8655,6 +8655,50 @@
|
|||
</comparators>
|
||||
</test>
|
||||
|
||||
<test> <!-- TESTED -->
|
||||
<description>count: file using -h option</description>
|
||||
<test-commands>
|
||||
<command>-fs NAMENODE -mkdir -p dir</command> <!-- make sure user home dir exists -->
|
||||
<command>-fs NAMENODE -put CLITEST_DATA/data15bytes file1</command>
|
||||
<command>-fs NAMENODE -put CLITEST_DATA/data1k file2</command>
|
||||
<command>-fs NAMENODE -count -h file1 file2</command>
|
||||
</test-commands>
|
||||
<cleanup-commands>
|
||||
<command>-fs NAMENODE -rm file1 file2</command>
|
||||
</cleanup-commands>
|
||||
<comparators>
|
||||
<comparator>
|
||||
<type>RegexpComparator</type>
|
||||
<expected-output>( |\t)*0( |\t)*1( |\t)*15 file1</expected-output>
|
||||
</comparator>
|
||||
</comparators>
|
||||
<comparators>
|
||||
<comparator>
|
||||
<type>RegexpComparator</type>
|
||||
<expected-output>( |\t)*0( |\t)*1( |\t)*1\.0 K file2</expected-output>
|
||||
</comparator>
|
||||
</comparators>
|
||||
</test>
|
||||
|
||||
<test> <!-- TESTED -->
|
||||
<description>count: directory using -q and -h options</description>
|
||||
<test-commands>
|
||||
<command>-fs NAMENODE -mkdir /dir1</command>
|
||||
<dfs-admin-command>-fs NAMENODE -setQuota 10 /dir1 </dfs-admin-command>
|
||||
<dfs-admin-command>-fs NAMENODE -setSpaceQuota 1m /dir1 </dfs-admin-command>
|
||||
<command>-fs NAMENODE -count -q -h /dir1</command>
|
||||
</test-commands>
|
||||
<cleanup-commands>
|
||||
<command>-fs NAMENODE -rm -r /dir1</command>
|
||||
</cleanup-commands>
|
||||
<comparators>
|
||||
<comparator>
|
||||
<type>RegexpComparator</type>
|
||||
<expected-output>( |\t)*10( |\t)*9( |\t)*1 M( |\t)*1 M( |\t)*1( |\t)*0( |\t)*0 /dir1</expected-output>
|
||||
</comparator>
|
||||
</comparators>
|
||||
</test>
|
||||
|
||||
<!-- Tests for chmod -->
|
||||
<test> <!-- TESTED -->
|
||||
<description>chmod: change permission(octal mode) of file in absolute path</description>
|
||||
|
|
Loading…
Reference in New Issue