Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-7713. dfs -count -q should label output column (Jonathan Allen via aw)

This commit is contained in:
Allen Wittenauer 2015-02-05 07:44:49 -08:00 committed by Arpit Agarwal
parent 7c072bf092
commit b7a052c84a
7 changed files with 226 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -16,7 +16,6 @@ Release 2.8.0 - UNRELEASED
HADOOP-11719. [Fsshell] Remove bin/hadoop reference from
GenericOptionsParser default help text.
(Brahma Reddy Battula via harsh)
HADOOP-11692. Improve authentication failure WARN message to avoid user
confusion. (Yongjun Zhang)
@ -48,6 +47,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11814. Reformat hadoop-annotations, o.a.h.classification.tools.
(Li Lu via wheat9)
HADOOP-7713. dfs -count -q should label output column (Jonathan Allen
via aw)
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp

57
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
View File

@ -238,28 +238,35 @@ public class ContentSummary implements Writable{
this.spaceConsumed = in.readLong();
this.spaceQuota = in.readLong();
}
/**
/**
* Output format:
* <----12----> <----12----> <-------18------->
* DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME
* DIR_COUNT FILE_COUNT CONTENT_SIZE
*/
private static final String STRING_FORMAT = "%12s %12s %18s ";
/**
private static final String SUMMARY_FORMAT = "%12s %12s %18s ";
/**
* Output format:
* <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18------->
* QUOTA REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME
* <----12----> <------15-----> <------15-----> <------15----->
* QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA
* <----12----> <----12----> <-------18------->
* DIR_COUNT FILE_COUNT CONTENT_SIZE
*/
private static final String QUOTA_STRING_FORMAT = "%12s %15s ";
private static final String SPACE_QUOTA_STRING_FORMAT = "%15s %15s ";
private static final String QUOTA_SUMMARY_FORMAT = "%12s %15s ";
private static final String SPACE_QUOTA_SUMMARY_FORMAT = "%15s %15s ";
private static final String[] HEADER_FIELDS = new String[] { "DIR_COUNT",
"FILE_COUNT", "CONTENT_SIZE"};
private static final String[] QUOTA_HEADER_FIELDS = new String[] { "QUOTA",
"REM_QUOTA", "SPACE_QUOTA", "REM_SPACE_QUOTA" };
/** The header string */
private static final String HEADER = String.format(
STRING_FORMAT.replace('d', 's'), "directories", "files", "bytes");
SUMMARY_FORMAT, (Object[]) HEADER_FIELDS);
private static final String QUOTA_HEADER = String.format(
QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
"name quota", "rem name quota", "space quota", "rem space quota") +
QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
(Object[]) QUOTA_HEADER_FIELDS) +
HEADER;
/** Return the header of the output.
@ -272,7 +279,25 @@ public class ContentSummary implements Writable{
public static String getHeader(boolean qOption) {
return qOption ? QUOTA_HEADER : HEADER;
}
/**
* Returns the names of the fields from the summary header.
*
* @return names of fields as displayed in the header
*/
public static String[] getHeaderFields() {
return HEADER_FIELDS;
}
/**
* Returns the names of the fields used in the quota summary.
*
* @return names of quota fields as displayed in the header
*/
public static String[] getQuotaHeaderFields() {
return QUOTA_HEADER_FIELDS;
}
@Override
public String toString() {
return toString(true);
@ -316,11 +341,11 @@ public class ContentSummary implements Writable{
spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
}
prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
prefix = String.format(QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem);
}
return prefix + String.format(STRING_FORMAT,
return prefix + String.format(SUMMARY_FORMAT,
formatSize(directoryCount, hOption),
formatSize(fileCount, hOption),
formatSize(length, hOption));

32
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.LinkedList;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@ -44,18 +45,26 @@ public class Count extends FsCommand {
private static final String OPTION_QUOTA = "q";
private static final String OPTION_HUMAN = "h";
private static final String OPTION_HEADER = "v";
public static final String NAME = "count";
public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] <path> ...";
public static final String DESCRIPTION =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
"that match the specified file pattern. The output columns are:\n" +
"DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" +
"QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" +
" DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" +
"The -h option shows file sizes in human readable format.";
"that match the specified file pattern. The output columns are:\n" +
StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
" PATHNAME\n" +
"or, with the -" + OPTION_QUOTA + " option:\n" +
StringUtils.join(ContentSummary.getQuotaHeaderFields(), ' ') + "\n" +
" " +
StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
" PATHNAME\n" +
"The -" + OPTION_HUMAN +
" option shows file sizes in human readable format.\n" +
"The -" + OPTION_HEADER + " option displays a header line.";
private boolean showQuotas;
private boolean humanReadable;
@ -65,7 +74,7 @@ public class Count extends FsCommand {
/** Constructor
* @deprecated invoke via {@link FsShell}
* @param cmd the count command
* @param pos the starting index of the arguments
* @param pos the starting index of the arguments
* @param conf configuration
*/
@Deprecated
@ -77,13 +86,16 @@ public class Count extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args) {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
OPTION_QUOTA, OPTION_HUMAN);
OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
args.add(".");
}
showQuotas = cf.getOpt(OPTION_QUOTA);
humanReadable = cf.getOpt(OPTION_HUMAN);
if (cf.getOpt(OPTION_HEADER)) {
out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME");
}
}
@Override

6
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
View File

@ -140,15 +140,15 @@ public class TestContentSummary {
// check the header with quotas
@Test
public void testGetHeaderWithQuota() {
String header = " name quota rem name quota space quota "
+ "rem space quota directories files bytes ";
String header = " QUOTA REM_QUOTA SPACE_QUOTA "
+ "REM_SPACE_QUOTA DIR_COUNT FILE_COUNT CONTENT_SIZE ";
assertEquals(header, ContentSummary.getHeader(true));
}
// check the header without quotas
@Test
public void testGetHeaderNoQuota() {
String header = " directories files bytes ";
String header = " DIR_COUNT FILE_COUNT CONTENT_SIZE ";
assertEquals(header, ContentSummary.getHeader(false));
}

110
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
View File

@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.CommandFormat.NotEnoughArgumentsException;
import org.junit.Test;
import org.junit.Before;
import org.junit.BeforeClass;
@ -47,7 +48,6 @@ public class TestCount {
private static Configuration conf;
private static FileSystem mockFs;
private static FileStatus fileStat;
private static ContentSummary mockCs;
@BeforeClass
public static void setup() {
@ -55,7 +55,6 @@ public class TestCount {
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
mockFs = mock(FileSystem.class);
fileStat = mock(FileStatus.class);
mockCs = mock(ContentSummary.class);
when(fileStat.isFile()).thenReturn(true);
}
@ -87,6 +86,85 @@ public class TestCount {
assertTrue(count.isHumanReadable());
}
// check no options is handled correctly
@Test
public void processOptionsNoOptions() {
LinkedList<String> options = new LinkedList<String>();
options.add("dummy");
Count count = new Count();
count.processOptions(options);
assertFalse(count.isShowQuotas());
}
// check -q is handled correctly
@Test
public void processOptionsShowQuotas() {
LinkedList<String> options = new LinkedList<String>();
options.add("-q");
options.add("dummy");
Count count = new Count();
count.processOptions(options);
assertTrue(count.isShowQuotas());
}
// check missing arguments is handled correctly
@Test
public void processOptionsMissingArgs() {
LinkedList<String> options = new LinkedList<String>();
Count count = new Count();
try {
count.processOptions(options);
fail("Count.processOptions - NotEnoughArgumentsException not thrown");
} catch (NotEnoughArgumentsException e) {
}
assertFalse(count.isShowQuotas());
}
// check the correct header is produced with no quotas (-v)
@Test
public void processOptionsHeaderNoQuotas() {
LinkedList<String> options = new LinkedList<String>();
options.add("-v");
options.add("dummy");
PrintStream out = mock(PrintStream.class);
Count count = new Count();
count.out = out;
count.processOptions(options);
String noQuotasHeader =
// <----12----> <----12----> <-------18------->
" DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME";
verify(out).println(noQuotasHeader);
verifyNoMoreInteractions(out);
}
// check the correct header is produced with quotas (-q -v)
@Test
public void processOptionsHeaderWithQuotas() {
LinkedList<String> options = new LinkedList<String>();
options.add("-q");
options.add("-v");
options.add("dummy");
PrintStream out = mock(PrintStream.class);
Count count = new Count();
count.out = out;
count.processOptions(options);
String withQuotasHeader =
// <----12----> <-----15------> <-----15------> <-----15------>
" QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA " +
// <----12----> <----12----> <-------18------->
" DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME";
verify(out).println(withQuotasHeader);
verifyNoMoreInteractions(out);
}
// check quotas are reported correctly
@Test
public void processPathShowQuotas() throws Exception {
@ -211,10 +289,28 @@ public class TestCount {
public void getUsage() {
Count count = new Count();
String actual = count.getUsage();
String expected = "-count [-q] [-h] <path> ...";
String expected = "-count [-q] [-h] [-v] <path> ...";
assertEquals("Count.getUsage", expected, actual);
}
// check the correct description is returned
@Test
public void getDescription() {
Count count = new Count();
String actual = count.getDescription();
String expected =
"Count the number of directories, files and bytes under the paths\n"
+ "that match the specified file pattern. The output columns are:\n"
+ "DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ "or, with the -q option:\n"
+ "QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA\n"
+ " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ "The -h option shows file sizes in human readable format.\n"
+ "The -v option displays a header line.";
assertEquals("Count.getDescription", expected, actual);
}
// mock content system
static class MockContentSummary extends ContentSummary {
@ -227,15 +323,15 @@ public class TestCount {
public String toString(boolean qOption, boolean hOption) {
if (qOption) {
if (hOption) {
return(HUMAN + WITH_QUOTAS);
return (HUMAN + WITH_QUOTAS);
} else {
return(BYTES + WITH_QUOTAS);
return (BYTES + WITH_QUOTAS);
}
} else {
if (hOption) {
return(HUMAN + NO_QUOTAS);
return (HUMAN + NO_QUOTAS);
} else {
return(BYTES + NO_QUOTAS);
return (BYTES + NO_QUOTAS);
}
}
}

16
hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
View File

@ -238,7 +238,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-count \[-q\] \[-h\] &lt;path&gt; \.\.\. :( )*</expected-output>
<expected-output>^-count \[-q\] \[-h\] \[-v\] &lt;path&gt; \.\.\. :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@ -250,20 +250,28 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or( )*</expected-output>
<expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^( |\t)*QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA( )*</expected-output>
<expected-output>^( |\t)*or, with the -q option:( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )*</expected-output>
<expected-output>^( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^( |\t)*The -h option shows file sizes in human readable format.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output>
</comparator>
</comparators>
</test>

42
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
View File

@ -8699,6 +8699,48 @@
</comparators>
</test>
<test> <!-- TESTED -->
<description>count: file using absolute path showing header record</description>
<test-commands>
<command>-fs NAMENODE -touchz /file1</command>
<command>-fs NAMENODE -count -v /file1</command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /file1</command>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output>
</comparator>
</comparators>
</test>
<test> <!-- TESTED -->
<description>count: file using absolute path with -q option and showing header record</description>
<test-commands>
<command>-fs NAMENODE -touchz /file1</command>
<command>-fs NAMENODE -count -q -v /file1</command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /file1</command>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>( |\t)*none( |\t)*inf( |\t)*none( |\t)*inf( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output>
</comparator>
</comparators>
</test>
<!-- Tests for chmod -->
<test> <!-- TESTED -->
<description>chmod: change permission(octal mode) of file in absolute path</description>