HADOOP-7713. dfs -count -q should label output column (Jonathan Allen via aw)

This commit is contained in:
Allen Wittenauer 2015-02-05 07:44:49 -08:00 committed by Arpit Agarwal
parent 7c072bf092
commit b7a052c84a
7 changed files with 226 additions and 41 deletions

View File

@ -16,7 +16,6 @@ Release 2.8.0 - UNRELEASED
HADOOP-11719. [Fsshell] Remove bin/hadoop reference from HADOOP-11719. [Fsshell] Remove bin/hadoop reference from
GenericOptionsParser default help text. GenericOptionsParser default help text.
(Brahma Reddy Battula via harsh)
HADOOP-11692. Improve authentication failure WARN message to avoid user HADOOP-11692. Improve authentication failure WARN message to avoid user
confusion. (Yongjun Zhang) confusion. (Yongjun Zhang)
@ -48,6 +47,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11814. Reformat hadoop-annotations, o.a.h.classification.tools. HADOOP-11814. Reformat hadoop-annotations, o.a.h.classification.tools.
(Li Lu via wheat9) (Li Lu via wheat9)
HADOOP-7713. dfs -count -q should label output column (Jonathan Allen
via aw)
OPTIMIZATIONS OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp HADOOP-11785. Reduce the number of listStatus operation in distcp

View File

@ -242,24 +242,31 @@ public class ContentSummary implements Writable{
/** /**
* Output format: * Output format:
* <----12----> <----12----> <-------18-------> * <----12----> <----12----> <-------18------->
* DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME * DIR_COUNT FILE_COUNT CONTENT_SIZE
*/ */
private static final String STRING_FORMAT = "%12s %12s %18s "; private static final String SUMMARY_FORMAT = "%12s %12s %18s ";
/** /**
* Output format: * Output format:
* <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18-------> * <----12----> <------15-----> <------15-----> <------15----->
* QUOTA REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME * QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA
* <----12----> <----12----> <-------18------->
* DIR_COUNT FILE_COUNT CONTENT_SIZE
*/ */
private static final String QUOTA_STRING_FORMAT = "%12s %15s "; private static final String QUOTA_SUMMARY_FORMAT = "%12s %15s ";
private static final String SPACE_QUOTA_STRING_FORMAT = "%15s %15s "; private static final String SPACE_QUOTA_SUMMARY_FORMAT = "%15s %15s ";
private static final String[] HEADER_FIELDS = new String[] { "DIR_COUNT",
"FILE_COUNT", "CONTENT_SIZE"};
private static final String[] QUOTA_HEADER_FIELDS = new String[] { "QUOTA",
"REM_QUOTA", "SPACE_QUOTA", "REM_SPACE_QUOTA" };
/** The header string */ /** The header string */
private static final String HEADER = String.format( private static final String HEADER = String.format(
STRING_FORMAT.replace('d', 's'), "directories", "files", "bytes"); SUMMARY_FORMAT, (Object[]) HEADER_FIELDS);
private static final String QUOTA_HEADER = String.format( private static final String QUOTA_HEADER = String.format(
QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
"name quota", "rem name quota", "space quota", "rem space quota") + (Object[]) QUOTA_HEADER_FIELDS) +
HEADER; HEADER;
/** Return the header of the output. /** Return the header of the output.
@ -273,6 +280,24 @@ public class ContentSummary implements Writable{
return qOption ? QUOTA_HEADER : HEADER; return qOption ? QUOTA_HEADER : HEADER;
} }
/**
* Returns the names of the fields from the summary header.
*
* @return names of fields as displayed in the header
*/
public static String[] getHeaderFields() {
return HEADER_FIELDS;
}
/**
* Returns the names of the fields used in the quota summary.
*
* @return names of quota fields as displayed in the header
*/
public static String[] getQuotaHeaderFields() {
return QUOTA_HEADER_FIELDS;
}
@Override @Override
public String toString() { public String toString() {
return toString(true); return toString(true);
@ -316,11 +341,11 @@ public class ContentSummary implements Writable{
spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption); spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
} }
prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, prefix = String.format(QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem); quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem);
} }
return prefix + String.format(STRING_FORMAT, return prefix + String.format(SUMMARY_FORMAT,
formatSize(directoryCount, hOption), formatSize(directoryCount, hOption),
formatSize(fileCount, hOption), formatSize(fileCount, hOption),
formatSize(length, hOption)); formatSize(length, hOption));

View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import java.util.LinkedList; import java.util.LinkedList;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -44,17 +45,25 @@ public class Count extends FsCommand {
private static final String OPTION_QUOTA = "q"; private static final String OPTION_QUOTA = "q";
private static final String OPTION_HUMAN = "h"; private static final String OPTION_HUMAN = "h";
private static final String OPTION_HEADER = "v";
public static final String NAME = "count"; public static final String NAME = "count";
public static final String USAGE = public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] <path> ..."; "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ "] <path> ...";
public static final String DESCRIPTION = public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" + "Count the number of directories, files and bytes under the paths\n" +
"that match the specified file pattern. The output columns are:\n" + "that match the specified file pattern. The output columns are:\n" +
"DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" + StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
"QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" + " PATHNAME\n" +
" DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" + "or, with the -" + OPTION_QUOTA + " option:\n" +
"The -h option shows file sizes in human readable format."; StringUtils.join(ContentSummary.getQuotaHeaderFields(), ' ') + "\n" +
" " +
StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
" PATHNAME\n" +
"The -" + OPTION_HUMAN +
" option shows file sizes in human readable format.\n" +
"The -" + OPTION_HEADER + " option displays a header line.";
private boolean showQuotas; private boolean showQuotas;
private boolean humanReadable; private boolean humanReadable;
@ -77,13 +86,16 @@ public class Count extends FsCommand {
@Override @Override
protected void processOptions(LinkedList<String> args) { protected void processOptions(LinkedList<String> args) {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
OPTION_QUOTA, OPTION_HUMAN); OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER);
cf.parse(args); cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory if (args.isEmpty()) { // default path is the current working directory
args.add("."); args.add(".");
} }
showQuotas = cf.getOpt(OPTION_QUOTA); showQuotas = cf.getOpt(OPTION_QUOTA);
humanReadable = cf.getOpt(OPTION_HUMAN); humanReadable = cf.getOpt(OPTION_HUMAN);
if (cf.getOpt(OPTION_HEADER)) {
out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME");
}
} }
@Override @Override

View File

@ -140,15 +140,15 @@ public class TestContentSummary {
// check the header with quotas // check the header with quotas
@Test @Test
public void testGetHeaderWithQuota() { public void testGetHeaderWithQuota() {
String header = " name quota rem name quota space quota " String header = " QUOTA REM_QUOTA SPACE_QUOTA "
+ "rem space quota directories files bytes "; + "REM_SPACE_QUOTA DIR_COUNT FILE_COUNT CONTENT_SIZE ";
assertEquals(header, ContentSummary.getHeader(true)); assertEquals(header, ContentSummary.getHeader(true));
} }
// check the header without quotas // check the header without quotas
@Test @Test
public void testGetHeaderNoQuota() { public void testGetHeaderNoQuota() {
String header = " directories files bytes "; String header = " DIR_COUNT FILE_COUNT CONTENT_SIZE ";
assertEquals(header, ContentSummary.getHeader(false)); assertEquals(header, ContentSummary.getHeader(false));
} }

View File

@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.CommandFormat.NotEnoughArgumentsException;
import org.junit.Test; import org.junit.Test;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
@ -47,7 +48,6 @@ public class TestCount {
private static Configuration conf; private static Configuration conf;
private static FileSystem mockFs; private static FileSystem mockFs;
private static FileStatus fileStat; private static FileStatus fileStat;
private static ContentSummary mockCs;
@BeforeClass @BeforeClass
public static void setup() { public static void setup() {
@ -55,7 +55,6 @@ public class TestCount {
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
mockFs = mock(FileSystem.class); mockFs = mock(FileSystem.class);
fileStat = mock(FileStatus.class); fileStat = mock(FileStatus.class);
mockCs = mock(ContentSummary.class);
when(fileStat.isFile()).thenReturn(true); when(fileStat.isFile()).thenReturn(true);
} }
@ -87,6 +86,85 @@ public class TestCount {
assertTrue(count.isHumanReadable()); assertTrue(count.isHumanReadable());
} }
// check no options is handled correctly
@Test
public void processOptionsNoOptions() {
LinkedList<String> options = new LinkedList<String>();
options.add("dummy");
Count count = new Count();
count.processOptions(options);
assertFalse(count.isShowQuotas());
}
// check -q is handled correctly
@Test
public void processOptionsShowQuotas() {
LinkedList<String> options = new LinkedList<String>();
options.add("-q");
options.add("dummy");
Count count = new Count();
count.processOptions(options);
assertTrue(count.isShowQuotas());
}
// check missing arguments is handled correctly
@Test
public void processOptionsMissingArgs() {
LinkedList<String> options = new LinkedList<String>();
Count count = new Count();
try {
count.processOptions(options);
fail("Count.processOptions - NotEnoughArgumentsException not thrown");
} catch (NotEnoughArgumentsException e) {
}
assertFalse(count.isShowQuotas());
}
// check the correct header is produced with no quotas (-v)
@Test
public void processOptionsHeaderNoQuotas() {
LinkedList<String> options = new LinkedList<String>();
options.add("-v");
options.add("dummy");
PrintStream out = mock(PrintStream.class);
Count count = new Count();
count.out = out;
count.processOptions(options);
String noQuotasHeader =
// <----12----> <----12----> <-------18------->
" DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME";
verify(out).println(noQuotasHeader);
verifyNoMoreInteractions(out);
}
// check the correct header is produced with quotas (-q -v)
@Test
public void processOptionsHeaderWithQuotas() {
LinkedList<String> options = new LinkedList<String>();
options.add("-q");
options.add("-v");
options.add("dummy");
PrintStream out = mock(PrintStream.class);
Count count = new Count();
count.out = out;
count.processOptions(options);
String withQuotasHeader =
// <----12----> <-----15------> <-----15------> <-----15------>
" QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA " +
// <----12----> <----12----> <-------18------->
" DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME";
verify(out).println(withQuotasHeader);
verifyNoMoreInteractions(out);
}
// check quotas are reported correctly // check quotas are reported correctly
@Test @Test
public void processPathShowQuotas() throws Exception { public void processPathShowQuotas() throws Exception {
@ -211,10 +289,28 @@ public class TestCount {
public void getUsage() { public void getUsage() {
Count count = new Count(); Count count = new Count();
String actual = count.getUsage(); String actual = count.getUsage();
String expected = "-count [-q] [-h] <path> ..."; String expected = "-count [-q] [-h] [-v] <path> ...";
assertEquals("Count.getUsage", expected, actual); assertEquals("Count.getUsage", expected, actual);
} }
// check the correct description is returned
@Test
public void getDescription() {
Count count = new Count();
String actual = count.getDescription();
String expected =
"Count the number of directories, files and bytes under the paths\n"
+ "that match the specified file pattern. The output columns are:\n"
+ "DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ "or, with the -q option:\n"
+ "QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA\n"
+ " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ "The -h option shows file sizes in human readable format.\n"
+ "The -v option displays a header line.";
assertEquals("Count.getDescription", expected, actual);
}
// mock content system // mock content system
static class MockContentSummary extends ContentSummary { static class MockContentSummary extends ContentSummary {
@ -227,15 +323,15 @@ public class TestCount {
public String toString(boolean qOption, boolean hOption) { public String toString(boolean qOption, boolean hOption) {
if (qOption) { if (qOption) {
if (hOption) { if (hOption) {
return(HUMAN + WITH_QUOTAS); return (HUMAN + WITH_QUOTAS);
} else { } else {
return(BYTES + WITH_QUOTAS); return (BYTES + WITH_QUOTAS);
} }
} else { } else {
if (hOption) { if (hOption) {
return(HUMAN + NO_QUOTAS); return (HUMAN + NO_QUOTAS);
} else { } else {
return(BYTES + NO_QUOTAS); return (BYTES + NO_QUOTAS);
} }
} }
} }

View File

@ -238,7 +238,7 @@
<comparators> <comparators>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
<expected-output>^-count \[-q\] \[-h\] &lt;path&gt; \.\.\. :( )*</expected-output> <expected-output>^-count \[-q\] \[-h\] \[-v\] &lt;path&gt; \.\.\. :( )*</expected-output>
</comparator> </comparator>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
@ -250,20 +250,28 @@
</comparator> </comparator>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
<expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or( )*</expected-output> <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output>
</comparator> </comparator>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
<expected-output>^( |\t)*QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA( )*</expected-output> <expected-output>^( |\t)*or, with the -q option:( )*</expected-output>
</comparator> </comparator>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
<expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )*</expected-output> <expected-output>^( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output>
</comparator> </comparator>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
<expected-output>^( |\t)*The -h option shows file sizes in human readable format.( )*</expected-output> <expected-output>^( |\t)*The -h option shows file sizes in human readable format.( )*</expected-output>
</comparator> </comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output>
</comparator>
</comparators> </comparators>
</test> </test>

View File

@ -8699,6 +8699,48 @@
</comparators> </comparators>
</test> </test>
<test> <!-- TESTED -->
<description>count: file using absolute path showing header record</description>
<test-commands>
<command>-fs NAMENODE -touchz /file1</command>
<command>-fs NAMENODE -count -v /file1</command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /file1</command>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output>
</comparator>
</comparators>
</test>
<test> <!-- TESTED -->
<description>count: file using absolute path with -q option and showing header record</description>
<test-commands>
<command>-fs NAMENODE -touchz /file1</command>
<command>-fs NAMENODE -count -q -v /file1</command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /file1</command>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>( |\t)*none( |\t)*inf( |\t)*none( |\t)*inf( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output>
</comparator>
</comparators>
</test>
<!-- Tests for chmod --> <!-- Tests for chmod -->
<test> <!-- TESTED --> <test> <!-- TESTED -->
<description>chmod: change permission(octal mode) of file in absolute path</description> <description>chmod: change permission(octal mode) of file in absolute path</description>