diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index aff33ee84f0..742a054d749 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -16,7 +16,6 @@ Release 2.8.0 - UNRELEASED HADOOP-11719. [Fsshell] Remove bin/hadoop reference from GenericOptionsParser default help text. - (Brahma Reddy Battula via harsh) HADOOP-11692. Improve authentication failure WARN message to avoid user confusion. (Yongjun Zhang) @@ -48,6 +47,9 @@ Release 2.8.0 - UNRELEASED HADOOP-11814. Reformat hadoop-annotations, o.a.h.classification.tools. (Li Lu via wheat9) + HADOOP-7713. dfs -count -q should label output column (Jonathan Allen + via aw) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java index dc4a7faac71..66137d02c16 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java @@ -238,28 +238,35 @@ public class ContentSummary implements Writable{ this.spaceConsumed = in.readLong(); this.spaceQuota = in.readLong(); } - - /** + + /** * Output format: * <----12----> <----12----> <-------18-------> - * DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME + * DIR_COUNT FILE_COUNT CONTENT_SIZE */ - private static final String STRING_FORMAT = "%12s %12s %18s "; - /** + private static final String SUMMARY_FORMAT = "%12s %12s %18s "; + /** * Output format: - * <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18-------> - * QUOTA REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME + * <----12----> <------15-----> <------15-----> <------15-----> + * QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA + * <----12----> <----12----> <-------18-------> + * DIR_COUNT FILE_COUNT CONTENT_SIZE */ - private static final String QUOTA_STRING_FORMAT = "%12s %15s "; - private static final String SPACE_QUOTA_STRING_FORMAT = "%15s %15s "; - + private static final String QUOTA_SUMMARY_FORMAT = "%12s %15s "; + private static final String SPACE_QUOTA_SUMMARY_FORMAT = "%15s %15s "; + + private static final String[] HEADER_FIELDS = new String[] { "DIR_COUNT", + "FILE_COUNT", "CONTENT_SIZE"}; + private static final String[] QUOTA_HEADER_FIELDS = new String[] { "QUOTA", + "REM_QUOTA", "SPACE_QUOTA", "REM_SPACE_QUOTA" }; + /** The header string */ private static final String HEADER = String.format( - STRING_FORMAT.replace('d', 's'), "directories", "files", "bytes"); + SUMMARY_FORMAT, (Object[]) HEADER_FIELDS); private static final String QUOTA_HEADER = String.format( - QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, - "name quota", "rem name quota", "space quota", "rem space quota") + + QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT, + (Object[]) QUOTA_HEADER_FIELDS) + HEADER; /** Return the header of the output. @@ -272,7 +279,25 @@ public class ContentSummary implements Writable{ public static String getHeader(boolean qOption) { return qOption ? QUOTA_HEADER : HEADER; } - + + /** + * Returns the names of the fields from the summary header. + * + * @return names of fields as displayed in the header + */ + public static String[] getHeaderFields() { + return HEADER_FIELDS; + } + + /** + * Returns the names of the fields used in the quota summary. + * + * @return names of quota fields as displayed in the header + */ + public static String[] getQuotaHeaderFields() { + return QUOTA_HEADER_FIELDS; + } + @Override public String toString() { return toString(true); @@ -316,11 +341,11 @@ public class ContentSummary implements Writable{ spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption); } - prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, + prefix = String.format(QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT, quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem); } - return prefix + String.format(STRING_FORMAT, + return prefix + String.format(SUMMARY_FORMAT, formatSize(directoryCount, hOption), formatSize(fileCount, hOption), formatSize(length, hOption)); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java index ff7a10f2975..dd7d1686ca7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.LinkedList; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -44,18 +45,26 @@ public class Count extends FsCommand { private static final String OPTION_QUOTA = "q"; private static final String OPTION_HUMAN = "h"; + private static final String OPTION_HEADER = "v"; public static final String NAME = "count"; public static final String USAGE = - "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] ..."; - public static final String DESCRIPTION = + "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER + + "] ..."; + public static final String DESCRIPTION = "Count the number of directories, files and bytes under the paths\n" + - "that match the specified file pattern. The output columns are:\n" + - "DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" + - "QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" + - " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" + - "The -h option shows file sizes in human readable format."; - + "that match the specified file pattern. The output columns are:\n" + + StringUtils.join(ContentSummary.getHeaderFields(), ' ') + + " PATHNAME\n" + + "or, with the -" + OPTION_QUOTA + " option:\n" + + StringUtils.join(ContentSummary.getQuotaHeaderFields(), ' ') + "\n" + + " " + + StringUtils.join(ContentSummary.getHeaderFields(), ' ') + + " PATHNAME\n" + + "The -" + OPTION_HUMAN + + " option shows file sizes in human readable format.\n" + + "The -" + OPTION_HEADER + " option displays a header line."; + private boolean showQuotas; private boolean humanReadable; @@ -65,7 +74,7 @@ public class Count extends FsCommand { /** Constructor * @deprecated invoke via {@link FsShell} * @param cmd the count command - * @param pos the starting index of the arguments + * @param pos the starting index of the arguments * @param conf configuration */ @Deprecated @@ -77,13 +86,16 @@ public class Count extends FsCommand { @Override protected void processOptions(LinkedList args) { CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, - OPTION_QUOTA, OPTION_HUMAN); + OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER); cf.parse(args); if (args.isEmpty()) { // default path is the current working directory args.add("."); } showQuotas = cf.getOpt(OPTION_QUOTA); humanReadable = cf.getOpt(OPTION_HUMAN); + if (cf.getOpt(OPTION_HEADER)) { + out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME"); + } } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java index b0ebe57590f..7cc7ae40949 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java @@ -140,15 +140,15 @@ public class TestContentSummary { // check the header with quotas @Test public void testGetHeaderWithQuota() { - String header = " name quota rem name quota space quota " - + "rem space quota directories files bytes "; + String header = " QUOTA REM_QUOTA SPACE_QUOTA " + + "REM_SPACE_QUOTA DIR_COUNT FILE_COUNT CONTENT_SIZE "; assertEquals(header, ContentSummary.getHeader(true)); } // check the header without quotas @Test public void testGetHeaderNoQuota() { - String header = " directories files bytes "; + String header = " DIR_COUNT FILE_COUNT CONTENT_SIZE "; assertEquals(header, ContentSummary.getHeader(false)); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java index 027b55ff342..11173b3d079 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.shell.CommandFormat.NotEnoughArgumentsException; import org.junit.Test; import org.junit.Before; import org.junit.BeforeClass; @@ -47,7 +48,6 @@ public class TestCount { private static Configuration conf; private static FileSystem mockFs; private static FileStatus fileStat; - private static ContentSummary mockCs; @BeforeClass public static void setup() { @@ -55,7 +55,6 @@ public class TestCount { conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); mockFs = mock(FileSystem.class); fileStat = mock(FileStatus.class); - mockCs = mock(ContentSummary.class); when(fileStat.isFile()).thenReturn(true); } @@ -87,6 +86,85 @@ public class TestCount { assertTrue(count.isHumanReadable()); } + // check no options is handled correctly + @Test + public void processOptionsNoOptions() { + LinkedList options = new LinkedList(); + options.add("dummy"); + Count count = new Count(); + count.processOptions(options); + assertFalse(count.isShowQuotas()); + } + + // check -q is handled correctly + @Test + public void processOptionsShowQuotas() { + LinkedList options = new LinkedList(); + options.add("-q"); + options.add("dummy"); + Count count = new Count(); + count.processOptions(options); + assertTrue(count.isShowQuotas()); + } + + // check missing arguments is handled correctly + @Test + public void processOptionsMissingArgs() { + LinkedList options = new LinkedList(); + Count count = new Count(); + try { + count.processOptions(options); + fail("Count.processOptions - NotEnoughArgumentsException not thrown"); + } catch (NotEnoughArgumentsException e) { + } + assertFalse(count.isShowQuotas()); + } + + // check the correct header is produced with no quotas (-v) + @Test + public void processOptionsHeaderNoQuotas() { + LinkedList options = new LinkedList(); + options.add("-v"); + options.add("dummy"); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + count.processOptions(options); + + String noQuotasHeader = + // <----12----> <----12----> <-------18-------> + " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME"; + verify(out).println(noQuotasHeader); + verifyNoMoreInteractions(out); + } + + // check the correct header is produced with quotas (-q -v) + @Test + public void processOptionsHeaderWithQuotas() { + LinkedList options = new LinkedList(); + options.add("-q"); + options.add("-v"); + options.add("dummy"); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + count.processOptions(options); + + String withQuotasHeader = + // <----12----> <-----15------> <-----15------> <-----15------> + " QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA " + + // <----12----> <----12----> <-------18-------> + " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME"; + verify(out).println(withQuotasHeader); + verifyNoMoreInteractions(out); + } + // check quotas are reported correctly @Test public void processPathShowQuotas() throws Exception { @@ -211,10 +289,28 @@ public class TestCount { public void getUsage() { Count count = new Count(); String actual = count.getUsage(); - String expected = "-count [-q] [-h] ..."; + String expected = "-count [-q] [-h] [-v] ..."; assertEquals("Count.getUsage", expected, actual); } + // check the correct description is returned + @Test + public void getDescription() { + Count count = new Count(); + String actual = count.getDescription(); + String expected = + "Count the number of directories, files and bytes under the paths\n" + + "that match the specified file pattern. The output columns are:\n" + + "DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n" + + "or, with the -q option:\n" + + "QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA\n" + + " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n" + + "The -h option shows file sizes in human readable format.\n" + + "The -v option displays a header line."; + + assertEquals("Count.getDescription", expected, actual); + } + // mock content system static class MockContentSummary extends ContentSummary { @@ -227,15 +323,15 @@ public class TestCount { public String toString(boolean qOption, boolean hOption) { if (qOption) { if (hOption) { - return(HUMAN + WITH_QUOTAS); + return (HUMAN + WITH_QUOTAS); } else { - return(BYTES + WITH_QUOTAS); + return (BYTES + WITH_QUOTAS); } } else { if (hOption) { - return(HUMAN + NO_QUOTAS); + return (HUMAN + NO_QUOTAS); } else { - return(BYTES + NO_QUOTAS); + return (BYTES + NO_QUOTAS); } } } diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index 52da00b17c2..36e138af4b1 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -238,7 +238,7 @@ RegexpComparator - ^-count \[-q\] \[-h\] <path> \.\.\. :( )* + ^-count \[-q\] \[-h\] \[-v\] <path> \.\.\. :( )* RegexpComparator @@ -250,20 +250,28 @@ RegexpComparator - ^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or( )* + ^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )* RegexpComparator - ^( |\t)*QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA( )* + ^( |\t)*or, with the -q option:( )* RegexpComparator - ^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )* + ^( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA( )* + + + RegexpComparator + ^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )* RegexpComparator ^( |\t)*The -h option shows file sizes in human readable format.( )* + + RegexpComparator + ^( |\t)*The -v option displays a header line.( )* + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 8e7af920494..ac26ce5efd8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -8699,6 +8699,48 @@ + + count: file using absolute path showing header record + + -fs NAMENODE -touchz /file1 + -fs NAMENODE -count -v /file1 + + + -fs NAMENODE -rm /file1 + + + + RegexpComparator + ( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME + + + RegexpComparator + ( |\t)*0( |\t)*1( |\t)*0 /file1 + + + + + + count: file using absolute path with -q option and showing header record + + -fs NAMENODE -touchz /file1 + -fs NAMENODE -count -q -v /file1 + + + -fs NAMENODE -rm /file1 + + + + RegexpComparator + ( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME + + + RegexpComparator + ( |\t)*none( |\t)*inf( |\t)*none( |\t)*inf( |\t)*0( |\t)*1( |\t)*0 /file1 + + + + chmod: change permission(octal mode) of file in absolute path