From 18a3dad44afd8061643fffc5bbe50fa66e47b72c Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 13 Apr 2015 21:01:15 -0700 Subject: [PATCH] HDFS-7701. Support reporting per storage type quota and usage with hadoop/hdfs shell. (Contributed by Peter Shi) --- .../org/apache/hadoop/fs/ContentSummary.java | 89 +++++++++-- .../apache/hadoop/fs/shell/CommandFormat.java | 49 +++++- .../org/apache/hadoop/fs/shell/Count.java | 75 ++++++++- .../org/apache/hadoop/fs/shell/TestCount.java | 142 +++++++++++++++++- .../src/test/resources/testConf.xml | 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + 6 files changed, 335 insertions(+), 25 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java index 66137d02c16..ccd6960dd0f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java @@ -20,8 +20,8 @@ package org.apache.hadoop.fs; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.List; -import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Writable; @@ -255,6 +255,8 @@ public class ContentSummary implements Writable{ private static final String QUOTA_SUMMARY_FORMAT = "%12s %15s "; private static final String SPACE_QUOTA_SUMMARY_FORMAT = "%15s %15s "; + private static final String STORAGE_TYPE_SUMMARY_FORMAT = "%13s %17s "; + private static final String[] HEADER_FIELDS = new String[] { "DIR_COUNT", "FILE_COUNT", "CONTENT_SIZE"}; private static final String[] QUOTA_HEADER_FIELDS = new String[] { "QUOTA", @@ -268,7 +270,11 @@ public class ContentSummary implements Writable{ QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT, (Object[]) QUOTA_HEADER_FIELDS) + HEADER; - + + /** default quota display string */ + private static final String QUOTA_NONE = "none"; + private static final String QUOTA_INF = "inf"; + /** Return the header of the output. * if qOption is false, output directory count, file count, and content size; * if qOption is true, output quota and remaining quota as well. @@ -280,6 +286,26 @@ public class ContentSummary implements Writable{ return qOption ? QUOTA_HEADER : HEADER; } + /** + * return the header of with the StorageTypes + * + * @param storageTypes + * @return storage header string + */ + public static String getStorageTypeHeader(List storageTypes) { + StringBuffer header = new StringBuffer(); + + for (StorageType st : storageTypes) { + /* the field length is 13/17 for quota and remain quota + * as the max length for quota name is ARCHIVE_QUOTA + * and remain quota name REM_ARCHIVE_QUOTA */ + String storageName = st.toString(); + header.append(String.format(STORAGE_TYPE_SUMMARY_FORMAT, storageName + "_QUOTA", + "REM_" + storageName + "_QUOTA")); + } + return header.toString(); + } + /** * Returns the names of the fields from the summary header. * @@ -325,13 +351,49 @@ public class ContentSummary implements Writable{ * @return the string representation of the object */ public String toString(boolean qOption, boolean hOption) { + return toString(qOption, hOption, false, null); + } + + /** + * Return the string representation of the object in the output format. + * if tOption is true, display the quota by storage types, + * Otherwise, same logic with #toString(boolean,boolean) + * + * @param qOption a flag indicating if quota needs to be printed or not + * @param hOption a flag indicating if human readable output if to be used + * @param tOption a flag indicating if display quota by storage types + * @param types Storage types to display + * @return the string representation of the object + */ + public String toString(boolean qOption, boolean hOption, + boolean tOption, List types) { String prefix = ""; + + if (tOption) { + StringBuffer content = new StringBuffer(); + for (StorageType st : types) { + long typeQuota = getTypeQuota(st); + long typeConsumed = getTypeConsumed(st); + String quotaStr = QUOTA_NONE; + String quotaRem = QUOTA_INF; + + if (typeQuota > 0) { + quotaStr = formatSize(typeQuota, hOption); + quotaRem = formatSize(typeQuota - typeConsumed, hOption); + } + + content.append(String.format(STORAGE_TYPE_SUMMARY_FORMAT, + quotaStr, quotaRem)); + } + return content.toString(); + } + if (qOption) { - String quotaStr = "none"; - String quotaRem = "inf"; - String spaceQuotaStr = "none"; - String spaceQuotaRem = "inf"; - + String quotaStr = QUOTA_NONE; + String quotaRem = QUOTA_INF; + String spaceQuotaStr = QUOTA_NONE; + String spaceQuotaRem = QUOTA_INF; + if (quota>0) { quotaStr = formatSize(quota, hOption); quotaRem = formatSize(quota-(directoryCount+fileCount), hOption); @@ -340,16 +402,17 @@ public class ContentSummary implements Writable{ spaceQuotaStr = formatSize(spaceQuota, hOption); spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption); } - + prefix = String.format(QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT, - quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem); + quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem); } - + return prefix + String.format(SUMMARY_FORMAT, - formatSize(directoryCount, hOption), - formatSize(fileCount, hOption), - formatSize(length, hOption)); + formatSize(directoryCount, hOption), + formatSize(fileCount, hOption), + formatSize(length, hOption)); } + /** * Formats a size to be human readable or in bytes * @param size value to be formatted diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java index e1aeea94acf..0f9aa388461 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java @@ -31,6 +31,7 @@ import java.util.Set; public class CommandFormat { final int minPar, maxPar; final Map options = new HashMap(); + final Map optionsWithValue = new HashMap(); boolean ignoreUnknownOpts = false; /** @@ -64,6 +65,18 @@ public class CommandFormat { } } + /** + * add option with value + * + * @param option option name + */ + public void addOptionWithValue(String option) { + if (options.containsKey(option)) { + throw new DuplicatedOptionException(option); + } + optionsWithValue.put(option, null); + } + /** Parse parameters starting from the given position * Consider using the variant that directly takes a List * @@ -99,6 +112,17 @@ public class CommandFormat { if (options.containsKey(opt)) { args.remove(pos); options.put(opt, Boolean.TRUE); + } else if (optionsWithValue.containsKey(opt)) { + args.remove(pos); + if (pos < args.size() && (args.size() > minPar)) { + arg = args.get(pos); + args.remove(pos); + } else { + arg = ""; + } + if (!arg.startsWith("-") || arg.equals("-")) { + optionsWithValue.put(opt, arg); + } } else if (ignoreUnknownOpts) { pos++; } else { @@ -122,7 +146,19 @@ public class CommandFormat { public boolean getOpt(String option) { return options.containsKey(option) ? options.get(option) : false; } - + + /** + * get the option's value + * + * @param option option name + * @return option value + * if option exists, but no value assigned, return "" + * if option not exists, return null + */ + public String getOptValue(String option) { + return optionsWithValue.get(option); + } + /** Returns all the options that are set * * @return Set of the enabled options @@ -203,4 +239,15 @@ public class CommandFormat { return option; } } + + /** + * Used when a duplicated option is supplied to a command. + */ + public static class DuplicatedOptionException extends IllegalArgumentException { + private static final long serialVersionUID = 0L; + + public DuplicatedOptionException(String duplicatedOption) { + super("option " + duplicatedOption + " already exsits!"); + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java index dd7d1686ca7..c615876a619 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java @@ -18,8 +18,10 @@ package org.apache.hadoop.fs.shell; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; +import java.util.List; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; @@ -27,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FsShell; +import org.apache.hadoop.fs.StorageType; /** * Count the number of directories, files, bytes, quota, and remaining quota. @@ -46,11 +49,12 @@ public class Count extends FsCommand { private static final String OPTION_QUOTA = "q"; private static final String OPTION_HUMAN = "h"; private static final String OPTION_HEADER = "v"; + private static final String OPTION_TYPE = "t"; public static final String NAME = "count"; public static final String USAGE = "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER - + "] ..."; + + "] [-" + OPTION_TYPE + " []] ..."; public static final String DESCRIPTION = "Count the number of directories, files and bytes under the paths\n" + "that match the specified file pattern. The output columns are:\n" + @@ -63,10 +67,19 @@ public class Count extends FsCommand { " PATHNAME\n" + "The -" + OPTION_HUMAN + " option shows file sizes in human readable format.\n" + - "The -" + OPTION_HEADER + " option displays a header line."; + "The -" + OPTION_HEADER + " option displays a header line.\n" + + "The -" + OPTION_TYPE + " option displays quota by storage types.\n" + + "It must be used with -" + OPTION_QUOTA + " option.\n" + + "If a comma-separated list of storage types is given after the -" + + OPTION_TYPE + " option, \n" + + "it displays the quota and usage for the specified types. \n" + + "Otherwise, it displays the quota and usage for all the storage \n" + + "types that support quota"; private boolean showQuotas; private boolean humanReadable; + private boolean showQuotabyType; + private List storageTypes = null; /** Constructor */ public Count() {} @@ -87,21 +100,54 @@ public class Count extends FsCommand { protected void processOptions(LinkedList args) { CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER); + cf.addOptionWithValue(OPTION_TYPE); cf.parse(args); if (args.isEmpty()) { // default path is the current working directory args.add("."); } showQuotas = cf.getOpt(OPTION_QUOTA); humanReadable = cf.getOpt(OPTION_HUMAN); - if (cf.getOpt(OPTION_HEADER)) { - out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME"); + + if (showQuotas) { + String types = cf.getOptValue(OPTION_TYPE); + + if (null != types) { + showQuotabyType = true; + storageTypes = getAndCheckStorageTypes(types); + } else { + showQuotabyType = false; + } } + + if (cf.getOpt(OPTION_HEADER)) { + if (showQuotabyType) { + out.println(ContentSummary.getStorageTypeHeader(storageTypes) + "PATHNAME"); + } else { + out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME"); + } + } + } + + private List getAndCheckStorageTypes(String types) { + if ("".equals(types) || "all".equalsIgnoreCase(types)) { + return StorageType.getTypesSupportingQuota(); + } + + String[] typeArray = StringUtils.split(types, ','); + List stTypes = new ArrayList<>(); + + for (String t : typeArray) { + stTypes.add(StorageType.parseStorageType(t)); + } + + return stTypes; } @Override protected void processPath(PathData src) throws IOException { ContentSummary summary = src.fs.getContentSummary(src.path); - out.println(summary.toString(showQuotas, isHumanReadable()) + src); + out.println(summary.toString(showQuotas, isHumanReadable(), + showQuotabyType, storageTypes) + src); } /** @@ -121,4 +167,23 @@ public class Count extends FsCommand { boolean isHumanReadable() { return humanReadable; } + + /** + * should print quota by storage types + * @return true if enables quota by storage types + */ + @InterfaceAudience.Private + boolean isShowQuotabyType() { + return showQuotabyType; + } + + /** + * show specified storage types + * @return specified storagetypes + */ + @InterfaceAudience.Private + List getStorageTypes() { + return storageTypes; + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java index d5f097d3865..22d9a21b975 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java @@ -24,13 +24,15 @@ import java.io.PrintStream; import java.io.IOException; import java.net.URI; import java.util.LinkedList; +import java.util.List; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.shell.CommandFormat.NotEnoughArgumentsException; import org.junit.Test; import org.junit.Before; @@ -79,11 +81,17 @@ public class TestCount { LinkedList options = new LinkedList(); options.add("-q"); options.add("-h"); + options.add("-t"); + options.add("SSD"); options.add("dummy"); Count count = new Count(); count.processOptions(options); assertTrue(count.isShowQuotas()); assertTrue(count.isHumanReadable()); + assertTrue(count.isShowQuotabyType()); + assertEquals(1, count.getStorageTypes().size()); + assertEquals(StorageType.SSD, count.getStorageTypes().get(0)); + } // check no options is handled correctly @@ -253,6 +261,112 @@ public class TestCount { verify(out).println(HUMAN + NO_QUOTAS + path.toString()); } + @Test + public void processPathWithQuotasByStorageTypesHeader() throws Exception { + Path path = new Path("mockfs:/test"); + + when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + LinkedList options = new LinkedList(); + options.add("-q"); + options.add("-v"); + options.add("-t"); + options.add("all"); + options.add("dummy"); + count.processOptions(options); + String withStorageTypeHeader = + // <----13---> <-------17------> <----13-----> <------17-------> + " DISK_QUOTA REM_DISK_QUOTA SSD_QUOTA REM_SSD_QUOTA " + + // <----13---> <-------17------> + "ARCHIVE_QUOTA REM_ARCHIVE_QUOTA " + + "PATHNAME"; + verify(out).println(withStorageTypeHeader); + verifyNoMoreInteractions(out); + } + + @Test + public void processPathWithQuotasBySSDStorageTypesHeader() throws Exception { + Path path = new Path("mockfs:/test"); + + when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + LinkedList options = new LinkedList(); + options.add("-q"); + options.add("-v"); + options.add("-t"); + options.add("SSD"); + options.add("dummy"); + count.processOptions(options); + String withStorageTypeHeader = + // <----13---> <-------17------> + " SSD_QUOTA REM_SSD_QUOTA " + + "PATHNAME"; + verify(out).println(withStorageTypeHeader); + verifyNoMoreInteractions(out); + } + + @Test + public void processPathWithQuotasByMultipleStorageTypesContent() throws Exception { + Path path = new Path("mockfs:/test"); + + when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat); + PathData pathData = new PathData(path.toString(), conf); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + LinkedList options = new LinkedList(); + options.add("-q"); + options.add("-t"); + options.add("SSD,DISK"); + options.add("dummy"); + count.processOptions(options); + count.processPath(pathData); + String withStorageType = BYTES + StorageType.SSD.toString() + + " " + StorageType.DISK.toString() + " " + pathData.toString(); + verify(out).println(withStorageType); + verifyNoMoreInteractions(out); + } + + @Test + public void processPathWithQuotasByMultipleStorageTypes() throws Exception { + Path path = new Path("mockfs:/test"); + + when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + LinkedList options = new LinkedList(); + options.add("-q"); + options.add("-v"); + options.add("-t"); + options.add("SSD,DISK"); + options.add("dummy"); + count.processOptions(options); + String withStorageTypeHeader = + // <----13---> <------17-------> + " SSD_QUOTA REM_SSD_QUOTA " + + " DISK_QUOTA REM_DISK_QUOTA " + + "PATHNAME"; + verify(out).println(withStorageTypeHeader); + verifyNoMoreInteractions(out); + } + @Test public void getCommandName() { Count count = new Count(); @@ -289,7 +403,7 @@ public class TestCount { public void getUsage() { Count count = new Count(); String actual = count.getUsage(); - String expected = "-count [-q] [-h] [-v] ..."; + String expected = "-count [-q] [-h] [-v] [-t []] ..."; assertEquals("Count.getUsage", expected, actual); } @@ -306,7 +420,13 @@ public class TestCount { + "QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA\n" + " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n" + "The -h option shows file sizes in human readable format.\n" - + "The -v option displays a header line."; + + "The -v option displays a header line.\n" + + "The -t option displays quota by storage types.\n" + + "It must be used with -q option.\n" + + "If a comma-separated list of storage types is given after the -t option, \n" + + "it displays the quota and usage for the specified types. \n" + + "Otherwise, it displays the quota and usage for all the storage \n" + + "types that support quota"; assertEquals("Count.getDescription", expected, actual); } @@ -321,7 +441,19 @@ public class TestCount { } @Override - public String toString(boolean qOption, boolean hOption) { + public String toString(boolean qOption, boolean hOption, + boolean tOption, List types) { + if (tOption) { + StringBuffer result = new StringBuffer(); + result.append(hOption ? HUMAN : BYTES); + + for (StorageType type : types) { + result.append(type.toString()); + result.append(" "); + } + return result.toString(); + } + if (qOption) { if (hOption) { return (HUMAN + WITH_QUOTAS); diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index ac28192d5d2..9b72960d637 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -262,7 +262,7 @@ RegexpComparator - ^-count \[-q\] \[-h\] \[-v\] <path> \.\.\. :( )* + ^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] <path> \.\.\. :( )* RegexpComparator diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1aaf42c02a6..7414d33dcb8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -491,6 +491,9 @@ Release 2.8.0 - UNRELEASED HDFS-8111. NPE thrown when invalid FSImage filename given for 'hdfs oiv_legacy' cmd ( surendra singh lilhore via vinayakumarb ) + HDFS-7701. Support reporting per storage type quota and usage + with hadoop/hdfs shell. (Peter Shi via Arpit Agarwal) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES