From f0efea490e5aa9dd629d2199aae9c5b1290a17ee Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 23 Aug 2016 04:13:48 -0700 Subject: [PATCH] HDFS-8986. Add option to -du to calculate directory space usage excluding snapshots. Contributed by Xiao Chen. --- .../org/apache/hadoop/fs/ContentSummary.java | 127 ++++++++-- .../org/apache/hadoop/fs/shell/Count.java | 20 +- .../org/apache/hadoop/fs/shell/FsUsage.java | 31 ++- .../src/site/markdown/FileSystemShell.md | 11 +- .../org/apache/hadoop/fs/shell/TestCount.java | 5 +- .../src/test/resources/testConf.xml | 12 +- .../hdfs/protocolPB/PBHelperClient.java | 8 + .../src/main/proto/hdfs.proto | 4 + .../ContentSummaryComputationContext.java | 6 + .../hadoop/hdfs/server/namenode/INode.java | 9 +- .../hdfs/server/namenode/INodeDirectory.java | 4 + .../org/apache/hadoop/hdfs/TestDFSShell.java | 235 +++++++++++++++++- 12 files changed, 432 insertions(+), 40 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java index 3dedbccde3b..3e759517c3d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java @@ -34,6 +34,11 @@ public class ContentSummary extends QuotaUsage implements Writable{ private long length; private long fileCount; private long directoryCount; + // These fields are to track the snapshot-related portion of the values. + private long snapshotLength; + private long snapshotFileCount; + private long snapshotDirectoryCount; + private long snapshotSpaceConsumed; /** We don't use generics. Instead override spaceConsumed and other methods in order to keep backward compatibility. */ @@ -56,6 +61,26 @@ public Builder directoryCount(long directoryCount) { return this; } + public Builder snapshotLength(long snapshotLength) { + this.snapshotLength = snapshotLength; + return this; + } + + public Builder snapshotFileCount(long snapshotFileCount) { + this.snapshotFileCount = snapshotFileCount; + return this; + } + + public Builder snapshotDirectoryCount(long snapshotDirectoryCount) { + this.snapshotDirectoryCount = snapshotDirectoryCount; + return this; + } + + public Builder snapshotSpaceConsumed(long snapshotSpaceConsumed) { + this.snapshotSpaceConsumed = snapshotSpaceConsumed; + return this; + } + @Override public Builder quota(long quota){ super.quota(quota); @@ -107,6 +132,10 @@ public ContentSummary build() { private long length; private long fileCount; private long directoryCount; + private long snapshotLength; + private long snapshotFileCount; + private long snapshotDirectoryCount; + private long snapshotSpaceConsumed; } /** Constructor deprecated by ContentSummary.Builder*/ @@ -142,17 +171,37 @@ private ContentSummary(Builder builder) { this.length = builder.length; this.fileCount = builder.fileCount; this.directoryCount = builder.directoryCount; + this.snapshotLength = builder.snapshotLength; + this.snapshotFileCount = builder.snapshotFileCount; + this.snapshotDirectoryCount = builder.snapshotDirectoryCount; + this.snapshotSpaceConsumed = builder.snapshotSpaceConsumed; } /** @return the length */ public long getLength() {return length;} + public long getSnapshotLength() { + return snapshotLength; + } + /** @return the directory count */ public long getDirectoryCount() {return directoryCount;} + public long getSnapshotDirectoryCount() { + return snapshotDirectoryCount; + } + /** @return the file count */ public long getFileCount() {return fileCount;} + public long getSnapshotFileCount() { + return snapshotFileCount; + } + + public long getSnapshotSpaceConsumed() { + return snapshotSpaceConsumed; + } + @Override @InterfaceAudience.Private public void write(DataOutput out) throws IOException { @@ -180,9 +229,14 @@ public boolean equals(Object to) { if (this == to) { return true; } else if (to instanceof ContentSummary) { - return getLength() == ((ContentSummary) to).getLength() && - getFileCount() == ((ContentSummary) to).getFileCount() && - getDirectoryCount() == ((ContentSummary) to).getDirectoryCount() && + ContentSummary right = (ContentSummary) to; + return getLength() == right.getLength() && + getFileCount() == right.getFileCount() && + getDirectoryCount() == right.getDirectoryCount() && + getSnapshotLength() == right.getSnapshotLength() && + getSnapshotFileCount() == right.getSnapshotFileCount() && + getSnapshotDirectoryCount() == right.getSnapshotDirectoryCount() && + getSnapshotSpaceConsumed() == right.getSnapshotSpaceConsumed() && super.equals(to); } else { return super.equals(to); @@ -191,7 +245,9 @@ public boolean equals(Object to) { @Override public int hashCode() { - long result = getLength() ^ getFileCount() ^ getDirectoryCount(); + long result = getLength() ^ getFileCount() ^ getDirectoryCount() + ^ getSnapshotLength() ^ getSnapshotFileCount() + ^ getSnapshotDirectoryCount() ^ getSnapshotSpaceConsumed(); return ((int) result) ^ super.hashCode(); } @@ -255,15 +311,14 @@ public String toString() { * @param qOption a flag indicating if quota needs to be printed or not * @return the string representation of the object */ + @Override public String toString(boolean qOption) { return toString(qOption, false); } /** Return the string representation of the object in the output format. - * if qOption is false, output directory count, file count, and content size; - * if qOption is true, output quota and remaining quota as well. - * if hOption is false file sizes are returned in bytes - * if hOption is true file sizes are returned in human readable + * For description of the options, + * @see #toString(boolean, boolean, boolean, boolean, List) * * @param qOption a flag indicating if quota needs to be printed or not * @param hOption a flag indicating if human readable output if to be used @@ -273,10 +328,24 @@ public String toString(boolean qOption, boolean hOption) { return toString(qOption, hOption, false, null); } + /** Return the string representation of the object in the output format. + * For description of the options, + * @see #toString(boolean, boolean, boolean, boolean, List) + * + * @param qOption a flag indicating if quota needs to be printed or not + * @param hOption a flag indicating if human readable output is to be used + * @param xOption a flag indicating if calculation from snapshots is to be + * included in the output + * @return the string representation of the object + */ + public String toString(boolean qOption, boolean hOption, boolean xOption) { + return toString(qOption, hOption, false, xOption, null); + } + /** * Return the string representation of the object in the output format. - * if tOption is true, display the quota by storage types, - * Otherwise, same logic with #toString(boolean,boolean) + * For description of the options, + * @see #toString(boolean, boolean, boolean, boolean, List) * * @param qOption a flag indicating if quota needs to be printed or not * @param hOption a flag indicating if human readable output if to be used @@ -286,6 +355,29 @@ public String toString(boolean qOption, boolean hOption) { */ public String toString(boolean qOption, boolean hOption, boolean tOption, List types) { + return toString(qOption, hOption, tOption, false, types); + } + + /** Return the string representation of the object in the output format. + * if qOption is false, output directory count, file count, and content size; + * if qOption is true, output quota and remaining quota as well. + * if hOption is false, file sizes are returned in bytes + * if hOption is true, file sizes are returned in human readable + * if tOption is true, display the quota by storage types + * if tOption is false, same logic with #toString(boolean,boolean) + * if xOption is false, output includes the calculation from snapshots + * if xOption is true, output excludes the calculation from snapshots + * + * @param qOption a flag indicating if quota needs to be printed or not + * @param hOption a flag indicating if human readable output is to be used + * @param tOption a flag indicating if display quota by storage types + * @param xOption a flag indicating if calculation from snapshots is to be + * included in the output + * @param types Storage types to display + * @return the string representation of the object + */ + public String toString(boolean qOption, boolean hOption, boolean tOption, + boolean xOption, List types) { String prefix = ""; if (tOption) { @@ -296,10 +388,17 @@ public String toString(boolean qOption, boolean hOption, prefix = getQuotaUsage(hOption); } - return prefix + String.format(SUMMARY_FORMAT, - formatSize(directoryCount, hOption), - formatSize(fileCount, hOption), - formatSize(length, hOption)); + if (xOption) { + return prefix + String.format(SUMMARY_FORMAT, + formatSize(directoryCount - snapshotDirectoryCount, hOption), + formatSize(fileCount - snapshotFileCount, hOption), + formatSize(length - snapshotLength, hOption)); + } else { + return prefix + String.format(SUMMARY_FORMAT, + formatSize(directoryCount, hOption), + formatSize(fileCount, hOption), + formatSize(length, hOption)); + } } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java index 2b5a4cc74de..16f52e35704 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java @@ -51,6 +51,8 @@ public static void registerCommands(CommandFactory factory) { private static final String OPTION_HUMAN = "h"; private static final String OPTION_HEADER = "v"; private static final String OPTION_TYPE = "t"; + // exclude snapshots from calculation. Only work on default columns. + private static final String OPTION_EXCLUDE_SNAPSHOT = "x"; //return the quota, namespace count and disk space usage. private static final String OPTION_QUOTA_AND_USAGE = "u"; @@ -58,7 +60,8 @@ public static void registerCommands(CommandFactory factory) { public static final String USAGE = "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER + "] [-" + OPTION_TYPE + " []] [-" + - OPTION_QUOTA_AND_USAGE + "] ..."; + OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT + + "] ..."; public static final String DESCRIPTION = "Count the number of directories, files and bytes under the paths\n" + "that match the specified file pattern. The output columns are:\n" + @@ -72,6 +75,8 @@ public static void registerCommands(CommandFactory factory) { "The -" + OPTION_HUMAN + " option shows file sizes in human readable format.\n" + "The -" + OPTION_HEADER + " option displays a header line.\n" + + "The -" + OPTION_EXCLUDE_SNAPSHOT + " option excludes snapshots " + + "from being calculated. \n" + "The -" + OPTION_TYPE + " option displays quota by storage types.\n" + "It must be used with -" + OPTION_QUOTA + " option.\n" + "If a comma-separated list of storage types is given after the -" + @@ -87,6 +92,7 @@ public static void registerCommands(CommandFactory factory) { private boolean showQuotabyType; private List storageTypes = null; private boolean showQuotasAndUsageOnly; + private boolean excludeSnapshots; /** Constructor */ public Count() {} @@ -106,7 +112,8 @@ public Count(String[] cmd, int pos, Configuration conf) { @Override protected void processOptions(LinkedList args) { CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, - OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE); + OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE, + OPTION_EXCLUDE_SNAPSHOT); cf.addOptionWithValue(OPTION_TYPE); cf.parse(args); if (args.isEmpty()) { // default path is the current working directory @@ -115,6 +122,7 @@ protected void processOptions(LinkedList args) { showQuotas = cf.getOpt(OPTION_QUOTA); humanReadable = cf.getOpt(OPTION_HUMAN); showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE); + excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT); if (showQuotas || showQuotasAndUsageOnly) { String types = cf.getOptValue(OPTION_TYPE); @@ -125,6 +133,11 @@ protected void processOptions(LinkedList args) { } else { showQuotabyType = false; } + if (excludeSnapshots) { + out.println(OPTION_QUOTA + " or " + OPTION_QUOTA_AND_USAGE + " option " + + "is given, the -x option is ignored."); + excludeSnapshots = false; + } } if (cf.getOpt(OPTION_HEADER)) { @@ -163,7 +176,8 @@ protected void processPath(PathData src) throws IOException { storageTypes) + src); } else { ContentSummary summary = src.fs.getContentSummary(src.path); - out.println(summary.toString(showQuotas, isHumanReadable()) + src); + out.println(summary. + toString(showQuotas, isHumanReadable(), excludeSnapshots) + src); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java index 765b181761c..197920fbd8b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java @@ -107,27 +107,30 @@ protected void processPath(PathData item) throws IOException { /** show disk usage */ public static class Du extends FsUsage { public static final String NAME = "du"; - public static final String USAGE = "[-s] [-h] ..."; + public static final String USAGE = "[-s] [-h] [-x] ..."; public static final String DESCRIPTION = - "Show the amount of space, in bytes, used by the files that " + - "match the specified file pattern. The following flags are optional:\n" + - "-s: Rather than showing the size of each individual file that" + - " matches the pattern, shows the total (summary) size.\n" + - "-h: Formats the sizes of files in a human-readable fashion" + - " rather than a number of bytes.\n\n" + - "Note that, even without the -s option, this only shows size summaries " + - "one level deep into a directory.\n\n" + - "The output is in the form \n" + - "\tsize\tdisk space consumed\tname(full path)\n"; + "Show the amount of space, in bytes, used by the files that match " + + "the specified file pattern. The following flags are optional:\n" + + "-s: Rather than showing the size of each individual file that" + + " matches the pattern, shows the total (summary) size.\n" + + "-h: Formats the sizes of files in a human-readable fashion" + + " rather than a number of bytes.\n" + + "-x: Excludes snapshots from being counted.\n\n" + + "Note that, even without the -s option, this only shows size " + + "summaries one level deep into a directory.\n\n" + + "The output is in the form \n" + + "\tsize\tdisk space consumed\tname(full path)\n"; protected boolean summary = false; + private boolean excludeSnapshots = false; @Override protected void processOptions(LinkedList args) throws IOException { - CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s"); + CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s", "x"); cf.parse(args); humanReadable = cf.getOpt("h"); summary = cf.getOpt("s"); + excludeSnapshots = cf.getOpt("x"); if (args.isEmpty()) args.add(Path.CUR_DIR); } @@ -156,6 +159,10 @@ protected void processPath(PathData item) throws IOException { ContentSummary contentSummary = item.fs.getContentSummary(item.path); long length = contentSummary.getLength(); long spaceConsumed = contentSummary.getSpaceConsumed(); + if (excludeSnapshots) { + length -= contentSummary.getSnapshotLength(); + spaceConsumed -= contentSummary.getSnapshotSpaceConsumed(); + } usagesTable.addRow(formatSize(length), formatSize(spaceConsumed), item); } } diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index 066cfe3462b..a8a192012c7 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -132,10 +132,12 @@ Similar to get command, except that the destination is restricted to a local fil count ----- -Usage: `hadoop fs -count [-q] [-h] [-v] [-t []] [-u] ` +Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t []] [-u] ` Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME +The -u and -q options control what columns the output contains. -q means show quotas, -u limits the output to show quotas and usage only. + The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME The output columns with -count -u are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA @@ -146,6 +148,8 @@ The -h option shows sizes in human readable format. The -v option displays a header line. +The -x option excludes snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path. The -x option is ignored if -u or -q option is given. + Example: * `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2` @@ -211,14 +215,15 @@ Example: du ---- -Usage: `hadoop fs -du [-s] [-h] URI [URI ...]` +Usage: `hadoop fs -du [-s] [-h] [-x] URI [URI ...]` Displays sizes of files and directories contained in the given directory or the length of a file in case its just a file. Options: -* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files. +* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files. Without the -s option, calculation is done by going 1-level deep from the given path. * The -h option will format file sizes in a "human-readable" fashion (e.g 64.0m instead of 67108864) +* The -x option will exclude snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path. The du returns three columns with the following format: diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java index 04174e01b0e..c7b30e49d6c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java @@ -447,7 +447,7 @@ public void getUsage() { Count count = new Count(); String actual = count.getUsage(); String expected = - "-count [-q] [-h] [-v] [-t []] [-u] ..."; + "-count [-q] [-h] [-v] [-t []] [-u] [-x] ..."; assertEquals("Count.getUsage", expected, actual); } @@ -465,6 +465,7 @@ public void getDescription() { + " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n" + "The -h option shows file sizes in human readable format.\n" + "The -v option displays a header line.\n" + + "The -x option excludes snapshots from being calculated. \n" + "The -t option displays quota by storage types.\n" + "It must be used with -q option.\n" + "If a comma-separated list of storage types is given after the -t option, \n" @@ -521,7 +522,7 @@ public MockContentSummary() { } @Override - public String toString(boolean qOption, boolean hOption) { + public String toString(boolean qOption, boolean hOption, boolean xOption) { if (qOption) { if (hOption) { return (HUMAN + WITH_QUOTAS); diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index 82bc7899ffa..a31c8280871 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -200,7 +200,7 @@ RegexpComparator - ^-du \[-s\] \[-h\] <path> \.\.\. :\s* + ^-du \[-s\] \[-h\] \[-x\] <path> \.\.\. :\s* RegexpComparator @@ -226,6 +226,10 @@ RegexpComparator \s*of bytes.\s* + + RegexpComparator + ^\s*-x\s*Excludes snapshots from being counted.\s* + RegexpComparator ^\s*Note that, even without the -s option, this only shows size summaries one level\s* @@ -274,7 +278,7 @@ RegexpComparator - ^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] <path> \.\.\. :( )* + ^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] \[-x\] <path> \.\.\. :( )* RegexpComparator @@ -308,6 +312,10 @@ RegexpComparator ^( |\t)*The -v option displays a header line.( )* + + RegexpComparator + ^( |\t)*The -x option excludes snapshots from being calculated.( )* + diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 3c5f583b3d9..7df68e743a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -1466,6 +1466,10 @@ public static ContentSummary convert(ContentSummaryProto cs) { builder.length(cs.getLength()). fileCount(cs.getFileCount()). directoryCount(cs.getDirectoryCount()). + snapshotLength(cs.getSnapshotLength()). + snapshotFileCount(cs.getSnapshotFileCount()). + snapshotDirectoryCount(cs.getSnapshotDirectoryCount()). + snapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()). quota(cs.getQuota()). spaceConsumed(cs.getSpaceConsumed()). spaceQuota(cs.getSpaceQuota()); @@ -2069,6 +2073,10 @@ public static ContentSummaryProto convert(ContentSummary cs) { builder.setLength(cs.getLength()). setFileCount(cs.getFileCount()). setDirectoryCount(cs.getDirectoryCount()). + setSnapshotLength(cs.getSnapshotLength()). + setSnapshotFileCount(cs.getSnapshotFileCount()). + setSnapshotDirectoryCount(cs.getSnapshotDirectoryCount()). + setSnapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()). setQuota(cs.getQuota()). setSpaceConsumed(cs.getSpaceConsumed()). setSpaceQuota(cs.getSpaceQuota()); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index f939ec34d23..7de8550d59b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -138,6 +138,10 @@ message ContentSummaryProto { required uint64 spaceConsumed = 5; required uint64 spaceQuota = 6; optional StorageTypeQuotaInfosProto typeQuotaInfos = 7; + optional uint64 snapshotLength = 8; + optional uint64 snapshotFileCount = 9; + optional uint64 snapshotDirectoryCount = 10; + optional uint64 snapshotSpaceConsumed = 11; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java index 5739835acf4..6df9e75d347 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java @@ -29,6 +29,7 @@ public class ContentSummaryComputationContext { private FSNamesystem fsn = null; private BlockStoragePolicySuite bsps = null; private ContentCounts counts = null; + private ContentCounts snapshotCounts = null; private long nextCountLimit = 0; private long limitPerRun = 0; private long yieldCount = 0; @@ -51,6 +52,7 @@ public ContentSummaryComputationContext(FSDirectory dir, this.limitPerRun = limitPerRun; this.nextCountLimit = limitPerRun; this.counts = new ContentCounts.Builder().build(); + this.snapshotCounts = new ContentCounts.Builder().build(); this.sleepMilliSec = sleepMicroSec/1000; this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000); } @@ -125,6 +127,10 @@ public ContentCounts getCounts() { return counts; } + public ContentCounts getSnapshotCounts() { + return snapshotCounts; + } + public BlockStoragePolicySuite getBlockStoragePolicySuite() { Preconditions.checkState((bsps != null || fsn != null), "BlockStoragePolicySuite must be either initialized or available via" + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index 0348c1fbe06..c6258a173b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -428,8 +428,9 @@ public final ContentSummary computeContentSummary(BlockStoragePolicySuite bsps) */ public final ContentSummary computeAndConvertContentSummary(int snapshotId, ContentSummaryComputationContext summary) { - ContentCounts counts = computeContentSummary(snapshotId, summary) - .getCounts(); + computeContentSummary(snapshotId, summary); + final ContentCounts counts = summary.getCounts(); + final ContentCounts snapshotCounts = summary.getSnapshotCounts(); final QuotaCounts q = getQuotaCounts(); return new ContentSummary.Builder(). length(counts.getLength()). @@ -440,6 +441,10 @@ public final ContentSummary computeAndConvertContentSummary(int snapshotId, spaceQuota(q.getStorageSpace()). typeConsumed(counts.getTypeSpaces()). typeQuota(q.getTypeSpaces().asArray()). + snapshotLength(snapshotCounts.getLength()). + snapshotFileCount(snapshotCounts.getFileCount()). + snapshotDirectoryCount(snapshotCounts.getDirectoryCount()). + snapshotSpaceConsumed(snapshotCounts.getStoragespace()). build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index d00c1362065..9a8f9b2c165 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -634,6 +634,10 @@ public ContentSummaryComputationContext computeContentSummary(int snapshotId, // computation should include all the deleted files/directories sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(), summary.getCounts()); + // Also compute ContentSummary for snapshotCounts (So we can extract it + // later from the ContentSummary of all). + sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(), + summary.getSnapshotCounts()); } final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature(); if (q != null && snapshotId == Snapshot.CURRENT_STATE_ID) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index dcf683e4c10..73ebf2854c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -113,6 +113,11 @@ static Path mkdir(FileSystem fs, Path p) throws IOException { return p; } + static void rmr(FileSystem fs, Path p) throws IOException { + assertTrue(fs.delete(p, true)); + assertFalse(fs.exists(p)); + } + static File createLocalFile(File f) throws IOException { assertTrue(!f.exists()); PrintWriter out = new PrintWriter(f); @@ -189,7 +194,7 @@ public void testRecursiveRm() throws IOException { MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); FileSystem fs = cluster.getFileSystem(); assertTrue("Not a HDFS: " + fs.getUri(), - fs instanceof DistributedFileSystem); + fs instanceof DistributedFileSystem); try { fs.mkdirs(new Path(new Path("parent"), "child")); try { @@ -224,6 +229,7 @@ public void testDu() throws IOException { shell.setConf(conf); try { + cluster.waitActive(); Path myPath = new Path("/test/dir"); assertTrue(fs.mkdirs(myPath)); assertTrue(fs.exists(myPath)); @@ -251,7 +257,7 @@ public void testDu() throws IOException { assertTrue(val == 0); String returnString = out.toString(); out.reset(); - // Check if size matchs as expected + // Check if size matches as expected assertThat(returnString, containsString(myFileLength.toString())); assertThat(returnString, containsString(myFileDiskUsed.toString())); assertThat(returnString, containsString(myFile2Length.toString())); @@ -308,7 +314,232 @@ public void testDu() throws IOException { System.setOut(psBackup); cluster.shutdown(); } + } + @Test (timeout = 180000) + public void testDuSnapshots() throws IOException { + final int replication = 2; + final Configuration conf = new HdfsConfiguration(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(replication).build(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final PrintStream psBackup = System.out; + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + final PrintStream psOut = new PrintStream(out); + final FsShell shell = new FsShell(); + shell.setConf(conf); + + try { + System.setOut(psOut); + cluster.waitActive(); + final Path parent = new Path("/test"); + final Path dir = new Path(parent, "dir"); + mkdir(dfs, dir); + final Path file = new Path(dir, "file"); + writeFile(dfs, file); + final Path file2 = new Path(dir, "file2"); + writeFile(dfs, file2); + final Long fileLength = dfs.getFileStatus(file).getLen(); + final Long fileDiskUsed = fileLength * replication; + final Long file2Length = dfs.getFileStatus(file2).getLen(); + final Long file2DiskUsed = file2Length * replication; + + /* + * Construct dir as follows: + * /test/dir/file <- this will later be deleted after snapshot taken. + * /test/dir/newfile <- this will be created after snapshot taken. + * /test/dir/file2 + * Snapshot enabled on /test + */ + + // test -du on /test/dir + int ret = -1; + try { + ret = shell.run(new String[] {"-du", dir.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, ret); + String returnString = out.toString(); + LOG.info("-du return is:\n" + returnString); + // Check if size matches as expected + assertTrue(returnString.contains(fileLength.toString())); + assertTrue(returnString.contains(fileDiskUsed.toString())); + assertTrue(returnString.contains(file2Length.toString())); + assertTrue(returnString.contains(file2DiskUsed.toString())); + out.reset(); + + // take a snapshot, then remove file and add newFile + final String snapshotName = "ss1"; + final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName); + dfs.allowSnapshot(parent); + assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath)); + rmr(dfs, file); + final Path newFile = new Path(dir, "newfile"); + writeFile(dfs, newFile); + final Long newFileLength = dfs.getFileStatus(newFile).getLen(); + final Long newFileDiskUsed = newFileLength * replication; + + // test -du -s on /test + ret = -1; + try { + ret = shell.run(new String[] {"-du", "-s", parent.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, ret); + returnString = out.toString(); + LOG.info("-du -s return is:\n" + returnString); + Long combinedLength = fileLength + file2Length + newFileLength; + Long combinedDiskUsed = fileDiskUsed + file2DiskUsed + newFileDiskUsed; + assertTrue(returnString.contains(combinedLength.toString())); + assertTrue(returnString.contains(combinedDiskUsed.toString())); + out.reset(); + + // test -du on /test + ret = -1; + try { + ret = shell.run(new String[] {"-du", parent.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, ret); + returnString = out.toString(); + LOG.info("-du return is:\n" + returnString); + assertTrue(returnString.contains(combinedLength.toString())); + assertTrue(returnString.contains(combinedDiskUsed.toString())); + out.reset(); + + // test -du -s -x on /test + ret = -1; + try { + ret = shell.run(new String[] {"-du", "-s", "-x", parent.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, ret); + returnString = out.toString(); + LOG.info("-du -s -x return is:\n" + returnString); + Long exludeSnapshotLength = file2Length + newFileLength; + Long excludeSnapshotDiskUsed = file2DiskUsed + newFileDiskUsed; + assertTrue(returnString.contains(exludeSnapshotLength.toString())); + assertTrue(returnString.contains(excludeSnapshotDiskUsed.toString())); + out.reset(); + + // test -du -x on /test + ret = -1; + try { + ret = shell.run(new String[] {"-du", "-x", parent.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, ret); + returnString = out.toString(); + LOG.info("-du -x return is:\n" + returnString); + assertTrue(returnString.contains(exludeSnapshotLength.toString())); + assertTrue(returnString.contains(excludeSnapshotDiskUsed.toString())); + out.reset(); + } finally { + System.setOut(psBackup); + cluster.shutdown(); + } + } + + @Test (timeout = 180000) + public void testCountSnapshots() throws IOException { + final int replication = 2; + final Configuration conf = new HdfsConfiguration(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(replication).build(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final PrintStream psBackup = System.out; + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + final PrintStream psOut = new PrintStream(out); + System.setOut(psOut); + final FsShell shell = new FsShell(); + shell.setConf(conf); + + try { + cluster.waitActive(); + final Path parent = new Path("/test"); + final Path dir = new Path(parent, "dir"); + mkdir(dfs, dir); + final Path file = new Path(dir, "file"); + writeFile(dfs, file); + final Path file2 = new Path(dir, "file2"); + writeFile(dfs, file2); + final long fileLength = dfs.getFileStatus(file).getLen(); + final long file2Length = dfs.getFileStatus(file2).getLen(); + final Path dir2 = new Path(parent, "dir2"); + mkdir(dfs, dir2); + + /* + * Construct dir as follows: + * /test/dir/file <- this will later be deleted after snapshot taken. + * /test/dir/newfile <- this will be created after snapshot taken. + * /test/dir/file2 + * /test/dir2 <- this will later be deleted after snapshot taken. + * Snapshot enabled on /test + */ + + // take a snapshot + // then create /test/dir/newfile and remove /test/dir/file, /test/dir2 + final String snapshotName = "s1"; + final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName); + dfs.allowSnapshot(parent); + assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath)); + rmr(dfs, file); + rmr(dfs, dir2); + final Path newFile = new Path(dir, "new file"); + writeFile(dfs, newFile); + final Long newFileLength = dfs.getFileStatus(newFile).getLen(); + + // test -count on /test. Include header for easier debugging. + int val = -1; + try { + val = shell.run(new String[] {"-count", "-v", parent.toString() }); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); + String returnString = out.toString(); + LOG.info("-count return is:\n" + returnString); + Scanner in = new Scanner(returnString); + in.nextLine(); + assertEquals(3, in.nextLong()); //DIR_COUNT + assertEquals(3, in.nextLong()); //FILE_COUNT + assertEquals(fileLength + file2Length + newFileLength, + in.nextLong()); //CONTENT_SIZE + out.reset(); + + // test -count -x on /test. Include header for easier debugging. + val = -1; + try { + val = + shell.run(new String[] {"-count", "-x", "-v", parent.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); + returnString = out.toString(); + LOG.info("-count -x return is:\n" + returnString); + in = new Scanner(returnString); + in.nextLine(); + assertEquals(2, in.nextLong()); //DIR_COUNT + assertEquals(2, in.nextLong()); //FILE_COUNT + assertEquals(file2Length + newFileLength, in.nextLong()); //CONTENT_SIZE + out.reset(); + } finally { + System.setOut(psBackup); + cluster.shutdown(); + } } @Test (timeout = 30000)