HDFS-8986. Add option to -du to calculate directory space usage excluding snapshots. Contributed by Xiao Chen.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java

@@ -34,6 +34,11 @@ public class ContentSummary extends QuotaUsage implements Writable{
   private long length;
   private long fileCount;
   private long directoryCount;
+  // These fields are to track the snapshot-related portion of the values.
+  private long snapshotLength;
+  private long snapshotFileCount;
+  private long snapshotDirectoryCount;
+  private long snapshotSpaceConsumed;
 
   /** We don't use generics. Instead override spaceConsumed and other methods
       in order to keep backward compatibility. */
@@ -56,6 +61,26 @@ public class ContentSummary extends QuotaUsage implements Writable{
       return this;
     }
 
+    public Builder snapshotLength(long snapshotLength) {
+      this.snapshotLength = snapshotLength;
+      return this;
+    }
+
+    public Builder snapshotFileCount(long snapshotFileCount) {
+      this.snapshotFileCount = snapshotFileCount;
+      return this;
+    }
+
+    public Builder snapshotDirectoryCount(long snapshotDirectoryCount) {
+      this.snapshotDirectoryCount = snapshotDirectoryCount;
+      return this;
+    }
+
+    public Builder snapshotSpaceConsumed(long snapshotSpaceConsumed) {
+      this.snapshotSpaceConsumed = snapshotSpaceConsumed;
+      return this;
+    }
+
     @Override
     public Builder quota(long quota){
       super.quota(quota);
@@ -107,6 +132,10 @@ public class ContentSummary extends QuotaUsage implements Writable{
     private long length;
     private long fileCount;
     private long directoryCount;
+    private long snapshotLength;
+    private long snapshotFileCount;
+    private long snapshotDirectoryCount;
+    private long snapshotSpaceConsumed;
   }
 
   /** Constructor deprecated by ContentSummary.Builder*/
@@ -142,17 +171,37 @@ public class ContentSummary extends QuotaUsage implements Writable{
     this.length = builder.length;
     this.fileCount = builder.fileCount;
     this.directoryCount = builder.directoryCount;
+    this.snapshotLength = builder.snapshotLength;
+    this.snapshotFileCount = builder.snapshotFileCount;
+    this.snapshotDirectoryCount = builder.snapshotDirectoryCount;
+    this.snapshotSpaceConsumed = builder.snapshotSpaceConsumed;
   }
 
   /** @return the length */
   public long getLength() {return length;}
 
+  public long getSnapshotLength() {
+    return snapshotLength;
+  }
+
   /** @return the directory count */
   public long getDirectoryCount() {return directoryCount;}
 
+  public long getSnapshotDirectoryCount() {
+    return snapshotDirectoryCount;
+  }
+
   /** @return the file count */
   public long getFileCount() {return fileCount;}
 
+  public long getSnapshotFileCount() {
+    return snapshotFileCount;
+  }
+
+  public long getSnapshotSpaceConsumed() {
+    return snapshotSpaceConsumed;
+  }
+
   @Override
   @InterfaceAudience.Private
   public void write(DataOutput out) throws IOException {
@@ -180,9 +229,14 @@ public class ContentSummary extends QuotaUsage implements Writable{
     if (this == to) {
       return true;
     } else if (to instanceof ContentSummary) {
-      return getLength() == ((ContentSummary) to).getLength() &&
+      ContentSummary right = (ContentSummary) to;
-          getFileCount() == ((ContentSummary) to).getFileCount() &&
+      return getLength() == right.getLength() &&
-          getDirectoryCount() == ((ContentSummary) to).getDirectoryCount() &&
+          getFileCount() == right.getFileCount() &&
+          getDirectoryCount() == right.getDirectoryCount() &&
+          getSnapshotLength() == right.getSnapshotLength() &&
+          getSnapshotFileCount() == right.getSnapshotFileCount() &&
+          getSnapshotDirectoryCount() == right.getSnapshotDirectoryCount() &&
+          getSnapshotSpaceConsumed() == right.getSnapshotSpaceConsumed() &&
           super.equals(to);
     } else {
       return super.equals(to);
@@ -191,7 +245,9 @@ public class ContentSummary extends QuotaUsage implements Writable{
 
   @Override
   public int hashCode() {
-    long result = getLength() ^ getFileCount() ^ getDirectoryCount();
+    long result = getLength() ^ getFileCount() ^ getDirectoryCount()
+        ^ getSnapshotLength() ^ getSnapshotFileCount()
+        ^ getSnapshotDirectoryCount() ^ getSnapshotSpaceConsumed();
     return ((int) result) ^ super.hashCode();
   }
 
@@ -255,15 +311,14 @@ public class ContentSummary extends QuotaUsage implements Writable{
    * @param qOption a flag indicating if quota needs to be printed or not
    * @return the string representation of the object
   */
+  @Override
   public String toString(boolean qOption) {
     return toString(qOption, false);
   }
 
   /** Return the string representation of the object in the output format.
-   * if qOption is false, output directory count, file count, and content size;
+   * For description of the options,
-   * if qOption is true, output quota and remaining quota as well.
+   * @see #toString(boolean, boolean, boolean, boolean, List)
-   * if hOption is false file sizes are returned in bytes
-   * if hOption is true file sizes are returned in human readable 
    * 
    * @param qOption a flag indicating if quota needs to be printed or not
    * @param hOption a flag indicating if human readable output if to be used
@@ -273,10 +328,24 @@ public class ContentSummary extends QuotaUsage implements Writable{
     return toString(qOption, hOption, false, null);
   }
 
+  /** Return the string representation of the object in the output format.
+   * For description of the options,
+   * @see #toString(boolean, boolean, boolean, boolean, List)
+   *
+   * @param qOption a flag indicating if quota needs to be printed or not
+   * @param hOption a flag indicating if human readable output is to be used
+   * @param xOption a flag indicating if calculation from snapshots is to be
+   *                included in the output
+   * @return the string representation of the object
+   */
+  public String toString(boolean qOption, boolean hOption, boolean xOption) {
+    return toString(qOption, hOption, false, xOption, null);
+  }
+
   /**
    * Return the string representation of the object in the output format.
-   * if tOption is true, display the quota by storage types,
+   * For description of the options,
-   * Otherwise, same logic with #toString(boolean,boolean)
+   * @see #toString(boolean, boolean, boolean, boolean, List)
    *
    * @param qOption a flag indicating if quota needs to be printed or not
    * @param hOption a flag indicating if human readable output if to be used
@@ -286,6 +355,29 @@ public class ContentSummary extends QuotaUsage implements Writable{
    */
   public String toString(boolean qOption, boolean hOption,
                          boolean tOption, List<StorageType> types) {
+    return toString(qOption, hOption, tOption, false, types);
+  }
+
+  /** Return the string representation of the object in the output format.
+   * if qOption is false, output directory count, file count, and content size;
+   * if qOption is true, output quota and remaining quota as well.
+   * if hOption is false, file sizes are returned in bytes
+   * if hOption is true, file sizes are returned in human readable
+   * if tOption is true, display the quota by storage types
+   * if tOption is false, same logic with #toString(boolean,boolean)
+   * if xOption is false, output includes the calculation from snapshots
+   * if xOption is true, output excludes the calculation from snapshots
+   *
+   * @param qOption a flag indicating if quota needs to be printed or not
+   * @param hOption a flag indicating if human readable output is to be used
+   * @param tOption a flag indicating if display quota by storage types
+   * @param xOption a flag indicating if calculation from snapshots is to be
+   *                included in the output
+   * @param types Storage types to display
+   * @return the string representation of the object
+   */
+  public String toString(boolean qOption, boolean hOption, boolean tOption,
+      boolean xOption, List<StorageType> types) {
     String prefix = "";
 
     if (tOption) {
@@ -296,10 +388,17 @@ public class ContentSummary extends QuotaUsage implements Writable{
       prefix = getQuotaUsage(hOption);
     }
 
-    return prefix + String.format(SUMMARY_FORMAT,
+    if (xOption) {
-        formatSize(directoryCount, hOption),
+      return prefix + String.format(SUMMARY_FORMAT,
-        formatSize(fileCount, hOption),
+          formatSize(directoryCount - snapshotDirectoryCount, hOption),
-        formatSize(length, hOption));
+          formatSize(fileCount - snapshotFileCount, hOption),
+          formatSize(length - snapshotLength, hOption));
+    } else {
+      return prefix + String.format(SUMMARY_FORMAT,
+          formatSize(directoryCount, hOption),
+          formatSize(fileCount, hOption),
+          formatSize(length, hOption));
+    }
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java

@@ -51,6 +51,8 @@ public class Count extends FsCommand {
   private static final String OPTION_HUMAN = "h";
   private static final String OPTION_HEADER = "v";
   private static final String OPTION_TYPE = "t";
+  // exclude snapshots from calculation. Only work on default columns.
+  private static final String OPTION_EXCLUDE_SNAPSHOT = "x";
   //return the quota, namespace count and disk space usage.
   private static final String OPTION_QUOTA_AND_USAGE = "u";
 
@@ -58,7 +60,8 @@ public class Count extends FsCommand {
   public static final String USAGE =
       "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
           + "] [-" + OPTION_TYPE + " [<storage type>]] [-" +
-          OPTION_QUOTA_AND_USAGE + "] <path> ...";
+          OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT
+          + "] <path> ...";
   public static final String DESCRIPTION =
       "Count the number of directories, files and bytes under the paths\n" +
           "that match the specified file pattern.  The output columns are:\n" +
@@ -72,6 +75,8 @@ public class Count extends FsCommand {
           "The -" + OPTION_HUMAN +
           " option shows file sizes in human readable format.\n" +
           "The -" + OPTION_HEADER + " option displays a header line.\n" +
+          "The -" + OPTION_EXCLUDE_SNAPSHOT + " option excludes snapshots " +
+          "from being calculated. \n" +
           "The -" + OPTION_TYPE + " option displays quota by storage types.\n" +
           "It must be used with -" + OPTION_QUOTA + " option.\n" +
           "If a comma-separated list of storage types is given after the -" +
@@ -87,6 +92,7 @@ public class Count extends FsCommand {
   private boolean showQuotabyType;
   private List<StorageType> storageTypes = null;
   private boolean showQuotasAndUsageOnly;
+  private boolean excludeSnapshots;
 
   /** Constructor */
   public Count() {}
@@ -106,7 +112,8 @@ public class Count extends FsCommand {
   @Override
   protected void processOptions(LinkedList<String> args) {
     CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
-        OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE);
+        OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE,
+        OPTION_EXCLUDE_SNAPSHOT);
     cf.addOptionWithValue(OPTION_TYPE);
     cf.parse(args);
     if (args.isEmpty()) { // default path is the current working directory
@@ -115,6 +122,7 @@ public class Count extends FsCommand {
     showQuotas = cf.getOpt(OPTION_QUOTA);
     humanReadable = cf.getOpt(OPTION_HUMAN);
     showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE);
+    excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT);
 
     if (showQuotas || showQuotasAndUsageOnly) {
       String types = cf.getOptValue(OPTION_TYPE);
@@ -125,6 +133,11 @@ public class Count extends FsCommand {
       } else {
         showQuotabyType = false;
       }
+      if (excludeSnapshots) {
+        out.println(OPTION_QUOTA + " or " + OPTION_QUOTA_AND_USAGE + " option "
+            + "is given, the -x option is ignored.");
+        excludeSnapshots = false;
+      }
     }
 
     if (cf.getOpt(OPTION_HEADER)) {
@@ -163,7 +176,8 @@ public class Count extends FsCommand {
           storageTypes) + src);
     } else {
       ContentSummary summary = src.fs.getContentSummary(src.path);
-      out.println(summary.toString(showQuotas, isHumanReadable()) + src);
+      out.println(summary.
+          toString(showQuotas, isHumanReadable(), excludeSnapshots) + src);
     }
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java

@@ -26,6 +26,7 @@ import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FsStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.StringUtils;
@@ -106,27 +107,30 @@ class FsUsage extends FsCommand {
   /** show disk usage */
   public static class Du extends FsUsage {
     public static final String NAME = "du";
-    public static final String USAGE = "[-s] [-h] <path> ...";
+    public static final String USAGE = "[-s] [-h] [-x] <path> ...";
     public static final String DESCRIPTION =
-    "Show the amount of space, in bytes, used by the files that " +
+        "Show the amount of space, in bytes, used by the files that match " +
-    "match the specified file pattern. The following flags are optional:\n" +
+            "the specified file pattern. The following flags are optional:\n" +
-    "-s: Rather than showing the size of each individual file that" +
+            "-s: Rather than showing the size of each individual file that" +
-    " matches the pattern, shows the total (summary) size.\n" +
+            " matches the pattern, shows the total (summary) size.\n" +
-    "-h: Formats the sizes of files in a human-readable fashion" +
+            "-h: Formats the sizes of files in a human-readable fashion" +
-    " rather than a number of bytes.\n\n" +
+            " rather than a number of bytes.\n" +
-    "Note that, even without the -s option, this only shows size summaries " +
+            "-x: Excludes snapshots from being counted.\n\n" +
-    "one level deep into a directory.\n\n" +
+            "Note that, even without the -s option, this only shows size " +
-    "The output is in the form \n" + 
+            "summaries one level deep into a directory.\n\n" +
-    "\tsize\tname(full path)\n"; 
+            "The output is in the form \n" +
+            "\tsize\tname(full path)\n";
 
     protected boolean summary = false;
+    private boolean excludeSnapshots = false;
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
-      CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s");
+      CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s", "x");
       cf.parse(args);
       humanReadable = cf.getOpt("h");
       summary = cf.getOpt("s");
+      excludeSnapshots = cf.getOpt("x");
       if (args.isEmpty()) args.add(Path.CUR_DIR);
     }
 
@@ -144,11 +148,10 @@ class FsUsage extends FsCommand {
 
     @Override
     protected void processPath(PathData item) throws IOException {
-      long length;
+      ContentSummary contentSummary = item.fs.getContentSummary(item.path);
-      if (item.stat.isDirectory()) {
+      long length = contentSummary.getLength();
-        length = item.fs.getContentSummary(item.path).getLength();
+      if (excludeSnapshots) {
-      } else {
+        length -= contentSummary.getSnapshotLength();
-        length = item.stat.getLen();
       }
       usagesTable.addRow(formatSize(length), item);
     }

hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md

@@ -132,10 +132,12 @@ Similar to get command, except that the destination is restricted to a local fil
 count
 -----
 
-Usage: `hadoop fs -count [-q] [-h] [-v] [-t [<storage type>]] [-u] <paths> `
+Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] <paths> `
 
 Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
 
+The -u and -q options control what columns the output contains.  -q means show quotas, -u limits the output to show quotas and usage only.
+
 The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
 
 The output columns with -count -u are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA
@@ -146,6 +148,8 @@ The -h option shows sizes in human readable format.
 
 The -v option displays a header line.
 
+The -x option excludes snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path. The -x option is ignored if -u or -q option is given.
+
 Example:
 
 * `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
@@ -211,14 +215,15 @@ Example:
 du
 ----
 
-Usage: `hadoop fs -du [-s] [-h] URI [URI ...]`
+Usage: `hadoop fs -du [-s] [-h] [-x] URI [URI ...]`
 
 Displays sizes of files and directories contained in the given directory or the length of a file in case its just a file.
 
 Options:
 
-* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files.
+* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files. Without the -s option, calculation is done by going 1-level deep from the given path.
 * The -h option will format file sizes in a "human-readable" fashion (e.g 64.0m instead of 67108864)
+* The -x option will exclude snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path.
 
 The du returns three columns with the following format:
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java

@@ -447,7 +447,7 @@ public class TestCount {
     Count count = new Count();
     String actual = count.getUsage();
     String expected =
-        "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] <path> ...";
+        "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] <path> ...";
     assertEquals("Count.getUsage", expected, actual);
   }
 
@@ -465,6 +465,7 @@ public class TestCount {
         + "      DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
         + "The -h option shows file sizes in human readable format.\n"
         + "The -v option displays a header line.\n"
+        + "The -x option excludes snapshots from being calculated. \n"
         + "The -t option displays quota by storage types.\n"
         + "It must be used with -q option.\n"
         + "If a comma-separated list of storage types is given after the -t option, \n"
@@ -520,7 +521,7 @@ public class TestCount {
     }
 
     @Override
-    public String toString(boolean qOption, boolean hOption) {
+    public String toString(boolean qOption, boolean hOption, boolean xOption) {
       if (qOption) {
         if (hOption) {
           return (HUMAN + WITH_QUOTAS);

hadoop-common-project/hadoop-common/src/test/resources/testConf.xml

@@ -200,7 +200,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-du \[-s\] \[-h\] &lt;path&gt; \.\.\. :\s*</expected-output>
+          <expected-output>^-du \[-s\] \[-h\] \[-x\] &lt;path&gt; \.\.\. :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -226,6 +226,10 @@
           <type>RegexpComparator</type>
           <expected-output>\s*of bytes.\s*</expected-output>
         </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-x\s*Excludes snapshots from being counted.\s*</expected-output>
+        </comparator>
         <comparator>
           <type>RegexpComparator</type>
           <expected-output>^\s*Note that, even without the -s option, this only shows size summaries one level\s*</expected-output>
@@ -274,7 +278,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] &lt;path&gt; \.\.\. :( )*</expected-output>
+          <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] \[-x\] &lt;path&gt; \.\.\. :( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -308,6 +312,10 @@
           <type>RegexpComparator</type>
           <expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output>
         </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*The -x option excludes snapshots from being calculated.( )*</expected-output>
+        </comparator>
       </comparators>
     </test>
 

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java

@@ -1364,6 +1364,10 @@ public class PBHelperClient {
     builder.length(cs.getLength()).
         fileCount(cs.getFileCount()).
         directoryCount(cs.getDirectoryCount()).
+        snapshotLength(cs.getSnapshotLength()).
+        snapshotFileCount(cs.getSnapshotFileCount()).
+        snapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
+        snapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
         quota(cs.getQuota()).
         spaceConsumed(cs.getSpaceConsumed()).
         spaceQuota(cs.getSpaceQuota());
@@ -1958,6 +1962,10 @@ public class PBHelperClient {
     builder.setLength(cs.getLength()).
         setFileCount(cs.getFileCount()).
         setDirectoryCount(cs.getDirectoryCount()).
+        setSnapshotLength(cs.getSnapshotLength()).
+        setSnapshotFileCount(cs.getSnapshotFileCount()).
+        setSnapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
+        setSnapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
         setQuota(cs.getQuota()).
         setSpaceConsumed(cs.getSpaceConsumed()).
         setSpaceQuota(cs.getSpaceQuota());

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto

@@ -136,6 +136,10 @@ message ContentSummaryProto {
   required uint64 spaceConsumed = 5;
   required uint64 spaceQuota = 6;
   optional StorageTypeQuotaInfosProto typeQuotaInfos = 7;
+  optional uint64 snapshotLength = 8;
+  optional uint64 snapshotFileCount = 9;
+  optional uint64 snapshotDirectoryCount = 10;
+  optional uint64 snapshotSpaceConsumed = 11;
 }
 
 /**

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java

@@ -29,6 +29,7 @@ public class ContentSummaryComputationContext {
   private FSNamesystem fsn = null;
   private BlockStoragePolicySuite bsps = null;
   private ContentCounts counts = null;
+  private ContentCounts snapshotCounts = null;
   private long nextCountLimit = 0;
   private long limitPerRun = 0;
   private long yieldCount = 0;
@@ -51,6 +52,7 @@ public class ContentSummaryComputationContext {
     this.limitPerRun = limitPerRun;
     this.nextCountLimit = limitPerRun;
     this.counts = new ContentCounts.Builder().build();
+    this.snapshotCounts = new ContentCounts.Builder().build();
     this.sleepMilliSec = sleepMicroSec/1000;
     this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000);
   }
@@ -125,6 +127,10 @@ public class ContentSummaryComputationContext {
     return counts;
   }
 
+  public ContentCounts getSnapshotCounts() {
+    return snapshotCounts;
+  }
+
   public BlockStoragePolicySuite getBlockStoragePolicySuite() {
     Preconditions.checkState((bsps != null || fsn != null),
         "BlockStoragePolicySuite must be either initialized or available via" +

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java

@@ -428,8 +428,9 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
    */
   public final ContentSummary computeAndConvertContentSummary(int snapshotId,
       ContentSummaryComputationContext summary) {
-    ContentCounts counts = computeContentSummary(snapshotId, summary)
+    computeContentSummary(snapshotId, summary);
-        .getCounts();
+    final ContentCounts counts = summary.getCounts();
+    final ContentCounts snapshotCounts = summary.getSnapshotCounts();
     final QuotaCounts q = getQuotaCounts();
     return new ContentSummary.Builder().
         length(counts.getLength()).
@@ -440,6 +441,10 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
         spaceQuota(q.getStorageSpace()).
         typeConsumed(counts.getTypeSpaces()).
         typeQuota(q.getTypeSpaces().asArray()).
+        snapshotLength(snapshotCounts.getLength()).
+        snapshotFileCount(snapshotCounts.getFileCount()).
+        snapshotDirectoryCount(snapshotCounts.getDirectoryCount()).
+        snapshotSpaceConsumed(snapshotCounts.getStoragespace()).
         build();
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java

@@ -634,6 +634,10 @@ public class INodeDirectory extends INodeWithAdditionalFields
       // computation should include all the deleted files/directories
       sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
           summary.getCounts());
+      // Also compute ContentSummary for snapshotCounts (So we can extract it
+      // later from the ContentSummary of all).
+      sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
+          summary.getSnapshotCounts());
     }
     final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
     if (q != null && snapshotId == Snapshot.CURRENT_STATE_ID) {

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java

@@ -105,6 +105,11 @@ public class TestDFSShell {
     return p;
   }
 
+  static void rmr(FileSystem fs, Path p) throws IOException {
+    assertTrue(fs.delete(p, true));
+    assertFalse(fs.exists(p));
+  }
+
   static File createLocalFile(File f) throws IOException {
     assertTrue(!f.exists());
     PrintWriter out = new PrintWriter(f);
@@ -214,6 +219,7 @@ public class TestDFSShell {
     shell.setConf(conf);
     
     try {
+      cluster.waitActive();
       Path myPath = new Path("/test/dir");
       assertTrue(fs.mkdirs(myPath));
       assertTrue(fs.exists(myPath));
@@ -239,7 +245,7 @@ public class TestDFSShell {
       assertTrue(val == 0);
       String returnString = out.toString();
       out.reset();
-      // Check if size matchs as expected
+      // Check if size matches as expected
       assertThat(returnString, containsString(myFileLength.toString()));
       assertThat(returnString, containsString(myFile2Length.toString()));
       
@@ -274,6 +280,221 @@ public class TestDFSShell {
                                   
   }
 
+  @Test (timeout = 180000)
+  public void testDuSnapshots() throws IOException {
+    final int replication = 2;
+    final Configuration conf = new HdfsConfiguration();
+    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(replication).build();
+    final DistributedFileSystem dfs = cluster.getFileSystem();
+    final PrintStream psBackup = System.out;
+    final ByteArrayOutputStream out = new ByteArrayOutputStream();
+    final PrintStream psOut = new PrintStream(out);
+    final FsShell shell = new FsShell();
+    shell.setConf(conf);
+
+    try {
+      System.setOut(psOut);
+      cluster.waitActive();
+      final Path parent = new Path("/test");
+      final Path dir = new Path(parent, "dir");
+      mkdir(dfs, dir);
+      final Path file = new Path(dir, "file");
+      writeFile(dfs, file);
+      final Path file2 = new Path(dir, "file2");
+      writeFile(dfs, file2);
+      final Long fileLength = dfs.getFileStatus(file).getLen();
+      final Long file2Length = dfs.getFileStatus(file2).getLen();
+
+      /*
+       * Construct dir as follows:
+       * /test/dir/file   <- this will later be deleted after snapshot taken.
+       * /test/dir/newfile <- this will be created after snapshot taken.
+       * /test/dir/file2
+       * Snapshot enabled on /test
+       */
+
+      // test -du on /test/dir
+      int ret = -1;
+      try {
+        ret = shell.run(new String[] {"-du", dir.toString()});
+      } catch (Exception e) {
+        System.err.println("Exception raised from DFSShell.run " +
+            e.getLocalizedMessage());
+      }
+      assertEquals(0, ret);
+      String returnString = out.toString();
+      LOG.info("-du return is:\n" + returnString);
+      // Check if size matches as expected
+      assertTrue(returnString.contains(fileLength.toString()));
+      assertTrue(returnString.contains(file2Length.toString()));
+      out.reset();
+
+      // take a snapshot, then remove file and add newFile
+      final String snapshotName = "ss1";
+      final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName);
+      dfs.allowSnapshot(parent);
+      assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath));
+      rmr(dfs, file);
+      final Path newFile = new Path(dir, "newfile");
+      writeFile(dfs, newFile);
+      final Long newFileLength = dfs.getFileStatus(newFile).getLen();
+
+      // test -du -s on /test
+      ret = -1;
+      try {
+        ret = shell.run(new String[] {"-du", "-s", parent.toString()});
+      } catch (Exception e) {
+        System.err.println("Exception raised from DFSShell.run " +
+            e.getLocalizedMessage());
+      }
+      assertEquals(0, ret);
+      returnString = out.toString();
+      LOG.info("-du -s return is:\n" + returnString);
+      Long combinedLength = fileLength + file2Length + newFileLength;
+      assertTrue(returnString.contains(combinedLength.toString()));
+      out.reset();
+
+      // test -du on /test
+      ret = -1;
+      try {
+        ret = shell.run(new String[] {"-du", parent.toString()});
+      } catch (Exception e) {
+        System.err.println("Exception raised from DFSShell.run " +
+            e.getLocalizedMessage());
+      }
+      assertEquals(0, ret);
+      returnString = out.toString();
+      LOG.info("-du return is:\n" + returnString);
+      assertTrue(returnString.contains(combinedLength.toString()));
+      out.reset();
+
+      // test -du -s -x on /test
+      ret = -1;
+      try {
+        ret = shell.run(new String[] {"-du", "-s", "-x", parent.toString()});
+      } catch (Exception e) {
+        System.err.println("Exception raised from DFSShell.run " +
+            e.getLocalizedMessage());
+      }
+      assertEquals(0, ret);
+      returnString = out.toString();
+      LOG.info("-du -s -x return is:\n" + returnString);
+      Long exludeSnapshotLength = file2Length + newFileLength;
+      assertTrue(returnString.contains(exludeSnapshotLength.toString()));
+      out.reset();
+
+      // test -du -x on /test
+      ret = -1;
+      try {
+        ret = shell.run(new String[] {"-du", "-x", parent.toString()});
+      } catch (Exception e) {
+        System.err.println("Exception raised from DFSShell.run " +
+            e.getLocalizedMessage());
+      }
+      assertEquals(0, ret);
+      returnString = out.toString();
+      LOG.info("-du -x return is:\n" + returnString);
+      assertTrue(returnString.contains(exludeSnapshotLength.toString()));
+      out.reset();
+    } finally {
+      System.setOut(psBackup);
+      cluster.shutdown();
+    }
+  }
+
+  @Test (timeout = 180000)
+  public void testCountSnapshots() throws IOException {
+    final int replication = 2;
+    final Configuration conf = new HdfsConfiguration();
+    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(replication).build();
+    final DistributedFileSystem dfs = cluster.getFileSystem();
+    final PrintStream psBackup = System.out;
+    final ByteArrayOutputStream out = new ByteArrayOutputStream();
+    final PrintStream psOut = new PrintStream(out);
+    System.setOut(psOut);
+    final FsShell shell = new FsShell();
+    shell.setConf(conf);
+
+    try {
+      cluster.waitActive();
+      final Path parent = new Path("/test");
+      final Path dir = new Path(parent, "dir");
+      mkdir(dfs, dir);
+      final Path file = new Path(dir, "file");
+      writeFile(dfs, file);
+      final Path file2 = new Path(dir, "file2");
+      writeFile(dfs, file2);
+      final long fileLength = dfs.getFileStatus(file).getLen();
+      final long file2Length = dfs.getFileStatus(file2).getLen();
+      final Path dir2 = new Path(parent, "dir2");
+      mkdir(dfs, dir2);
+
+      /*
+       * Construct dir as follows:
+       * /test/dir/file   <- this will later be deleted after snapshot taken.
+       * /test/dir/newfile <- this will be created after snapshot taken.
+       * /test/dir/file2
+       * /test/dir2       <- this will later be deleted after snapshot taken.
+       * Snapshot enabled on /test
+       */
+
+      // take a snapshot
+      // then create /test/dir/newfile and remove /test/dir/file, /test/dir2
+      final String snapshotName = "s1";
+      final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName);
+      dfs.allowSnapshot(parent);
+      assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath));
+      rmr(dfs, file);
+      rmr(dfs, dir2);
+      final Path newFile = new Path(dir, "new file");
+      writeFile(dfs, newFile);
+      final Long newFileLength = dfs.getFileStatus(newFile).getLen();
+
+      // test -count on /test. Include header for easier debugging.
+      int val = -1;
+      try {
+        val = shell.run(new String[] {"-count", "-v", parent.toString() });
+      } catch (Exception e) {
+        System.err.println("Exception raised from DFSShell.run " +
+            e.getLocalizedMessage());
+      }
+      assertEquals(0, val);
+      String returnString = out.toString();
+      LOG.info("-count return is:\n" + returnString);
+      Scanner in = new Scanner(returnString);
+      in.nextLine();
+      assertEquals(3, in.nextLong()); //DIR_COUNT
+      assertEquals(3, in.nextLong()); //FILE_COUNT
+      assertEquals(fileLength + file2Length + newFileLength,
+          in.nextLong()); //CONTENT_SIZE
+      out.reset();
+
+      // test -count -x on /test. Include header for easier debugging.
+      val = -1;
+      try {
+        val =
+            shell.run(new String[] {"-count", "-x", "-v", parent.toString()});
+      } catch (Exception e) {
+        System.err.println("Exception raised from DFSShell.run " +
+            e.getLocalizedMessage());
+      }
+      assertEquals(0, val);
+      returnString = out.toString();
+      LOG.info("-count -x return is:\n" + returnString);
+      in = new Scanner(returnString);
+      in.nextLine();
+      assertEquals(2, in.nextLong()); //DIR_COUNT
+      assertEquals(2, in.nextLong()); //FILE_COUNT
+      assertEquals(file2Length + newFileLength, in.nextLong()); //CONTENT_SIZE
+      out.reset();
+    } finally {
+      System.setOut(psBackup);
+      cluster.shutdown();
+    }
+  }
+
   @Test (timeout = 30000)
   public void testPut() throws IOException {
     Configuration conf = new HdfsConfiguration();