HADOOP-10691. Improve the readability of 'hadoop fs -help'. Contributed by Lei Xu.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1602331 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -77,6 +77,9 @@ Release 2.5.0 - UNRELEASED
 
     HADOOP-6350. Documenting Hadoop metrics. (Akira Ajisaka via Arpit Agarwal)
 
+    HADOOP-10691. Improve the readability of 'hadoop fs -help'.
+    (Lei Xu via wang)
+
   OPTIMIZATIONS
 
   BUG FIXES 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java

@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.LinkedList;
 
+import org.apache.commons.lang.WordUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -31,6 +32,7 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.shell.Command;
 import org.apache.hadoop.fs.shell.CommandFactory;
 import org.apache.hadoop.fs.shell.FsCommand;
+import org.apache.hadoop.tools.TableListing;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
@@ -40,6 +42,8 @@ public class FsShell extends Configured implements Tool {
   
   static final Log LOG = LogFactory.getLog(FsShell.class);
 
+  private static final int MAX_LINE_WIDTH = 80;
+
   private FileSystem fs;
   private Trash trash;
   protected CommandFactory commandFactory;
@@ -117,7 +121,7 @@ public class FsShell extends Configured implements Tool {
     public static final String NAME = "usage";
     public static final String USAGE = "[cmd ...]";
     public static final String DESCRIPTION =
-      "Displays the usage for given command or all commands if none\n" +
+      "Displays the usage for given command or all commands if none " +
       "is specified.";
     
     @Override
@@ -137,7 +141,7 @@ public class FsShell extends Configured implements Tool {
     public static final String NAME = "help";
     public static final String USAGE = "[cmd ...]";
     public static final String DESCRIPTION =
-      "Displays help for given command or all commands if none\n" +
+      "Displays help for given command or all commands if none " +
       "is specified.";
     
     @Override
@@ -197,7 +201,7 @@ public class FsShell extends Configured implements Tool {
       for (String name : commandFactory.getNames()) {
         Command instance = commandFactory.getInstance(name);
         if (!instance.isDeprecated()) {
-          System.out.println("\t[" + instance.getUsage() + "]");
+          out.println("\t[" + instance.getUsage() + "]");
           instances.add(instance);
         }
       }
@@ -217,20 +221,48 @@ public class FsShell extends Configured implements Tool {
     out.println(usagePrefix + " " + instance.getUsage());
   }
 
-  // TODO: will eventually auto-wrap the text, but this matches the expected
-  // output for the hdfs tests...
   private void printInstanceHelp(PrintStream out, Command instance) {
-    boolean firstLine = true;
+    out.println(instance.getUsage() + " :");
+    TableListing listing = null;
+    final String prefix = "  ";
     for (String line : instance.getDescription().split("\n")) {
-      String prefix;
+      if (line.matches("^[ \t]*[-<].*$")) {
-      if (firstLine) {
+        String[] segments = line.split(":");
-        prefix = instance.getUsage() + ":\t";
+        if (segments.length == 2) {
-        firstLine = false;
+          if (listing == null) {
-      } else {
+            listing = createOptionTableListing();
-        prefix = "\t\t";
+          }
+          listing.addRow(segments[0].trim(), segments[1].trim());
+          continue;
+        }
+      }
+
+      // Normal literal description.
+      if (listing != null) {
+        for (String listingLine : listing.toString().split("\n")) {
+          out.println(prefix + listingLine);
+        }
+        listing = null;
+      }
+
+      for (String descLine : WordUtils.wrap(
+          line, MAX_LINE_WIDTH, "\n", true).split("\n")) {
+        out.println(prefix + descLine);
       }
-      System.out.println(prefix + line);
     }
+
+    if (listing != null) {
+      for (String listingLine : listing.toString().split("\n")) {
+        out.println(prefix + listingLine);
+      }
+    }
+  }
+
+  // Creates a two-row table, the first row is for the command line option,
+  // the second row is for the option description.
+  private TableListing createOptionTableListing() {
+    return new TableListing.Builder().addField("").addField("", true)
+        .wrapWidth(MAX_LINE_WIDTH).build();
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShellPermissions.java

@@ -63,18 +63,18 @@ public class FsShellPermissions extends FsCommand {
     public static final String NAME = "chmod";
     public static final String USAGE = "[-R] <MODE[,MODE]... | OCTALMODE> PATH...";
     public static final String DESCRIPTION =
-      "Changes permissions of a file.\n" +
+      "Changes permissions of a file. " +
-      "\tThis works similar to shell's chmod with a few exceptions.\n\n" +
+      "This works similar to the shell's chmod command with a few exceptions.\n" +
-      "-R\tmodifies the files recursively. This is the only option\n" +
+      "-R: modifies the files recursively. This is the only option" +
-      "\tcurrently supported.\n\n" +
+      " currently supported.\n" +
-      "MODE\tMode is same as mode used for chmod shell command.\n" +
+      "<MODE>: Mode is the same as mode used for the shell's command. " +
-      "\tOnly letters recognized are 'rwxXt'. E.g. +t,a+r,g-w,+rwx,o=r\n\n" +
+      "The only letters recognized are 'rwxXt', e.g. +t,a+r,g-w,+rwx,o=r.\n" +
-      "OCTALMODE Mode specifed in 3 or 4 digits. If 4 digits, the first may\n" +
+      "<OCTALMODE>: Mode specifed in 3 or 4 digits. If 4 digits, the first " +
-      "be 1 or 0 to turn the sticky bit on or off, respectively.  Unlike " +
+      "may be 1 or 0 to turn the sticky bit on or off, respectively.  Unlike " +
-      "shell command, it is not possible to specify only part of the mode\n" +
+      "the shell command, it is not possible to specify only part of the " +
-      "\tE.g. 754 is same as u=rwx,g=rx,o=r\n\n" +
+      "mode, e.g. 754 is same as u=rwx,g=rx,o=r.\n\n" +
-      "\tIf none of 'augo' is specified, 'a' is assumed and unlike\n" +
+      "If none of 'augo' is specified, 'a' is assumed and unlike the " +
-      "\tshell command, no umask is applied.";
+      "shell command, no umask is applied.";
 
     protected ChmodParser pp;
 
@@ -121,18 +121,18 @@ public class FsShellPermissions extends FsCommand {
     public static final String NAME = "chown";
     public static final String USAGE = "[-R] [OWNER][:[GROUP]] PATH...";
     public static final String DESCRIPTION =
-      "Changes owner and group of a file.\n" +
+      "Changes owner and group of a file. " +
-      "\tThis is similar to shell's chown with a few exceptions.\n\n" +
+      "This is similar to the shell's chown command with a few exceptions.\n" +
-      "\t-R\tmodifies the files recursively. This is the only option\n" +
+      "-R: modifies the files recursively. This is the only option " +
-      "\tcurrently supported.\n\n" +
+      "currently supported.\n\n" +
-      "\tIf only owner or group is specified then only owner or\n" +
+      "If only the owner or group is specified, then only the owner or " +
-      "\tgroup is modified.\n\n" +
+      "group is modified. " +
-      "\tThe owner and group names may only consist of digits, alphabet,\n"+
+      "The owner and group names may only consist of digits, alphabet, "+
-      "\tand any of " + allowedChars + ". The names are case sensitive.\n\n" +
+      "and any of " + allowedChars + ". The names are case sensitive.\n\n" +
-      "\tWARNING: Avoid using '.' to separate user name and group though\n" +
+      "WARNING: Avoid using '.' to separate user name and group though " +
-      "\tLinux allows it. If user names have dots in them and you are\n" +
+      "Linux allows it. If user names have dots in them and you are " +
-      "\tusing local file system, you might see surprising results since\n" +
+      "using local file system, you might see surprising results since " +
-      "\tshell command 'chown' is used for local files.";
+      "the shell command 'chown' is used for local files.";
 
     ///allows only "allowedChars" above in names for owner and group
     static private final Pattern chownPattern = Pattern.compile(

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java

@@ -59,8 +59,8 @@ class AclCommands extends FsCommand {
     public static String DESCRIPTION = "Displays the Access Control Lists"
         + " (ACLs) of files and directories. If a directory has a default ACL,"
         + " then getfacl also displays the default ACL.\n"
-        + "-R: List the ACLs of all files and directories recursively.\n"
+        + "  -R: List the ACLs of all files and directories recursively.\n"
-        + "<path>: File or directory to list.\n";
+        + "  <path>: File or directory to list.\n";
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
@@ -153,19 +153,19 @@ class AclCommands extends FsCommand {
     public static String DESCRIPTION = "Sets Access Control Lists (ACLs)"
         + " of files and directories.\n" 
         + "Options:\n"
-        + "-b :Remove all but the base ACL entries. The entries for user,"
+        + "  -b :Remove all but the base ACL entries. The entries for user,"
         + " group and others are retained for compatibility with permission "
         + "bits.\n" 
-        + "-k :Remove the default ACL.\n"
+        + "  -k :Remove the default ACL.\n"
-        + "-R :Apply operations to all files and directories recursively.\n"
+        + "  -R :Apply operations to all files and directories recursively.\n"
-        + "-m :Modify ACL. New entries are added to the ACL, and existing"
+        + "  -m :Modify ACL. New entries are added to the ACL, and existing"
         + " entries are retained.\n"
-        + "-x :Remove specified ACL entries. Other ACL entries are retained.\n"
+        + "  -x :Remove specified ACL entries. Other ACL entries are retained.\n"
-        + "--set :Fully replace the ACL, discarding all existing entries."
+        + "  --set :Fully replace the ACL, discarding all existing entries."
         + " The <acl_spec> must include entries for user, group, and others"
         + " for compatibility with permission bits.\n"
-        + "<acl_spec>: Comma separated list of ACL entries.\n"
+        + "  <acl_spec>: Comma separated list of ACL entries.\n"
-        + "<path>: File or directory to modify.\n";
+        + "  <path>: File or directory to modify.\n";
 
     CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "b", "k", "R",
         "m", "x", "-set");

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java

@@ -55,10 +55,10 @@ class CopyCommands {
     public static final String NAME = "getmerge";    
     public static final String USAGE = "[-nl] <src> <localdst>";
     public static final String DESCRIPTION =
-      "Get all the files in the directories that\n" +
+      "Get all the files in the directories that " +
-      "match the source file pattern and merge and sort them to only\n" +
+      "match the source file pattern and merge and sort them to only " +
       "one file on local fs. <src> is kept.\n" +
-      "  -nl   Add a newline character at the end of each file.";
+      "-nl: Add a newline character at the end of each file.";
 
     protected PathData dst = null;
     protected String delimiter = null;
@@ -135,12 +135,12 @@ class CopyCommands {
     public static final String NAME = "cp";
     public static final String USAGE = "[-f] [-p | -p[topx]] <src> ... <dst>";
     public static final String DESCRIPTION =
-      "Copy files that match the file pattern <src> to a\n" +
+      "Copy files that match the file pattern <src> to a " +
-      "destination.  When copying multiple files, the destination\n" +
+      "destination.  When copying multiple files, the destination " +
-      "must be a directory. Passing -p preserves status\n" +
+      "must be a directory. Passing -p preserves status " +
-      "[topx] (timestamps, ownership, permission, XAttr).\n" +
+      "[topx] (timestamps, ownership, permission, XAttr). " +
-      "If -p is specified with no <arg>, then preserves\n" +
+      "If -p is specified with no <arg>, then preserves " +
-      "timestamps, ownership, permission. Passing -f\n" +
+      "timestamps, ownership, permission. Passing -f " +
       "overwrites the destination if it already exists.\n";
     
     @Override
@@ -184,10 +184,10 @@ class CopyCommands {
     public static final String USAGE =
       "[-p] [-ignoreCrc] [-crc] <src> ... <localdst>";
     public static final String DESCRIPTION =
-      "Copy files that match the file pattern <src>\n" +
+      "Copy files that match the file pattern <src> " +
-      "to the local name.  <src> is kept.  When copying multiple,\n" +
+      "to the local name.  <src> is kept.  When copying multiple " +
-      "files, the destination must be a directory. Passing\n" +
+      "files, the destination must be a directory. Passing " +
-      "-p preserves access and modification times,\n" +
+      "-p preserves access and modification times, " +
       "ownership and the mode.\n";
 
     @Override
@@ -211,11 +211,11 @@ class CopyCommands {
     public static final String NAME = "put";
     public static final String USAGE = "[-f] [-p] <localsrc> ... <dst>";
     public static final String DESCRIPTION =
-      "Copy files from the local file system\n" +
+      "Copy files from the local file system " +
-      "into fs. Copying fails if the file already\n" +
+      "into fs. Copying fails if the file already " +
-      "exists, unless the -f flag is given. Passing\n" +
+      "exists, unless the -f flag is given. Passing " +
-      "-p preserves access and modification times,\n" +
+      "-p preserves access and modification times, " +
-      "ownership and the mode. Passing -f overwrites\n" +
+      "ownership and the mode. Passing -f overwrites " +
       "the destination if it already exists.\n";
 
     @Override
@@ -278,9 +278,9 @@ class CopyCommands {
     public static final String NAME = "appendToFile";
     public static final String USAGE = "<localsrc> ... <dst>";
     public static final String DESCRIPTION =
-        "Appends the contents of all the given local files to the\n" +
+        "Appends the contents of all the given local files to the " +
-            "given dst file. The dst file will be created if it does\n" +
+            "given dst file. The dst file will be created if it does " +
-            "not exist. If <localSrc> is -, then the input is read\n" +
+            "not exist. If <localSrc> is -, then the input is read " +
             "from stdin.";
 
     private static final int DEFAULT_IO_LENGTH = 1024 * 1024;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java

@@ -51,13 +51,13 @@ class Delete {
     public static final String NAME = "rm";
     public static final String USAGE = "[-f] [-r|-R] [-skipTrash] <src> ...";
     public static final String DESCRIPTION =
-      "Delete all files that match the specified file pattern.\n" +
+      "Delete all files that match the specified file pattern. " +
       "Equivalent to the Unix command \"rm <src>\"\n" +
-      "-skipTrash option bypasses trash, if enabled, and immediately\n" +
+      "-skipTrash: option bypasses trash, if enabled, and immediately " +
       "deletes <src>\n" +
-      "  -f     If the file does not exist, do not display a diagnostic\n" +
+      "-f: If the file does not exist, do not display a diagnostic " +
-      "         message or modify the exit status to reflect an error.\n" +
+      "message or modify the exit status to reflect an error.\n" +
-      "  -[rR]  Recursively deletes directories";
+      "-[rR]:  Recursively deletes directories";
 
     private boolean skipTrash = false;
     private boolean deleteDirs = false;
@@ -147,7 +147,7 @@ class Delete {
     public static final String USAGE =
       "[--ignore-fail-on-non-empty] <dir> ...";
     public static final String DESCRIPTION =
-      "Removes the directory entry specified by each directory argument,\n" +
+      "Removes the directory entry specified by each directory argument, " +
       "provided it is empty.\n"; 
     
     private boolean ignoreNonEmpty = false;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java

@@ -75,7 +75,7 @@ class Display extends FsCommand {
     public static final String NAME = "cat";
     public static final String USAGE = "[-ignoreCrc] <src> ...";
     public static final String DESCRIPTION =
-      "Fetch all files that match the file pattern <src> \n" +
+      "Fetch all files that match the file pattern <src> " +
       "and display their content on stdout.\n";
 
     private boolean verifyChecksum = true;
@@ -170,11 +170,11 @@ class Display extends FsCommand {
     public static final String NAME = "checksum";
     public static final String USAGE = "<src> ...";
     public static final String DESCRIPTION =
-      "Dump checksum information for files that match the file\n" +
+      "Dump checksum information for files that match the file " +
-      "pattern <src> to stdout. Note that this requires a round-trip\n" +
+      "pattern <src> to stdout. Note that this requires a round-trip " +
-      "to a datanode storing each block of the file, and thus is not\n" +
+      "to a datanode storing each block of the file, and thus is not " +
-      "efficient to run on a large number of files. The checksum of a\n" +
+      "efficient to run on a large number of files. The checksum of a " +
-      "file depends on its content, block size and the checksum\n" +
+      "file depends on its content, block size and the checksum " +
       "algorithm and parameters used for creating the file.";
 
     @Override

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java

@@ -57,12 +57,12 @@ class FsUsage extends FsCommand {
     public static final String NAME = "df";
     public static final String USAGE = "[-h] [<path> ...]";
     public static final String DESCRIPTION =
-      "Shows the capacity, free and used space of the filesystem.\n"+
+      "Shows the capacity, free and used space of the filesystem. "+
-      "If the filesystem has multiple partitions, and no path to a\n" +
+      "If the filesystem has multiple partitions, and no path to a " +
-      "particular partition is specified, then the status of the root\n" +
+      "particular partition is specified, then the status of the root " +
       "partitions will be shown.\n" +
-      "  -h   Formats the sizes of files in a human-readable fashion\n" +
+      "-h: Formats the sizes of files in a human-readable fashion " +
-      "       rather than a number of bytes.\n\n";
+      "rather than a number of bytes.";
     
     @Override
     protected void processOptions(LinkedList<String> args)
@@ -108,14 +108,14 @@ class FsUsage extends FsCommand {
     public static final String NAME = "du";
     public static final String USAGE = "[-s] [-h] <path> ...";
     public static final String DESCRIPTION =
-    "Show the amount of space, in bytes, used by the files that\n" +
+    "Show the amount of space, in bytes, used by the files that " +
     "match the specified file pattern. The following flags are optional:\n" +
-    "  -s   Rather than showing the size of each individual file that\n" +
+    "-s: Rather than showing the size of each individual file that" +
-    "       matches the pattern, shows the total (summary) size.\n" +
+    " matches the pattern, shows the total (summary) size.\n" +
-    "  -h   Formats the sizes of files in a human-readable fashion\n" +
+    "-h: Formats the sizes of files in a human-readable fashion" +
-    "       rather than a number of bytes.\n\n" +
+    " rather than a number of bytes.\n\n" +
-    "Note that, even without the -s option, this only shows size summaries\n" +
+    "Note that, even without the -s option, this only shows size summaries " +
-    "one level deep into a directory.\n" +
+    "one level deep into a directory.\n\n" +
     "The output is in the form \n" + 
     "\tsize\tname(full path)\n"; 
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java

@@ -49,16 +49,16 @@ class Ls extends FsCommand {
   public static final String NAME = "ls";
   public static final String USAGE = "[-d] [-h] [-R] [<path> ...]";
   public static final String DESCRIPTION =
-		    "List the contents that match the specified file pattern. If\n" + 
+		    "List the contents that match the specified file pattern. If " +
-		    "path is not specified, the contents of /user/<currentUser>\n" +
+		    "path is not specified, the contents of /user/<currentUser> " +
-		    "will be listed. Directory entries are of the form \n" +
+		    "will be listed. Directory entries are of the form:\n" +
-		    "\tpermissions - userid groupid size_of_directory(in bytes) modification_date(yyyy-MM-dd HH:mm) directoryName \n" +
+		    "\tpermissions - userId groupId sizeOfDirectory(in bytes) modificationDate(yyyy-MM-dd HH:mm) directoryName\n\n" +
-		    "and file entries are of the form \n" + 
+		    "and file entries are of the form:\n" +
-		    "\tpermissions number_of_replicas userid groupid size_of_file(in bytes) modification_date(yyyy-MM-dd HH:mm) fileName \n" +
+		    "\tpermissions numberOfReplicas userId groupId sizeOfFile(in bytes) modificationDate(yyyy-MM-dd HH:mm) fileName\n" +
-		    "  -d  Directories are listed as plain files.\n" +
+		    "-d:  Directories are listed as plain files.\n" +
-		    "  -h  Formats the sizes of files in a human-readable fashion\n" +
+		    "-h:  Formats the sizes of files in a human-readable fashion " +
-		    "      rather than a number of bytes.\n" +
+		    "rather than a number of bytes.\n" +
-		    "  -R  Recursively list the contents of directories.";
+		    "-R:  Recursively list the contents of directories.";
 		  
   
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java

@@ -44,7 +44,7 @@ class Mkdir extends FsCommand {
   public static final String USAGE = "[-p] <path> ...";
   public static final String DESCRIPTION =
     "Create a directory in specified location.\n" +
-    "  -p  Do not fail if the directory already exists";
+    "-p: Do not fail if the directory already exists";
 
   private boolean createParents;
   

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java

@@ -45,7 +45,7 @@ class MoveCommands {
     public static final String NAME = "moveFromLocal";
     public static final String USAGE = "<localsrc> ... <dst>";
     public static final String DESCRIPTION = 
-      "Same as -put, except that the source is\n" +
+      "Same as -put, except that the source is " +
       "deleted after it's copied.";
 
     @Override
@@ -87,8 +87,8 @@ class MoveCommands {
     public static final String NAME = "mv";
     public static final String USAGE = "<src> ... <dst>";
     public static final String DESCRIPTION = 
-      "Move files that match the specified file pattern <src>\n" +
+      "Move files that match the specified file pattern <src> " +
-      "to a destination <dst>.  When moving multiple files, the\n" +
+      "to a destination <dst>.  When moving multiple files, the " +
       "destination must be a directory.";
 
     @Override

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java

@@ -41,12 +41,12 @@ class SetReplication extends FsCommand {
   public static final String NAME = "setrep";
   public static final String USAGE = "[-R] [-w] <rep> <path> ...";
   public static final String DESCRIPTION =
-    "Set the replication level of a file. If <path> is a directory\n" +
+    "Set the replication level of a file. If <path> is a directory " +
-    "then the command recursively changes the replication factor of\n" +
+    "then the command recursively changes the replication factor of " +
     "all files under the directory tree rooted at <path>.\n" +
-    "The -w flag requests that the command wait for the replication\n" +
+    "-w: It requests that the command waits for the replication " +
     "to complete. This can potentially take a very long time.\n" +
-    "The -R flag is accepted for backwards compatibility. It has no effect.";
+    "-R: It is accepted for backwards compatibility. It has no effect.";
   
   protected short newRep = 0;
   protected List<PathData> waitList = new LinkedList<PathData>();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java

@@ -51,8 +51,8 @@ class Stat extends FsCommand {
   public static final String NAME = "stat";
   public static final String USAGE = "[format] <path> ...";
   public static final String DESCRIPTION =
-    "Print statistics about the file/directory at <path>\n" +
+    "Print statistics about the file/directory at <path> " +
-    "in the specified format. Format accepts filesize in blocks (%b), group name of owner(%g),\n" +
+    "in the specified format. Format accepts filesize in blocks (%b), group name of owner(%g), " +
     "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y)\n";
 
   protected static final SimpleDateFormat timeFmt;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java

@@ -43,7 +43,7 @@ class Tail extends FsCommand {
   public static final String USAGE = "[-f] <file>";
   public static final String DESCRIPTION =
     "Show the last 1KB of the file.\n" +
-    "\t\tThe -f option shows appended data as the file grows.\n";
+    "-f: Shows appended data as the file grows.\n";
 
   private long startingOffset = -1024;
   private boolean follow = false;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java

@@ -43,8 +43,7 @@ class Test extends FsCommand {
     "  -e  return 0 if <path> exists.\n" +
     "  -f  return 0 if <path> is a file.\n" +
     "  -s  return 0 if file <path> is greater than zero bytes in size.\n" +
-    "  -z  return 0 if file <path> is zero bytes in size.\n" +
+    "  -z  return 0 if file <path> is zero bytes in size, else return 1.";
-    "else, return 1.";
 
   private char flag;
   

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java

@@ -47,8 +47,8 @@ class Touch extends FsCommand {
     public static final String NAME = "touchz";
     public static final String USAGE = "<path> ...";
     public static final String DESCRIPTION =
-      "Creates a file of zero length\n" +
+      "Creates a file of zero length " +
-      "at <path> with current time as the timestamp of that <path>.\n" +
+      "at <path> with current time as the timestamp of that <path>. " +
       "An error is returned if the file exists with non-zero length\n";
 
     @Override

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java

@@ -59,10 +59,10 @@ class XAttrCommands extends FsCommand {
       "-R: Recursively list the attributes for all files and directories.\n" +
       "-n name: Dump the named extended attribute value.\n" +
       "-d: Dump all extended attribute values associated with pathname.\n" +
-      "-e <encoding>: Encode values after retrieving them.\n" +
+      "-e <encoding>: Encode values after retrieving them." +
-      "Valid encodings are \"text\", \"hex\", and \"base64\".\n" +
+      "Valid encodings are \"text\", \"hex\", and \"base64\". " +
-      "Values encoded as text strings are enclosed in double quotes (\"),\n" +
+      "Values encoded as text strings are enclosed in double quotes (\")," +
-      " and values encoded as hexadecimal and base64 are prefixed with\n" +
+      " and values encoded as hexadecimal and base64 are prefixed with " +
       "0x and 0s, respectively.\n" +
       "<path>: The file or directory.\n";
     private final static Function<String, XAttrCodec> enValueOfFunc =
@@ -137,11 +137,11 @@ class XAttrCommands extends FsCommand {
     public static final String DESCRIPTION =
       "Sets an extended attribute name and value for a file or directory.\n" +
       "-n name: The extended attribute name.\n" +
-      "-v value: The extended attribute value. There are three different\n" +
+      "-v value: The extended attribute value. There are three different " +
-      "encoding methods for the value. If the argument is enclosed in double\n" +
+      "encoding methods for the value. If the argument is enclosed in double " +
-      "quotes, then the value is the string inside the quotes. If the\n" +
+      "quotes, then the value is the string inside the quotes. If the " +
-      "argument is prefixed with 0x or 0X, then it is taken as a hexadecimal\n" +
+      "argument is prefixed with 0x or 0X, then it is taken as a hexadecimal " +
-      "number. If the argument begins with 0s or 0S, then it is taken as a\n" +
+      "number. If the argument begins with 0s or 0S, then it is taken as a " +
       "base64 encoding.\n" +
       "-x name: Remove the extended attribute.\n" +
       "<path>: The file or directory.\n";

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hdfs.tools;
+package org.apache.hadoop.tools;
 
 import java.util.ArrayList;
 import java.util.LinkedList;

hadoop-common-project/hadoop-common/src/test/resources/testConf.xml

@@ -54,47 +54,55 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-ls \[-d\] \[-h\] \[-R\] \[&lt;path&gt; \.\.\.\]:( |\t)*List the contents that match the specified file pattern. If( )*</expected-output>
+          <expected-output>^-ls \[-d\] \[-h\] \[-R\] \[&lt;path&gt; \.\.\.\] :( |\t)*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*path is not specified, the contents of /user/&lt;currentUser&gt;( )*</expected-output>
+          <expected-output>^\s*List the contents that match the specified file pattern. If path is not</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*path is not specified, the contents of /user/&lt;currentUser&gt;( )*</expected-output>
+          <expected-output>^\s*specified, the contents of /user/&lt;currentUser&gt; will be listed. Directory entries( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*will be listed. Directory entries are of the form( )*</expected-output>
+          <expected-output>^\s*are of the form:( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*permissions - userid groupid size_of_directory\(in bytes\) modification_date\(yyyy-MM-dd HH:mm\) directoryName( )*</expected-output>
+          <expected-output>^\s*permissions - userId groupId sizeOfDirectory\(in bytes\)( )*</expected-output>
-        </comparator>
-         <comparator>
-          <type>RegexpComparator</type>
-          <expected-output>^( |\t)*and file entries are of the form( )*</expected-output>
-        </comparator>
-          <comparator>
-          <type>RegexpComparator</type>
-          <expected-output>^( |\t)*permissions number_of_replicas userid groupid size_of_file\(in bytes\) modification_date\(yyyy-MM-dd HH:mm\) fileName( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*-d\s+Directories are listed as plain files\.</expected-output>
+          <expected-output>^\s*modificationDate\(yyyy-MM-dd HH:mm\) directoryName( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*-h\s+Formats the sizes of files in a human-readable fashion( )*</expected-output>
+          <expected-output>^\s*and file entries are of the form:( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*rather than a number of bytes\.( )*</expected-output>
+          <expected-output>^\s*permissions numberOfReplicas userId groupId sizeOfFile\(in bytes\)( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*-R\s+Recursively list the contents of directories\.</expected-output>
+          <expected-output>^\s*modificationDate\(yyyy-MM-dd HH:mm\) fileName( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-d\s+Directories are listed as plain files\.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-h\s+Formats the sizes of files in a human-readable fashion rather than a number( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*of bytes\.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-R\s+Recursively list the contents of directories\.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -109,7 +117,11 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-lsr:\s+\(DEPRECATED\) Same as 'ls -R'</expected-output>
+          <expected-output>^-lsr :\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s+\(DEPRECATED\) Same as 'ls -R'</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -125,23 +137,19 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-get( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt;( )*</expected-output>
+          <expected-output>^-get( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*to the local name.( )*&lt;src&gt; is kept.( )*When copying multiple,( )*</expected-output>
+          <expected-output>\s*Copy files that match the file pattern &lt;src&gt; to the local name.  &lt;src&gt; is kept.\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*files, the destination must be a directory.( )*Passing( )*</expected-output>
+          <expected-output>\s*When copying multiple files, the destination must be a directory. Passing -p\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*-p preserves access and modification times,( )*</expected-output>
+          <expected-output>^( |\t)*preserves access and modification times, ownership and the mode.*</expected-output>
-        </comparator>
-        <comparator>
-          <type>RegexpComparator</type>
-          <expected-output>^( |\t)*ownership and the mode.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -156,35 +164,39 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-du \[-s\] \[-h\] &lt;path&gt; \.\.\.:\s+Show the amount of space, in bytes, used by the files that\s*</expected-output>
+          <expected-output>^-du \[-s\] \[-h\] &lt;path&gt; \.\.\. :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*match the specified file pattern. The following flags are optional:</expected-output>
+          <expected-output>^\s*Show the amount of space, in bytes, used by the files that match the specified\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*-s\s*Rather than showing the size of each individual file that</expected-output>
+          <expected-output>^\s*file pattern. The following flags are optional:\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*matches the pattern, shows the total \(summary\) size.</expected-output>
+          <expected-output>^\s*-s\s*Rather than showing the size of each individual file that matches the\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*-h\s*Formats the sizes of files in a human-readable fashion</expected-output>
+          <expected-output>^\s*pattern, shows the total \(summary\) size.\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>\s*rather than a number of bytes.</expected-output>
+          <expected-output>^\s*-h\s*Formats the sizes of files in a human-readable fashion rather than a number\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*Note that, even without the -s option, this only shows size summaries</expected-output>
+          <expected-output>\s*of bytes.\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*one level deep into a directory.</expected-output>
+          <expected-output>^\s*Note that, even without the -s option, this only shows size summaries one level\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*deep into a directory.</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -207,7 +219,11 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-dus:\s+\(DEPRECATED\) Same as 'du -s'</expected-output>
+          <expected-output>^-dus :</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*\(DEPRECATED\) Same as 'du -s'</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -222,7 +238,11 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-count \[-q\] &lt;path&gt; \.\.\.:( |\t)*Count the number of directories, files and bytes under the paths( )*</expected-output>
+          <expected-output>^-count \[-q\] &lt;path&gt; \.\.\. :\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*Count the number of directories, files and bytes under the paths( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -253,15 +273,15 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-mv &lt;src&gt; \.\.\. &lt;dst&gt;:( |\t)*Move files that match the specified file pattern &lt;src&gt;( )*</expected-output>
+          <expected-output>^-mv &lt;src&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*to a destination &lt;dst&gt;.  When moving multiple files, the( )*</expected-output>
+          <expected-output>\s*Move files that match the specified file pattern &lt;src&gt; to a destination &lt;dst&gt;.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*destination must be a directory.( )*</expected-output>
+          <expected-output>^( |\t)*When moving multiple files, the destination must be a directory.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -276,31 +296,27 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-cp \[-f\] \[-p \| -p\[topx\]\] &lt;src&gt; \.\.\. &lt;dst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt; to a( )*</expected-output>
+          <expected-output>^-cp \[-f\] \[-p \| -p\[topx\]\] &lt;src&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*destination.  When copying multiple files, the destination( )*</expected-output>
+          <expected-output>^\s*Copy files that match the file pattern &lt;src&gt; to a destination.  When copying( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*must be a directory.( )*Passing -p preserves status( )*</expected-output>
+          <expected-output>^( |\t)*multiple files, the destination must be a directory.( )*Passing -p preserves status( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*\[topx\] \(timestamps, ownership, permission, XAttr\).( )*</expected-output>
+          <expected-output>^( |\t)*\[topx\] \(timestamps, ownership, permission, XAttr\). If -p is specified with no( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*If -p is specified with no &lt;arg&gt;, then preserves( )*</expected-output>
+          <expected-output>^( |\t)*&lt;arg&gt;, then preserves timestamps, ownership, permission. Passing -f overwrites( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*timestamps, ownership, permission. Passing -f( )*</expected-output>
+          <expected-output>^\s*the destination if it already exists.( )*</expected-output>
-        </comparator>
-        <comparator>
-          <type>RegexpComparator</type>
-          <expected-output>^( |\t)*overwrites the destination if it already exists.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -315,31 +331,31 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-rm \[-f\] \[-r\|-R\] \[-skipTrash\] &lt;src&gt; \.\.\.:( |\t)*Delete all files that match the specified file pattern.( )*</expected-output>
+          <expected-output>^-rm \[-f\] \[-r\|-R\] \[-skipTrash\] &lt;src&gt; \.\.\. :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*Equivalent to the Unix command "rm &lt;src&gt;"( )*</expected-output>
+          <expected-output>^\s*Delete all files that match the specified file pattern. Equivalent to the Unix( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*-skipTrash option bypasses trash, if enabled, and immediately( )*</expected-output>
+          <expected-output>^\s*command "rm &lt;src&gt;"( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*deletes &lt;src&gt;( )*</expected-output>
+          <expected-output>^\s*-skipTrash\s+option bypasses trash, if enabled, and immediately deletes &lt;src&gt;( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s+-f\s+If the file does not exist, do not display a diagnostic</expected-output>
+          <expected-output>^\s+-f\s+If the file does not exist, do not display a diagnostic message or\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s+message or modify the exit status to reflect an error\.</expected-output>
+          <expected-output>^\s+modify the exit status to reflect an error\.\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s+-\[rR\]\s+Recursively deletes directories</expected-output>
+          <expected-output>^\s+-\[rR\]\s+Recursively deletes directories\s*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -354,11 +370,15 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-rmdir \[--ignore-fail-on-non-empty\] &lt;dir&gt; \.\.\.:\s+Removes the directory entry specified by each directory argument,</expected-output>
+          <expected-output>^-rmdir \[--ignore-fail-on-non-empty\] &lt;dir&gt; \.\.\. :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>\s+provided it is empty.</expected-output>
+          <expected-output>\s+Removes the directory entry specified by each directory argument, provided it is\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>\s+empty\.\s*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -373,7 +393,11 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-rmr:\s+\(DEPRECATED\) Same as 'rm -r'</expected-output>
+          <expected-output>^-rmr :\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*\(DEPRECATED\) Same as 'rm -r'\s*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -388,27 +412,23 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-put \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt;:\s+Copy files from the local file system</expected-output>
+          <expected-output>^-put \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*into fs.( )*Copying fails if the file already( )*</expected-output>
+          <expected-output>^\s*Copy files from the local file system into fs.( )*Copying fails if the file already( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*exists, unless the -f flag is given.( )*Passing( )*</expected-output>
+          <expected-output>^\s*exists, unless the -f flag is given.( )*Passing -p preserves access and( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*-p preserves access and modification times,( )*</expected-output>
+          <expected-output>^\s*modification times, ownership and the mode. Passing -f overwrites the( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*ownership and the mode. Passing -f overwrites( )*</expected-output>
+          <expected-output>^( |\t)*destination if it already exists.( )*</expected-output>
-        </comparator>
-        <comparator>
-          <type>RegexpComparator</type>
-          <expected-output>^( |\t)*the destination if it already exists.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -423,7 +443,11 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-copyFromLocal \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt;:\s+Identical to the -put command\.</expected-output>
+          <expected-output>^-copyFromLocal \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*Identical to the -put command\.\s*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -438,11 +462,11 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-moveFromLocal &lt;localsrc&gt; \.\.\. &lt;dst&gt;:\s+Same as -put, except that the source is</expected-output>
+          <expected-output>^-moveFromLocal &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*deleted after it's copied.</expected-output>
+          <expected-output>^( |\t)*Same as -put, except that the source is deleted after it's copied.</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -458,23 +482,19 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-get( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt;( )*</expected-output>
+          <expected-output>^-get( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*to the local name.( )*&lt;src&gt; is kept.( )*When copying multiple,( )*</expected-output>
+          <expected-output>^( |\t)*Copy files that match the file pattern &lt;src&gt; to the local name.( )*&lt;src&gt; is kept.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*files, the destination must be a directory.( )*Passing( )*</expected-output>
+          <expected-output>^( |\t)*When copying multiple files, the destination must be a directory. Passing -p( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*-p preserves access and modification times,( )*</expected-output>
+          <expected-output>^( |\t)*preserves access and modification times, ownership and the mode.( )*</expected-output>
-        </comparator>
-        <comparator>
-          <type>RegexpComparator</type>
-          <expected-output>^( |\t)*ownership and the mode.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -489,19 +509,19 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-getmerge \[-nl\] &lt;src&gt; &lt;localdst&gt;:( |\t)*Get all the files in the directories that( )*</expected-output>
+          <expected-output>^-getmerge \[-nl\] &lt;src&gt; &lt;localdst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*match the source file pattern and merge and sort them to only( )*</expected-output>
+          <expected-output>^( |\t)*Get all the files in the directories that match the source file pattern and( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*one file on local fs. &lt;src&gt; is kept.( )*</expected-output>
+          <expected-output>^( |\t)*merge and sort them to only one file on local fs. &lt;src&gt; is kept.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*-nl   Add a newline character at the end of each file.( )*</expected-output>
+          <expected-output>^( |\t)*-nl\s+Add a newline character at the end of each file.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -517,11 +537,15 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-cat \[-ignoreCrc\] &lt;src&gt; \.\.\.:( |\t)*Fetch all files that match the file pattern &lt;src&gt;( )*</expected-output>
+          <expected-output>^-cat \[-ignoreCrc\] &lt;src&gt; \.\.\. :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*and display their content on stdout.</expected-output>
+          <expected-output>^\s*Fetch all files that match the file pattern &lt;src&gt; and display their content on\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*stdout.</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -537,7 +561,27 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-checksum &lt;src&gt; \.\.\.:( |\t)*Dump checksum information for files.*</expected-output>
+          <expected-output>^-checksum &lt;src&gt; \.\.\. :\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*Dump checksum information for files that match the file pattern &lt;src&gt; to stdout\.\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*Note that this requires a round-trip to a datanode storing each block of the\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*file, and thus is not efficient to run on a large number of files\. The checksum\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*of a file depends on its content, block size and the checksum algorithm and\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*parameters used for creating the file\.\s*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -552,7 +596,11 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-copyToLocal \[-p\] \[-ignoreCrc\] \[-crc\] &lt;src&gt; \.\.\. &lt;localdst&gt;:\s+Identical to the -get command.</expected-output>
+          <expected-output>^-copyToLocal \[-p\] \[-ignoreCrc\] \[-crc\] &lt;src&gt; \.\.\. &lt;localdst&gt; :\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*Identical to the -get command.\s*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -567,7 +615,11 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-moveToLocal &lt;src&gt; &lt;localdst&gt;:\s+Not implemented yet</expected-output>
+          <expected-output>^-moveToLocal &lt;src&gt; &lt;localdst&gt; :\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*Not implemented yet</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -582,7 +634,11 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-mkdir \[-p\] &lt;path&gt; \.\.\.:( |\t)*Create a directory in specified location.( )*</expected-output>
+          <expected-output>^-mkdir \[-p\] &lt;path&gt; \.\.\. :\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*Create a directory in specified location.( )*</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
@@ -601,27 +657,31 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-setrep \[-R\] \[-w\] &lt;rep&gt; &lt;path&gt; \.\.\.:( |\t)*Set the replication level of a file. If &lt;path&gt; is a directory( )*</expected-output>
+          <expected-output>^-setrep \[-R\] \[-w\] &lt;rep&gt; &lt;path&gt; \.\.\. :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*then the command recursively changes the replication factor of( )*</expected-output>
+          <expected-output>^\s*Set the replication level of a file. If &lt;path&gt; is a directory then the command( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*all files under the directory tree rooted at &lt;path&gt;\.( )*</expected-output>
+          <expected-output>^\s*recursively changes the replication factor of all files under the directory tree( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*rooted at &lt;path&gt;\.( )*</expected-output>
         </comparator>
         <comparator>
             <type>RegexpComparator</type>
-            <expected-output>^( |\t)*The -w flag requests that the command wait for the replication( )*</expected-output>
+            <expected-output>^\s*-w\s+It requests that the command waits for the replication to complete\. This( )*</expected-output>
         </comparator>
         <comparator>
             <type>RegexpComparator</type>
-            <expected-output>^( |\t)*to complete. This can potentially take a very long time\.( )*</expected-output>
+            <expected-output>^( |\t)*can potentially take a very long time\.( )*</expected-output>
         </comparator>
           <comparator>
               <type>RegexpComparator</type>
-              <expected-output>^( |\t)*The -R flag is accepted for backwards compatibility\. It has no effect\.( )*</expected-output>
+              <expected-output>^( |\t)*-R\s+It is accepted for backwards compatibility\. It has no effect\.( )*</expected-output>
           </comparator>
       </comparators>
     </test>
@@ -636,15 +696,15 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-touchz &lt;path&gt; \.\.\.:( |\t)*Creates a file of zero length( )*</expected-output>
+          <expected-output>^-touchz &lt;path&gt; \.\.\. :( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*at &lt;path&gt; with current time as the timestamp of that &lt;path&gt;.( )*</expected-output>
+          <expected-output>^( |\t)*Creates a file of zero length at &lt;path&gt; with current time as the timestamp of( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*An error is returned if the file exists with non-zero length( )*</expected-output>
+          <expected-output>^( |\t)* that &lt;path&gt;\. An error is returned if the file exists with non-zero length( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -659,11 +719,15 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-test -\[defsz\] &lt;path&gt;:\sAnswer various questions about &lt;path&gt;, with result via exit status.</expected-output>
+          <expected-output>^-test -\[defsz\] &lt;path&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*else, return 1.( )*</expected-output>
+          <expected-output>^\s*Answer various questions about &lt;path&gt;, with result via exit status.</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-[defsz]\s+return 0 if .*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -678,15 +742,23 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-stat \[format\] &lt;path&gt; \.\.\.:( |\t)*Print statistics about the file/directory at &lt;path&gt;( )*</expected-output>
+          <expected-output>^-stat \[format\] &lt;path&gt; \.\.\. :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*in the specified format. Format accepts filesize in blocks \(%b\), group name of owner\(%g\),( )*</expected-output>
+          <expected-output>^( |\t)*Print statistics about the file/directory at &lt;path&gt; in the specified format.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*filename \(%n\), block size \(%o\), replication \(%r\), user name of owner\(%u\), modification date \(%y, %Y\)( )*</expected-output>
+          <expected-output>^( |\t)*Format accepts filesize in blocks \(%b\), group name of owner\(%g\), filename \(%n\),( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*block size \(%o\), replication \(%r\), user name of owner\(%u\), modification date( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*\(%y, %Y\)( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -701,11 +773,15 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-tail \[-f\] &lt;file&gt;:( |\t)+Show the last 1KB of the file.( )*</expected-output>
+          <expected-output>^-tail \[-f\] &lt;file&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*The -f option shows appended data as the file grows.( )*</expected-output>
+          <expected-output>^\s*Show the last 1KB of the file.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*-f\s+Shows appended data as the file grows.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -720,47 +796,55 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-chmod \[-R\] &lt;MODE\[,MODE\]... \| OCTALMODE&gt; PATH...:( |\t)*Changes permissions of a file.( )*</expected-output>
+          <expected-output>^-chmod \[-R\] &lt;MODE\[,MODE\]... \| OCTALMODE&gt; PATH... :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*This works similar to shell's chmod with a few exceptions.( )*</expected-output>
+          <expected-output>^( |\t)*Changes permissions of a file. This works similar to the shell's chmod command( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*-R( |\t)*modifies the files recursively. This is the only option( )*</expected-output>
+          <expected-output>^( |\t)*with a few exceptions.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*currently supported.( )*</expected-output>
+          <expected-output>^( |\t)*-R\s*modifies the files recursively. This is the only option currently( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*MODE( |\t)*Mode is same as mode used for chmod shell command.( )*</expected-output>
+          <expected-output>^( |\t)*supported.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*Only letters recognized are 'rwxXt'. E.g. \+t,a\+r,g-w,\+rwx,o=r( )*</expected-output>
+          <expected-output>^( |\t)*&lt;MODE&gt;\s*Mode is the same as mode used for the shell's command. The only( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*OCTALMODE Mode specifed in 3 or 4 digits. If 4 digits, the first may( )*</expected-output>
+          <expected-output>^( |\t)*letters recognized are 'rwxXt', e\.g\. \+t,a\+r,g-w,\+rwx,o=r\.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*be 1 or 0 to turn the sticky bit on or off, respectively.( )*Unlike( |\t)*shell command, it is not possible to specify only part of the mode( )*</expected-output>
+          <expected-output>^( |\t)*&lt;OCTALMODE&gt;\s+Mode specifed in 3 or 4 digits. If 4 digits, the first may be 1 or( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*E.g. 754 is same as u=rwx,g=rx,o=r( )*</expected-output>
+          <expected-output>^( |\t)*0 to turn the sticky bit on or off, respectively.( )*Unlike( |\t)*the( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*If none of 'augo' is specified, 'a' is assumed and unlike( )*</expected-output>
+          <expected-output>^( |\t)*shell command, it is not possible to specify only part of the( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*shell command, no umask is applied.( )*</expected-output>
+          <expected-output>^( |\t)*mode, e\.g\. 754 is same as u=rwx,g=rx,o=r\.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*If none of 'augo' is specified, 'a' is assumed and unlike the shell command, no( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*umask is applied.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -775,51 +859,47 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-chown \[-R\] \[OWNER\]\[:\[GROUP\]\] PATH...:( |\t)*Changes owner and group of a file.( )*</expected-output>
+          <expected-output>^-chown \[-R\] \[OWNER\]\[:\[GROUP\]\] PATH... :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*This is similar to shell's chown with a few exceptions.( )*</expected-output>
+          <expected-output>^\s*Changes owner and group of a file\. This is similar to the shell's chown command( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*-R( |\t)*modifies the files recursively. This is the only option( )*</expected-output>
+          <expected-output>^( |\t)*with a few exceptions.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*currently supported.( )*</expected-output>
+          <expected-output>^( |\t)*-R( |\t)*modifies the files recursively. This is the only option currently( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*If only owner or group is specified then only owner or( )*</expected-output>
+          <expected-output>^( |\t)*supported.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*group is modified.( )*</expected-output>
+          <expected-output>^( |\t)*If only the owner or group is specified, then only the owner or group is( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*The owner and group names may only consist of digits, alphabet,( )*</expected-output>
+          <expected-output>^( |\t)*modified. The owner and group names may only consist of digits, alphabet, and( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*and any of .+?. The names are case sensitive.( )*</expected-output>
+          <expected-output>^( |\t)*any of .+?. The names are case sensitive.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*WARNING: Avoid using '.' to separate user name and group though( )*</expected-output>
+          <expected-output>^( |\t)*WARNING: Avoid using '.' to separate user name and group though Linux allows it.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*Linux allows it. If user names have dots in them and you are( )*</expected-output>
+          <expected-output>^( |\t)*If user names have dots in them and you are using local file system, you might( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*using local file system, you might see surprising results since( )*</expected-output>
+          <expected-output>^( |\t)*see surprising results since the shell command 'chown' is used for local files.( )*</expected-output>
-        </comparator>
-        <comparator>
-          <type>RegexpComparator</type>
-          <expected-output>^( |\t)*shell command 'chown' is used for local files.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -834,7 +914,11 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-chgrp \[-R\] GROUP PATH...:( |\t)*This is equivalent to -chown ... :GROUP ...( )*</expected-output>
+          <expected-output>^-chgrp \[-R\] GROUP PATH... :( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*This is equivalent to -chown ... :GROUP ...( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -849,11 +933,11 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-help \[cmd ...\]:( |\t)*Displays help for given command or all commands if none( )*</expected-output>
+          <expected-output>^-help \[cmd ...\] :( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*is specified.( )*</expected-output>
+          <expected-output>^( |\t)*Displays help for given command or all commands if none is specified.( )*</expected-output>
         </comparator>
       </comparators>
     </test>

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java

@@ -40,7 +40,8 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats;
 import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.CachePoolStats;
-import org.apache.hadoop.hdfs.tools.TableListing.Justification;
+import org.apache.hadoop.tools.TableListing;
+import org.apache.hadoop.tools.TableListing.Justification;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;