HDFS-14188. Make hdfs ec -verifyClusterSetup command accept an erasure coding policy as a parameter. Contributed by Kitti Nanasi.

Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java

@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.shell.CommandFormat;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -612,24 +613,32 @@ public class ECAdmin extends Configured implements Tool {
 
     @Override
     public String getShortUsage() {
-      return "[" + getName() + "]\n";
+      return "[" + getName() + " [-policy <policy>...<policy>]]\n";
     }
 
     @Override
     public String getLongUsage() {
+      TableListing listing = AdminHelper.getOptionDescriptionListing();
+      listing.addRow("<policy>", "The name of the erasure coding policy");
       return getShortUsage() + "\n"
-          + "Verify the cluster setup can support all enabled erasure coding"
+          + "Verify if the cluster setup can support all enabled erasure " +
-          + " policies.\n";
+          "coding policies. If optional parameter -policy is specified, " +
+          "verify if the cluster setup can support the given policy.\n";
     }
 
     @Override
     public int run(Configuration conf, List<String> args) throws IOException {
-      if (args.size() > 0) {
+      boolean isPolicyOption = StringUtils.popOption("-policy", args);
-        System.err.println(getName() + ": Too many arguments");
-        return 1;
-      }
       final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
-      ECTopologyVerifierResult result = getECTopologyVerifierResult(dfs);
+      ECTopologyVerifierResult result;
+      if (isPolicyOption) {
+        CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
+        c.parse(args);
+        String[] parameters = args.toArray(new String[args.size()]);
+        result = getECTopologyResultForPolicies(dfs, parameters);
+      } else {
+        result = getECTopologyVerifierResult(dfs);
+      }
       System.out.println(result.getResultMessage());
       if (result.isSupported()) {
         return 0;
@@ -649,6 +658,21 @@ public class ECAdmin extends Configured implements Tool {
         getEnabledPolicies(policies));
   }
 
+  private static ECTopologyVerifierResult getECTopologyResultForPolicies(
+      final DistributedFileSystem dfs, final String... policyNames)
+      throws IOException {
+    ErasureCodingPolicy[] policies =
+        new ErasureCodingPolicy[policyNames.length];
+    for (int i = 0; i < policyNames.length; i++) {
+      policies[i] =
+        getPolicy(dfs.getClient().getNamenode().getErasureCodingPolicies(),
+            policyNames[i]);
+    }
+    final DatanodeInfo[] report = dfs.getClient().getNamenode()
+        .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL);
+    return ECTopologyVerifier.getECTopologyVerifierResult(report, policies);
+  }
+
   private static ECTopologyVerifierResult getECTopologyVerifierResultForPolicy(
       final DistributedFileSystem dfs, final String policyName)
       throws IOException {

hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md

@@ -488,6 +488,7 @@ Usage:
          [-listCodecs]
          [-enablePolicy -policy <policyName>]
          [-disablePolicy -policy <policyName>]
+         [-verifyClusterSetup -policy <policyName>...<policyName>]
          [-help [cmd ...]]
 
 | COMMAND\_OPTION | Description |
@@ -500,6 +501,7 @@ Usage:
 |-listCodecs| Get the list of supported erasure coding codecs and coders in system|
 |-enablePolicy| Enable an ErasureCoding policy in system|
 |-disablePolicy| Disable an ErasureCoding policy in system|
+|-verifyClusterSetup| Verify if the cluster setup can support a list of erasure coding policies|
 
 Runs the ErasureCoding CLI. See [HDFS ErasureCoding](./HDFSErasureCoding.html#Administrative_commands) for more information on this command.
 

hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md

@@ -170,6 +170,7 @@ Deployment
          [-listCodecs]
          [-enablePolicy -policy <policyName>]
          [-disablePolicy -policy <policyName>]
+         [-verifyClusterSetup -policy <policyName>...<policyName>]
          [-help [cmd ...]]
 
 Below are the details about each command.
@@ -221,6 +222,10 @@ Below are the details about each command.
 
      Disable an erasure coding policy.
 
+*  `[-verifyClusterSetup -policy <policyName>...<policyName>]`
+
+     Verify if the cluster setup can support all enabled erasure coding policies. If optional parameter -policy is specified, verify if the cluster setup can support the given policy or policies.
+
 Limitations
 -----------
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java

@@ -68,8 +68,7 @@ public class TestECAdmin {
     try {
       System.out.flush();
       System.err.flush();
-      out.reset();
+      resetOutputs();
-      err.reset();
     } finally {
       System.setOut(OLD_OUT);
       System.setErr(OLD_ERR);
@@ -242,4 +241,59 @@ public class TestECAdmin {
         err.toString().contains("RemoteException: The policy name " +
             "NonExistentPolicy does not exist"));
   }
+
+  @Test
+  public void testVerifyClusterSetupWithGivenPolicies() throws Exception {
+    cluster = DFSTestUtil.setupCluster(conf, 5, 2, 0);
+
+    String[] args = new String[]{"-verifyClusterSetup", "-policy",
+        "RS-3-2-1024k"};
+    int ret = admin.run(args);
+    LOG.info("Command stdout: {}", out.toString());
+    LOG.info("Command stderr: {}", err.toString());
+    assertEquals("Return value of the command is not successful", 2, ret);
+    assertTrue("Result of cluster topology verify " +
+        "should be logged correctly", out.toString()
+        .contains("less than the minimum required number of racks (3) " +
+            "for the erasure coding policies: RS-3-2-1024k"));
+    assertTrue("Error output should be empty", err.toString().isEmpty());
+
+    resetOutputs();
+    args = new String[]{"-verifyClusterSetup", "-policy",
+        "RS-10-4-1024k", "RS-3-2-1024k"};
+    ret = admin.run(args);
+    LOG.info("Command stdout: {}", out.toString());
+    LOG.info("Command stderr: {}", err.toString());
+    assertEquals("Return value of the command is not successful", 2, ret);
+    assertTrue("Result of cluster topology verify " +
+        "should be logged correctly", out.toString()
+        .contains(
+            "for the erasure coding policies: RS-10-4-1024k, RS-3-2-1024k"));
+    assertTrue("Error output should be empty", err.toString().isEmpty());
+
+    resetOutputs();
+    args = new String[]{"-verifyClusterSetup", "-policy", "invalidPolicy"};
+    ret = admin.run(args);
+    LOG.info("Command stdout: {}", out.toString());
+    LOG.info("Command stderr: {}", err.toString());
+    assertEquals("Return value of the command is not successful", -1, ret);
+    assertTrue("Error message should be logged", err.toString()
+        .contains("The given erasure coding policy invalidPolicy " +
+            "does not exist."));
+
+    resetOutputs();
+    args = new String[]{"-verifyClusterSetup", "-policy"};
+    ret = admin.run(args);
+    LOG.info("Command stdout: {}", out.toString());
+    LOG.info("Command stderr: {}", err.toString());
+    assertEquals("Return value of the command is not successful", -1, ret);
+    assertTrue("Error message should be logged", err.toString()
+        .contains("NotEnoughArgumentsException: Not enough arguments: " +
+            "expected 1 but got 0"));
+  }
+
+  private void resetOutputs() {
+    out.reset();
+    err.reset();
+  }
 }