YARN-4290. Add -showDetails option to YARN Nodes CLI to print all nodes reports information. (Sunil G via wangda)

(cherry picked from commit a0de702851)

hadoop-yarn-project/CHANGES.txt

@@ -582,6 +582,9 @@ Release 2.8.0 - UNRELEASED
 
     YARN-4480. Clean up some inappropriate imports. (Kai Zheng via umamahesh)
 
+    YARN-4290. Add -showDetails option to YARN Nodes CLI to print all nodes reports
+    information. (Sunil G via wangda)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java

@@ -54,6 +54,7 @@ public class NodeCLI extends YarnCLI {
 
   private static final String NODE_STATE_CMD = "states";
   private static final String NODE_ALL = "all";
+  private static final String NODE_SHOW_DETAILS = "showDetails";
 
   public static void main(String[] args) throws Exception {
     NodeCLI cli = new NodeCLI();
@@ -72,7 +73,8 @@ public class NodeCLI extends YarnCLI {
     opts.addOption(STATUS_CMD, true, "Prints the status report of the node.");
     opts.addOption(LIST_CMD, false, "List all running nodes. " +
         "Supports optional use of -states to filter nodes " +
-        "based on node state, all -all to list all nodes.");
+        "based on node state, all -all to list all nodes, " +
+        "-showDetails to display more details about each node.");
     Option nodeStateOpt = new Option(NODE_STATE_CMD, true,
         "Works with -list to filter nodes based on input comma-separated list of node states.");
     nodeStateOpt.setValueSeparator(',');
@@ -82,6 +84,9 @@ public class NodeCLI extends YarnCLI {
     Option allOpt = new Option(NODE_ALL, false,
         "Works with -list to list all nodes.");
     opts.addOption(allOpt);
+    Option showDetailsOpt = new Option(NODE_SHOW_DETAILS, false,
+        "Works with -list to show more details about each node.");
+    opts.addOption(showDetailsOpt);
     opts.getOption(STATUS_CMD).setArgName("NodeId");
 
     int exitCode = -1;
@@ -119,7 +124,13 @@ public class NodeCLI extends YarnCLI {
       } else {
         nodeStates.add(NodeState.RUNNING);
       }
-      listClusterNodes(nodeStates);
+
+      // List all node details with more information.
+      if (cliParser.hasOption(NODE_SHOW_DETAILS)) {
+        listDetailedClusterNodes(nodeStates);
+      } else {
+        listClusterNodes(nodeStates);
+      }
     } else if (cliParser.hasOption(HELP_CMD)) {
       printUsage(opts);
       return 0;
@@ -163,6 +174,67 @@ public class NodeCLI extends YarnCLI {
     writer.flush();
   }
 
+  /**
+   * Lists the nodes which are matching the given node states along with
+   * detailed node informations such as resource usage etc.
+   *
+   * @param nodeStates
+   * @throws YarnException
+   * @throws IOException
+   */
+  private void listDetailedClusterNodes(Set<NodeState> nodeStates)
+      throws YarnException, IOException {
+    PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout,
+        Charset.forName("UTF-8")));
+    List<NodeReport> nodesReport = client.getNodeReports(nodeStates
+        .toArray(new NodeState[0]));
+    writer.println("Total Nodes:" + nodesReport.size());
+    writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address",
+        "Number-of-Running-Containers");
+    for (NodeReport nodeReport : nodesReport) {
+      writer.printf(NODES_PATTERN, nodeReport.getNodeId(),
+          nodeReport.getNodeState(), nodeReport.getHttpAddress(),
+          nodeReport.getNumContainers());
+      writer.println("Detailed Node Information :");
+      writer.print("\tConfigured Resources : ");
+      writer.println(nodeReport.getCapability());
+      writer.print("\tAllocated Resources : ");
+      if (nodeReport.getUsed() != null) {
+        writer.print(nodeReport.getUsed());
+      }
+      writer.println();
+
+      writer.print("\tResource Utilization by Node : ");
+      if (nodeReport.getNodeUtilization() != null) {
+        writer.print("PMem:"
+            + nodeReport.getNodeUtilization().getPhysicalMemory()
+            + " MB, VMem:" + nodeReport.getNodeUtilization().getVirtualMemory()
+            + " MB, VCores:" + nodeReport.getNodeUtilization().getCPU());
+      }
+      writer.println();
+
+      writer.print("\tResource Utilization by Containers : ");
+      if (nodeReport.getAggregatedContainersUtilization() != null) {
+        writer.print("PMem:"
+            + nodeReport.getAggregatedContainersUtilization()
+                .getPhysicalMemory()
+            + " MB, VMem:"
+            + nodeReport.getAggregatedContainersUtilization()
+                .getVirtualMemory() + " MB, VCores:"
+            + nodeReport.getAggregatedContainersUtilization().getCPU());
+      }
+      writer.println();
+
+      writer.print("\tNode-Labels : ");
+      // Create a List for node labels since we need it get sorted
+      List<String> nodeLabelsList = new ArrayList<String>(
+          nodeReport.getNodeLabels());
+      Collections.sort(nodeLabelsList);
+      writer.println(StringUtils.join(nodeLabelsList.iterator(), ','));
+    }
+    writer.flush();
+  }
+
   /**
    * Prints the node report for node id.
    * 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java

@@ -992,6 +992,35 @@ public class TestYarnCLI {
     Assert.assertEquals(nodesReportStr, sysOutStream.toString());
     verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt());
 
+    sysOutStream.reset();
+    result = cli.run(new String[] { "-list", "-showDetails" });
+    assertEquals(0, result);
+    baos = new ByteArrayOutputStream();
+    pw = new PrintWriter(baos);
+    pw.println("Total Nodes:2");
+    pw.print("         Node-Id\t     Node-State\tNode-Http-Address\t");
+    pw.println("Number-of-Running-Containers");
+    pw.print("         host0:0\t        RUNNING\t       host1:8888\t");
+    pw.println("                           0");
+    pw.println("Detailed Node Information :");
+    pw.println("\tConfigured Resources : <memory:0, vCores:0>");
+    pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+    pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
+    pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
+    pw.println("\tNode-Labels : ");
+    pw.print("         host1:0\t        RUNNING\t       host1:8888\t");
+    pw.println("                           0");
+    pw.println("Detailed Node Information :");
+    pw.println("\tConfigured Resources : <memory:0, vCores:0>");
+    pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+    pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
+    pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
+    pw.println("\tNode-Labels : ");
+    pw.close();
+    nodesReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+    verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
+
     sysOutStream.reset();
     nodeStates.clear();
     nodeStates.add(NodeState.UNHEALTHY);
@@ -1011,7 +1040,7 @@ public class TestYarnCLI {
     pw.close();
     nodesReportStr = baos.toString("UTF-8");
     Assert.assertEquals(nodesReportStr, sysOutStream.toString());
-    verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
+    verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
 
     sysOutStream.reset();
     nodeStates.clear();
@@ -1032,7 +1061,7 @@ public class TestYarnCLI {
     pw.close();
     nodesReportStr = baos.toString("UTF-8");
     Assert.assertEquals(nodesReportStr, sysOutStream.toString());
-    verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
+    verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
 
     sysOutStream.reset();
     nodeStates.clear();
@@ -1053,7 +1082,7 @@ public class TestYarnCLI {
     pw.close();
     nodesReportStr = baos.toString("UTF-8");
     Assert.assertEquals(nodesReportStr, sysOutStream.toString());
-    verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
+    verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt());
 
     sysOutStream.reset();
     nodeStates.clear();
@@ -1074,7 +1103,7 @@ public class TestYarnCLI {
     pw.close();
     nodesReportStr = baos.toString("UTF-8");
     Assert.assertEquals(nodesReportStr, sysOutStream.toString());
-    verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt());
+    verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt());
 
     sysOutStream.reset();
     nodeStates.clear();
@@ -1107,7 +1136,7 @@ public class TestYarnCLI {
     pw.close();
     nodesReportStr = baos.toString("UTF-8");
     Assert.assertEquals(nodesReportStr, sysOutStream.toString());
-    verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt());
+    verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt());
 
     sysOutStream.reset();
     nodeStates.clear();
@@ -1142,7 +1171,7 @@ public class TestYarnCLI {
     pw.close();
     nodesReportStr = baos.toString("UTF-8");
     Assert.assertEquals(nodesReportStr, sysOutStream.toString());
-    verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt());
+    verify(sysOut, times(10)).write(any(byte[].class), anyInt(), anyInt());
   }
 
   private List<NodeReport> getNodeReports(
@@ -1727,7 +1756,9 @@ public class TestYarnCLI {
     pw.println(" -help              Displays help for all commands.");
     pw.println(" -list              List all running nodes. Supports optional use of");
     pw.println("                    -states to filter nodes based on node state, all -all");
-    pw.println("                    to list all nodes.");
+    pw.println("                    to list all nodes, -showDetails to display more");
+    pw.println("                    details about each node.");
+    pw.println(" -showDetails       Works with -list to show more details about each node.");
     pw.println(" -states <States>   Works with -list to filter nodes based on input");
     pw.println("                    comma-separated list of node states.");
     pw.println(" -status <NodeId>   Prints the status report of the node.");