Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4290. Add -showDetails option to YARN Nodes CLI to print all nodes reports information. (Sunil G via wangda) 

(cherry picked from commit a0de702851)
This commit is contained in:
Wangda Tan 2015-12-21 11:36:22 -08:00
parent f16daa59a4
commit daf5fae369
3 changed files with 115 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -582,6 +582,9 @@ Release 2.8.0 - UNRELEASED
YARN-4480. Clean up some inappropriate imports. (Kai Zheng via umamahesh)
YARN-4290. Add -showDetails option to YARN Nodes CLI to print all nodes reports
information. (Sunil G via wangda)
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not

76
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
View File

@ -54,6 +54,7 @@ public class NodeCLI extends YarnCLI {
private static final String NODE_STATE_CMD = "states";
private static final String NODE_ALL = "all";
private static final String NODE_SHOW_DETAILS = "showDetails";
public static void main(String[] args) throws Exception {
NodeCLI cli = new NodeCLI();
@ -72,7 +73,8 @@ public class NodeCLI extends YarnCLI {
opts.addOption(STATUS_CMD, true, "Prints the status report of the node.");
opts.addOption(LIST_CMD, false, "List all running nodes. " +
"Supports optional use of -states to filter nodes " +
"based on node state, all -all to list all nodes.");
"based on node state, all -all to list all nodes, " +
"-showDetails to display more details about each node.");
Option nodeStateOpt = new Option(NODE_STATE_CMD, true,
"Works with -list to filter nodes based on input comma-separated list of node states.");
nodeStateOpt.setValueSeparator(',');
@ -82,6 +84,9 @@ public class NodeCLI extends YarnCLI {
Option allOpt = new Option(NODE_ALL, false,
"Works with -list to list all nodes.");
opts.addOption(allOpt);
Option showDetailsOpt = new Option(NODE_SHOW_DETAILS, false,
"Works with -list to show more details about each node.");
opts.addOption(showDetailsOpt);
opts.getOption(STATUS_CMD).setArgName("NodeId");
int exitCode = -1;
@ -119,7 +124,13 @@ public class NodeCLI extends YarnCLI {
} else {
nodeStates.add(NodeState.RUNNING);
}
listClusterNodes(nodeStates);
// List all node details with more information.
if (cliParser.hasOption(NODE_SHOW_DETAILS)) {
listDetailedClusterNodes(nodeStates);
} else {
listClusterNodes(nodeStates);
}
} else if (cliParser.hasOption(HELP_CMD)) {
printUsage(opts);
return 0;
@ -163,6 +174,67 @@ public class NodeCLI extends YarnCLI {
writer.flush();
}
/**
* Lists the nodes which are matching the given node states along with
* detailed node informations such as resource usage etc.
*
* @param nodeStates
* @throws YarnException
* @throws IOException
*/
private void listDetailedClusterNodes(Set<NodeState> nodeStates)
throws YarnException, IOException {
PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout,
Charset.forName("UTF-8")));
List<NodeReport> nodesReport = client.getNodeReports(nodeStates
.toArray(new NodeState[0]));
writer.println("Total Nodes:" + nodesReport.size());
writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address",
"Number-of-Running-Containers");
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(),
nodeReport.getNodeState(), nodeReport.getHttpAddress(),
nodeReport.getNumContainers());
writer.println("Detailed Node Information :");
writer.print("\tConfigured Resources : ");
writer.println(nodeReport.getCapability());
writer.print("\tAllocated Resources : ");
if (nodeReport.getUsed() != null) {
writer.print(nodeReport.getUsed());
}
writer.println();
writer.print("\tResource Utilization by Node : ");
if (nodeReport.getNodeUtilization() != null) {
writer.print("PMem:"
+ nodeReport.getNodeUtilization().getPhysicalMemory()
+ " MB, VMem:" + nodeReport.getNodeUtilization().getVirtualMemory()
+ " MB, VCores:" + nodeReport.getNodeUtilization().getCPU());
}
writer.println();
writer.print("\tResource Utilization by Containers : ");
if (nodeReport.getAggregatedContainersUtilization() != null) {
writer.print("PMem:"
+ nodeReport.getAggregatedContainersUtilization()
.getPhysicalMemory()
+ " MB, VMem:"
+ nodeReport.getAggregatedContainersUtilization()
.getVirtualMemory() + " MB, VCores:"
+ nodeReport.getAggregatedContainersUtilization().getCPU());
}
writer.println();
writer.print("\tNode-Labels : ");
// Create a List for node labels since we need it get sorted
List<String> nodeLabelsList = new ArrayList<String>(
nodeReport.getNodeLabels());
Collections.sort(nodeLabelsList);
writer.println(StringUtils.join(nodeLabelsList.iterator(), ','));
}
writer.flush();
}
/**
* Prints the node report for node id.
*

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -992,6 +992,35 @@ public class TestYarnCLI {
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
result = cli.run(new String[] { "-list", "-showDetails" });
assertEquals(0, result);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total Nodes:2");
pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
pw.println("Number-of-Running-Containers");
pw.print(" host0:0\t RUNNING\t host1:8888\t");
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
pw.println("\tAllocated Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
pw.print(" host1:0\t RUNNING\t host1:8888\t");
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
pw.println("\tAllocated Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
nodeStates.clear();
nodeStates.add(NodeState.UNHEALTHY);
@ -1011,7 +1040,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
nodeStates.clear();
@ -1032,7 +1061,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
nodeStates.clear();
@ -1053,7 +1082,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
nodeStates.clear();
@ -1074,7 +1103,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt());
verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
nodeStates.clear();
@ -1107,7 +1136,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt());
verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
nodeStates.clear();
@ -1142,7 +1171,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt());
verify(sysOut, times(10)).write(any(byte[].class), anyInt(), anyInt());
}
private List<NodeReport> getNodeReports(
@ -1727,7 +1756,9 @@ public class TestYarnCLI {
pw.println(" -help Displays help for all commands.");
pw.println(" -list List all running nodes. Supports optional use of");
pw.println(" -states to filter nodes based on node state, all -all");
pw.println(" to list all nodes.");
pw.println(" to list all nodes, -showDetails to display more");
pw.println(" details about each node.");
pw.println(" -showDetails Works with -list to show more details about each node.");
pw.println(" -states <States> Works with -list to filter nodes based on input");
pw.println(" comma-separated list of node states.");
pw.println(" -status <NodeId> Prints the status report of the node.");