From 7dc272199ffea28c8597151e17d4606928a7f9f3 Mon Sep 17 00:00:00 2001 From: Suma Shivaprasad Date: Wed, 28 Nov 2018 00:42:00 -0800 Subject: [PATCH] YARN-9034. ApplicationCLI should have option to take clusterId. Contributed by Rohith Sharma K S. --- .../yarn/client/cli/ApplicationCLI.java | 17 ++++ .../hadoop/yarn/client/cli/ClusterCLI.java | 2 + .../hadoop/yarn/client/cli/NodeCLI.java | 2 + .../hadoop/yarn/client/cli/QueueCLI.java | 2 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 1 + .../hadoop/yarn/client/cli/YarnCLI.java | 9 +- .../yarn/client/cli/TestClusterCLI.java | 41 ++++----- .../hadoop/yarn/client/cli/TestYarnCLI.java | 89 +++++++++---------- .../api/impl/TimelineReaderClientImpl.java | 2 + 9 files changed, 92 insertions(+), 73 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 2b3415413ce..ff8142ab79b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.api.records.ShellContainerCommand; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.AppAdminClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; @@ -113,6 +114,7 @@ public class ApplicationCLI extends YarnCLI { public static final String VERSION = "version"; public static final String STATES = "states"; public static final String SHELL_CMD = "shell"; + public static final String CLUSTER_ID_OPTION = "clusterId"; private static String firstArg = null; @@ -278,6 +280,8 @@ public class ApplicationCLI extends YarnCLI { "the ability to finalize the upgrade automatically."); opts.addOption(UPGRADE_CANCEL, false, "Works with -upgrade option to " + "cancel current upgrade."); + opts.addOption(CLUSTER_ID_OPTION, true, "ClusterId. " + + "By default, it will take default cluster id from the RM"); opts.getOption(LAUNCH_CMD).setArgName("Application Name> ()); - ClusterCLI cli = new ClusterCLI(); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + ClusterCLI cli = createAndGetClusterCLI(); int rc = cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD }); @@ -154,9 +140,7 @@ public class TestClusterCLI { @Test public void testHelp() throws Exception { - ClusterCLI cli = new ClusterCLI(); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + ClusterCLI cli = createAndGetClusterCLI(); int rc = cli.run(new String[] { "cluster", "--help" }); @@ -192,4 +176,15 @@ public class TestClusterCLI { pw.close(); verify(sysOut).println(baos.toString("UTF-8")); } + + private ClusterCLI createAndGetClusterCLI() { + ClusterCLI cli = new ClusterCLI() { + @Override protected void createAndStartYarnClient() { + } + }; + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + return cli; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 672e3d7e43a..8c59a6133a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -993,10 +993,7 @@ public class TestYarnCLI { @Test (timeout = 5000) public void testNodesHelpCommand() throws Exception { - NodeCLI nodeCLI = new NodeCLI(); - nodeCLI.setClient(client); - nodeCLI.setSysOutPrintStream(sysOut); - nodeCLI.setSysErrPrintStream(sysErr); + NodeCLI nodeCLI = createAndGetNodeCLI(); nodeCLI.run(new String[] {}); Assert.assertEquals(createNodeCLIHelpMessage(), sysOutStream.toString()); @@ -1290,9 +1287,7 @@ public class TestYarnCLI { nodeReports.addAll(getNodeReports(1, NodeState.REBOOTED)); nodeReports.addAll(getNodeReports(1, NodeState.LOST)); - NodeCLI cli = new NodeCLI(); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); + NodeCLI cli = createAndGetNodeCLI(); Set nodeStates = new HashSet(); nodeStates.add(NodeState.NEW); @@ -1545,12 +1540,9 @@ public class TestYarnCLI { @Test public void testNodeStatus() throws Exception { NodeId nodeId = NodeId.newInstance("host0", 0); - NodeCLI cli = new NodeCLI(); when(client.getNodeReports()) .thenReturn(getNodeReports(3, NodeState.RUNNING, false, false, false)); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + NodeCLI cli = createAndGetNodeCLI(); int result = cli.run(new String[] { "-status", nodeId.toString() }); assertEquals(0, result); verify(client).getNodeReports(); @@ -1583,12 +1575,9 @@ public class TestYarnCLI { @Test public void testNodeStatusWithEmptyNodeLabels() throws Exception { NodeId nodeId = NodeId.newInstance("host0", 0); - NodeCLI cli = new NodeCLI(); when(client.getNodeReports()).thenReturn( getNodeReports(3, NodeState.RUNNING)); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + NodeCLI cli = createAndGetNodeCLI(); int result = cli.run(new String[] { "-status", nodeId.toString() }); assertEquals(0, result); verify(client).getNodeReports(); @@ -1620,12 +1609,9 @@ public class TestYarnCLI { @Test public void testNodeStatusWithEmptyResourceUtilization() throws Exception { NodeId nodeId = NodeId.newInstance("host0", 0); - NodeCLI cli = new NodeCLI(); when(client.getNodeReports()) .thenReturn(getNodeReports(3, NodeState.RUNNING, false, true, true)); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + NodeCLI cli = createAndGetNodeCLI(); int result = cli.run(new String[] { "-status", nodeId.toString() }); assertEquals(0, result); verify(client).getNodeReports(); @@ -1657,12 +1643,10 @@ public class TestYarnCLI { @Test public void testAbsentNodeStatus() throws Exception { NodeId nodeId = NodeId.newInstance("Absenthost0", 0); - NodeCLI cli = new NodeCLI(); + when(client.getNodeReports()).thenReturn( getNodeReports(0, NodeState.RUNNING)); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + NodeCLI cli = createAndGetNodeCLI(); int result = cli.run(new String[] { "-status", nodeId.toString() }); assertEquals(0, result); verify(client).getNodeReports(); @@ -1702,10 +1686,7 @@ public class TestYarnCLI { createContainerCLIHelpMessage()), normalize(sysOutStream.toString())); sysOutStream.reset(); - NodeCLI nodeCLI = new NodeCLI(); - nodeCLI.setClient(client); - nodeCLI.setSysOutPrintStream(sysOut); - nodeCLI.setSysErrPrintStream(sysErr); + NodeCLI nodeCLI = createAndGetNodeCLI(); result = nodeCLI.run(new String[] { "-status" }); Assert.assertEquals(result, -1); Assert.assertEquals(String.format("Missing argument for options%n%1s", @@ -1774,10 +1755,7 @@ public class TestYarnCLI { yarnClient.init(yarnConf); yarnClient.start(); - QueueCLI cli = new QueueCLI(); - cli.setClient(yarnClient); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + QueueCLI cli = createAndGetQueueCLI(yarnClient); sysOutStream.reset(); // Get status for the root.a queue int result = cli.run(new String[] { "-status", "a" }); @@ -1788,10 +1766,7 @@ public class TestYarnCLI { // In-queue preemption is disabled at the "root.a" queue level Assert.assertTrue(queueStatusOut .contains("Intra-queue Preemption : disabled")); - cli = new QueueCLI(); - cli.setClient(yarnClient); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + cli = createAndGetQueueCLI(yarnClient); sysOutStream.reset(); // Get status for the root.a.a1 queue result = cli.run(new String[] { "-status", "a1" }); @@ -1836,10 +1811,7 @@ public class TestYarnCLI { yarnClient.init(yarnConf); yarnClient.start(); - QueueCLI cli = new QueueCLI(); - cli.setClient(yarnClient); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + QueueCLI cli = createAndGetQueueCLI(yarnClient); sysOutStream.reset(); int result = cli.run(new String[] { "-status", "a1" }); assertEquals(0, result); @@ -1880,10 +1852,7 @@ public class TestYarnCLI { yarnClient.init(yarnConf); yarnClient.start(); - QueueCLI cli = new QueueCLI(); - cli.setClient(yarnClient); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + QueueCLI cli = createAndGetQueueCLI(yarnClient); sysOutStream.reset(); int result = cli.run(new String[] { "-status", "a1" }); assertEquals(0, result); @@ -2101,14 +2070,36 @@ public class TestYarnCLI { } private ApplicationCLI createAndGetAppCLI() { - ApplicationCLI cli = new ApplicationCLI(); + ApplicationCLI cli = new ApplicationCLI() { + @Override protected void createAndStartYarnClient() { + } + }; cli.setClient(client); cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); return cli; } - + private QueueCLI createAndGetQueueCLI() { - QueueCLI cli = new QueueCLI(); + return createAndGetQueueCLI(client); + } + + private QueueCLI createAndGetQueueCLI(YarnClient client) { + QueueCLI cli = new QueueCLI() { + @Override protected void createAndStartYarnClient() { + } + }; + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + return cli; + } + + private NodeCLI createAndGetNodeCLI() { + NodeCLI cli = new NodeCLI() { + @Override protected void createAndStartYarnClient() { + } + }; cli.setClient(client); cli.setSysOutPrintStream(sysOut); cli.setSysErrPrintStream(sysErr); @@ -2152,6 +2143,9 @@ public class TestYarnCLI { pw.println(" deprecated, this new command"); pw.println(" 'changeQueue' performs same"); pw.println(" functionality."); + pw.println(" -clusterId ClusterId. By default, it will"); + pw.println(" take default cluster id from the"); + pw.println(" RM"); pw.println(" -component Works with -flex option to"); pw.println(" change the number of"); pw.println(" components/containers running"); @@ -2298,6 +2292,8 @@ public class TestYarnCLI { ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); pw.println("usage: applicationattempt"); + pw.println(" -clusterId ClusterId. By default, it will take"); + pw.println(" default cluster id from the RM"); pw.println(" -fail Fails application attempt."); pw.println(" -help Displays help for all commands."); pw.println(" -list List application attempts for"); @@ -2314,6 +2310,7 @@ public class TestYarnCLI { PrintWriter pw = new PrintWriter(baos); pw.println("usage: container"); pw.println(" -appTypes Works with -list to specify the app type when application name is provided."); + pw.println(" -clusterId ClusterId. By default, it will take default cluster id from the RM "); pw.println(" -components Works with -list to filter instances based on input comma-separated list of component names."); pw.println(" -help Displays help for all commands."); pw.println(" -list List containers for application attempt when application attempt ID is provided. When application name is provided, then it finds the instances of the application based on app's own implementation, and -appTypes option must be specified unless it is the default yarn-service type. With app name, it supports optional use of -version to filter instances based on app version, -components to filter instances based on component names, -states to filter instances based on instance state."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java index b138a54af7c..db53f93136f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java @@ -94,6 +94,8 @@ public class TimelineReaderClientImpl extends TimelineReaderClient { conf, timelineReaderWebAppAddress, RESOURCE_URI_STR_V2); clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID, YarnConfiguration.DEFAULT_RM_CLUSTER_ID); + LOG.info("Initialized TimelineReader URI=" + baseUri + ", clusterId=" + + clusterId); super.serviceInit(conf); }