From 9c8882546af27abb97a16c2dae4595a16865fedd Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 5 Mar 2015 10:56:28 -0800 Subject: [PATCH] YARN-2786. Created a yarn cluster CLI and seeded with one command for listing node-labels collection. Contributed by Wangda Tan. --- hadoop-yarn-project/CHANGES.txt | 3 + hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 + hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 8 +- .../hadoop/yarn/client/cli/ClusterCLI.java | 157 +++++++++++++++++ .../yarn/client/cli/TestClusterCLI.java | 158 ++++++++++++++++++ 5 files changed, 329 insertions(+), 1 deletion(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a9d237f96da..027075ec62f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -52,6 +52,9 @@ Release 2.7.0 - UNRELEASED YARN-3249. Add a 'kill application' button to Resource Manager's Web UI. (Ryu Kobayashi via ozawa) + YARN-2786. Created a yarn cluster CLI and seeded with one command for listing + node-labels collection. (Wangda Tan via vinodkv) + IMPROVEMENTS YARN-3005. [JDK7] Use switch statement for String instead of if-else diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 061fc6eac9d..872394c4a87 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -89,6 +89,7 @@ function print_usage(){ echo " classpath prints the class path needed to" echo " get the Hadoop jar and the" echo " required libraries" + echo " cluster prints cluster information" echo " daemonlog get/set the log level for each" echo " daemon" echo "" @@ -291,6 +292,9 @@ elif [ "$COMMAND" = "logs" ] ; then elif [ "$COMMAND" = "daemonlog" ] ; then CLASS=org.apache.hadoop.log.LogLevel YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" +elif [ "$COMMAND" = "cluster" ] ; then + CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI + YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" else CLASS=$COMMAND fi diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index d594957034c..7125836476c 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -150,7 +150,7 @@ if "%1" == "--loglevel" ( ) set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^ - application applicationattempt container node logs daemonlog historyserver ^ + application applicationattempt cluster container node logs daemonlog historyserver ^ timelineserver classpath for %%i in ( %yarncommands% ) do ( if %yarn-command% == %%i set yarncommand=true @@ -192,6 +192,11 @@ goto :eof set yarn-command-arguments=%yarn-command% %yarn-command-arguments% goto :eof +:cluster + set CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + :container set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% @@ -312,6 +317,7 @@ goto :eof @echo jar ^ run a jar file @echo application prints application(s) report/kill application @echo applicationattempt prints applicationattempt(s) report + @echo cluster prints cluster information @echo container prints container(s) report @echo node prints node report(s) @echo queue prints queue information diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java new file mode 100644 index 00000000000..39248039ed1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.cli; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Options; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Cluster CLI used to get over all information of the cluster + */ +@Private +public class ClusterCLI extends YarnCLI { + private static final String TITLE = "yarn cluster"; + public static final String LIST_LABELS_CMD = "list-node-labels"; + public static final String DIRECTLY_ACCESS_NODE_LABEL_STORE = + "directly-access-node-label-store"; + public static final String CMD = "cluster"; + private boolean accessLocal = false; + static CommonNodeLabelsManager localNodeLabelsManager = null; + + public static void main(String[] args) throws Exception { + ClusterCLI cli = new ClusterCLI(); + cli.setSysOutPrintStream(System.out); + cli.setSysErrPrintStream(System.err); + int res = ToolRunner.run(cli, args); + cli.stop(); + System.exit(res); + } + + @Override + public int run(String[] args) throws Exception { + Options opts = new Options(); + + opts.addOption("lnl", LIST_LABELS_CMD, false, + "List cluster node-label collection"); + opts.addOption("h", HELP_CMD, false, "Displays help for all commands."); + opts.addOption("dnl", DIRECTLY_ACCESS_NODE_LABEL_STORE, false, + "Directly access node label store, " + + "with this option, all node label related operations" + + " will NOT connect RM. Instead, they will" + + " access/modify stored node labels directly." + + " By default, it is false (access via RM)." + + " AND PLEASE NOTE: if you configured " + + YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR + + " to a local directory" + + " (instead of NFS or HDFS), this option will only work" + + " when the command run on the machine where RM is running." + + " Also, this option is UNSTABLE, could be removed in future" + + " releases."); + + int exitCode = -1; + CommandLine parsedCli = null; + try { + parsedCli = new GnuParser().parse(opts, args); + } catch (MissingArgumentException ex) { + sysout.println("Missing argument for options"); + printUsage(opts); + return exitCode; + } + + if (parsedCli.hasOption(DIRECTLY_ACCESS_NODE_LABEL_STORE)) { + accessLocal = true; + } + + if (parsedCli.hasOption(LIST_LABELS_CMD)) { + printClusterNodeLabels(); + } else if (parsedCli.hasOption(HELP_CMD)) { + printUsage(opts); + return 0; + } else { + syserr.println("Invalid Command Usage : "); + printUsage(opts); + } + return 0; + } + + private List sortStrSet(Set labels) { + List list = new ArrayList(); + list.addAll(labels); + Collections.sort(list); + return list; + } + + void printClusterNodeLabels() throws YarnException, IOException { + Set nodeLabels = null; + if (accessLocal) { + nodeLabels = + getNodeLabelManagerInstance(getConf()).getClusterNodeLabels(); + } else { + nodeLabels = client.getClusterNodeLabels(); + } + sysout.println(String.format("Node Labels: %s", + StringUtils.join(sortStrSet(nodeLabels).iterator(), ","))); + } + + @VisibleForTesting + static synchronized CommonNodeLabelsManager + getNodeLabelManagerInstance(Configuration conf) { + if (localNodeLabelsManager == null) { + localNodeLabelsManager = new CommonNodeLabelsManager(); + localNodeLabelsManager.init(conf); + localNodeLabelsManager.start(); + } + return localNodeLabelsManager; + } + + @VisibleForTesting + void printUsage(Options opts) throws UnsupportedEncodingException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = + new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8"))); + new HelpFormatter().printHelp(pw, HelpFormatter.DEFAULT_WIDTH, TITLE, null, + opts, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, + null); + pw.close(); + sysout.println(baos.toString("UTF-8")); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java new file mode 100644 index 00000000000..f9ccf877048 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.cli; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.HashSet; + +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.ImmutableSet; + +public class TestClusterCLI { + ByteArrayOutputStream sysOutStream; + private PrintStream sysOut; + ByteArrayOutputStream sysErrStream; + private PrintStream sysErr; + + @Before + public void setup() { + sysOutStream = new ByteArrayOutputStream(); + sysOut = spy(new PrintStream(sysOutStream)); + sysErrStream = new ByteArrayOutputStream(); + sysErr = spy(new PrintStream(sysErrStream)); + System.setOut(sysOut); + } + + @Test + public void testGetClusterNodeLabels() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn( + ImmutableSet.of("label1", "label2")); + ClusterCLI cli = new ClusterCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = + cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.print("Node Labels: label1,label2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetClusterNodeLabelsWithLocalAccess() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn( + ImmutableSet.of("remote1", "remote2")); + ClusterCLI cli = new ClusterCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + ClusterCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class); + when(ClusterCLI.localNodeLabelsManager.getClusterNodeLabels()) + .thenReturn(ImmutableSet.of("local1", "local2")); + + int rc = + cli.run(new String[] { ClusterCLI.CMD, + "-" + ClusterCLI.LIST_LABELS_CMD, + "-" + ClusterCLI.DIRECTLY_ACCESS_NODE_LABEL_STORE }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + // it should return local* instead of remote* + pw.print("Node Labels: local1,local2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetEmptyClusterNodeLabels() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn(new HashSet()); + ClusterCLI cli = new ClusterCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = + cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.print("Node Labels: "); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testHelp() throws Exception { + ClusterCLI cli = new ClusterCLI(); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = + cli.run(new String[] { "cluster", "--help" }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("usage: yarn cluster"); + pw.println(" -dnl,--directly-access-node-label-store Directly access node label"); + pw.println(" store, with this option, all"); + pw.println(" node label related operations"); + pw.println(" will NOT connect RM. Instead,"); + pw.println(" they will access/modify stored"); + pw.println(" node labels directly. By"); + pw.println(" default, it is false (access"); + pw.println(" via RM). AND PLEASE NOTE: if"); + pw.println(" you configured"); + pw.println(" yarn.node-labels.fs-store.root-"); + pw.println(" dir to a local directory"); + pw.println(" (instead of NFS or HDFS), this"); + pw.println(" option will only work when the"); + pw.println(" command run on the machine"); + pw.println(" where RM is running. Also, this"); + pw.println(" option is UNSTABLE, could be"); + pw.println(" removed in future releases."); + pw.println(" -h,--help Displays help for all commands."); + pw.println(" -lnl,--list-node-labels List cluster node-label"); + pw.println(" collection"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } +} \ No newline at end of file