From 8c3a0d5a98993081b6085ddfa208f05e6fd1303e Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 6 Nov 2014 20:48:31 -0800 Subject: [PATCH] YARN-2647. Added a queue CLI for getting queue information. Contributed by Sunil Govind. (cherry picked from commit 61effcbebc819ee1bd3d60f5444c39c972494a32) Conflicts: hadoop-yarn-project/hadoop-yarn/bin/yarn (cherry picked from commit b1521c7c3ce7dfaabdb3fc1d74c10970dda57fe5) --- hadoop-yarn-project/CHANGES.txt | 3 + hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 + hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 6 + .../hadoop/yarn/client/cli/QueueCLI.java | 148 ++++++++++++++++++ .../hadoop/yarn/client/cli/TestYarnCLI.java | 61 ++++++++ 5 files changed, 222 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5872583132d..f50793dd31c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -120,6 +120,9 @@ Release 2.6.0 - UNRELEASED YARN-2198. Remove the need to run NodeManager as privileged account for Windows Secure Container Executor. (Remus Rusanu via jianhe) + YARN-2647. Added a queue CLI for getting queue information. (Sunil Govind via + vinodkv) + IMPROVEMENTS YARN-2242. Improve exception information on AM launch crashes. (Li Lu diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index a739dac8e31..a4b5f1cc16a 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -74,6 +74,7 @@ function print_usage(){ echo " report" echo " container prints container(s) report" echo " node prints node report(s)" + echo " queue prints queue information" echo " logs dump container logs" echo " classpath prints the class path needed to" echo " get the Hadoop jar and the" @@ -213,6 +214,9 @@ elif [ "$COMMAND" = "application" ] || elif [ "$COMMAND" = "node" ] ; then CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" +elif [ "COMMAND" = "queue" ] ; then + CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI + YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" elif [ "$COMMAND" = "resourcemanager" ] ; then CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 0cf0b40c76d..7b7fdc4db49 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -196,6 +196,11 @@ goto :eof set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% goto :eof +:queue + set CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + :resourcemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager @@ -298,6 +303,7 @@ goto :eof @echo applicationattempt prints applicationattempt(s) report @echo container prints container(s) report @echo node prints node report(s) + @echo queue prints queue information @echo logs dump container logs @echo classpath prints the class path needed to get the @echo Hadoop jar and the required libraries diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java new file mode 100644 index 00000000000..3c74f13f3a8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.client.cli; + +import java.io.IOException; +import java.io.PrintWriter; +import java.text.DecimalFormat; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Options; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.exceptions.YarnException; + +import com.google.common.annotations.VisibleForTesting; + +@Private +@Unstable +public class QueueCLI extends YarnCLI { + public static final String QUEUE = "queue"; + + public static void main(String[] args) throws Exception { + QueueCLI cli = new QueueCLI(); + cli.setSysOutPrintStream(System.out); + cli.setSysErrPrintStream(System.err); + int res = ToolRunner.run(cli, args); + cli.stop(); + System.exit(res); + } + + @Override + public int run(String[] args) throws Exception { + Options opts = new Options(); + int exitCode = -1; + if (args.length > 0) { + opts.addOption(STATUS_CMD, true, + "List queue information about given queue."); + opts.addOption(HELP_CMD, false, "Displays help for all commands."); + opts.getOption(STATUS_CMD).setArgName("Queue Name"); + } else { + syserr.println("Invalid Command usage. Command must start with 'queue'"); + return exitCode; + } + + CommandLine cliParser = null; + try { + cliParser = new GnuParser().parse(opts, args); + } catch (MissingArgumentException ex) { + sysout.println("Missing argument for options"); + printUsage(opts); + return exitCode; + } + + if (cliParser.hasOption(STATUS_CMD)) { + if (args.length != 3) { + printUsage(opts); + return exitCode; + } + listQueue(cliParser.getOptionValue(STATUS_CMD)); + } else if (cliParser.hasOption(HELP_CMD)) { + printUsage(opts); + return 0; + } else { + syserr.println("Invalid Command Usage : "); + printUsage(opts); + } + return 0; + } + + /** + * It prints the usage of the command + * + * @param opts + */ + @VisibleForTesting + void printUsage(Options opts) { + new HelpFormatter().printHelp(QUEUE, opts); + } + + /** + * Lists the Queue Information matching the given queue name + * + * @param queueName + * @throws YarnException + * @throws IOException + */ + private void listQueue(String queueName) throws YarnException, IOException { + PrintWriter writer = new PrintWriter(sysout); + + QueueInfo queueInfo = client.getQueueInfo(queueName); + writer.println("Queue Information : "); + printQueueInfo(writer, queueInfo); + writer.flush(); + } + + private void printQueueInfo(PrintWriter writer, QueueInfo queueInfo) { + writer.print("Queue Name : "); + writer.println(queueInfo.getQueueName()); + + writer.print("\tState : "); + writer.println(queueInfo.getQueueState()); + DecimalFormat df = new DecimalFormat("#.0"); + writer.print("\tCapacity : "); + writer.println(df.format(queueInfo.getCapacity() * 100) + "%"); + writer.print("\tCurrent Capacity : "); + writer.println(df.format(queueInfo.getCurrentCapacity() * 100) + "%"); + writer.print("\tMaximum Capacity : "); + writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%"); + writer.print("\tDefault Node Label expression : "); + if (null != queueInfo.getDefaultNodeLabelExpression()) { + writer.println(queueInfo.getDefaultNodeLabelExpression()); + } else { + writer.println(); + } + + Set nodeLabels = queueInfo.getAccessibleNodeLabels(); + StringBuilder labelList = new StringBuilder(); + writer.print("\tAccessible Node Labels : "); + for (String nodeLabel : nodeLabels) { + if (labelList.length() > 0) { + labelList.append(','); + } + labelList.append(nodeLabel); + } + writer.println(labelList.toString()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 19ded9c3483..adb58e86f52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -1235,6 +1237,57 @@ public class TestYarnCLI { Assert.assertEquals(String.format("Missing argument for options%n%1s", createNodeCLIHelpMessage()), sysOutStream.toString()); } + + @Test + public void testGetQueueInfo() throws Exception { + QueueCLI cli = createAndGetQueueCLI(); + Set nodeLabels = new HashSet(); + nodeLabels.add("GPU"); + nodeLabels.add("JDK_7"); + QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f, + null, null, QueueState.RUNNING, nodeLabels, "GPU"); + when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo); + int result = cli.run(new String[] { "queue", "-status", "queueA" }); + assertEquals(0, result); + verify(client).getQueueInfo("queueA"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Queue Information : "); + pw.println("Queue Name : " + "queueA"); + pw.println("\tState : " + "RUNNING"); + pw.println("\tCapacity : " + "40.0%"); + pw.println("\tCurrent Capacity : " + "50.0%"); + pw.println("\tMaximum Capacity : " + "80.0%"); + pw.println("\tDefault Node Label expression : " + "GPU"); + pw.println("\tAccessible Node Labels : " + "JDK_7,GPU"); + pw.close(); + String queueInfoStr = baos.toString("UTF-8"); + Assert.assertEquals(queueInfoStr, sysOutStream.toString()); + } + + @Test + public void testGetQueueInfoWithEmptyNodeLabel() throws Exception { + QueueCLI cli = createAndGetQueueCLI(); + QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f, + null, null, QueueState.RUNNING, null, null); + when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo); + int result = cli.run(new String[] { "queue", "-status", "queueA" }); + assertEquals(0, result); + verify(client).getQueueInfo("queueA"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Queue Information : "); + pw.println("Queue Name : " + "queueA"); + pw.println("\tState : " + "RUNNING"); + pw.println("\tCapacity : " + "40.0%"); + pw.println("\tCurrent Capacity : " + "50.0%"); + pw.println("\tMaximum Capacity : " + "80.0%"); + pw.println("\tDefault Node Label expression : "); + pw.println("\tAccessible Node Labels : "); + pw.close(); + String queueInfoStr = baos.toString("UTF-8"); + Assert.assertEquals(queueInfoStr, sysOutStream.toString()); + } private void verifyUsageInfo(YarnCLI cli) throws Exception { cli.setSysErrPrintStream(sysErr); @@ -1272,6 +1325,14 @@ public class TestYarnCLI { cli.setSysOutPrintStream(sysOut); return cli; } + + private QueueCLI createAndGetQueueCLI() { + QueueCLI cli = new QueueCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + return cli; + } private String createApplicationCLIHelpMessage() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream();