YARN-2647. Added a queue CLI for getting queue information. Contributed by Sunil Govind.
(cherry picked from commit61effcbebc
) Conflicts: hadoop-yarn-project/hadoop-yarn/bin/yarn (cherry picked from commitb1521c7c3c
)
This commit is contained in:
parent
a081bdcf63
commit
8c3a0d5a98
|
@ -120,6 +120,9 @@ Release 2.6.0 - UNRELEASED
|
||||||
YARN-2198. Remove the need to run NodeManager as privileged account for
|
YARN-2198. Remove the need to run NodeManager as privileged account for
|
||||||
Windows Secure Container Executor. (Remus Rusanu via jianhe)
|
Windows Secure Container Executor. (Remus Rusanu via jianhe)
|
||||||
|
|
||||||
|
YARN-2647. Added a queue CLI for getting queue information. (Sunil Govind via
|
||||||
|
vinodkv)
|
||||||
|
|
||||||
IMPROVEMENTS
|
IMPROVEMENTS
|
||||||
|
|
||||||
YARN-2242. Improve exception information on AM launch crashes. (Li Lu
|
YARN-2242. Improve exception information on AM launch crashes. (Li Lu
|
||||||
|
|
|
@ -74,6 +74,7 @@ function print_usage(){
|
||||||
echo " report"
|
echo " report"
|
||||||
echo " container prints container(s) report"
|
echo " container prints container(s) report"
|
||||||
echo " node prints node report(s)"
|
echo " node prints node report(s)"
|
||||||
|
echo " queue prints queue information"
|
||||||
echo " logs dump container logs"
|
echo " logs dump container logs"
|
||||||
echo " classpath prints the class path needed to"
|
echo " classpath prints the class path needed to"
|
||||||
echo " get the Hadoop jar and the"
|
echo " get the Hadoop jar and the"
|
||||||
|
@ -213,6 +214,9 @@ elif [ "$COMMAND" = "application" ] ||
|
||||||
elif [ "$COMMAND" = "node" ] ; then
|
elif [ "$COMMAND" = "node" ] ; then
|
||||||
CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
|
CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
|
||||||
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
|
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
|
||||||
|
elif [ "COMMAND" = "queue" ] ; then
|
||||||
|
CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI
|
||||||
|
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
|
||||||
elif [ "$COMMAND" = "resourcemanager" ] ; then
|
elif [ "$COMMAND" = "resourcemanager" ] ; then
|
||||||
CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties
|
CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties
|
||||||
CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
|
CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
|
||||||
|
|
|
@ -196,6 +196,11 @@ goto :eof
|
||||||
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
|
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
|
||||||
goto :eof
|
goto :eof
|
||||||
|
|
||||||
|
:queue
|
||||||
|
set CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI
|
||||||
|
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
|
||||||
|
goto :eof
|
||||||
|
|
||||||
:resourcemanager
|
:resourcemanager
|
||||||
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties
|
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties
|
||||||
set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
|
set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
|
||||||
|
@ -298,6 +303,7 @@ goto :eof
|
||||||
@echo applicationattempt prints applicationattempt(s) report
|
@echo applicationattempt prints applicationattempt(s) report
|
||||||
@echo container prints container(s) report
|
@echo container prints container(s) report
|
||||||
@echo node prints node report(s)
|
@echo node prints node report(s)
|
||||||
|
@echo queue prints queue information
|
||||||
@echo logs dump container logs
|
@echo logs dump container logs
|
||||||
@echo classpath prints the class path needed to get the
|
@echo classpath prints the class path needed to get the
|
||||||
@echo Hadoop jar and the required libraries
|
@echo Hadoop jar and the required libraries
|
||||||
|
|
|
@ -0,0 +1,148 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.client.cli;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
import java.text.DecimalFormat;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import org.apache.commons.cli.CommandLine;
|
||||||
|
import org.apache.commons.cli.GnuParser;
|
||||||
|
import org.apache.commons.cli.HelpFormatter;
|
||||||
|
import org.apache.commons.cli.MissingArgumentException;
|
||||||
|
import org.apache.commons.cli.Options;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
|
import org.apache.hadoop.util.ToolRunner;
|
||||||
|
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
|
||||||
|
@Private
|
||||||
|
@Unstable
|
||||||
|
public class QueueCLI extends YarnCLI {
|
||||||
|
public static final String QUEUE = "queue";
|
||||||
|
|
||||||
|
public static void main(String[] args) throws Exception {
|
||||||
|
QueueCLI cli = new QueueCLI();
|
||||||
|
cli.setSysOutPrintStream(System.out);
|
||||||
|
cli.setSysErrPrintStream(System.err);
|
||||||
|
int res = ToolRunner.run(cli, args);
|
||||||
|
cli.stop();
|
||||||
|
System.exit(res);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int run(String[] args) throws Exception {
|
||||||
|
Options opts = new Options();
|
||||||
|
int exitCode = -1;
|
||||||
|
if (args.length > 0) {
|
||||||
|
opts.addOption(STATUS_CMD, true,
|
||||||
|
"List queue information about given queue.");
|
||||||
|
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
|
||||||
|
opts.getOption(STATUS_CMD).setArgName("Queue Name");
|
||||||
|
} else {
|
||||||
|
syserr.println("Invalid Command usage. Command must start with 'queue'");
|
||||||
|
return exitCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
CommandLine cliParser = null;
|
||||||
|
try {
|
||||||
|
cliParser = new GnuParser().parse(opts, args);
|
||||||
|
} catch (MissingArgumentException ex) {
|
||||||
|
sysout.println("Missing argument for options");
|
||||||
|
printUsage(opts);
|
||||||
|
return exitCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cliParser.hasOption(STATUS_CMD)) {
|
||||||
|
if (args.length != 3) {
|
||||||
|
printUsage(opts);
|
||||||
|
return exitCode;
|
||||||
|
}
|
||||||
|
listQueue(cliParser.getOptionValue(STATUS_CMD));
|
||||||
|
} else if (cliParser.hasOption(HELP_CMD)) {
|
||||||
|
printUsage(opts);
|
||||||
|
return 0;
|
||||||
|
} else {
|
||||||
|
syserr.println("Invalid Command Usage : ");
|
||||||
|
printUsage(opts);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* It prints the usage of the command
|
||||||
|
*
|
||||||
|
* @param opts
|
||||||
|
*/
|
||||||
|
@VisibleForTesting
|
||||||
|
void printUsage(Options opts) {
|
||||||
|
new HelpFormatter().printHelp(QUEUE, opts);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lists the Queue Information matching the given queue name
|
||||||
|
*
|
||||||
|
* @param queueName
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
private void listQueue(String queueName) throws YarnException, IOException {
|
||||||
|
PrintWriter writer = new PrintWriter(sysout);
|
||||||
|
|
||||||
|
QueueInfo queueInfo = client.getQueueInfo(queueName);
|
||||||
|
writer.println("Queue Information : ");
|
||||||
|
printQueueInfo(writer, queueInfo);
|
||||||
|
writer.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void printQueueInfo(PrintWriter writer, QueueInfo queueInfo) {
|
||||||
|
writer.print("Queue Name : ");
|
||||||
|
writer.println(queueInfo.getQueueName());
|
||||||
|
|
||||||
|
writer.print("\tState : ");
|
||||||
|
writer.println(queueInfo.getQueueState());
|
||||||
|
DecimalFormat df = new DecimalFormat("#.0");
|
||||||
|
writer.print("\tCapacity : ");
|
||||||
|
writer.println(df.format(queueInfo.getCapacity() * 100) + "%");
|
||||||
|
writer.print("\tCurrent Capacity : ");
|
||||||
|
writer.println(df.format(queueInfo.getCurrentCapacity() * 100) + "%");
|
||||||
|
writer.print("\tMaximum Capacity : ");
|
||||||
|
writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%");
|
||||||
|
writer.print("\tDefault Node Label expression : ");
|
||||||
|
if (null != queueInfo.getDefaultNodeLabelExpression()) {
|
||||||
|
writer.println(queueInfo.getDefaultNodeLabelExpression());
|
||||||
|
} else {
|
||||||
|
writer.println();
|
||||||
|
}
|
||||||
|
|
||||||
|
Set<String> nodeLabels = queueInfo.getAccessibleNodeLabels();
|
||||||
|
StringBuilder labelList = new StringBuilder();
|
||||||
|
writer.print("\tAccessible Node Labels : ");
|
||||||
|
for (String nodeLabel : nodeLabels) {
|
||||||
|
if (labelList.length() > 0) {
|
||||||
|
labelList.append(',');
|
||||||
|
}
|
||||||
|
labelList.append(nodeLabel);
|
||||||
|
}
|
||||||
|
writer.println(labelList.toString());
|
||||||
|
}
|
||||||
|
}
|
|
@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeState;
|
import org.apache.hadoop.yarn.api.records.NodeState;
|
||||||
import org.apache.hadoop.yarn.api.records.Priority;
|
import org.apache.hadoop.yarn.api.records.Priority;
|
||||||
|
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
||||||
|
import org.apache.hadoop.yarn.api.records.QueueState;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
|
@ -1235,6 +1237,57 @@ public class TestYarnCLI {
|
||||||
Assert.assertEquals(String.format("Missing argument for options%n%1s",
|
Assert.assertEquals(String.format("Missing argument for options%n%1s",
|
||||||
createNodeCLIHelpMessage()), sysOutStream.toString());
|
createNodeCLIHelpMessage()), sysOutStream.toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetQueueInfo() throws Exception {
|
||||||
|
QueueCLI cli = createAndGetQueueCLI();
|
||||||
|
Set<String> nodeLabels = new HashSet<String>();
|
||||||
|
nodeLabels.add("GPU");
|
||||||
|
nodeLabels.add("JDK_7");
|
||||||
|
QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
|
||||||
|
null, null, QueueState.RUNNING, nodeLabels, "GPU");
|
||||||
|
when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
|
||||||
|
int result = cli.run(new String[] { "queue", "-status", "queueA" });
|
||||||
|
assertEquals(0, result);
|
||||||
|
verify(client).getQueueInfo("queueA");
|
||||||
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
PrintWriter pw = new PrintWriter(baos);
|
||||||
|
pw.println("Queue Information : ");
|
||||||
|
pw.println("Queue Name : " + "queueA");
|
||||||
|
pw.println("\tState : " + "RUNNING");
|
||||||
|
pw.println("\tCapacity : " + "40.0%");
|
||||||
|
pw.println("\tCurrent Capacity : " + "50.0%");
|
||||||
|
pw.println("\tMaximum Capacity : " + "80.0%");
|
||||||
|
pw.println("\tDefault Node Label expression : " + "GPU");
|
||||||
|
pw.println("\tAccessible Node Labels : " + "JDK_7,GPU");
|
||||||
|
pw.close();
|
||||||
|
String queueInfoStr = baos.toString("UTF-8");
|
||||||
|
Assert.assertEquals(queueInfoStr, sysOutStream.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetQueueInfoWithEmptyNodeLabel() throws Exception {
|
||||||
|
QueueCLI cli = createAndGetQueueCLI();
|
||||||
|
QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
|
||||||
|
null, null, QueueState.RUNNING, null, null);
|
||||||
|
when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
|
||||||
|
int result = cli.run(new String[] { "queue", "-status", "queueA" });
|
||||||
|
assertEquals(0, result);
|
||||||
|
verify(client).getQueueInfo("queueA");
|
||||||
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
PrintWriter pw = new PrintWriter(baos);
|
||||||
|
pw.println("Queue Information : ");
|
||||||
|
pw.println("Queue Name : " + "queueA");
|
||||||
|
pw.println("\tState : " + "RUNNING");
|
||||||
|
pw.println("\tCapacity : " + "40.0%");
|
||||||
|
pw.println("\tCurrent Capacity : " + "50.0%");
|
||||||
|
pw.println("\tMaximum Capacity : " + "80.0%");
|
||||||
|
pw.println("\tDefault Node Label expression : ");
|
||||||
|
pw.println("\tAccessible Node Labels : ");
|
||||||
|
pw.close();
|
||||||
|
String queueInfoStr = baos.toString("UTF-8");
|
||||||
|
Assert.assertEquals(queueInfoStr, sysOutStream.toString());
|
||||||
|
}
|
||||||
|
|
||||||
private void verifyUsageInfo(YarnCLI cli) throws Exception {
|
private void verifyUsageInfo(YarnCLI cli) throws Exception {
|
||||||
cli.setSysErrPrintStream(sysErr);
|
cli.setSysErrPrintStream(sysErr);
|
||||||
|
@ -1272,6 +1325,14 @@ public class TestYarnCLI {
|
||||||
cli.setSysOutPrintStream(sysOut);
|
cli.setSysOutPrintStream(sysOut);
|
||||||
return cli;
|
return cli;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private QueueCLI createAndGetQueueCLI() {
|
||||||
|
QueueCLI cli = new QueueCLI();
|
||||||
|
cli.setClient(client);
|
||||||
|
cli.setSysOutPrintStream(sysOut);
|
||||||
|
cli.setSysErrPrintStream(sysErr);
|
||||||
|
return cli;
|
||||||
|
}
|
||||||
|
|
||||||
private String createApplicationCLIHelpMessage() throws IOException {
|
private String createApplicationCLIHelpMessage() throws IOException {
|
||||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
|
Loading…
Reference in New Issue