YARN-2786. Created a yarn cluster CLI and seeded with one command for listing node-labels collection. Contributed by Wangda Tan.

This commit is contained in:
Vinod Kumar Vavilapalli 2015-03-05 10:54:34 -08:00
parent 28b85a2116
commit 138c9cadee
5 changed files with 330 additions and 0 deletions

View File

@ -97,6 +97,9 @@ Release 2.7.0 - UNRELEASED
YARN-3249. Add a 'kill application' button to Resource Manager's Web UI.
(Ryu Kobayashi via ozawa)
YARN-2786. Created a yarn cluster CLI and seeded with one command for listing
node-labels collection. (Wangda Tan via vinodkv)
IMPROVEMENTS
YARN-3005. [JDK7] Use switch statement for String instead of if-else

View File

@ -25,6 +25,7 @@ function hadoop_usage
echo " applicationattempt prints applicationattempt(s) report"
echo " classpath prints the class path needed to get the"
echo " Hadoop jar and the required libraries"
echo " cluster prints cluster information"
echo " container prints container(s) report"
echo " daemonlog get/set the log level for each daemon"
echo " jar <jar> run a jar file"
@ -83,6 +84,11 @@ case "${COMMAND}" in
classpath)
hadoop_do_classpath_subcommand "$@"
;;
cluster)
CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI
hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
;;
daemonlog)
CLASS=org.apache.hadoop.log.LogLevel
hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"

View File

@ -192,6 +192,11 @@ goto :eof
set yarn-command-arguments=%yarn-command% %yarn-command-arguments%
goto :eof
:cluster
set CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
:container
set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
@ -312,6 +317,7 @@ goto :eof
@echo jar ^<jar^> run a jar file
@echo application prints application(s) report/kill application
@echo applicationattempt prints applicationattempt(s) report
@echo cluster prints cluster information
@echo container prints container(s) report
@echo node prints node report(s)
@echo queue prints queue information

View File

@ -0,0 +1,157 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.client.cli;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.Options;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import com.google.common.annotations.VisibleForTesting;
/**
* Cluster CLI used to get over all information of the cluster
*/
@Private
public class ClusterCLI extends YarnCLI {
private static final String TITLE = "yarn cluster";
public static final String LIST_LABELS_CMD = "list-node-labels";
public static final String DIRECTLY_ACCESS_NODE_LABEL_STORE =
"directly-access-node-label-store";
public static final String CMD = "cluster";
private boolean accessLocal = false;
static CommonNodeLabelsManager localNodeLabelsManager = null;
public static void main(String[] args) throws Exception {
ClusterCLI cli = new ClusterCLI();
cli.setSysOutPrintStream(System.out);
cli.setSysErrPrintStream(System.err);
int res = ToolRunner.run(cli, args);
cli.stop();
System.exit(res);
}
@Override
public int run(String[] args) throws Exception {
Options opts = new Options();
opts.addOption("lnl", LIST_LABELS_CMD, false,
"List cluster node-label collection");
opts.addOption("h", HELP_CMD, false, "Displays help for all commands.");
opts.addOption("dnl", DIRECTLY_ACCESS_NODE_LABEL_STORE, false,
"Directly access node label store, "
+ "with this option, all node label related operations"
+ " will NOT connect RM. Instead, they will"
+ " access/modify stored node labels directly."
+ " By default, it is false (access via RM)."
+ " AND PLEASE NOTE: if you configured "
+ YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR
+ " to a local directory"
+ " (instead of NFS or HDFS), this option will only work"
+ " when the command run on the machine where RM is running."
+ " Also, this option is UNSTABLE, could be removed in future"
+ " releases.");
int exitCode = -1;
CommandLine parsedCli = null;
try {
parsedCli = new GnuParser().parse(opts, args);
} catch (MissingArgumentException ex) {
sysout.println("Missing argument for options");
printUsage(opts);
return exitCode;
}
if (parsedCli.hasOption(DIRECTLY_ACCESS_NODE_LABEL_STORE)) {
accessLocal = true;
}
if (parsedCli.hasOption(LIST_LABELS_CMD)) {
printClusterNodeLabels();
} else if (parsedCli.hasOption(HELP_CMD)) {
printUsage(opts);
return 0;
} else {
syserr.println("Invalid Command Usage : ");
printUsage(opts);
}
return 0;
}
private List<String> sortStrSet(Set<String> labels) {
List<String> list = new ArrayList<String>();
list.addAll(labels);
Collections.sort(list);
return list;
}
void printClusterNodeLabels() throws YarnException, IOException {
Set<String> nodeLabels = null;
if (accessLocal) {
nodeLabels =
getNodeLabelManagerInstance(getConf()).getClusterNodeLabels();
} else {
nodeLabels = client.getClusterNodeLabels();
}
sysout.println(String.format("Node Labels: %s",
StringUtils.join(sortStrSet(nodeLabels).iterator(), ",")));
}
@VisibleForTesting
static synchronized CommonNodeLabelsManager
getNodeLabelManagerInstance(Configuration conf) {
if (localNodeLabelsManager == null) {
localNodeLabelsManager = new CommonNodeLabelsManager();
localNodeLabelsManager.init(conf);
localNodeLabelsManager.start();
}
return localNodeLabelsManager;
}
@VisibleForTesting
void printUsage(Options opts) throws UnsupportedEncodingException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw =
new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8")));
new HelpFormatter().printHelp(pw, HelpFormatter.DEFAULT_WIDTH, TITLE, null,
opts, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD,
null);
pw.close();
sysout.println(baos.toString("UTF-8"));
}
}

View File

@ -0,0 +1,158 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.client.cli;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.HashSet;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.junit.Before;
import org.junit.Test;
import com.google.common.collect.ImmutableSet;
public class TestClusterCLI {
ByteArrayOutputStream sysOutStream;
private PrintStream sysOut;
ByteArrayOutputStream sysErrStream;
private PrintStream sysErr;
@Before
public void setup() {
sysOutStream = new ByteArrayOutputStream();
sysOut = spy(new PrintStream(sysOutStream));
sysErrStream = new ByteArrayOutputStream();
sysErr = spy(new PrintStream(sysErrStream));
System.setOut(sysOut);
}
@Test
public void testGetClusterNodeLabels() throws Exception {
YarnClient client = mock(YarnClient.class);
when(client.getClusterNodeLabels()).thenReturn(
ImmutableSet.of("label1", "label2"));
ClusterCLI cli = new ClusterCLI();
cli.setClient(client);
cli.setSysOutPrintStream(sysOut);
cli.setSysErrPrintStream(sysErr);
int rc =
cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD });
assertEquals(0, rc);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.print("Node Labels: label1,label2");
pw.close();
verify(sysOut).println(baos.toString("UTF-8"));
}
@Test
public void testGetClusterNodeLabelsWithLocalAccess() throws Exception {
YarnClient client = mock(YarnClient.class);
when(client.getClusterNodeLabels()).thenReturn(
ImmutableSet.of("remote1", "remote2"));
ClusterCLI cli = new ClusterCLI();
cli.setClient(client);
cli.setSysOutPrintStream(sysOut);
cli.setSysErrPrintStream(sysErr);
ClusterCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class);
when(ClusterCLI.localNodeLabelsManager.getClusterNodeLabels())
.thenReturn(ImmutableSet.of("local1", "local2"));
int rc =
cli.run(new String[] { ClusterCLI.CMD,
"-" + ClusterCLI.LIST_LABELS_CMD,
"-" + ClusterCLI.DIRECTLY_ACCESS_NODE_LABEL_STORE });
assertEquals(0, rc);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
// it should return local* instead of remote*
pw.print("Node Labels: local1,local2");
pw.close();
verify(sysOut).println(baos.toString("UTF-8"));
}
@Test
public void testGetEmptyClusterNodeLabels() throws Exception {
YarnClient client = mock(YarnClient.class);
when(client.getClusterNodeLabels()).thenReturn(new HashSet<String>());
ClusterCLI cli = new ClusterCLI();
cli.setClient(client);
cli.setSysOutPrintStream(sysOut);
cli.setSysErrPrintStream(sysErr);
int rc =
cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD });
assertEquals(0, rc);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.print("Node Labels: ");
pw.close();
verify(sysOut).println(baos.toString("UTF-8"));
}
@Test
public void testHelp() throws Exception {
ClusterCLI cli = new ClusterCLI();
cli.setSysOutPrintStream(sysOut);
cli.setSysErrPrintStream(sysErr);
int rc =
cli.run(new String[] { "cluster", "--help" });
assertEquals(0, rc);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("usage: yarn cluster");
pw.println(" -dnl,--directly-access-node-label-store Directly access node label");
pw.println(" store, with this option, all");
pw.println(" node label related operations");
pw.println(" will NOT connect RM. Instead,");
pw.println(" they will access/modify stored");
pw.println(" node labels directly. By");
pw.println(" default, it is false (access");
pw.println(" via RM). AND PLEASE NOTE: if");
pw.println(" you configured");
pw.println(" yarn.node-labels.fs-store.root-");
pw.println(" dir to a local directory");
pw.println(" (instead of NFS or HDFS), this");
pw.println(" option will only work when the");
pw.println(" command run on the machine");
pw.println(" where RM is running. Also, this");
pw.println(" option is UNSTABLE, could be");
pw.println(" removed in future releases.");
pw.println(" -h,--help Displays help for all commands.");
pw.println(" -lnl,--list-node-labels List cluster node-label");
pw.println(" collection");
pw.close();
verify(sysOut).println(baos.toString("UTF-8"));
}
}