YARN-9899. Migration tool that help to generate CS config based on FS config [Phase 2]. Contributed by Peter Bacsko
This commit is contained in:
Szilard Nemeth
parent
3161813482
commit
8c9018d5c7
|
|
@ -47,6 +47,7 @@ function hadoop_usage
|
|||
hadoop_add_subcommand "queue" client "prints queue information"
|
||||
hadoop_add_subcommand "registrydns" daemon "run the registry DNS server"
|
||||
hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager"
|
||||
hadoop_add_subcommand "fs2cs" client "converts Fair Scheduler configuration to Capacity Scheduler (EXPERIMENTAL)"
|
||||
hadoop_add_subcommand "rmadmin" admin "admin tools"
|
||||
hadoop_add_subcommand "router" daemon "run the Router daemon"
|
||||
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
|
||||
|
|
@ -165,6 +166,9 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
|
|||
hadoop_translate_cygwin_path sld
|
||||
hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
|
||||
;;
|
||||
fs2cs)
|
||||
HADOOP_CLASSNAME="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverterMain"
|
||||
;;
|
||||
rmadmin)
|
||||
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -21,10 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager;
|
|||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.sun.jersey.spi.container.servlet.ServletContainer;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigArgumentHandler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigArgumentHandler.CliOption;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverter;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.slf4j.Marker;
|
||||
|
|
@ -232,13 +228,6 @@ public class ResourceManager extends CompositeService
|
|||
private Configuration conf;
|
||||
|
||||
private UserGroupInformation rmLoginUGI;
|
||||
private static FSConfigToCSConfigArgumentHandler
|
||||
fsConfigConversionArgumentHandler;
|
||||
|
||||
static {
|
||||
FSConfigToCSConfigConverter converter = initFSConfigConverter();
|
||||
initFSArgumentHandler(converter);
|
||||
}
|
||||
|
||||
public ResourceManager() {
|
||||
super("ResourceManager");
|
||||
|
|
@ -1576,22 +1565,6 @@ public class ResourceManager extends CompositeService
|
|||
} else if (argv[0].equals("-remove-application-from-state-store")
|
||||
&& argv.length == 2) {
|
||||
removeApplication(conf, argv[1]);
|
||||
} else if (argv[0].equals("-convert-fs-configuration")) {
|
||||
String[] args = Arrays.copyOfRange(argv, 1, argv.length);
|
||||
try {
|
||||
int exitCode =
|
||||
fsConfigConversionArgumentHandler.parseAndConvert(args);
|
||||
if (exitCode != 0) {
|
||||
LOG.error(FATAL,
|
||||
"Error while starting FS configuration conversion, " +
|
||||
"see previous error messages for details!");
|
||||
System.exit(exitCode);
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
LOG.error(FATAL,
|
||||
"Error while starting FS configuration conversion!", t);
|
||||
System.exit(-1);
|
||||
}
|
||||
} else {
|
||||
printUsage(System.err);
|
||||
}
|
||||
|
|
@ -1744,12 +1717,6 @@ public class ResourceManager extends CompositeService
|
|||
out.println(" "
|
||||
+ "[-format-conf-store]" + "\n");
|
||||
|
||||
out.println("[-convert-fs-configuration ");
|
||||
out.println(FSConfigToCSConfigConverter.WARNING_TEXT);
|
||||
for (CliOption cliOption : CliOption.values()) {
|
||||
out.println(" " + cliOption.getAsArgumentString());
|
||||
}
|
||||
out.println("]");
|
||||
}
|
||||
|
||||
protected RMAppLifetimeMonitor createRMAppLifetimeMonitor() {
|
||||
|
|
@ -1767,17 +1734,4 @@ public class ResourceManager extends CompositeService
|
|||
public boolean isSecurityEnabled() {
|
||||
return UserGroupInformation.isSecurityEnabled();
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static void initFSArgumentHandler(FSConfigToCSConfigConverter converter) {
|
||||
ResourceManager.fsConfigConversionArgumentHandler =
|
||||
new FSConfigToCSConfigArgumentHandler(converter);
|
||||
}
|
||||
|
||||
private static FSConfigToCSConfigConverter initFSConfigConverter() {
|
||||
FSConfigToCSConfigRuleHandler ruleHandler =
|
||||
new FSConfigToCSConfigRuleHandler();
|
||||
return new FSConfigToCSConfigConverter(ruleHandler);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,9 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
|
|||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.GnuParser;
|
||||
import org.apache.commons.cli.MissingArgumentException;
|
||||
import org.apache.commons.cli.HelpFormatter;
|
||||
import org.apache.commons.cli.Option;
|
||||
import org.apache.commons.cli.Options;
|
||||
import org.apache.commons.cli.ParseException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
|
@ -47,69 +48,76 @@ public class FSConfigToCSConfigArgumentHandler {
|
|||
*/
|
||||
public enum CliOption {
|
||||
YARN_SITE("yarn-site.xml", "y", "yarnsiteconfig",
|
||||
"Path to a valid yarn-site.xml config file", true, true),
|
||||
"Path to a valid yarn-site.xml config file", true),
|
||||
|
||||
// fair-scheduler.xml is not mandatory
|
||||
// if FairSchedulerConfiguration.ALLOCATION_FILE is defined in yarn-site.xml
|
||||
FAIR_SCHEDULER("fair-scheduler.xml", "f", "fsconfig",
|
||||
"Path to a valid fair-scheduler.xml config file", false, true),
|
||||
"Path to a valid fair-scheduler.xml config file", true),
|
||||
CONVERSION_RULES("conversion rules config file", "r", "rulesconfig",
|
||||
"Optional parameter. If given, should specify a valid path to the " +
|
||||
"conversion rules file (property format).", false, true),
|
||||
"conversion rules file (property format).", true),
|
||||
CONSOLE_MODE("console mode", "p", "print",
|
||||
"If defined, the converted configuration will " +
|
||||
"only be emitted to the console.", false, false),
|
||||
"only be emitted to the console.", false),
|
||||
CLUSTER_RESOURCE("cluster resource", "c", "cluster-resource",
|
||||
"Needs to be given if maxResources is defined as percentages " +
|
||||
"for any queue, otherwise this parameter can be omitted.",
|
||||
false, true),
|
||||
true),
|
||||
OUTPUT_DIR("output directory", "o", "output-directory",
|
||||
"Output directory for yarn-site.xml and" +
|
||||
" capacity-scheduler.xml files." +
|
||||
"Must have write permission for user who is running this script.",
|
||||
true, true);
|
||||
true),
|
||||
HELP("help", "h", "help", "Displays the list of options", false);
|
||||
|
||||
private final String name;
|
||||
private final String shortSwitch;
|
||||
private final String longSwitch;
|
||||
private final String description;
|
||||
private final boolean required;
|
||||
private final boolean hasArg;
|
||||
|
||||
CliOption(String name, String shortSwitch, String longSwitch,
|
||||
String description, boolean required, boolean hasArg) {
|
||||
String description, boolean hasArg) {
|
||||
this.name = name;
|
||||
this.shortSwitch = shortSwitch;
|
||||
this.longSwitch = longSwitch;
|
||||
this.description = description;
|
||||
this.required = required;
|
||||
this.hasArg = hasArg;
|
||||
}
|
||||
|
||||
public Option createCommonsCliOption() {
|
||||
Option option = new Option(shortSwitch, longSwitch, hasArg, description);
|
||||
option.setRequired(required);
|
||||
return option;
|
||||
}
|
||||
|
||||
public String getAsArgumentString() {
|
||||
return shortSwitch + "|" + longSwitch + ": " + description;
|
||||
}
|
||||
}
|
||||
|
||||
public int parseAndConvert(String[] args) throws Exception {
|
||||
int parseAndConvert(String[] args) throws Exception {
|
||||
Options opts = createOptions();
|
||||
|
||||
try {
|
||||
if (args.length == 0) {
|
||||
LOG.info("Missing command line arguments");
|
||||
printHelp(opts);
|
||||
return 0;
|
||||
}
|
||||
|
||||
CommandLine cliParser = new GnuParser().parse(opts, args);
|
||||
|
||||
if (cliParser.hasOption(CliOption.HELP.shortSwitch)) {
|
||||
printHelp(opts);
|
||||
return 0;
|
||||
}
|
||||
|
||||
checkOptionPresent(cliParser, CliOption.YARN_SITE);
|
||||
checkOptionPresent(cliParser, CliOption.OUTPUT_DIR);
|
||||
checkOutputDefined(cliParser);
|
||||
|
||||
FSConfigToCSConfigConverterParams params = validateInputFiles(cliParser);
|
||||
converter.convert(params);
|
||||
} catch (MissingArgumentException e) {
|
||||
String msg = "Missing argument for options" + e.getMessage();
|
||||
} catch (ParseException e) {
|
||||
String msg = "Options parsing failed: " + e.getMessage();
|
||||
logAndStdErr(e, msg);
|
||||
printHelp(opts);
|
||||
return -1;
|
||||
} catch (PreconditionException e) {
|
||||
String msg = "Cannot start FS config conversion due to the following"
|
||||
|
|
@ -131,7 +139,8 @@ public class FSConfigToCSConfigArgumentHandler {
|
|||
}
|
||||
|
||||
private void logAndStdErr(Exception e, String msg) {
|
||||
LOG.error(msg, e);
|
||||
LOG.debug("Stack trace", e);
|
||||
LOG.error(msg);
|
||||
System.err.println(msg);
|
||||
}
|
||||
|
||||
|
|
@ -172,6 +181,11 @@ public class FSConfigToCSConfigArgumentHandler {
|
|||
.build();
|
||||
}
|
||||
|
||||
private void printHelp(Options opts) {
|
||||
HelpFormatter formatter = new HelpFormatter();
|
||||
formatter.printHelp("General options are: ", opts);
|
||||
}
|
||||
|
||||
private static void checkOptionPresent(CommandLine cliParser,
|
||||
CliOption cliOption) {
|
||||
if (!cliParser.hasOption(cliOption.shortSwitch)) {
|
||||
|
|
@ -181,6 +195,20 @@ public class FSConfigToCSConfigArgumentHandler {
|
|||
}
|
||||
}
|
||||
|
||||
private static void checkOutputDefined(CommandLine cliParser) {
|
||||
boolean hasOutputDir =
|
||||
cliParser.hasOption(CliOption.OUTPUT_DIR.shortSwitch);
|
||||
|
||||
boolean console =
|
||||
cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch);
|
||||
|
||||
if (!console && !hasOutputDir) {
|
||||
throw new PreconditionException(
|
||||
"Output directory or console mode was not defined. Please" +
|
||||
" use -h or --help to see command line switches");
|
||||
}
|
||||
}
|
||||
|
||||
private static void checkFile(CliOption cliOption, String filePath) {
|
||||
checkFileInternal(cliOption, filePath, true);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.slf4j.Marker;
|
||||
import org.slf4j.MarkerFactory;
|
||||
|
||||
/**
|
||||
* Main class that invokes the FS->CS converter.
|
||||
*
|
||||
*/
|
||||
@SuppressWarnings("checkstyle:hideutilityclassconstructor")
|
||||
public class FSConfigToCSConfigConverterMain {
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(FSConfigToCSConfigConverterMain.class);
|
||||
private static final Marker FATAL =
|
||||
MarkerFactory.getMarker("FATAL");
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
FSConfigToCSConfigRuleHandler ruleHandler =
|
||||
new FSConfigToCSConfigRuleHandler();
|
||||
FSConfigToCSConfigConverter converter =
|
||||
new FSConfigToCSConfigConverter(ruleHandler);
|
||||
FSConfigToCSConfigArgumentHandler fsConfigConversionArgumentHandler =
|
||||
new FSConfigToCSConfigArgumentHandler(converter);
|
||||
int exitCode =
|
||||
fsConfigConversionArgumentHandler.parseAndConvert(args);
|
||||
if (exitCode != 0) {
|
||||
LOG.error(FATAL,
|
||||
"Error while starting FS configuration conversion, " +
|
||||
"see previous error messages for details!");
|
||||
System.exit(exitCode);
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
LOG.error(FATAL,
|
||||
"Error while starting FS configuration conversion!", t);
|
||||
System.exit(-1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.FSPlacementRule;
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.PrimaryGroupPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.RejectPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.SecondaryGroupExistingPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule;
|
||||
|
|
@ -51,15 +52,15 @@ class QueuePlacementConverter {
|
|||
ruleCount++;
|
||||
if (rule instanceof UserPlacementRule) {
|
||||
UserPlacementRule userRule = (UserPlacementRule) rule;
|
||||
if (mapping.length() > 0) {
|
||||
mapping.append(";");
|
||||
}
|
||||
|
||||
// nested rule
|
||||
if (userRule.getParentRule() != null) {
|
||||
handleNestedRule(mapping, userRule);
|
||||
} else {
|
||||
if (!userAsDefaultQueue) {
|
||||
if (mapping.length() > 0) {
|
||||
mapping.append(";");
|
||||
}
|
||||
mapping.append("u:" + USER + ":" + USER);
|
||||
}
|
||||
}
|
||||
|
|
@ -82,8 +83,11 @@ class QueuePlacementConverter {
|
|||
mapping.append("u:" + USER + ":").append(defaultRule.defaultQueueName);
|
||||
} else if (rule instanceof SecondaryGroupExistingPlacementRule) {
|
||||
// TODO: wait for YARN-9840
|
||||
if (mapping.length() > 0) {
|
||||
mapping.append(";");
|
||||
}
|
||||
mapping.append("u:" + USER + ":" + SECONDARY_GROUP);
|
||||
} else {
|
||||
} else if (!(rule instanceof RejectPlacementRule)) {
|
||||
throw new IllegalArgumentException("Unknown placement rule: " + rule);
|
||||
}
|
||||
}
|
||||
|
|
@ -99,6 +103,9 @@ class QueuePlacementConverter {
|
|||
private void handleNestedRule(StringBuilder mapping,
|
||||
UserPlacementRule userRule) {
|
||||
PlacementRule pr = userRule.getParentRule();
|
||||
if (mapping.length() > 0) {
|
||||
mapping.append(";");
|
||||
}
|
||||
if (pr instanceof PrimaryGroupPlacementRule) {
|
||||
// TODO: wait for YARN-9841
|
||||
mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER);
|
||||
|
|
|
|||
|
|
@ -18,6 +18,13 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager;
|
||||
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||
import org.apache.hadoop.http.lib.StaticUserWebFilter;
|
||||
|
|
@ -39,8 +46,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptR
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverter;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverterParams;
|
||||
import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer;
|
||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||
import org.junit.After;
|
||||
|
|
@ -49,23 +54,8 @@ import org.junit.Before;
|
|||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.ExpectedException;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.CONVERSION_RULES_FILE;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.FS_ALLOC_FILE;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.OUTPUT_DIR;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
public class TestResourceManager {
|
||||
private static final Logger LOG =
|
||||
|
|
@ -372,71 +362,4 @@ public class TestResourceManager {
|
|||
dummyResourceManager.stop();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Example command: <br>
|
||||
* opt/hadoop/bin/yarn resourcemanager -convert-fs-configuration<br>
|
||||
* -o /tmp/output<br>
|
||||
* -y /opt/hadoop/etc/hadoop/yarn-site.xml<br>
|
||||
* -f /opt/hadoop/etc/hadoop/fair-scheduler.xml<br>
|
||||
* -r /home/systest/sample-rules-config.properties<br>
|
||||
*/
|
||||
@Test
|
||||
@SuppressWarnings("checkstyle:javadocstyle")
|
||||
public void testResourceManagerConvertFSConfigurationDefaults()
|
||||
throws Exception {
|
||||
setupFSConfigConversionFiles();
|
||||
|
||||
ArgumentCaptor<FSConfigToCSConfigConverterParams> conversionParams =
|
||||
ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class);
|
||||
|
||||
final String mainSwitch = "-convert-fs-configuration";
|
||||
FSConfigToCSConfigConverter mockConverter =
|
||||
mock(FSConfigToCSConfigConverter.class);
|
||||
|
||||
ResourceManager.initFSArgumentHandler(mockConverter);
|
||||
ResourceManager.main(new String[] {mainSwitch, "-o", OUTPUT_DIR,
|
||||
"-y", YARN_SITE_XML, "-f", FS_ALLOC_FILE, "-r",
|
||||
CONVERSION_RULES_FILE});
|
||||
|
||||
// validate params
|
||||
verify(mockConverter).convert(conversionParams.capture());
|
||||
FSConfigToCSConfigConverterParams params = conversionParams.getValue();
|
||||
LOG.info("FS config converter parameters: " + params);
|
||||
|
||||
assertThat(params.getYarnSiteXmlConfig()).isEqualTo(YARN_SITE_XML);
|
||||
assertThat(params.getFairSchedulerXmlConfig()).isEqualTo(FS_ALLOC_FILE);
|
||||
assertThat(params.getConversionRulesConfig())
|
||||
.isEqualTo(CONVERSION_RULES_FILE);
|
||||
assertThat(params.isConsole()).isEqualTo(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testResourceManagerConvertFSConfigurationWithConsoleParam()
|
||||
throws Exception {
|
||||
setupFSConfigConversionFiles();
|
||||
|
||||
ArgumentCaptor<FSConfigToCSConfigConverterParams> conversionParams =
|
||||
ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class);
|
||||
|
||||
final String mainSwitch = "-convert-fs-configuration";
|
||||
FSConfigToCSConfigConverter mockConverter =
|
||||
mock(FSConfigToCSConfigConverter.class);
|
||||
|
||||
ResourceManager.initFSArgumentHandler(mockConverter);
|
||||
ResourceManager.main(new String[] {mainSwitch, "-o", OUTPUT_DIR,
|
||||
"-p", "-y", YARN_SITE_XML, "-f", FS_ALLOC_FILE, "-r",
|
||||
CONVERSION_RULES_FILE});
|
||||
|
||||
// validate params
|
||||
verify(mockConverter).convert(conversionParams.capture());
|
||||
FSConfigToCSConfigConverterParams params = conversionParams.getValue();
|
||||
LOG.info("FS config converter parameters: " + params);
|
||||
|
||||
assertThat(params.getYarnSiteXmlConfig()).isEqualTo(YARN_SITE_XML);
|
||||
assertThat(params.getFairSchedulerXmlConfig()).isEqualTo(FS_ALLOC_FILE);
|
||||
assertThat(params.getConversionRulesConfig())
|
||||
.isEqualTo(CONVERSION_RULES_FILE);
|
||||
assertThat(params.isConsole()).isEqualTo(true);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -88,7 +88,11 @@ public class FSConfigConverterTestCommons {
|
|||
return errContent;
|
||||
}
|
||||
|
||||
private void deleteTestFiles() {
|
||||
ByteArrayOutputStream getStdOutContent() {
|
||||
return outContent;
|
||||
}
|
||||
|
||||
public void deleteTestFiles() {
|
||||
//Files may not be created so we are not strict here!
|
||||
deleteFile(FS_ALLOC_FILE, false);
|
||||
deleteFile(YARN_SITE_XML, false);
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@
|
|||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import org.apache.commons.cli.MissingOptionException;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
|
|
@ -116,13 +115,13 @@ public class TestFSConfigToCSConfigArgumentHandler {
|
|||
String[] args = new String[] {"-o",
|
||||
FSConfigConverterTestCommons.OUTPUT_DIR};
|
||||
|
||||
expectedException.expect(MissingOptionException.class);
|
||||
expectedException.expectMessage("Missing required option: y");
|
||||
int retVal = argumentHandler.parseAndConvert(args);
|
||||
assertEquals("Return value", -1, retVal);
|
||||
|
||||
argumentHandler.parseAndConvert(args);
|
||||
assertTrue("Error content missing", fsTestCommons.getErrContent()
|
||||
.toString().contains("Missing yarn-site.xml parameter"));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testMissingFairSchedulerXmlArgument() throws Exception {
|
||||
setupFSConfigConversionFiles(true);
|
||||
|
|
@ -142,10 +141,12 @@ public class TestFSConfigToCSConfigArgumentHandler {
|
|||
String[] args = new String[] {"-y",
|
||||
FSConfigConverterTestCommons.YARN_SITE_XML};
|
||||
|
||||
expectedException.expect(MissingOptionException.class);
|
||||
expectedException.expectMessage("Missing required option: o");
|
||||
int retVal = argumentHandler.parseAndConvert(args);
|
||||
assertEquals("Return value", -1, retVal);
|
||||
|
||||
argumentHandler.parseAndConvert(args);
|
||||
assertTrue("Error content missing", fsTestCommons.getErrContent()
|
||||
.toString()
|
||||
.contains("Output directory or console mode was not defined"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -183,8 +184,8 @@ public class TestFSConfigToCSConfigArgumentHandler {
|
|||
FSConfigConverterTestCommons.YARN_SITE_XML, "-o",
|
||||
FSConfigConverterTestCommons.YARN_SITE_XML);
|
||||
|
||||
argumentHandler.parseAndConvert(args);
|
||||
System.out.println(fsTestCommons.getErrContent());
|
||||
int retVal = argumentHandler.parseAndConvert(args);
|
||||
assertEquals("Return value", -1, retVal);
|
||||
assertTrue("Error content missing", fsTestCommons.getErrContent()
|
||||
.toString()
|
||||
.contains("Cannot start FS config conversion due to the following " +
|
||||
|
|
@ -355,7 +356,8 @@ public class TestFSConfigToCSConfigArgumentHandler {
|
|||
Mockito.doThrow(UnsupportedPropertyException.class)
|
||||
.when(mockConverter)
|
||||
.convert(ArgumentMatchers.any(FSConfigToCSConfigConverterParams.class));
|
||||
argumentHandler.parseAndConvert(args);
|
||||
int retVal = argumentHandler.parseAndConvert(args);
|
||||
assertEquals("Return value", -1, retVal);
|
||||
assertTrue("Error content missing", fsTestCommons.getErrContent()
|
||||
.toString().contains("Unsupported property/setting encountered"));
|
||||
}
|
||||
|
|
@ -372,7 +374,8 @@ public class TestFSConfigToCSConfigArgumentHandler {
|
|||
|
||||
Mockito.doThrow(ConversionException.class).when(mockConverter)
|
||||
.convert(ArgumentMatchers.any(FSConfigToCSConfigConverterParams.class));
|
||||
argumentHandler.parseAndConvert(args);
|
||||
int retVal = argumentHandler.parseAndConvert(args);
|
||||
assertEquals("Return value", -1, retVal);
|
||||
assertTrue("Error content missing", fsTestCommons.getErrContent()
|
||||
.toString().contains("Fatal error during FS config conversion"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,135 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
|
||||
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.CONVERSION_RULES_FILE;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.FS_ALLOC_FILE;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.OUTPUT_DIR;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
|
||||
/**
|
||||
* Unit tests for TestFSConfigToCSConfigConverterMain.
|
||||
*
|
||||
*/
|
||||
public class TestFSConfigToCSConfigConverterMain {
|
||||
private FSConfigConverterTestCommons converterTestCommons;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
converterTestCommons = new FSConfigConverterTestCommons();
|
||||
converterTestCommons.setUp();
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
converterTestCommons.tearDown();
|
||||
}
|
||||
|
||||
/*
|
||||
* Example command:
|
||||
* opt/hadoop/bin/yarn fs2cs
|
||||
* -o /tmp/output
|
||||
* -y /opt/hadoop/etc/hadoop/yarn-site.xml
|
||||
* -f /opt/hadoop/etc/hadoop/fair-scheduler.xml
|
||||
* -r /home/systest/sample-rules-config.properties
|
||||
*/
|
||||
@Test
|
||||
public void testConvertFSConfigurationDefaults()
|
||||
throws Exception {
|
||||
setupFSConfigConversionFiles();
|
||||
|
||||
FSConfigToCSConfigConverterMain.main(new String[] {
|
||||
"-o", OUTPUT_DIR,
|
||||
"-y", YARN_SITE_XML,
|
||||
"-f", FS_ALLOC_FILE,
|
||||
"-r", CONVERSION_RULES_FILE});
|
||||
|
||||
boolean csConfigExists =
|
||||
new File(OUTPUT_DIR, "capacity-scheduler.xml").exists();
|
||||
boolean yarnSiteConfigExists =
|
||||
new File(OUTPUT_DIR, "yarn-site.xml").exists();
|
||||
|
||||
assertTrue("capacity-scheduler.xml was not generated", csConfigExists);
|
||||
assertTrue("yarn-site.xml was not generated", yarnSiteConfigExists);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertFSConfigurationWithConsoleParam()
|
||||
throws Exception {
|
||||
setupFSConfigConversionFiles();
|
||||
|
||||
FSConfigToCSConfigConverterMain.main(new String[] {
|
||||
"-p",
|
||||
"-y", YARN_SITE_XML,
|
||||
"-f", FS_ALLOC_FILE,
|
||||
"-r", CONVERSION_RULES_FILE});
|
||||
|
||||
String stdout = converterTestCommons.getStdOutContent().toString();
|
||||
assertTrue("Stdout doesn't contain yarn-site.xml",
|
||||
stdout.contains("======= yarn-site.xml ======="));
|
||||
assertTrue("Stdout doesn't contain capacity-scheduler.xml",
|
||||
stdout.contains("======= capacity-scheduler.xml ======="));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShortHelpSwitch() {
|
||||
FSConfigToCSConfigConverterMain.main(new String[] {"-h"});
|
||||
|
||||
verifyHelpText();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLongHelpSwitch() {
|
||||
FSConfigToCSConfigConverterMain.main(new String[] {"--help"});
|
||||
|
||||
verifyHelpText();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertFSConfigurationWithLongSwitches()
|
||||
throws IOException {
|
||||
setupFSConfigConversionFiles();
|
||||
|
||||
FSConfigToCSConfigConverterMain.main(new String[] {
|
||||
"--print",
|
||||
"--yarnsiteconfig", YARN_SITE_XML,
|
||||
"--fsconfig", FS_ALLOC_FILE,
|
||||
"--rulesconfig", CONVERSION_RULES_FILE});
|
||||
|
||||
String stdout = converterTestCommons.getStdOutContent().toString();
|
||||
assertTrue("Stdout doesn't contain yarn-site.xml",
|
||||
stdout.contains("======= yarn-site.xml ======="));
|
||||
assertTrue("Stdout doesn't contain capacity-scheduler.xml",
|
||||
stdout.contains("======= capacity-scheduler.xml ======="));
|
||||
}
|
||||
|
||||
private void verifyHelpText() {
|
||||
String stdout = converterTestCommons.getStdOutContent().toString();
|
||||
assertTrue("Help was not displayed",
|
||||
stdout.contains("General options are:"));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,265 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
|
||||
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ENABLE_QUEUE_MAPPING_OVERRIDE;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QUEUE_MAPPING;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyZeroInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.DefaultPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.FSPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.PrimaryGroupPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.RejectPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.SecondaryGroupExistingPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.MockitoJUnitRunner;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
/**
|
||||
* Unit tests for QueuePlacementConverter.
|
||||
*
|
||||
*/
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class TestQueuePlacementConverter {
|
||||
@Mock
|
||||
private PlacementManager placementManager;
|
||||
|
||||
@Mock
|
||||
private FSConfigToCSConfigRuleHandler ruleHandler;
|
||||
|
||||
private QueuePlacementConverter converter;
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
this.converter = new QueuePlacementConverter();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertUserAsDefaultQueue() {
|
||||
Map<String, String> properties = convert(true);
|
||||
|
||||
verifyMapping(properties, "u:%user:%user");
|
||||
verifyZeroInteractions(ruleHandler);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertUserPlacementRuleWithoutUserAsDefaultQueue() {
|
||||
testConvertUserPlacementRule(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertUserPlacementRuleWithUserAsDefaultQueue() {
|
||||
testConvertUserPlacementRule(true);
|
||||
}
|
||||
|
||||
private void testConvertUserPlacementRule(boolean userAsDefaultQueue) {
|
||||
PlacementRule rule = mock(UserPlacementRule.class);
|
||||
initPlacementManagerMock(rule);
|
||||
|
||||
Map<String, String> properties = convert(userAsDefaultQueue);
|
||||
|
||||
verifyMapping(properties, "u:%user:%user");
|
||||
verifyZeroInteractions(ruleHandler);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertSpecifiedPlacementRule() {
|
||||
PlacementRule rule = mock(SpecifiedPlacementRule.class);
|
||||
initPlacementManagerMock(rule);
|
||||
|
||||
Map<String, String> properties = convert(false);
|
||||
|
||||
verifyMappingNoOverride(properties, 1);
|
||||
verifyZeroInteractions(ruleHandler);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertSpecifiedPlacementRuleAtSecondPlace() {
|
||||
PlacementRule rule = mock(UserPlacementRule.class);
|
||||
PlacementRule rule2 = mock(SpecifiedPlacementRule.class);
|
||||
initPlacementManagerMock(rule, rule2);
|
||||
|
||||
Map<String, String> properties = convert(false);
|
||||
|
||||
verifyMappingNoOverride(properties, 2);
|
||||
verify(ruleHandler).handleSpecifiedNotFirstRule();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertPrimaryGroupPlacementRule() {
|
||||
PlacementRule rule = mock(PrimaryGroupPlacementRule.class);
|
||||
initPlacementManagerMock(rule);
|
||||
|
||||
Map<String, String> properties = convert(false);
|
||||
|
||||
verifyMapping(properties, "u:%user:%primary_group");
|
||||
verifyZeroInteractions(ruleHandler);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertSecondaryGroupPlacementRule() {
|
||||
PlacementRule rule = mock(SecondaryGroupExistingPlacementRule.class);
|
||||
initPlacementManagerMock(rule);
|
||||
|
||||
Map<String, String> properties = convert(false);
|
||||
|
||||
verifyMapping(properties, "u:%user:%secondary_group");
|
||||
verifyZeroInteractions(ruleHandler);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertDefaultPlacementRule() {
|
||||
DefaultPlacementRule rule = mock(DefaultPlacementRule.class);
|
||||
rule.defaultQueueName = "abc";
|
||||
initPlacementManagerMock(rule);
|
||||
|
||||
Map<String, String> properties = convert(false);
|
||||
|
||||
verifyMapping(properties, "u:%user:abc");
|
||||
verifyZeroInteractions(ruleHandler);
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void testConvertUnsupportedPlacementRule() {
|
||||
PlacementRule rule = mock(TestPlacementRule.class);
|
||||
initPlacementManagerMock(rule);
|
||||
|
||||
// throws exception
|
||||
convert(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertRejectPlacementRule() {
|
||||
PlacementRule rule = mock(RejectPlacementRule.class);
|
||||
initPlacementManagerMock(rule);
|
||||
|
||||
Map<String, String> properties = convert(false);
|
||||
|
||||
assertEquals("Map is not empty", 0, properties.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertNestedPrimaryGroupRule() {
|
||||
UserPlacementRule rule = mock(UserPlacementRule.class);
|
||||
PrimaryGroupPlacementRule parent = mock(PrimaryGroupPlacementRule.class);
|
||||
when(rule.getParentRule()).thenReturn(parent);
|
||||
initPlacementManagerMock(rule);
|
||||
|
||||
Map<String, String> properties = convert(false);
|
||||
|
||||
verifyMapping(properties, "u:%user:%primary_group.%user");
|
||||
verifyZeroInteractions(ruleHandler);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertNestedSecondaryGroupRule() {
|
||||
UserPlacementRule rule = mock(UserPlacementRule.class);
|
||||
SecondaryGroupExistingPlacementRule parent =
|
||||
mock(SecondaryGroupExistingPlacementRule.class);
|
||||
when(rule.getParentRule()).thenReturn(parent);
|
||||
initPlacementManagerMock(rule);
|
||||
|
||||
Map<String, String> properties = convert(false);
|
||||
|
||||
verifyMapping(properties, "u:%user:%secondary_group.%user");
|
||||
verifyZeroInteractions(ruleHandler);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertNestedDefaultRule() {
|
||||
UserPlacementRule rule = mock(UserPlacementRule.class);
|
||||
DefaultPlacementRule parent =
|
||||
mock(DefaultPlacementRule.class);
|
||||
parent.defaultQueueName = "abc";
|
||||
when(rule.getParentRule()).thenReturn(parent);
|
||||
initPlacementManagerMock(rule);
|
||||
|
||||
Map<String, String> properties = convert(false);
|
||||
|
||||
verifyMapping(properties, "u:%user:abc.%user");
|
||||
verifyZeroInteractions(ruleHandler);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConvertMultiplePlacementRules() {
|
||||
UserPlacementRule rule1 = mock(UserPlacementRule.class);
|
||||
PrimaryGroupPlacementRule rule2 =
|
||||
mock(PrimaryGroupPlacementRule.class);
|
||||
SecondaryGroupExistingPlacementRule rule3 =
|
||||
mock(SecondaryGroupExistingPlacementRule.class);
|
||||
initPlacementManagerMock(rule1, rule2, rule3);
|
||||
|
||||
Map<String, String> properties = convert(false);
|
||||
|
||||
verifyMapping(properties,
|
||||
"u:%user:%user;u:%user:%primary_group;u:%user:%secondary_group");
|
||||
verifyZeroInteractions(ruleHandler);
|
||||
}
|
||||
|
||||
private void initPlacementManagerMock(
|
||||
PlacementRule... rules) {
|
||||
List<PlacementRule> listOfRules = Lists.newArrayList(rules);
|
||||
when(placementManager.getPlacementRules()).thenReturn(listOfRules);
|
||||
}
|
||||
|
||||
private Map<String, String> convert(boolean userAsDefaultQueue) {
|
||||
return converter.convertPlacementPolicy(placementManager, ruleHandler,
|
||||
userAsDefaultQueue);
|
||||
}
|
||||
|
||||
private void verifyMapping(Map<String, String> properties,
|
||||
String expectedValue) {
|
||||
assertEquals("Map size", 1, properties.size());
|
||||
String value = properties.get(QUEUE_MAPPING);
|
||||
assertNotNull("No mapping property found", value);
|
||||
assertEquals("Mapping", expectedValue, value);
|
||||
}
|
||||
|
||||
private void verifyMappingNoOverride(Map<String, String> properties,
|
||||
int expectedSize) {
|
||||
assertEquals("Map size", expectedSize, properties.size());
|
||||
String value = properties.get(ENABLE_QUEUE_MAPPING_OVERRIDE);
|
||||
assertNotNull("No mapping property found", value);
|
||||
assertEquals("Override mapping", "false", value);
|
||||
}
|
||||
|
||||
private class TestPlacementRule extends FSPlacementRule {
|
||||
@Override
|
||||
public ApplicationPlacementContext getPlacementForApp(
|
||||
ApplicationSubmissionContext asc, String user) throws YarnException {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue