YARN-10507. Add the capability to fs2cs to write the converted placement rules inside capacity-scheduler.xml. Contributed by Peter Bacsko
This commit is contained in:
parent
580a6a75a3
commit
4176759909
|
@ -112,6 +112,9 @@ public class FSConfigToCSConfigArgumentHandler {
|
||||||
ENABLE_ASYNC_SCHEDULER("enable asynchronous scheduler", "a", "enable-async-scheduler",
|
ENABLE_ASYNC_SCHEDULER("enable asynchronous scheduler", "a", "enable-async-scheduler",
|
||||||
"Enables the Asynchronous scheduler which decouples the CapacityScheduler" +
|
"Enables the Asynchronous scheduler which decouples the CapacityScheduler" +
|
||||||
" scheduling from Node Heartbeats.", false),
|
" scheduling from Node Heartbeats.", false),
|
||||||
|
RULES_TO_FILE("rules to external file", "e", "rules-to-file",
|
||||||
|
"Generates the converted placement rules to an external JSON file " +
|
||||||
|
"called mapping-rules.json", false),
|
||||||
HELP("help", "h", "help", "Displays the list of options", false);
|
HELP("help", "h", "help", "Displays the list of options", false);
|
||||||
|
|
||||||
private final String name;
|
private final String name;
|
||||||
|
@ -254,6 +257,13 @@ public class FSConfigToCSConfigArgumentHandler {
|
||||||
checkDirectory(CliOption.OUTPUT_DIR, outputDir);
|
checkDirectory(CliOption.OUTPUT_DIR, outputDir);
|
||||||
checkOutputDirDoesNotContainXmls(yarnSiteXmlFile, outputDir);
|
checkOutputDirDoesNotContainXmls(yarnSiteXmlFile, outputDir);
|
||||||
|
|
||||||
|
// check mapping-rules.json if we intend to generate it
|
||||||
|
if (!cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch) &&
|
||||||
|
cliParser.hasOption(CliOption.RULES_TO_FILE.shortSwitch)) {
|
||||||
|
checkFileNotInOutputDir(new File(outputDir),
|
||||||
|
FSConfigToCSConfigConverter.MAPPING_RULES_JSON);
|
||||||
|
}
|
||||||
|
|
||||||
return FSConfigToCSConfigConverterParams.Builder.create()
|
return FSConfigToCSConfigConverterParams.Builder.create()
|
||||||
.withYarnSiteXmlConfig(yarnSiteXmlFile)
|
.withYarnSiteXmlConfig(yarnSiteXmlFile)
|
||||||
.withFairSchedulerXmlConfig(fairSchedulerXmlFile)
|
.withFairSchedulerXmlConfig(fairSchedulerXmlFile)
|
||||||
|
@ -263,6 +273,8 @@ public class FSConfigToCSConfigArgumentHandler {
|
||||||
.withConsole(cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch))
|
.withConsole(cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch))
|
||||||
.withOutputDirectory(outputDir)
|
.withOutputDirectory(outputDir)
|
||||||
.withConvertPlacementRules(convertPlacementRules)
|
.withConvertPlacementRules(convertPlacementRules)
|
||||||
|
.withPlacementRulesToFile(
|
||||||
|
cliParser.hasOption(CliOption.RULES_TO_FILE.shortSwitch))
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -17,13 +17,18 @@
|
||||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
|
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
|
||||||
|
|
||||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
|
||||||
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT;
|
||||||
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_JSON;
|
||||||
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT_JSON;
|
||||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSQueueConverter.QUEUE_MAX_AM_SHARE_DISABLED;
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSQueueConverter.QUEUE_MAX_AM_SHARE_DISABLED;
|
||||||
|
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
@ -55,6 +60,7 @@ import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.fasterxml.jackson.databind.ObjectWriter;
|
import com.fasterxml.jackson.databind.ObjectWriter;
|
||||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||||
|
import com.fasterxml.jackson.core.JsonGenerator;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Converts Fair Scheduler configuration (site and fair-scheduler.xml)
|
* Converts Fair Scheduler configuration (site and fair-scheduler.xml)
|
||||||
|
@ -64,13 +70,13 @@ import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTest
|
||||||
public class FSConfigToCSConfigConverter {
|
public class FSConfigToCSConfigConverter {
|
||||||
public static final Logger LOG = LoggerFactory.getLogger(
|
public static final Logger LOG = LoggerFactory.getLogger(
|
||||||
FSConfigToCSConfigConverter.class.getName());
|
FSConfigToCSConfigConverter.class.getName());
|
||||||
|
public static final String MAPPING_RULES_JSON =
|
||||||
|
"mapping-rules.json";
|
||||||
private static final String YARN_SITE_XML = "yarn-site.xml";
|
private static final String YARN_SITE_XML = "yarn-site.xml";
|
||||||
private static final String CAPACITY_SCHEDULER_XML =
|
private static final String CAPACITY_SCHEDULER_XML =
|
||||||
"capacity-scheduler.xml";
|
"capacity-scheduler.xml";
|
||||||
private static final String FAIR_SCHEDULER_XML =
|
private static final String FAIR_SCHEDULER_XML =
|
||||||
"fair-scheduler.xml";
|
"fair-scheduler.xml";
|
||||||
private static final String MAPPING_RULES_JSON =
|
|
||||||
"mapping-rules.json";
|
|
||||||
|
|
||||||
public static final String WARNING_TEXT =
|
public static final String WARNING_TEXT =
|
||||||
"WARNING: This feature is experimental and not intended " +
|
"WARNING: This feature is experimental and not intended " +
|
||||||
|
@ -99,6 +105,7 @@ public class FSConfigToCSConfigConverter {
|
||||||
private boolean consoleMode = false;
|
private boolean consoleMode = false;
|
||||||
private boolean convertPlacementRules = true;
|
private boolean convertPlacementRules = true;
|
||||||
private String outputDirectory;
|
private String outputDirectory;
|
||||||
|
private boolean rulesToFile;
|
||||||
|
|
||||||
public FSConfigToCSConfigConverter(FSConfigToCSConfigRuleHandler
|
public FSConfigToCSConfigConverter(FSConfigToCSConfigRuleHandler
|
||||||
ruleHandler, ConversionOptions conversionOptions) {
|
ruleHandler, ConversionOptions conversionOptions) {
|
||||||
|
@ -106,7 +113,6 @@ public class FSConfigToCSConfigConverter {
|
||||||
this.conversionOptions = conversionOptions;
|
this.conversionOptions = conversionOptions;
|
||||||
this.yarnSiteOutputStream = System.out;
|
this.yarnSiteOutputStream = System.out;
|
||||||
this.capacitySchedulerOutputStream = System.out;
|
this.capacitySchedulerOutputStream = System.out;
|
||||||
this.mappingRulesOutputStream = System.out;
|
|
||||||
this.placementConverter = new QueuePlacementConverter();
|
this.placementConverter = new QueuePlacementConverter();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -116,6 +122,7 @@ public class FSConfigToCSConfigConverter {
|
||||||
this.clusterResource = getClusterResource(params);
|
this.clusterResource = getClusterResource(params);
|
||||||
this.convertPlacementRules = params.isConvertPlacementRules();
|
this.convertPlacementRules = params.isConvertPlacementRules();
|
||||||
this.outputDirectory = params.getOutputDirectory();
|
this.outputDirectory = params.getOutputDirectory();
|
||||||
|
this.rulesToFile = params.isPlacementRulesToFile();
|
||||||
prepareOutputFiles(params.isConsole());
|
prepareOutputFiles(params.isConsole());
|
||||||
loadConversionRules(params.getConversionRulesConfig());
|
loadConversionRules(params.getConversionRulesConfig());
|
||||||
Configuration inputYarnSiteConfig = getInputYarnSiteConfig(params);
|
Configuration inputYarnSiteConfig = getInputYarnSiteConfig(params);
|
||||||
|
@ -127,9 +134,9 @@ public class FSConfigToCSConfigConverter {
|
||||||
private void prepareOutputFiles(boolean console)
|
private void prepareOutputFiles(boolean console)
|
||||||
throws FileNotFoundException {
|
throws FileNotFoundException {
|
||||||
if (console) {
|
if (console) {
|
||||||
LOG.info("Console mode is enabled, " + YARN_SITE_XML + " and" +
|
LOG.info("Console mode is enabled, {}, {} and {} will be only emitted " +
|
||||||
" " + CAPACITY_SCHEDULER_XML + " will be only emitted " +
|
"to the console!",
|
||||||
"to the console!");
|
YARN_SITE_XML, CAPACITY_SCHEDULER_XML, MAPPING_RULES_JSON);
|
||||||
this.consoleMode = true;
|
this.consoleMode = true;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -253,6 +260,10 @@ public class FSConfigToCSConfigConverter {
|
||||||
convertYarnSiteXml(inputYarnSiteConfig);
|
convertYarnSiteXml(inputYarnSiteConfig);
|
||||||
convertCapacitySchedulerXml(fs);
|
convertCapacitySchedulerXml(fs);
|
||||||
|
|
||||||
|
if (convertPlacementRules) {
|
||||||
|
performRuleConversion(fs);
|
||||||
|
}
|
||||||
|
|
||||||
if (consoleMode) {
|
if (consoleMode) {
|
||||||
System.out.println("======= " + CAPACITY_SCHEDULER_XML + " =======");
|
System.out.println("======= " + CAPACITY_SCHEDULER_XML + " =======");
|
||||||
}
|
}
|
||||||
|
@ -263,10 +274,6 @@ public class FSConfigToCSConfigConverter {
|
||||||
System.out.println("======= " + YARN_SITE_XML + " =======");
|
System.out.println("======= " + YARN_SITE_XML + " =======");
|
||||||
}
|
}
|
||||||
convertedYarnSiteConfig.writeXml(yarnSiteOutputStream);
|
convertedYarnSiteConfig.writeXml(yarnSiteOutputStream);
|
||||||
|
|
||||||
if (convertPlacementRules) {
|
|
||||||
performRuleConversion(fs);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void convertYarnSiteXml(Configuration inputYarnSiteConfig) {
|
private void convertYarnSiteXml(Configuration inputYarnSiteConfig) {
|
||||||
|
@ -313,28 +320,58 @@ public class FSConfigToCSConfigConverter {
|
||||||
fs.getRMContext().getQueuePlacementManager();
|
fs.getRMContext().getQueuePlacementManager();
|
||||||
|
|
||||||
if (placementManager.getPlacementRules().size() > 0) {
|
if (placementManager.getPlacementRules().size() > 0) {
|
||||||
if (!consoleMode) {
|
mappingRulesOutputStream = getOutputStreamForJson();
|
||||||
File mappingRulesFile = new File(outputDirectory,
|
|
||||||
MAPPING_RULES_JSON);
|
|
||||||
this.mappingRulesOutputStream =
|
|
||||||
new FileOutputStream(mappingRulesFile);
|
|
||||||
} else {
|
|
||||||
System.out.println("======= " + MAPPING_RULES_JSON + " =======");
|
|
||||||
}
|
|
||||||
|
|
||||||
MappingRulesDescription desc =
|
MappingRulesDescription desc =
|
||||||
placementConverter.convertPlacementPolicy(placementManager,
|
placementConverter.convertPlacementPolicy(placementManager,
|
||||||
ruleHandler, capacitySchedulerConfig);
|
ruleHandler, capacitySchedulerConfig);
|
||||||
|
|
||||||
ObjectMapper mapper = new ObjectMapper();
|
ObjectMapper mapper = new ObjectMapper();
|
||||||
|
// close output stream if we write to a file, leave it open otherwise
|
||||||
|
if (!consoleMode && rulesToFile) {
|
||||||
|
mapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, true);
|
||||||
|
} else {
|
||||||
|
mapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false);
|
||||||
|
}
|
||||||
ObjectWriter writer = mapper.writer(new DefaultPrettyPrinter());
|
ObjectWriter writer = mapper.writer(new DefaultPrettyPrinter());
|
||||||
|
|
||||||
|
if (consoleMode && rulesToFile) {
|
||||||
|
System.out.println("======= " + MAPPING_RULES_JSON + " =======");
|
||||||
|
}
|
||||||
writer.writeValue(mappingRulesOutputStream, desc);
|
writer.writeValue(mappingRulesOutputStream, desc);
|
||||||
|
|
||||||
|
capacitySchedulerConfig.set(MAPPING_RULE_FORMAT,
|
||||||
|
MAPPING_RULE_FORMAT_JSON);
|
||||||
|
if (!rulesToFile) {
|
||||||
|
String json =
|
||||||
|
((ByteArrayOutputStream)mappingRulesOutputStream)
|
||||||
|
.toString(StandardCharsets.UTF_8.displayName());
|
||||||
|
capacitySchedulerConfig.set(MAPPING_RULE_JSON, json);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
LOG.info("No rules to convert");
|
LOG.info("No rules to convert");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Console RulesToFile OutputStream
|
||||||
|
* true true System.out / PrintStream
|
||||||
|
* true false ByteArrayOutputStream
|
||||||
|
* false true FileOutputStream
|
||||||
|
* false false ByteArrayOutputStream
|
||||||
|
*/
|
||||||
|
private OutputStream getOutputStreamForJson() throws FileNotFoundException {
|
||||||
|
if (consoleMode && rulesToFile) {
|
||||||
|
return System.out;
|
||||||
|
} else if (rulesToFile) {
|
||||||
|
File mappingRulesFile = new File(outputDirectory,
|
||||||
|
MAPPING_RULES_JSON);
|
||||||
|
return new FileOutputStream(mappingRulesFile);
|
||||||
|
} else {
|
||||||
|
return new ByteArrayOutputStream();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void emitDefaultQueueMaxParallelApplications() {
|
private void emitDefaultQueueMaxParallelApplications() {
|
||||||
if (queueMaxAppsDefault != Integer.MAX_VALUE) {
|
if (queueMaxAppsDefault != Integer.MAX_VALUE) {
|
||||||
capacitySchedulerConfig.set(
|
capacitySchedulerConfig.set(
|
||||||
|
@ -467,11 +504,6 @@ public class FSConfigToCSConfigConverter {
|
||||||
this.placementConverter = converter;
|
this.placementConverter = converter;
|
||||||
}
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
void setMappingRulesOutputStream(OutputStream outputStream) {
|
|
||||||
this.mappingRulesOutputStream = outputStream;
|
|
||||||
}
|
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
void setConsoleMode(boolean console) {
|
void setConsoleMode(boolean console) {
|
||||||
this.consoleMode = console;
|
this.consoleMode = console;
|
||||||
|
|
|
@ -16,6 +16,9 @@
|
||||||
|
|
||||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
|
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
|
||||||
|
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
|
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.slf4j.Marker;
|
import org.slf4j.Marker;
|
||||||
|
@ -25,12 +28,16 @@ import org.slf4j.MarkerFactory;
|
||||||
* Main class that invokes the FS->CS converter.
|
* Main class that invokes the FS->CS converter.
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("checkstyle:hideutilityclassconstructor")
|
public final class FSConfigToCSConfigConverterMain {
|
||||||
public class FSConfigToCSConfigConverterMain {
|
private FSConfigToCSConfigConverterMain() {
|
||||||
|
// no instances
|
||||||
|
}
|
||||||
|
|
||||||
private static final Logger LOG =
|
private static final Logger LOG =
|
||||||
LoggerFactory.getLogger(FSConfigToCSConfigConverterMain.class);
|
LoggerFactory.getLogger(FSConfigToCSConfigConverterMain.class);
|
||||||
private static final Marker FATAL =
|
private static final Marker FATAL =
|
||||||
MarkerFactory.getMarker("FATAL");
|
MarkerFactory.getMarker("FATAL");
|
||||||
|
private static Consumer<Integer> exitFunction = System::exit;
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
try {
|
try {
|
||||||
|
@ -44,11 +51,16 @@ public class FSConfigToCSConfigConverterMain {
|
||||||
"see previous error messages for details!");
|
"see previous error messages for details!");
|
||||||
}
|
}
|
||||||
|
|
||||||
System.exit(exitCode);
|
exitFunction.accept(exitCode);
|
||||||
} catch (Throwable t) {
|
} catch (Throwable t) {
|
||||||
LOG.error(FATAL,
|
LOG.error(FATAL,
|
||||||
"Error while starting FS configuration conversion!", t);
|
"Error while starting FS configuration conversion!", t);
|
||||||
System.exit(-1);
|
exitFunction.accept(-1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
static void setExit(Consumer<Integer> exitFunc) {
|
||||||
|
exitFunction = exitFunc;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,7 +28,7 @@ public final class FSConfigToCSConfigConverterParams {
|
||||||
private String clusterResource;
|
private String clusterResource;
|
||||||
private String outputDirectory;
|
private String outputDirectory;
|
||||||
private boolean convertPlacementRules;
|
private boolean convertPlacementRules;
|
||||||
|
private boolean placementRulesToFile;
|
||||||
|
|
||||||
|
|
||||||
private FSConfigToCSConfigConverterParams() {
|
private FSConfigToCSConfigConverterParams() {
|
||||||
|
@ -63,6 +63,10 @@ public final class FSConfigToCSConfigConverterParams {
|
||||||
return convertPlacementRules;
|
return convertPlacementRules;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public boolean isPlacementRulesToFile() {
|
||||||
|
return placementRulesToFile;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "FSConfigToCSConfigConverterParams{" +
|
return "FSConfigToCSConfigConverterParams{" +
|
||||||
|
@ -72,6 +76,7 @@ public final class FSConfigToCSConfigConverterParams {
|
||||||
", clusterResource='" + clusterResource + '\'' +
|
", clusterResource='" + clusterResource + '\'' +
|
||||||
", console=" + console + '\'' +
|
", console=" + console + '\'' +
|
||||||
", convertPlacementRules=" + convertPlacementRules +
|
", convertPlacementRules=" + convertPlacementRules +
|
||||||
|
", placementRulesToFile=" + placementRulesToFile +
|
||||||
'}';
|
'}';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -87,6 +92,7 @@ public final class FSConfigToCSConfigConverterParams {
|
||||||
private String clusterResource;
|
private String clusterResource;
|
||||||
private String outputDirectory;
|
private String outputDirectory;
|
||||||
private boolean convertPlacementRules;
|
private boolean convertPlacementRules;
|
||||||
|
private boolean placementRulesToFile;
|
||||||
|
|
||||||
private Builder() {
|
private Builder() {
|
||||||
}
|
}
|
||||||
|
@ -130,6 +136,11 @@ public final class FSConfigToCSConfigConverterParams {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Builder withPlacementRulesToFile(boolean rulesToFile) {
|
||||||
|
this.placementRulesToFile = rulesToFile;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
public FSConfigToCSConfigConverterParams build() {
|
public FSConfigToCSConfigConverterParams build() {
|
||||||
FSConfigToCSConfigConverterParams params =
|
FSConfigToCSConfigConverterParams params =
|
||||||
new FSConfigToCSConfigConverterParams();
|
new FSConfigToCSConfigConverterParams();
|
||||||
|
@ -140,6 +151,7 @@ public final class FSConfigToCSConfigConverterParams {
|
||||||
params.conversionRulesConfig = this.conversionRulesConfig;
|
params.conversionRulesConfig = this.conversionRulesConfig;
|
||||||
params.outputDirectory = this.outputDirectory;
|
params.outputDirectory = this.outputDirectory;
|
||||||
params.convertPlacementRules = this.convertPlacementRules;
|
params.convertPlacementRules = this.convertPlacementRules;
|
||||||
|
params.placementRulesToFile = this.placementRulesToFile;
|
||||||
return params;
|
return params;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,4 +28,7 @@ public class VerificationException extends RuntimeException {
|
||||||
public VerificationException(String message, Throwable cause) {
|
public VerificationException(String message, Throwable cause) {
|
||||||
super(message, cause);
|
super(message, cause);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public VerificationException() {
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertFalse;
|
import static org.junit.Assert.assertFalse;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.Mockito.doThrow;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.verifyZeroInteractions;
|
import static org.mockito.Mockito.verifyZeroInteractions;
|
||||||
|
|
||||||
|
@ -29,6 +31,7 @@ import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
|
@ -45,8 +48,6 @@ import org.mockito.junit.MockitoJUnitRunner;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Unit tests for FSConfigToCSConfigArgumentHandler.
|
* Unit tests for FSConfigToCSConfigArgumentHandler.
|
||||||
*
|
*
|
||||||
|
@ -229,6 +230,28 @@ public class TestFSConfigToCSConfigArgumentHandler {
|
||||||
"precondition error"));
|
"precondition error"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testVerificationException() throws Exception {
|
||||||
|
setupFSConfigConversionFiles(true);
|
||||||
|
ConversionOptions mockOptions = Mockito.mock(ConversionOptions.class);
|
||||||
|
FSConfigToCSConfigArgumentHandler argumentHandler =
|
||||||
|
new FSConfigToCSConfigArgumentHandler(mockOptions, mockValidator);
|
||||||
|
argumentHandler.setConverterSupplier(this::getMockConverter);
|
||||||
|
|
||||||
|
String[] args = getArgumentsAsArrayWithDefaults("-f",
|
||||||
|
FSConfigConverterTestCommons.FS_ALLOC_FILE,
|
||||||
|
"-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE);
|
||||||
|
|
||||||
|
doThrow(new VerificationException("test", new Exception("test")))
|
||||||
|
.when(mockConverter)
|
||||||
|
.convert(any(FSConfigToCSConfigConverterParams.class));
|
||||||
|
|
||||||
|
argumentHandler.parseAndConvert(args);
|
||||||
|
|
||||||
|
verify(mockOptions).handleVerificationFailure(any(Exception.class),
|
||||||
|
any(String.class));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testFairSchedulerXmlIsNotDefinedIfItsDefinedInYarnSiteXml()
|
public void testFairSchedulerXmlIsNotDefinedIfItsDefinedInYarnSiteXml()
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
@ -534,7 +557,8 @@ public class TestFSConfigToCSConfigArgumentHandler {
|
||||||
|
|
||||||
String[] args = new String[] {
|
String[] args = new String[] {
|
||||||
"-y", FSConfigConverterTestCommons.YARN_SITE_XML,
|
"-y", FSConfigConverterTestCommons.YARN_SITE_XML,
|
||||||
"-o", FSConfigConverterTestCommons.OUTPUT_DIR};
|
"-o", FSConfigConverterTestCommons.OUTPUT_DIR,
|
||||||
|
"-e"};
|
||||||
|
|
||||||
int retVal = argumentHandler.parseAndConvert(args);
|
int retVal = argumentHandler.parseAndConvert(args);
|
||||||
assertEquals("Return value", -1, retVal);
|
assertEquals("Return value", -1, retVal);
|
||||||
|
@ -564,6 +588,13 @@ public class TestFSConfigToCSConfigArgumentHandler {
|
||||||
YarnConfiguration.CS_CONFIGURATION_FILE);
|
YarnConfiguration.CS_CONFIGURATION_FILE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMappingRulesJsonExistsInOutputFolder()
|
||||||
|
throws Exception {
|
||||||
|
testFileExistsInOutputFolder(
|
||||||
|
"mapping-rules.json");
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testPlacementRulesConversionEnabled() throws Exception {
|
public void testPlacementRulesConversionEnabled() throws Exception {
|
||||||
testPlacementRuleConversion(true);
|
testPlacementRuleConversion(true);
|
||||||
|
|
|
@ -36,7 +36,6 @@ import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.verifyZeroInteractions;
|
import static org.mockito.Mockito.verifyZeroInteractions;
|
||||||
|
|
||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.ByteArrayOutputStream;
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
@ -463,6 +462,7 @@ public class TestFSConfigToCSConfigConverter {
|
||||||
FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
|
FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
|
||||||
.withClusterResource(CLUSTER_RESOURCE_STRING)
|
.withClusterResource(CLUSTER_RESOURCE_STRING)
|
||||||
.withConvertPlacementRules(true)
|
.withConvertPlacementRules(true)
|
||||||
|
.withPlacementRulesToFile(true)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
converter.convert(params);
|
converter.convert(params);
|
||||||
|
@ -612,17 +612,17 @@ public class TestFSConfigToCSConfigConverter {
|
||||||
config.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE,
|
config.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE,
|
||||||
true);
|
true);
|
||||||
|
|
||||||
ByteArrayOutputStream jsonOutStream = new ByteArrayOutputStream();
|
|
||||||
converter.setConvertPlacementRules(true);
|
converter.setConvertPlacementRules(true);
|
||||||
converter.setMappingRulesOutputStream(jsonOutStream);
|
|
||||||
converter.setConsoleMode(true);
|
converter.setConsoleMode(true);
|
||||||
converter.convert(config);
|
converter.convert(config);
|
||||||
|
String json = converter.getCapacitySchedulerConfig()
|
||||||
|
.get(CapacitySchedulerConfiguration.MAPPING_RULE_JSON);
|
||||||
|
|
||||||
MappingRulesDescription description =
|
MappingRulesDescription description =
|
||||||
new ObjectMapper()
|
new ObjectMapper()
|
||||||
.reader()
|
.reader()
|
||||||
.forType(MappingRulesDescription.class)
|
.forType(MappingRulesDescription.class)
|
||||||
.readValue(jsonOutStream.toByteArray());
|
.readValue(json);
|
||||||
|
|
||||||
if (hasPlacementRules) {
|
if (hasPlacementRules) {
|
||||||
// fs.xml defines 5 rules
|
// fs.xml defines 5 rules
|
||||||
|
|
|
@ -21,17 +21,17 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.conve
|
||||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.OUTPUT_DIR;
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.OUTPUT_DIR;
|
||||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML;
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML;
|
||||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles;
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles;
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.contrib.java.lang.system.ExpectedSystemExit;
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -40,19 +40,20 @@ import org.junit.contrib.java.lang.system.ExpectedSystemExit;
|
||||||
*/
|
*/
|
||||||
public class TestFSConfigToCSConfigConverterMain {
|
public class TestFSConfigToCSConfigConverterMain {
|
||||||
private FSConfigConverterTestCommons converterTestCommons;
|
private FSConfigConverterTestCommons converterTestCommons;
|
||||||
|
private ExitFunc exitFunc;
|
||||||
@Rule
|
|
||||||
public final ExpectedSystemExit exit = ExpectedSystemExit.none();
|
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
|
exitFunc = new ExitFunc();
|
||||||
converterTestCommons = new FSConfigConverterTestCommons();
|
converterTestCommons = new FSConfigConverterTestCommons();
|
||||||
converterTestCommons.setUp();
|
converterTestCommons.setUp();
|
||||||
|
FSConfigToCSConfigConverterMain.setExit(exitFunc);
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
public void tearDown() throws Exception {
|
public void tearDown() throws Exception {
|
||||||
QueueMetrics.clearQueueMetrics();
|
QueueMetrics.clearQueueMetrics();
|
||||||
|
FSConfigToCSConfigConverterMain.setExit(System::exit);
|
||||||
converterTestCommons.tearDown();
|
converterTestCommons.tearDown();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -68,7 +69,6 @@ public class TestFSConfigToCSConfigConverterMain {
|
||||||
public void testConvertFSConfigurationDefaults()
|
public void testConvertFSConfigurationDefaults()
|
||||||
throws Exception {
|
throws Exception {
|
||||||
setupFSConfigConversionFiles();
|
setupFSConfigConversionFiles();
|
||||||
exit.expectSystemExitWithStatus(0);
|
|
||||||
|
|
||||||
FSConfigToCSConfigConverterMain.main(new String[] {
|
FSConfigToCSConfigConverterMain.main(new String[] {
|
||||||
"-o", OUTPUT_DIR,
|
"-o", OUTPUT_DIR,
|
||||||
|
@ -83,17 +83,18 @@ public class TestFSConfigToCSConfigConverterMain {
|
||||||
|
|
||||||
assertTrue("capacity-scheduler.xml was not generated", csConfigExists);
|
assertTrue("capacity-scheduler.xml was not generated", csConfigExists);
|
||||||
assertTrue("yarn-site.xml was not generated", yarnSiteConfigExists);
|
assertTrue("yarn-site.xml was not generated", yarnSiteConfigExists);
|
||||||
|
assertEquals("Exit code", 0, exitFunc.exitCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testConvertFSConfigurationWithConsoleParam()
|
public void testConvertFSConfigurationWithConsoleParam()
|
||||||
throws Exception {
|
throws Exception {
|
||||||
setupFSConfigConversionFiles();
|
setupFSConfigConversionFiles();
|
||||||
exit.expectSystemExitWithStatus(0);
|
|
||||||
|
|
||||||
FSConfigToCSConfigConverterMain.main(new String[] {
|
FSConfigToCSConfigConverterMain.main(new String[] {
|
||||||
"-p",
|
"-p",
|
||||||
"-m",
|
"-m",
|
||||||
|
"-e",
|
||||||
"-y", YARN_SITE_XML,
|
"-y", YARN_SITE_XML,
|
||||||
"-f", FS_ALLOC_FILE,
|
"-f", FS_ALLOC_FILE,
|
||||||
"-r", CONVERSION_RULES_FILE});
|
"-r", CONVERSION_RULES_FILE});
|
||||||
|
@ -105,35 +106,42 @@ public class TestFSConfigToCSConfigConverterMain {
|
||||||
stdout.contains("======= capacity-scheduler.xml ======="));
|
stdout.contains("======= capacity-scheduler.xml ======="));
|
||||||
assertTrue("Stdout doesn't contain mapping-rules.json",
|
assertTrue("Stdout doesn't contain mapping-rules.json",
|
||||||
stdout.contains("======= mapping-rules.json ======="));
|
stdout.contains("======= mapping-rules.json ======="));
|
||||||
|
assertEquals("Exit code", 0, exitFunc.exitCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testShortHelpSwitch() {
|
public void testShortHelpSwitch() {
|
||||||
exit.expectSystemExitWithStatus(0);
|
|
||||||
|
|
||||||
FSConfigToCSConfigConverterMain.main(new String[] {"-h"});
|
FSConfigToCSConfigConverterMain.main(new String[] {"-h"});
|
||||||
|
|
||||||
verifyHelpText();
|
verifyHelpText();
|
||||||
|
assertEquals("Exit code", 0, exitFunc.exitCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testLongHelpSwitch() {
|
public void testLongHelpSwitch() {
|
||||||
exit.expectSystemExitWithStatus(0);
|
|
||||||
|
|
||||||
FSConfigToCSConfigConverterMain.main(new String[] {"--help"});
|
FSConfigToCSConfigConverterMain.main(new String[] {"--help"});
|
||||||
|
|
||||||
verifyHelpText();
|
verifyHelpText();
|
||||||
|
assertEquals("Exit code", 0, exitFunc.exitCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testHelpDisplayedWithoutArgs() {
|
||||||
|
FSConfigToCSConfigConverterMain.main(new String[] {});
|
||||||
|
|
||||||
|
verifyHelpText();
|
||||||
|
assertEquals("Exit code", 0, exitFunc.exitCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testConvertFSConfigurationWithLongSwitches()
|
public void testConvertFSConfigurationWithLongSwitches()
|
||||||
throws IOException {
|
throws IOException {
|
||||||
exit.expectSystemExitWithStatus(0);
|
|
||||||
setupFSConfigConversionFiles();
|
setupFSConfigConversionFiles();
|
||||||
|
|
||||||
FSConfigToCSConfigConverterMain.main(new String[] {
|
FSConfigToCSConfigConverterMain.main(new String[] {
|
||||||
"--print",
|
"--print",
|
||||||
"--convert-placement-rules",
|
"--convert-placement-rules",
|
||||||
|
"--rules-to-file",
|
||||||
"--yarnsiteconfig", YARN_SITE_XML,
|
"--yarnsiteconfig", YARN_SITE_XML,
|
||||||
"--fsconfig", FS_ALLOC_FILE,
|
"--fsconfig", FS_ALLOC_FILE,
|
||||||
"--rulesconfig", CONVERSION_RULES_FILE});
|
"--rulesconfig", CONVERSION_RULES_FILE});
|
||||||
|
@ -145,6 +153,16 @@ public class TestFSConfigToCSConfigConverterMain {
|
||||||
stdout.contains("======= capacity-scheduler.xml ======="));
|
stdout.contains("======= capacity-scheduler.xml ======="));
|
||||||
assertTrue("Stdout doesn't contain mapping-rules.json",
|
assertTrue("Stdout doesn't contain mapping-rules.json",
|
||||||
stdout.contains("======= mapping-rules.json ======="));
|
stdout.contains("======= mapping-rules.json ======="));
|
||||||
|
assertEquals("Exit code", 0, exitFunc.exitCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNegativeReturnValueOnError() {
|
||||||
|
FSConfigToCSConfigConverterMain.main(new String[] {
|
||||||
|
"--print",
|
||||||
|
"--yarnsiteconfig"});
|
||||||
|
|
||||||
|
assertEquals("Exit code", -1, exitFunc.exitCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void verifyHelpText() {
|
private void verifyHelpText() {
|
||||||
|
@ -152,4 +170,14 @@ public class TestFSConfigToCSConfigConverterMain {
|
||||||
assertTrue("Help was not displayed",
|
assertTrue("Help was not displayed",
|
||||||
stdout.contains("General options are:"));
|
stdout.contains("General options are:"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("checkstyle:visibilitymodifier")
|
||||||
|
class ExitFunc implements Consumer<Integer> {
|
||||||
|
int exitCode;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void accept(Integer t) {
|
||||||
|
this.exitCode = t.intValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue