YARN-10099. FS-CS converter: handle allow-undeclared-pools and user-as-default-queue properly and fix misc issues. Contributed by Peter Bacsko
This commit is contained in:
parent
5977360878
commit
a7d72c523a
|
@ -96,7 +96,7 @@ public class AllocationFileLoaderService extends AbstractService {
|
||||||
private Thread reloadThread;
|
private Thread reloadThread;
|
||||||
private volatile boolean running = true;
|
private volatile boolean running = true;
|
||||||
|
|
||||||
AllocationFileLoaderService(FairScheduler scheduler) {
|
public AllocationFileLoaderService(FairScheduler scheduler) {
|
||||||
this(SystemClock.getInstance(), scheduler);
|
this(SystemClock.getInstance(), scheduler);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -186,7 +186,7 @@ public class AllocationFileLoaderService extends AbstractService {
|
||||||
* classpath, but loaded like a regular File.
|
* classpath, but loaded like a regular File.
|
||||||
*/
|
*/
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
Path getAllocationFile(Configuration conf)
|
public Path getAllocationFile(Configuration conf)
|
||||||
throws UnsupportedFileSystemException {
|
throws UnsupportedFileSystemException {
|
||||||
String allocFilePath = conf.get(FairSchedulerConfiguration.ALLOCATION_FILE,
|
String allocFilePath = conf.get(FairSchedulerConfiguration.ALLOCATION_FILE,
|
||||||
FairSchedulerConfiguration.DEFAULT_ALLOCATION_FILE);
|
FairSchedulerConfiguration.DEFAULT_ALLOCATION_FILE);
|
||||||
|
|
|
@ -26,26 +26,37 @@ import java.io.OutputStream;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
import javax.xml.parsers.DocumentBuilder;
|
||||||
|
import javax.xml.parsers.DocumentBuilderFactory;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.security.authorize.AccessControlList;
|
import org.apache.hadoop.security.authorize.AccessControlList;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.security.AccessType;
|
import org.apache.hadoop.yarn.security.AccessType;
|
||||||
|
import org.apache.hadoop.yarn.security.ConfiguredYarnAuthorizer;
|
||||||
|
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
|
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.AllocationFileParser;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.w3c.dom.Document;
|
||||||
|
import org.w3c.dom.Element;
|
||||||
|
import org.w3c.dom.NodeList;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
|
||||||
|
@ -77,7 +88,7 @@ public class FSConfigToCSConfigConverter {
|
||||||
private ConversionOptions conversionOptions;
|
private ConversionOptions conversionOptions;
|
||||||
private boolean drfUsed = false;
|
private boolean drfUsed = false;
|
||||||
|
|
||||||
private Configuration yarnSiteConfig;
|
private Configuration convertedYarnSiteConfig;
|
||||||
private Configuration capacitySchedulerConfig;
|
private Configuration capacitySchedulerConfig;
|
||||||
private FSConfigToCSConfigRuleHandler ruleHandler;
|
private FSConfigToCSConfigRuleHandler ruleHandler;
|
||||||
|
|
||||||
|
@ -98,11 +109,11 @@ public class FSConfigToCSConfigConverter {
|
||||||
validateParams(params);
|
validateParams(params);
|
||||||
prepareOutputFiles(params.getOutputDirectory(), params.isConsole());
|
prepareOutputFiles(params.getOutputDirectory(), params.isConsole());
|
||||||
loadConversionRules(params.getConversionRulesConfig());
|
loadConversionRules(params.getConversionRulesConfig());
|
||||||
Configuration conf = createConfiguration(params);
|
Configuration inputYarnSiteConfig = getInputYarnSiteConfig(params);
|
||||||
handleFairSchedulerConfig(params, conf);
|
handleFairSchedulerConfig(params, inputYarnSiteConfig);
|
||||||
|
|
||||||
this.clusterResource = getClusterResource(params);
|
this.clusterResource = getClusterResource(params);
|
||||||
convert(conf);
|
convert(inputYarnSiteConfig);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void prepareOutputFiles(String outputDirectory, boolean console)
|
private void prepareOutputFiles(String outputDirectory, boolean console)
|
||||||
|
@ -162,13 +173,10 @@ public class FSConfigToCSConfigConverter {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Configuration createConfiguration(
|
private Configuration getInputYarnSiteConfig(
|
||||||
FSConfigToCSConfigConverterParams params) {
|
FSConfigToCSConfigConverterParams params) {
|
||||||
Configuration conf = new YarnConfiguration();
|
Configuration conf = new YarnConfiguration();
|
||||||
conf.addResource(new Path(params.getYarnSiteXmlConfig()));
|
conf.addResource(new Path(params.getYarnSiteXmlConfig()));
|
||||||
conf.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
|
|
||||||
conf.setBoolean(FairSchedulerConfiguration.NO_TERMINAL_RULE_CHECK,
|
|
||||||
conversionOptions.isNoRuleTerminalCheck());
|
|
||||||
return conf;
|
return conf;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -199,7 +207,7 @@ public class FSConfigToCSConfigConverter {
|
||||||
}
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
void convert(Configuration conf) throws Exception {
|
void convert(Configuration inputYarnSiteConfig) throws Exception {
|
||||||
System.out.println(WARNING_TEXT);
|
System.out.println(WARNING_TEXT);
|
||||||
|
|
||||||
// initialize Fair Scheduler
|
// initialize Fair Scheduler
|
||||||
|
@ -207,9 +215,20 @@ public class FSConfigToCSConfigConverter {
|
||||||
PlacementManager placementManager = new PlacementManager();
|
PlacementManager placementManager = new PlacementManager();
|
||||||
ctx.setQueuePlacementManager(placementManager);
|
ctx.setQueuePlacementManager(placementManager);
|
||||||
|
|
||||||
|
// Prepare a separate config for the FS instance
|
||||||
|
// to force the use of ConfiguredYarnAuthorizer, otherwise
|
||||||
|
// it might use that of Ranger
|
||||||
|
Configuration fsConfig = new Configuration(inputYarnSiteConfig);
|
||||||
|
fsConfig.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
|
||||||
|
fsConfig.setBoolean(FairSchedulerConfiguration.NO_TERMINAL_RULE_CHECK,
|
||||||
|
conversionOptions.isNoRuleTerminalCheck());
|
||||||
|
fsConfig.setClass(YarnConfiguration.YARN_AUTHORIZATION_PROVIDER,
|
||||||
|
ConfiguredYarnAuthorizer.class, YarnAuthorizationProvider.class);
|
||||||
FairScheduler fs = new FairScheduler();
|
FairScheduler fs = new FairScheduler();
|
||||||
fs.setRMContext(ctx);
|
fs.setRMContext(ctx);
|
||||||
fs.init(conf);
|
fs.init(fsConfig);
|
||||||
|
boolean havePlacementPolicies =
|
||||||
|
checkPlacementPoliciesPresent(fs, inputYarnSiteConfig);
|
||||||
|
|
||||||
drfUsed = isDrfUsed(fs);
|
drfUsed = isDrfUsed(fs);
|
||||||
|
|
||||||
|
@ -217,13 +236,13 @@ public class FSConfigToCSConfigConverter {
|
||||||
queueMaxAppsDefault = allocConf.getQueueMaxAppsDefault();
|
queueMaxAppsDefault = allocConf.getQueueMaxAppsDefault();
|
||||||
queueMaxAMShareDefault = allocConf.getQueueMaxAMShareDefault();
|
queueMaxAMShareDefault = allocConf.getQueueMaxAMShareDefault();
|
||||||
|
|
||||||
yarnSiteConfig = new Configuration(false);
|
convertedYarnSiteConfig = new Configuration(false);
|
||||||
capacitySchedulerConfig = new Configuration(false);
|
capacitySchedulerConfig = new Configuration(false);
|
||||||
|
|
||||||
checkUserMaxApps(allocConf);
|
checkUserMaxApps(allocConf);
|
||||||
checkUserMaxAppsDefault(allocConf);
|
checkUserMaxAppsDefault(allocConf);
|
||||||
|
|
||||||
convertYarnSiteXml(conf);
|
convertYarnSiteXml(inputYarnSiteConfig, havePlacementPolicies);
|
||||||
convertCapacitySchedulerXml(fs);
|
convertCapacitySchedulerXml(fs);
|
||||||
|
|
||||||
if (consoleMode) {
|
if (consoleMode) {
|
||||||
|
@ -235,7 +254,7 @@ public class FSConfigToCSConfigConverter {
|
||||||
System.out.println();
|
System.out.println();
|
||||||
System.out.println("======= " + YARN_SITE_XML + " =======");
|
System.out.println("======= " + YARN_SITE_XML + " =======");
|
||||||
}
|
}
|
||||||
yarnSiteConfig.writeXml(yarnSiteOutputStream);
|
convertedYarnSiteConfig.writeXml(yarnSiteOutputStream);
|
||||||
}
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
|
@ -248,17 +267,23 @@ public class FSConfigToCSConfigConverter {
|
||||||
this.capacitySchedulerOutputStream = out;
|
this.capacitySchedulerOutputStream = out;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void convertYarnSiteXml(Configuration conf) {
|
private void convertYarnSiteXml(Configuration inputYarnSiteConfig,
|
||||||
|
boolean havePlacementPolicies) {
|
||||||
FSYarnSiteConverter siteConverter =
|
FSYarnSiteConverter siteConverter =
|
||||||
new FSYarnSiteConverter();
|
new FSYarnSiteConverter();
|
||||||
siteConverter.convertSiteProperties(conf, yarnSiteConfig, drfUsed);
|
siteConverter.convertSiteProperties(inputYarnSiteConfig,
|
||||||
|
convertedYarnSiteConfig, drfUsed);
|
||||||
|
|
||||||
autoCreateChildQueues = siteConverter.isAutoCreateChildQueues();
|
// See docs: "allow-undeclared-pools" and "user-as-default-queue" are
|
||||||
|
// ignored if we have placement rules
|
||||||
|
autoCreateChildQueues =
|
||||||
|
!havePlacementPolicies && siteConverter.isAutoCreateChildQueues();
|
||||||
|
userAsDefaultQueue =
|
||||||
|
!havePlacementPolicies && siteConverter.isUserAsDefaultQueue();
|
||||||
preemptionEnabled = siteConverter.isPreemptionEnabled();
|
preemptionEnabled = siteConverter.isPreemptionEnabled();
|
||||||
sizeBasedWeight = siteConverter.isSizeBasedWeight();
|
sizeBasedWeight = siteConverter.isSizeBasedWeight();
|
||||||
userAsDefaultQueue = siteConverter.isUserAsDefaultQueue();
|
|
||||||
|
|
||||||
checkReservationSystem(conf);
|
checkReservationSystem(inputYarnSiteConfig);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void convertCapacitySchedulerXml(FairScheduler fs) {
|
private void convertCapacitySchedulerXml(FairScheduler fs) {
|
||||||
|
@ -402,6 +427,46 @@ public class FSConfigToCSConfigConverter {
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
Configuration getYarnSiteConfig() {
|
Configuration getYarnSiteConfig() {
|
||||||
return yarnSiteConfig;
|
return convertedYarnSiteConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Determines whether <queuePlacementPolicy> is present
|
||||||
|
* in the allocation file or not.
|
||||||
|
*
|
||||||
|
* Note that placementManager.getPlacementRules.size()
|
||||||
|
* doesn't work - by default, "allow-undeclared-pools" and
|
||||||
|
* "user-as-default-queue" are translated to policies internally
|
||||||
|
* inside QueuePlacementPolicy.fromConfiguration().
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private boolean checkPlacementPoliciesPresent(FairScheduler scheduler,
|
||||||
|
Configuration inputYarnSiteConfig)
|
||||||
|
throws RuntimeException {
|
||||||
|
|
||||||
|
try (AllocationFileLoaderService loader =
|
||||||
|
new AllocationFileLoaderService(scheduler)){
|
||||||
|
|
||||||
|
Path allocFilePath = loader.getAllocationFile(inputYarnSiteConfig);
|
||||||
|
FileSystem fs = allocFilePath.getFileSystem(inputYarnSiteConfig);
|
||||||
|
|
||||||
|
DocumentBuilderFactory docBuilderFactory =
|
||||||
|
DocumentBuilderFactory.newInstance();
|
||||||
|
DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
|
||||||
|
Document doc = builder.parse(fs.open(allocFilePath));
|
||||||
|
Element root = doc.getDocumentElement();
|
||||||
|
|
||||||
|
NodeList elements = root.getChildNodes();
|
||||||
|
|
||||||
|
AllocationFileParser allocationFileParser =
|
||||||
|
new AllocationFileParser(elements);
|
||||||
|
allocationFileParser.parse();
|
||||||
|
docBuilderFactory.setIgnoringComments(true);
|
||||||
|
|
||||||
|
return
|
||||||
|
allocationFileParser.getQueuePlacementPolicy().isPresent();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new PreconditionException("Unable to parse allocation file", e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,6 +30,10 @@ public class PreconditionException extends RuntimeException {
|
||||||
super(message);
|
super(message);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public PreconditionException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
|
||||||
public PreconditionException(String message, MissingArgumentException ex) {
|
public PreconditionException(String message, MissingArgumentException ex) {
|
||||||
super(message, ex);
|
super(message, ex);
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,6 +34,8 @@ class QueuePlacementConverter {
|
||||||
private static final String PRIMARY_GROUP = "%primary_group";
|
private static final String PRIMARY_GROUP = "%primary_group";
|
||||||
private static final String SECONDARY_GROUP = "%secondary_group";
|
private static final String SECONDARY_GROUP = "%secondary_group";
|
||||||
|
|
||||||
|
private static final String RULE_SEPARATOR = ",";
|
||||||
|
|
||||||
Map<String, String> convertPlacementPolicy(PlacementManager placementManager,
|
Map<String, String> convertPlacementPolicy(PlacementManager placementManager,
|
||||||
FSConfigToCSConfigRuleHandler ruleHandler, boolean userAsDefaultQueue) {
|
FSConfigToCSConfigRuleHandler ruleHandler, boolean userAsDefaultQueue) {
|
||||||
StringBuilder mapping = new StringBuilder();
|
StringBuilder mapping = new StringBuilder();
|
||||||
|
@ -59,7 +61,7 @@ class QueuePlacementConverter {
|
||||||
} else {
|
} else {
|
||||||
if (!userAsDefaultQueue) {
|
if (!userAsDefaultQueue) {
|
||||||
if (mapping.length() > 0) {
|
if (mapping.length() > 0) {
|
||||||
mapping.append(";");
|
mapping.append(RULE_SEPARATOR);
|
||||||
}
|
}
|
||||||
mapping.append("u:" + USER + ":" + USER);
|
mapping.append("u:" + USER + ":" + USER);
|
||||||
}
|
}
|
||||||
|
@ -72,18 +74,18 @@ class QueuePlacementConverter {
|
||||||
"yarn.scheduler.capacity.queue-mappings-override.enable", "false");
|
"yarn.scheduler.capacity.queue-mappings-override.enable", "false");
|
||||||
} else if (rule instanceof PrimaryGroupPlacementRule) {
|
} else if (rule instanceof PrimaryGroupPlacementRule) {
|
||||||
if (mapping.length() > 0) {
|
if (mapping.length() > 0) {
|
||||||
mapping.append(";");
|
mapping.append(RULE_SEPARATOR);
|
||||||
}
|
}
|
||||||
mapping.append("u:" + USER + ":" + PRIMARY_GROUP);
|
mapping.append("u:" + USER + ":" + PRIMARY_GROUP);
|
||||||
} else if (rule instanceof DefaultPlacementRule) {
|
} else if (rule instanceof DefaultPlacementRule) {
|
||||||
DefaultPlacementRule defaultRule = (DefaultPlacementRule) rule;
|
DefaultPlacementRule defaultRule = (DefaultPlacementRule) rule;
|
||||||
if (mapping.length() > 0) {
|
if (mapping.length() > 0) {
|
||||||
mapping.append(";");
|
mapping.append(RULE_SEPARATOR);
|
||||||
}
|
}
|
||||||
mapping.append("u:" + USER + ":").append(defaultRule.defaultQueueName);
|
mapping.append("u:" + USER + ":").append(defaultRule.defaultQueueName);
|
||||||
} else if (rule instanceof SecondaryGroupExistingPlacementRule) {
|
} else if (rule instanceof SecondaryGroupExistingPlacementRule) {
|
||||||
if (mapping.length() > 0) {
|
if (mapping.length() > 0) {
|
||||||
mapping.append(";");
|
mapping.append(RULE_SEPARATOR);
|
||||||
}
|
}
|
||||||
mapping.append("u:" + USER + ":" + SECONDARY_GROUP);
|
mapping.append("u:" + USER + ":" + SECONDARY_GROUP);
|
||||||
} else if (!(rule instanceof RejectPlacementRule)) {
|
} else if (!(rule instanceof RejectPlacementRule)) {
|
||||||
|
@ -103,7 +105,7 @@ class QueuePlacementConverter {
|
||||||
UserPlacementRule userRule) {
|
UserPlacementRule userRule) {
|
||||||
PlacementRule pr = userRule.getParentRule();
|
PlacementRule pr = userRule.getParentRule();
|
||||||
if (mapping.length() > 0) {
|
if (mapping.length() > 0) {
|
||||||
mapping.append(";");
|
mapping.append(RULE_SEPARATOR);
|
||||||
}
|
}
|
||||||
if (pr instanceof PrimaryGroupPlacementRule) {
|
if (pr instanceof PrimaryGroupPlacementRule) {
|
||||||
mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER);
|
mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER);
|
||||||
|
|
|
@ -27,6 +27,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.conve
|
||||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS;
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS;
|
||||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.ABORT;
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.ABORT;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertNull;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
|
|
||||||
|
@ -74,6 +75,9 @@ public class TestFSConfigToCSConfigConverter {
|
||||||
prepareFileName("fair-scheduler-onlyfairpolicy.xml");
|
prepareFileName("fair-scheduler-onlyfairpolicy.xml");
|
||||||
private static final String FS_MIXED_POLICY_XML =
|
private static final String FS_MIXED_POLICY_XML =
|
||||||
prepareFileName("fair-scheduler-orderingpolicy-mixed.xml");
|
prepareFileName("fair-scheduler-orderingpolicy-mixed.xml");
|
||||||
|
private static final String FS_NO_PLACEMENT_RULES_XML =
|
||||||
|
prepareFileName("fair-scheduler-noplacementrules.xml");
|
||||||
|
|
||||||
|
|
||||||
@Mock
|
@Mock
|
||||||
private FSConfigToCSConfigRuleHandler ruleHandler;
|
private FSConfigToCSConfigRuleHandler ruleHandler;
|
||||||
|
@ -504,6 +508,112 @@ public class TestFSConfigToCSConfigConverter {
|
||||||
CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null));
|
CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("checkstyle:linelength")
|
||||||
|
public void testUserAsDefaultQueueWithPlacementRules() throws Exception {
|
||||||
|
config = new Configuration(false);
|
||||||
|
config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
|
||||||
|
config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
|
||||||
|
FAIR_SCHEDULER_XML);
|
||||||
|
|
||||||
|
converter.convert(config);
|
||||||
|
|
||||||
|
Configuration convertedConf = getConvertedCSConfig();
|
||||||
|
|
||||||
|
String expectedMappingRules =
|
||||||
|
"u:%user:root.admins.devs.%user,u:%user:root.users.%user,u:%user:root.default";
|
||||||
|
String mappingRules =
|
||||||
|
convertedConf.get(CapacitySchedulerConfiguration.QUEUE_MAPPING);
|
||||||
|
assertEquals("Mapping rules", expectedMappingRules, mappingRules);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUserAsDefaultQueueTrueWithoutPlacementRules()
|
||||||
|
throws Exception {
|
||||||
|
testUserAsDefaultQueueWithoutPlacementRules(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUserAsDefaultQueueFalseWithoutPlacementRules()
|
||||||
|
throws Exception {
|
||||||
|
testUserAsDefaultQueueWithoutPlacementRules(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void testUserAsDefaultQueueWithoutPlacementRules(boolean
|
||||||
|
userAsDefaultQueue) throws Exception {
|
||||||
|
config = new Configuration(false);
|
||||||
|
config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
|
||||||
|
config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
|
||||||
|
FS_NO_PLACEMENT_RULES_XML);
|
||||||
|
config.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE,
|
||||||
|
userAsDefaultQueue);
|
||||||
|
|
||||||
|
converter.convert(config);
|
||||||
|
|
||||||
|
Configuration convertedConf = getConvertedCSConfig();
|
||||||
|
String mappingRules =
|
||||||
|
convertedConf.get(CapacitySchedulerConfiguration.QUEUE_MAPPING);
|
||||||
|
|
||||||
|
if (userAsDefaultQueue) {
|
||||||
|
assertEquals("Mapping rules", "u:%user:%user", mappingRules);
|
||||||
|
} else {
|
||||||
|
assertEquals("Mapping rules", "u:%user:root.default", mappingRules);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testAutoCreateChildQueuesWithPlacementRules() throws Exception {
|
||||||
|
config = new Configuration(false);
|
||||||
|
config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
|
||||||
|
config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
|
||||||
|
FAIR_SCHEDULER_XML);
|
||||||
|
config.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS,
|
||||||
|
true);
|
||||||
|
|
||||||
|
converter.convert(config);
|
||||||
|
|
||||||
|
Configuration convertedConf = getConvertedCSConfig();
|
||||||
|
String property =
|
||||||
|
"yarn.scheduler.capacity.root.auto-create-child-queue.enabled";
|
||||||
|
assertNull("Auto-create queue shouldn't be set",
|
||||||
|
convertedConf.get(property));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testAutoCreateChildQueuesTrueWithoutPlacementRules()
|
||||||
|
throws Exception {
|
||||||
|
testAutoCreateChildQueuesWithoutPlacementRules(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testAutoCreateChildQueuesFalseWithoutPlacementRules()
|
||||||
|
throws Exception {
|
||||||
|
testAutoCreateChildQueuesWithoutPlacementRules(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void testAutoCreateChildQueuesWithoutPlacementRules(
|
||||||
|
boolean allowUndeclaredPools) throws Exception {
|
||||||
|
config = new Configuration(false);
|
||||||
|
config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
|
||||||
|
config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
|
||||||
|
FS_NO_PLACEMENT_RULES_XML);
|
||||||
|
config.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS,
|
||||||
|
allowUndeclaredPools);
|
||||||
|
|
||||||
|
converter.convert(config);
|
||||||
|
|
||||||
|
Configuration convertedConf = getConvertedCSConfig();
|
||||||
|
String property =
|
||||||
|
"yarn.scheduler.capacity.root.auto-create-child-queue.enabled";
|
||||||
|
|
||||||
|
if (allowUndeclaredPools) {
|
||||||
|
assertEquals("Auto-create queue wasn't enabled", true,
|
||||||
|
convertedConf.getBoolean(property, false));
|
||||||
|
} else {
|
||||||
|
assertNull("Auto-create queue shouldn't be set",
|
||||||
|
convertedConf.get(property));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private Configuration getConvertedCSConfig() {
|
private Configuration getConvertedCSConfig() {
|
||||||
ByteArrayInputStream input =
|
ByteArrayInputStream input =
|
||||||
new ByteArrayInputStream(csConfigOut.toByteArray());
|
new ByteArrayInputStream(csConfigOut.toByteArray());
|
||||||
|
|
|
@ -334,7 +334,6 @@ public class TestFSQueueConverter {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testQueueAutoCreateChildQueue() {
|
public void testQueueAutoCreateChildQueue() {
|
||||||
config.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, true);
|
|
||||||
converter = builder
|
converter = builder
|
||||||
.withCapacitySchedulerConfig(csConfig)
|
.withCapacitySchedulerConfig(csConfig)
|
||||||
.withAutoCreateChildQueues(true)
|
.withAutoCreateChildQueues(true)
|
||||||
|
@ -346,6 +345,19 @@ public class TestFSQueueConverter {
|
||||||
csConfig);
|
csConfig);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testQueueWithNoAutoCreateChildQueue() {
|
||||||
|
converter = builder
|
||||||
|
.withCapacitySchedulerConfig(csConfig)
|
||||||
|
.withAutoCreateChildQueues(false)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
converter.convertQueueHierarchy(rootQueue);
|
||||||
|
|
||||||
|
assertNoValueForQueues(ALL_QUEUES, ".auto-create-child-queue.enabled",
|
||||||
|
csConfig);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testQueueSizeBasedWeightEnabled() {
|
public void testQueueSizeBasedWeightEnabled() {
|
||||||
converter = builder.withSizeBasedWeight(true).build();
|
converter = builder.withSizeBasedWeight(true).build();
|
||||||
|
|
|
@ -224,7 +224,7 @@ public class TestQueuePlacementConverter {
|
||||||
Map<String, String> properties = convert(false);
|
Map<String, String> properties = convert(false);
|
||||||
|
|
||||||
verifyMapping(properties,
|
verifyMapping(properties,
|
||||||
"u:%user:%user;u:%user:%primary_group;u:%user:%secondary_group");
|
"u:%user:%user,u:%user:%primary_group,u:%user:%secondary_group");
|
||||||
verifyZeroInteractions(ruleHandler);
|
verifyZeroInteractions(ruleHandler);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,84 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
or more contributor license agreements. See the NOTICE file
|
||||||
|
distributed with this work for additional information
|
||||||
|
regarding copyright ownership. The ASF licenses this file
|
||||||
|
to you under the Apache License, Version 2.0 (the
|
||||||
|
"License"); you may not use this file except in compliance
|
||||||
|
with the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<allocations>
|
||||||
|
<queue name="root">
|
||||||
|
<weight>1.0</weight>
|
||||||
|
<schedulingPolicy>drf</schedulingPolicy>
|
||||||
|
<aclSubmitApps>alice,bob,joe,john hadoop_users</aclSubmitApps>
|
||||||
|
<aclAdministerApps>alice,bob,joe,john hadoop_users</aclAdministerApps>
|
||||||
|
<queue name="default">
|
||||||
|
<weight>1.0</weight>
|
||||||
|
<schedulingPolicy>drf</schedulingPolicy>
|
||||||
|
</queue>
|
||||||
|
<queue name="users" type="parent">
|
||||||
|
<weight>1.0</weight>
|
||||||
|
<schedulingPolicy>drf</schedulingPolicy>
|
||||||
|
<queue name="john">
|
||||||
|
<weight>1.0</weight>
|
||||||
|
<schedulingPolicy>drf</schedulingPolicy>
|
||||||
|
<aclSubmitApps>john </aclSubmitApps>
|
||||||
|
<aclAdministerApps>john </aclAdministerApps>
|
||||||
|
<maxContainerAllocation>vcores=2,memory-mb=8192</maxContainerAllocation>
|
||||||
|
</queue>
|
||||||
|
<queue name="joe">
|
||||||
|
<maxResources>memory-mb=50.0%, vcores=50.0%</maxResources>
|
||||||
|
<weight>3.0</weight>
|
||||||
|
<allowPreemptionFrom>false</allowPreemptionFrom>
|
||||||
|
<schedulingPolicy>drf</schedulingPolicy>
|
||||||
|
<aclSubmitApps>joe </aclSubmitApps>
|
||||||
|
<aclAdministerApps>joe </aclAdministerApps>
|
||||||
|
</queue>
|
||||||
|
</queue>
|
||||||
|
<queue name="admins" type="parent">
|
||||||
|
<maxChildResources>memory-mb=8192, vcores=1</maxChildResources>
|
||||||
|
<weight>1.0</weight>
|
||||||
|
<schedulingPolicy>drf</schedulingPolicy>
|
||||||
|
<maxContainerAllocation>vcores=3,memory-mb=4096</maxContainerAllocation>
|
||||||
|
<queue name="alice">
|
||||||
|
<maxResources>memory-mb=16384, vcores=4</maxResources>
|
||||||
|
<maxRunningApps>2</maxRunningApps>
|
||||||
|
<weight>3.0</weight>
|
||||||
|
<allowPreemptionFrom>false</allowPreemptionFrom>
|
||||||
|
<schedulingPolicy>drf</schedulingPolicy>
|
||||||
|
<aclSubmitApps>alice </aclSubmitApps>
|
||||||
|
<aclAdministerApps>alice </aclAdministerApps>
|
||||||
|
<maxAMShare>0.15</maxAMShare>
|
||||||
|
<reservation>memory-mb=16384, vcores=4</reservation>
|
||||||
|
</queue>
|
||||||
|
<queue name="bob">
|
||||||
|
<maxResources>memory-mb=8192, vcores=2</maxResources>
|
||||||
|
<weight>1.0</weight>
|
||||||
|
<schedulingPolicy>drf</schedulingPolicy>
|
||||||
|
<aclSubmitApps>bob </aclSubmitApps>
|
||||||
|
<aclAdministerApps>bob </aclAdministerApps>
|
||||||
|
<maxAMShare>-1.0</maxAMShare>
|
||||||
|
</queue>
|
||||||
|
</queue>
|
||||||
|
</queue>
|
||||||
|
<user name="alice">
|
||||||
|
<maxRunningApps>30</maxRunningApps>
|
||||||
|
</user>
|
||||||
|
<userMaxAppsDefault>10</userMaxAppsDefault>
|
||||||
|
<defaultFairSharePreemptionTimeout>23</defaultFairSharePreemptionTimeout>
|
||||||
|
<defaultMinSharePreemptionTimeout>24</defaultMinSharePreemptionTimeout>
|
||||||
|
<defaultFairSharePreemptionThreshold>0.12</defaultFairSharePreemptionThreshold>
|
||||||
|
<queueMaxAppsDefault>15</queueMaxAppsDefault>
|
||||||
|
<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>
|
||||||
|
<queueMaxAMShareDefault>0.16</queueMaxAMShareDefault>
|
||||||
|
</allocations>
|
Loading…
Reference in New Issue