YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok
This commit is contained in:
Szilard Nemeth
parent
d7c5400fbc
commit
f544ef4a51
|
|
@ -24,6 +24,7 @@
|
|||
import javax.xml.bind.annotation.XmlTransient;
|
||||
import javax.xml.bind.annotation.XmlType;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
|
|
@ -49,6 +50,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
|
|||
protected String queueName;
|
||||
private String queuePath;
|
||||
protected int maxParallelApps;
|
||||
private boolean isAbsoluteResource;
|
||||
protected CapacitySchedulerQueueInfoList queues;
|
||||
protected QueueCapacitiesInfo capacities;
|
||||
protected CapacitySchedulerHealthInfo health;
|
||||
|
|
@ -90,6 +92,9 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) {
|
|||
health = new CapacitySchedulerHealthInfo(cs);
|
||||
maximumAllocation = new ResourceInfo(parent.getMaximumAllocation());
|
||||
|
||||
isAbsoluteResource = parent.getCapacityConfigType() ==
|
||||
AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
|
||||
|
||||
CapacitySchedulerConfiguration conf = cs.getConfiguration();
|
||||
queueAcls = new QueueAclsInfo();
|
||||
queueAcls.addAll(getSortedQueueAclInfoList(queueName, conf));
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@
|
|||
import org.apache.hadoop.yarn.security.AccessType;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
|
||||
|
|
@ -43,11 +44,6 @@
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper;
|
||||
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
|
||||
CapacitySchedulerConfiguration.RESOURCE_PATTERN;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
|
||||
CapacitySchedulerConfiguration.CAPACITY;
|
||||
|
||||
@XmlRootElement
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
@XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class})
|
||||
|
|
@ -179,10 +175,8 @@ public class CapacitySchedulerQueueInfo {
|
|||
.getLeafOnlyProperties());
|
||||
}
|
||||
|
||||
String configuredCapacity = conf.get(
|
||||
CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY);
|
||||
isAbsoluteResource = (configuredCapacity != null)
|
||||
&& RESOURCE_PATTERN.matcher(configuredCapacity).find();
|
||||
isAbsoluteResource = q.getCapacityConfigType() ==
|
||||
AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
|
||||
|
||||
autoCreateChildQueueEnabled = conf.
|
||||
isAutoCreateChildQueueEnabled(queuePath);
|
||||
|
|
|
|||
|
|
@ -136,6 +136,22 @@ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
|
|||
"webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
|
||||
throws Exception {
|
||||
Configuration config = CSConfigGenerator
|
||||
.createAbsoluteConfigLegacyAutoCreation();
|
||||
config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
|
||||
YarnConfiguration.MEMORY_CONFIGURATION_STORE);
|
||||
|
||||
initResourceManager(config);
|
||||
initAutoQueueHandler(8192 * GB);
|
||||
createQueue("root.managed.queue1");
|
||||
|
||||
assertJsonResponse(sendRequest(),
|
||||
"webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSchedulerResponseAbsoluteMode()
|
||||
throws Exception {
|
||||
|
|
@ -189,7 +205,7 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues()
|
|||
"maximum-applications", 300);
|
||||
|
||||
initResourceManager(config);
|
||||
initAutoQueueHandler();
|
||||
initAutoQueueHandler(1200 * GB);
|
||||
|
||||
// same as webapp/scheduler-response-WeightMode.json, but with effective resources filled in
|
||||
assertJsonResponse(sendRequest(),
|
||||
|
|
@ -212,10 +228,10 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues()
|
|||
"webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json");
|
||||
}
|
||||
|
||||
private void initAutoQueueHandler() throws Exception {
|
||||
private void initAutoQueueHandler(int nodeMemory) throws Exception {
|
||||
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
|
||||
autoQueueHandler = cs.getCapacitySchedulerQueueManager();
|
||||
rm.registerNode("h1:1234", 1200 * GB); // label = x
|
||||
rm.registerNode("h1:1234", nodeMemory); // label = x
|
||||
}
|
||||
|
||||
private void createQueue(String queuePath) throws YarnException,
|
||||
|
|
@ -255,6 +271,19 @@ public static Configuration createPercentageConfigLegacyAutoCreation() {
|
|||
return createConfiguration(conf);
|
||||
}
|
||||
|
||||
public static Configuration createAbsoluteConfigLegacyAutoCreation() {
|
||||
Map<String, String> conf = new HashMap<>();
|
||||
conf.put("yarn.scheduler.capacity.root.queues", "default, managed");
|
||||
conf.put("yarn.scheduler.capacity.root.default.state", "STOPPED");
|
||||
conf.put("yarn.scheduler.capacity.root.managed.capacity", "[memory=4096,vcores=4]");
|
||||
conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.capacity",
|
||||
"[memory=2048,vcores=2]");
|
||||
conf.put("yarn.scheduler.capacity.root.managed.state", "RUNNING");
|
||||
conf.put("yarn.scheduler.capacity.root.managed." +
|
||||
"auto-create-child-queue.enabled", "true");
|
||||
return createConfiguration(conf);
|
||||
}
|
||||
|
||||
public static Configuration createAbsoluteConfig() {
|
||||
Map<String, String> conf = new HashMap<>();
|
||||
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
|
||||
|
|
|
|||
|
|
@ -574,7 +574,7 @@ private void verifySchedulerInfoJson(JSONObject json)
|
|||
JSONObject info = json.getJSONObject("scheduler");
|
||||
assertEquals("incorrect number of elements", 1, info.length());
|
||||
info = info.getJSONObject("schedulerInfo");
|
||||
assertEquals("incorrect number of elements", 23, info.length());
|
||||
assertEquals("incorrect number of elements", 24, info.length());
|
||||
JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
|
||||
JSONArray partitionsCapsArray =
|
||||
capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
"queueName": "root",
|
||||
"queuePath": "root",
|
||||
"maxParallelApps": 2147483647,
|
||||
"isAbsoluteResource": true,
|
||||
"queues": {"queue": [
|
||||
{
|
||||
"type": "capacitySchedulerLeafQueueInfo",
|
||||
|
|
@ -1735,4 +1736,4 @@
|
|||
"autoQueueTemplateProperties": {},
|
||||
"autoQueueParentTemplateProperties": {},
|
||||
"autoQueueLeafTemplateProperties": {}
|
||||
}}}
|
||||
}}}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -8,6 +8,7 @@
|
|||
<queueName>root</queueName>
|
||||
<queuePath>root</queuePath>
|
||||
<maxParallelApps>2147483647</maxParallelApps>
|
||||
<isAbsoluteResource>false</isAbsoluteResource>
|
||||
<queues>
|
||||
<queue>
|
||||
<queuePath>root.a</queuePath>
|
||||
|
|
@ -4546,4 +4547,4 @@
|
|||
<autoQueueParentTemplateProperties/>
|
||||
<autoQueueLeafTemplateProperties/>
|
||||
</schedulerInfo>
|
||||
</scheduler>
|
||||
</scheduler>
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
"queueName": "root",
|
||||
"queuePath": "root",
|
||||
"maxParallelApps": 2147483647,
|
||||
"isAbsoluteResource": false,
|
||||
"queues": {"queue": [
|
||||
{
|
||||
"queuePath": "root.a",
|
||||
|
|
@ -4953,4 +4954,4 @@
|
|||
"autoQueueTemplateProperties": {},
|
||||
"autoQueueParentTemplateProperties": {},
|
||||
"autoQueueLeafTemplateProperties": {}
|
||||
}}}
|
||||
}}}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
<queueName>root</queueName>
|
||||
<queuePath>root</queuePath>
|
||||
<maxParallelApps>2147483647</maxParallelApps>
|
||||
<isAbsoluteResource>false</isAbsoluteResource>
|
||||
<queues>
|
||||
<queue>
|
||||
<queuePath>root.a</queuePath>
|
||||
|
|
@ -4992,4 +4993,4 @@
|
|||
<autoQueueParentTemplateProperties/>
|
||||
<autoQueueLeafTemplateProperties/>
|
||||
</schedulerInfo>
|
||||
</scheduler>
|
||||
</scheduler>
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
"queueName": "root",
|
||||
"queuePath": "root",
|
||||
"maxParallelApps": 2147483647,
|
||||
"isAbsoluteResource": false,
|
||||
"queues": {"queue": [
|
||||
{
|
||||
"type": "capacitySchedulerLeafQueueInfo",
|
||||
|
|
@ -1735,4 +1736,4 @@
|
|||
"autoQueueTemplateProperties": {},
|
||||
"autoQueueParentTemplateProperties": {},
|
||||
"autoQueueLeafTemplateProperties": {}
|
||||
}}}
|
||||
}}}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
"queueName": "root",
|
||||
"queuePath": "root",
|
||||
"maxParallelApps": 2147483647,
|
||||
"isAbsoluteResource": false,
|
||||
"queues": {"queue": [
|
||||
{
|
||||
"type": "capacitySchedulerLeafQueueInfo",
|
||||
|
|
@ -1576,4 +1577,4 @@
|
|||
"autoQueueTemplateProperties": {},
|
||||
"autoQueueParentTemplateProperties": {},
|
||||
"autoQueueLeafTemplateProperties": {}
|
||||
}}}
|
||||
}}}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
"queueName": "root",
|
||||
"queuePath": "root",
|
||||
"maxParallelApps": 2147483647,
|
||||
"isAbsoluteResource": false,
|
||||
"queues": {"queue": [
|
||||
{
|
||||
"type": "capacitySchedulerLeafQueueInfo",
|
||||
|
|
@ -1735,4 +1736,4 @@
|
|||
"autoQueueTemplateProperties": {},
|
||||
"autoQueueParentTemplateProperties": {},
|
||||
"autoQueueLeafTemplateProperties": {}
|
||||
}}}
|
||||
}}}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
"queueName": "root",
|
||||
"queuePath": "root",
|
||||
"maxParallelApps": 2147483647,
|
||||
"isAbsoluteResource": false,
|
||||
"queues": {"queue": [
|
||||
{
|
||||
"type": "capacitySchedulerLeafQueueInfo",
|
||||
|
|
@ -4003,4 +4004,4 @@
|
|||
"autoQueueTemplateProperties": {},
|
||||
"autoQueueParentTemplateProperties": {},
|
||||
"autoQueueLeafTemplateProperties": {}
|
||||
}}}
|
||||
}}}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
"queueName": "root",
|
||||
"queuePath": "root",
|
||||
"maxParallelApps": 2147483647,
|
||||
"isAbsoluteResource": false,
|
||||
"queues": {"queue": [
|
||||
{
|
||||
"type": "capacitySchedulerLeafQueueInfo",
|
||||
|
|
@ -1735,4 +1736,4 @@
|
|||
"autoQueueTemplateProperties": {},
|
||||
"autoQueueParentTemplateProperties": {},
|
||||
"autoQueueLeafTemplateProperties": {}
|
||||
}}}
|
||||
}}}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
"queueName": "root",
|
||||
"queuePath": "root",
|
||||
"maxParallelApps": 2147483647,
|
||||
"isAbsoluteResource": false,
|
||||
"queues": {"queue": [
|
||||
{
|
||||
"queuePath": "root.a",
|
||||
|
|
@ -4500,4 +4501,4 @@
|
|||
"autoQueueTemplateProperties": {},
|
||||
"autoQueueParentTemplateProperties": {},
|
||||
"autoQueueLeafTemplateProperties": {}
|
||||
}}}
|
||||
}}}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
<queueName>root</queueName>
|
||||
<queuePath>root</queuePath>
|
||||
<maxParallelApps>2147483647</maxParallelApps>
|
||||
<isAbsoluteResource>false</isAbsoluteResource>
|
||||
<queues>
|
||||
<queue>
|
||||
<queuePath>root.a</queuePath>
|
||||
|
|
@ -4535,4 +4536,4 @@
|
|||
<autoQueueParentTemplateProperties/>
|
||||
<autoQueueLeafTemplateProperties/>
|
||||
</schedulerInfo>
|
||||
</scheduler>
|
||||
</scheduler>
|
||||
|
|
|
|||
Loading…
Reference in New Issue