Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok

This commit is contained in:
Szilard Nemeth 2021-12-10 15:09:53 +01:00
parent d7c5400fbc
commit f544ef4a51
16 changed files with 1645 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
View File

@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@ -49,6 +50,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected String queueName;
private String queuePath;
protected int maxParallelApps;
private boolean isAbsoluteResource;
protected CapacitySchedulerQueueInfoList queues;
protected QueueCapacitiesInfo capacities;
protected CapacitySchedulerHealthInfo health;
@ -90,6 +92,9 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
health = new CapacitySchedulerHealthInfo(cs);
maximumAllocation = new ResourceInfo(parent.getMaximumAllocation());
isAbsoluteResource = parent.getCapacityConfigType() ==
AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
CapacitySchedulerConfiguration conf = cs.getConfiguration();
queueAcls = new QueueAclsInfo();
queueAcls.addAll(getSortedQueueAclInfoList(queueName, conf));

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
@ -43,11 +44,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueu
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
CapacitySchedulerConfiguration.RESOURCE_PATTERN;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
CapacitySchedulerConfiguration.CAPACITY;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class})
@ -179,10 +175,8 @@ public class CapacitySchedulerQueueInfo {
.getLeafOnlyProperties());
}
String configuredCapacity = conf.get(
CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY);
isAbsoluteResource = (configuredCapacity != null)
&& RESOURCE_PATTERN.matcher(configuredCapacity).find();
isAbsoluteResource = q.getCapacityConfigType() ==
AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
autoCreateChildQueueEnabled = conf.
isAutoCreateChildQueueEnabled(queuePath);

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
View File

@ -136,6 +136,22 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
"webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
}
@Test
public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
throws Exception {
Configuration config = CSConfigGenerator
.createAbsoluteConfigLegacyAutoCreation();
config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
initAutoQueueHandler(8192 * GB);
createQueue("root.managed.queue1");
assertJsonResponse(sendRequest(),
"webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json");
}
@Test
public void testSchedulerResponseAbsoluteMode()
throws Exception {
@ -189,7 +205,7 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
"maximum-applications", 300);
initResourceManager(config);
initAutoQueueHandler();
initAutoQueueHandler(1200 * GB);
// same as webapp/scheduler-response-WeightMode.json, but with effective resources filled in
assertJsonResponse(sendRequest(),
@ -212,10 +228,10 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
"webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json");
}
private void initAutoQueueHandler() throws Exception {
private void initAutoQueueHandler(int nodeMemory) throws Exception {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
autoQueueHandler = cs.getCapacitySchedulerQueueManager();
rm.registerNode("h1:1234", 1200 * GB); // label = x
rm.registerNode("h1:1234", nodeMemory); // label = x
}
private void createQueue(String queuePath) throws YarnException,
@ -255,6 +271,19 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
return createConfiguration(conf);
}
public static Configuration createAbsoluteConfigLegacyAutoCreation() {
Map<String, String> conf = new HashMap<>();
conf.put("yarn.scheduler.capacity.root.queues", "default, managed");
conf.put("yarn.scheduler.capacity.root.default.state", "STOPPED");
conf.put("yarn.scheduler.capacity.root.managed.capacity", "[memory=4096,vcores=4]");
conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.capacity",
"[memory=2048,vcores=2]");
conf.put("yarn.scheduler.capacity.root.managed.state", "RUNNING");
conf.put("yarn.scheduler.capacity.root.managed." +
"auto-create-child-queue.enabled", "true");
return createConfiguration(conf);
}
public static Configuration createAbsoluteConfig() {
Map<String, String> conf = new HashMap<>();
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
View File

@ -574,7 +574,7 @@ public class TestRMWebServicesForCSWithPartitions extends JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements", 23, info.length());
assertEquals("incorrect number of elements", 24, info.length());
JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
JSONArray partitionsCapsArray =
capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteMode.json
View File

@ -8,6 +8,7 @@
"queueName": "root",
"queuePath": "root",
"maxParallelApps": 2147483647,
"isAbsoluteResource": true,
"queues": {"queue": [
{
"type": "capacitySchedulerLeafQueueInfo",
@ -1735,4 +1736,4 @@
"autoQueueTemplateProperties": {},
"autoQueueParentTemplateProperties": {},
"autoQueueLeafTemplateProperties": {}
}}}
}}}

1582
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json Normal file
View File

File diff suppressed because it is too large Load Diff

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
View File

@ -8,6 +8,7 @@
<queueName>root</queueName>
<queuePath>root</queuePath>
<maxParallelApps>2147483647</maxParallelApps>
<isAbsoluteResource>false</isAbsoluteResource>
<queues>
<queue>
<queuePath>root.a</queuePath>
@ -4546,4 +4547,4 @@
<autoQueueParentTemplateProperties/>
<autoQueueLeafTemplateProperties/>
</schedulerInfo>
</scheduler>
</scheduler>

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json
View File

@ -8,6 +8,7 @@
"queueName": "root",
"queuePath": "root",
"maxParallelApps": 2147483647,
"isAbsoluteResource": false,
"queues": {"queue": [
{
"queuePath": "root.a",
@ -4953,4 +4954,4 @@
"autoQueueTemplateProperties": {},
"autoQueueParentTemplateProperties": {},
"autoQueueLeafTemplateProperties": {}
}}}
}}}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml
View File

@ -8,6 +8,7 @@
<queueName>root</queueName>
<queuePath>root</queuePath>
<maxParallelApps>2147483647</maxParallelApps>
<isAbsoluteResource>false</isAbsoluteResource>
<queues>
<queue>
<queuePath>root.a</queuePath>
@ -4992,4 +4993,4 @@
<autoQueueParentTemplateProperties/>
<autoQueueLeafTemplateProperties/>
</schedulerInfo>
</scheduler>
</scheduler>

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageMode.json
View File

@ -8,6 +8,7 @@
"queueName": "root",
"queuePath": "root",
"maxParallelApps": 2147483647,
"isAbsoluteResource": false,
"queues": {"queue": [
{
"type": "capacitySchedulerLeafQueueInfo",
@ -1735,4 +1736,4 @@
"autoQueueTemplateProperties": {},
"autoQueueParentTemplateProperties": {},
"autoQueueLeafTemplateProperties": {}
}}}
}}}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json
View File

@ -8,6 +8,7 @@
"queueName": "root",
"queuePath": "root",
"maxParallelApps": 2147483647,
"isAbsoluteResource": false,
"queues": {"queue": [
{
"type": "capacitySchedulerLeafQueueInfo",
@ -1576,4 +1577,4 @@
"autoQueueTemplateProperties": {},
"autoQueueParentTemplateProperties": {},
"autoQueueLeafTemplateProperties": {}
}}}
}}}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightMode.json
View File

@ -8,6 +8,7 @@
"queueName": "root",
"queuePath": "root",
"maxParallelApps": 2147483647,
"isAbsoluteResource": false,
"queues": {"queue": [
{
"type": "capacitySchedulerLeafQueueInfo",
@ -1735,4 +1736,4 @@
"autoQueueTemplateProperties": {},
"autoQueueParentTemplateProperties": {},
"autoQueueLeafTemplateProperties": {}
}}}
}}}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json
View File

@ -8,6 +8,7 @@
"queueName": "root",
"queuePath": "root",
"maxParallelApps": 2147483647,
"isAbsoluteResource": false,
"queues": {"queue": [
{
"type": "capacitySchedulerLeafQueueInfo",
@ -4003,4 +4004,4 @@
"autoQueueTemplateProperties": {},
"autoQueueParentTemplateProperties": {},
"autoQueueLeafTemplateProperties": {}
}}}
}}}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json
View File

@ -8,6 +8,7 @@
"queueName": "root",
"queuePath": "root",
"maxParallelApps": 2147483647,
"isAbsoluteResource": false,
"queues": {"queue": [
{
"type": "capacitySchedulerLeafQueueInfo",
@ -1735,4 +1736,4 @@
"autoQueueTemplateProperties": {},
"autoQueueParentTemplateProperties": {},
"autoQueueLeafTemplateProperties": {}
}}}
}}}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json
View File

@ -8,6 +8,7 @@
"queueName": "root",
"queuePath": "root",
"maxParallelApps": 2147483647,
"isAbsoluteResource": false,
"queues": {"queue": [
{
"queuePath": "root.a",
@ -4500,4 +4501,4 @@
"autoQueueTemplateProperties": {},
"autoQueueParentTemplateProperties": {},
"autoQueueLeafTemplateProperties": {}
}}}
}}}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml
View File

@ -8,6 +8,7 @@
<queueName>root</queueName>
<queuePath>root</queuePath>
<maxParallelApps>2147483647</maxParallelApps>
<isAbsoluteResource>false</isAbsoluteResource>
<queues>
<queue>
<queuePath>root.a</queuePath>
@ -4535,4 +4536,4 @@
<autoQueueParentTemplateProperties/>
<autoQueueLeafTemplateProperties/>
</schedulerInfo>
</scheduler>
</scheduler>