Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-11006. Allow overriding user limit factor and maxAMResourcePercent with AQCv2 templates. Contributed by Benjamin Teke

This commit is contained in:
Szilard Nemeth 2021-12-07 15:43:55 +01:00
parent 60440a836a
commit 7284d23476
2 changed files with 17 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -1707,12 +1707,12 @@ public class LeafQueue extends AbstractCSQueue {
@Override @Override
protected void setDynamicQueueProperties( protected void setDynamicQueueProperties(
CapacitySchedulerConfiguration configuration) { CapacitySchedulerConfiguration configuration) {
super.setDynamicQueueProperties(configuration);
// set to -1, to disable it // set to -1, to disable it
configuration.setUserLimitFactor(getQueuePath(), -1); configuration.setUserLimitFactor(getQueuePath(), -1);
// Set Max AM percentage to a higher value // Set Max AM percentage to a higher value
configuration.setMaximumApplicationMasterResourcePerQueuePercent( configuration.setMaximumApplicationMasterResourcePerQueuePercent(
getQueuePath(), 1f); getQueuePath(), 1f);
super.setDynamicQueueProperties(configuration);
} }
private void updateSchedulerHealthForCompletedContainer( private void updateSchedulerHealthForCompletedContainer(

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
View File

@ -688,11 +688,27 @@ public class TestCapacitySchedulerNewQueueAutoCreation
LeafQueue a2 = createQueue("root.a.a-auto.a2"); LeafQueue a2 = createQueue("root.a.a-auto.a2");
Assert.assertEquals("weight is not set by template", 6f, Assert.assertEquals("weight is not set by template", 6f,
a2.getQueueCapacities().getWeight(), 1e-6); a2.getQueueCapacities().getWeight(), 1e-6);
Assert.assertEquals("user limit factor should be disabled with dynamic queues",
-1f, a2.getUserLimitFactor(), 1e-6);
Assert.assertEquals("maximum AM resource percent should be 1 with dynamic queues",
1f, a2.getMaxAMResourcePerQueuePercent(), 1e-6);
// Set the user-limit-factor and maximum-am-resource-percent via templates to ensure their
// modified defaults are indeed overridden
csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
"root.a.*") + "user-limit-factor", "10");
csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
"root.a.*") + "maximum-am-resource-percent", "0.8");
cs.reinitialize(csConf, mockRM.getRMContext()); cs.reinitialize(csConf, mockRM.getRMContext());
a2 = (LeafQueue) cs.getQueue("root.a.a-auto.a2"); a2 = (LeafQueue) cs.getQueue("root.a.a-auto.a2");
Assert.assertEquals("weight is overridden", 6f, Assert.assertEquals("weight is overridden", 6f,
a2.getQueueCapacities().getWeight(), 1e-6); a2.getQueueCapacities().getWeight(), 1e-6);
Assert.assertEquals("user limit factor should be modified by templates",
10f, a2.getUserLimitFactor(), 1e-6);
Assert.assertEquals("maximum AM resource percent should be modified by templates",
0.8f, a2.getMaxAMResourcePerQueuePercent(), 1e-6);
csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f); csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f);
cs.reinitialize(csConf, mockRM.getRMContext()); cs.reinitialize(csConf, mockRM.getRMContext());