Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-11408. Add a check of autoQueueCreation is disabled for emitDefaultUserLimitFactor method (#5278) 

* added a check of autoQueueCreationV2Disabled

Change-Id: If1e36c5969d270c1b81a4bbd2e883fa819c81f20

* added check of AutoCreateChildQueueDisabled

Change-Id: Ia011b4393ccd8d4d419a2e46b06a5237d050851c

* removed auto-create-child-queue-enabled check and emit

Change-Id: I7a154124519ecbd81379b46a238707c16db1e82a
This commit is contained in:
susheel-gupta 2023-02-22 14:16:42 +05:30 committed by GitHub
parent fe5bb49ad9
commit 49b8ac19f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 14 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java
View File

@ -16,6 +16,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_ENABLED;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DEFAULT_AUTO_QUEUE_CREATION_ENABLED;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_LIMIT_FACTOR; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_LIMIT_FACTOR;
@ -82,13 +84,13 @@ public class FSQueueConverter {
emitMaxParallelApps(queueName, queue); emitMaxParallelApps(queueName, queue);
emitMaxAllocations(queueName, queue); emitMaxAllocations(queueName, queue);
emitPreemptionDisabled(queueName, queue); emitPreemptionDisabled(queueName, queue);
emitDefaultUserLimitFactor(queueName, children);
emitChildCapacity(queue); emitChildCapacity(queue);
emitMaximumCapacity(queueName, queue); emitMaximumCapacity(queueName, queue);
emitSizeBasedWeight(queueName); emitSizeBasedWeight(queueName);
emitOrderingPolicy(queueName, queue); emitOrderingPolicy(queueName, queue);
checkMaxChildCapacitySetting(queue); checkMaxChildCapacitySetting(queue);
emitDefaultUserLimitFactor(queueName, children);
for (FSQueue childQueue : children) { for (FSQueue childQueue : children) {
convertQueueHierarchy(childQueue); convertQueueHierarchy(childQueue);
@ -220,7 +222,7 @@ public class FSQueueConverter {
} }
public void emitDefaultUserLimitFactor(String queueName, List<FSQueue> children) { public void emitDefaultUserLimitFactor(String queueName, List<FSQueue> children) {
if (children.isEmpty()) { if (children.isEmpty() && checkAutoQueueCreationV2Disabled(queueName)) {
capacitySchedulerConfig.setFloat( capacitySchedulerConfig.setFloat(
CapacitySchedulerConfiguration. CapacitySchedulerConfiguration.
PREFIX + queueName + DOT + USER_LIMIT_FACTOR, PREFIX + queueName + DOT + USER_LIMIT_FACTOR,
@ -309,6 +311,12 @@ public class FSQueueConverter {
} }
} }
private boolean checkAutoQueueCreationV2Disabled(String queueName) {
return !capacitySchedulerConfig.getBoolean(
PREFIX + queueName + DOT + AUTO_QUEUE_CREATION_V2_ENABLED,
DEFAULT_AUTO_QUEUE_CREATION_ENABLED);
}
private String getQueueShortName(String queueName) { private String getQueueShortName(String queueName) {
int lastDot = queueName.lastIndexOf("."); int lastDot = queueName.lastIndexOf(".");
return queueName.substring(lastDot + 1); return queueName.substring(lastDot + 1);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
View File

@ -194,6 +194,8 @@ public class TestFSConfigToCSConfigConverter {
assertNull("root.users user-limit-factor should be null", assertNull("root.users user-limit-factor should be null",
conf.get(PREFIX + "root.users." + USER_LIMIT_FACTOR)); conf.get(PREFIX + "root.users." + USER_LIMIT_FACTOR));
assertEquals("root.users auto-queue-creation-v2.enabled", "true",
conf.get(PREFIX + "root.users.auto-queue-creation-v2.enabled"));
assertEquals("root.default user-limit-factor", "-1.0", assertEquals("root.default user-limit-factor", "-1.0",
conf.get(PREFIX + "root.default.user-limit-factor")); conf.get(PREFIX + "root.default.user-limit-factor"));
@ -203,6 +205,8 @@ public class TestFSConfigToCSConfigConverter {
assertEquals("root.admins.bob user-limit-factor", "-1.0", assertEquals("root.admins.bob user-limit-factor", "-1.0",
conf.get(PREFIX + "root.admins.bob.user-limit-factor")); conf.get(PREFIX + "root.admins.bob.user-limit-factor"));
assertNull("root.admin.bob auto-queue-creation-v2.enabled should be null",
conf.get(PREFIX + "root.admin.bob.auto-queue-creation-v2.enabled"));
} }
@Test @Test