From 9191e08f0ad4ebc2a3b776c4cc71d0fc5c053beb Mon Sep 17 00:00:00 2001 From: Suma Shivaprasad Date: Mon, 10 Jun 2019 14:33:24 -0700 Subject: [PATCH] YARN-9569. Auto-created leaf queues do not honor cluster-wide min/max memory/vcores. Contributed by Craig Condit. --- .../capacity/AbstractManagedParentQueue.java | 8 ++++++ ...CapacitySchedulerAutoCreatedQueueBase.java | 27 +++++++++++++++++-- ...estCapacitySchedulerAutoQueueCreation.java | 11 ++++---- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java index 3e90863e5df..7626c66d555 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common @@ -200,6 +201,13 @@ public abstract class AbstractManagedParentQueue extends ParentQueue { CapacitySchedulerConfiguration leafQueueConfigs = new CapacitySchedulerConfiguration(new Configuration(false), false); + String prefix = YarnConfiguration.RESOURCE_TYPES + "."; + Map rtProps = csContext + .getConfiguration().getPropsWithPrefix(prefix); + for (Map.Entry entry : rtProps.entrySet()) { + leafQueueConfigs.set(prefix + entry.getKey(), entry.getValue()); + } + SortedMap sortedConfigs = sortCSConfigurations(); SortedMap templateConfigs = getConfigurationsWithPrefix (sortedConfigs, configPrefix); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index c64582ef21f..d0cacde21e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.RandomUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.GroupMappingServiceProvider; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; @@ -189,7 +190,7 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { @Before public void setUp() throws Exception { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CapacitySchedulerConfiguration conf = setupSchedulerConfiguration(); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); @@ -494,8 +495,22 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { return queueMappings; } + protected CapacitySchedulerConfiguration setupSchedulerConfiguration() { + Configuration schedConf = new Configuration(); + schedConf.setInt(YarnConfiguration.RESOURCE_TYPES + + ".vcores.minimum-allocation", 1); + schedConf.setInt(YarnConfiguration.RESOURCE_TYPES + + ".vcores.maximum-allocation", 8); + schedConf.setInt(YarnConfiguration.RESOURCE_TYPES + + ".memory-mb.minimum-allocation", 1024); + schedConf.setInt(YarnConfiguration.RESOURCE_TYPES + + ".memory-mb.maximum-allocation", 16384); + + return new CapacitySchedulerConfiguration(schedConf); + } + protected MockRM setupSchedulerInstance() throws Exception { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CapacitySchedulerConfiguration conf = setupSchedulerConfiguration(); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); @@ -579,6 +594,14 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { autoCreatedLeafQueue.getMaxApplicationsPerUser()); } + protected void validateContainerLimits( + AutoCreatedLeafQueue autoCreatedLeafQueue) { + assertEquals(8, + autoCreatedLeafQueue.getMaximumAllocation().getVirtualCores()); + assertEquals(16384, + autoCreatedLeafQueue.getMaximumAllocation().getMemorySize()); + } + protected void validateInitialQueueEntitlement(CSQueue parentQueue, String leafQueueName, Map expectedTotalChildQueueAbsCapacityByLabel, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index f859ca73285..5f32e0ddff4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -132,6 +132,7 @@ public class TestCapacitySchedulerAutoQueueCreation expectedChildQueueAbsCapacity, accessibleNodeLabelsOnC); validateUserAndAppLimits(autoCreatedLeafQueue, 1000, 1000); + validateContainerLimits(autoCreatedLeafQueue); assertTrue(autoCreatedLeafQueue .getOrderingPolicy() instanceof FairOrderingPolicy); @@ -256,8 +257,7 @@ public class TestCapacitySchedulerAutoQueueCreation throws Exception { CapacityScheduler newCS = new CapacityScheduler(); try { - CapacitySchedulerConfiguration newConf = - new CapacitySchedulerConfiguration(); + CapacitySchedulerConfiguration newConf = setupSchedulerConfiguration(); setupQueueConfiguration(newConf); newConf.setAutoCreateChildQueueEnabled(C, false); @@ -285,8 +285,7 @@ public class TestCapacitySchedulerAutoQueueCreation throws Exception { CapacityScheduler newCS = new CapacityScheduler(); try { - CapacitySchedulerConfiguration newConf = - new CapacitySchedulerConfiguration(); + CapacitySchedulerConfiguration newConf = setupSchedulerConfiguration(); setupQueueConfiguration(newConf); newConf.setAutoCreatedLeafQueueConfigCapacity(A1, A1_CAPACITY / 10); newConf.setAutoCreateChildQueueEnabled(A1, true); @@ -315,8 +314,7 @@ public class TestCapacitySchedulerAutoQueueCreation throws Exception { CapacityScheduler newCS = new CapacityScheduler(); try { - CapacitySchedulerConfiguration newConf = - new CapacitySchedulerConfiguration(); + CapacitySchedulerConfiguration newConf = setupSchedulerConfiguration(); setupQueueConfiguration(newConf); newConf.setAutoCreatedLeafQueueConfigCapacity(A, A_CAPACITY / 10); newConf.setAutoCreateChildQueueEnabled(A, true); @@ -773,6 +771,7 @@ public class TestCapacitySchedulerAutoQueueCreation validateCapacities(user3Queue, 0.3f, 0.09f, 0.4f,0.2f); validateUserAndAppLimits(user3Queue, 900, 900); + validateContainerLimits(user3Queue); GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy = (GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue)