From f473473355a76a477be15b21896ba5834d8f43ff Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Wed, 15 Apr 2020 07:24:04 +0200 Subject: [PATCH] YARN-10234. FS-CS converter: don't enable auto-create queue property for root. Contributed by Peter Bacsko (cherry picked from commit 55fcbcb5c2a096f98f273fda52ae25ecaa1d8bb6) --- .../fair/converter/FSQueueConverter.java | 4 +++- .../TestFSConfigToCSConfigConverter.java | 23 +++++++++++++++---- .../fair/converter/TestFSQueueConverter.java | 2 +- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java index a84b6cbe0d1..2b3f28495d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -27,6 +27,7 @@ import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; @@ -225,7 +226,8 @@ public class FSQueueConverter { * @param queueName */ private void emitAutoCreateChildQueue(String queueName, FSQueue queue) { - if (autoCreateChildQueues && !queue.getChildQueues().isEmpty()) { + if (autoCreateChildQueues && !queue.getChildQueues().isEmpty() + && !queueName.equals(CapacitySchedulerConfiguration.ROOT)) { capacitySchedulerConfig.setBoolean(PREFIX + queueName + ".auto-create-child-queue.enabled", true); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 4ceb8c85725..dd3e6d45888 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -649,6 +649,7 @@ public class TestFSConfigToCSConfigConverter { testAutoCreateChildQueuesWithoutPlacementRules(false); } + @SuppressWarnings("checkstyle:linelength") private void testAutoCreateChildQueuesWithoutPlacementRules( boolean allowUndeclaredPools) throws Exception { config = new Configuration(false); @@ -661,15 +662,27 @@ public class TestFSConfigToCSConfigConverter { converter.convert(config); Configuration convertedConf = converter.getCapacitySchedulerConfig(); - String property = + String rootUserAutoCreate = + "yarn.scheduler.capacity.root.users.auto-create-child-queue.enabled"; + String rootAutoCreate = "yarn.scheduler.capacity.root.auto-create-child-queue.enabled"; + String leafQueueAutoCreate = + "yarn.scheduler.capacity.root.users.joe.auto-create-child-queue.enabled"; if (allowUndeclaredPools) { - assertEquals("Auto-create queue wasn't enabled", true, - convertedConf.getBoolean(property, false)); + assertEquals("Auto-create queue wasn't enabled for root.users", true, + convertedConf.getBoolean(rootUserAutoCreate, false)); + assertNull("Auto-create queue shouldn't be set for root", + convertedConf.get(rootAutoCreate)); + assertNull("Auto-create queue shouldn't be set for leaf", + convertedConf.get(leafQueueAutoCreate)); } else { - assertNull("Auto-create queue shouldn't be set", - convertedConf.get(property)); + assertNull("Auto-create queue shouldn't be set for root.users", + convertedConf.get(rootUserAutoCreate)); + assertNull("Auto-create queue shouldn't be set for root", + convertedConf.get(rootAutoCreate)); + assertNull("Auto-create queue shouldn't be set for leaf", + convertedConf.get(leafQueueAutoCreate)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java index 41018623e89..0ed6bbf2024 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java @@ -316,7 +316,7 @@ public class TestFSQueueConverter { converter.convertQueueHierarchy(rootQueue); - Set parentQueues = Sets.newHashSet("root", + Set parentQueues = Sets.newHashSet( "root.admins", "root.users");