From f1766e5bb4bccee9b6b88daa1483c01b50db330d Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Tue, 26 Jan 2021 16:22:35 +0100 Subject: [PATCH] YARN-10596. Allow static definition of childless ParentQueues with auto-queue-creation-v2 enabled. Contributed by Andras Gyori --- .../CapacitySchedulerQueueManager.java | 9 ++++- ...CapacitySchedulerNewQueueAutoCreation.java | 40 +++++++++++++++++++ ...WebServicesCapacitySchedDynamicConfig.java | 2 - 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index c5ce700eef5..00d1cda3088 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -238,14 +238,21 @@ static CSQueue parseQueue( boolean isReservableQueue = conf.isReservable(fullQueueName); boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled( fullQueueName); + // if a queue is eligible for auto queue creation v2 + // it must be a ParentQueue (even if it is empty) + boolean isAutoQueueCreationV2Enabled = conf.isAutoQueueCreationV2Enabled( + fullQueueName); boolean isDynamicParent = false; + // Auto created parent queues might not have static children, but they + // must be kept as a ParentQueue CSQueue oldQueue = oldQueues.get(fullQueueName); if (oldQueue instanceof ParentQueue) { isDynamicParent = ((ParentQueue) oldQueue).isDynamicQueue(); } - if (childQueueNames.size() == 0 && !isDynamicParent) { + if (childQueueNames.size() == 0 && !isDynamicParent && + !isAutoQueueCreationV2Enabled) { if (null == parent) { throw new IllegalStateException( "Queue configuration missing child queue names for " + queueName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index 98b6d3f9882..eada112039a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -422,6 +422,46 @@ public void testAutoQueueCreationOnAppSubmission() throws Exception { Assert.assertTrue(user0.isDynamicQueue()); } + @Test + public void testChildlessParentQueueWhenAutoQueueCreationEnabled() + throws Exception { + startScheduler(); + csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"}); + csConf.setNonLabeledQueueWeight("root", 1f); + csConf.setNonLabeledQueueWeight("root.a", 1f); + csConf.setNonLabeledQueueWeight("root.b", 1f); + csConf.setQueues("root.a", new String[]{"a1"}); + csConf.setNonLabeledQueueWeight("root.a.a1", 1f); + csConf.setAutoQueueCreationV2Enabled("root", true); + csConf.setAutoQueueCreationV2Enabled("root.a", true); + cs.reinitialize(csConf, mockRM.getRMContext()); + + CSQueue empty = cs.getQueue("root.empty-auto-parent"); + Assert.assertTrue("empty-auto-parent is not a LeafQueue", + empty instanceof LeafQueue); + empty.stopQueue(); + + csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"}); + csConf.setNonLabeledQueueWeight("root", 1f); + csConf.setNonLabeledQueueWeight("root.a", 1f); + csConf.setNonLabeledQueueWeight("root.b", 1f); + csConf.setQueues("root.a", new String[]{"a1"}); + csConf.setNonLabeledQueueWeight("root.a.a1", 1f); + csConf.setAutoQueueCreationV2Enabled("root", true); + csConf.setAutoQueueCreationV2Enabled("root.a", true); + csConf.setAutoQueueCreationV2Enabled("root.empty-auto-parent", true); + cs.reinitialize(csConf, mockRM.getRMContext()); + + empty = cs.getQueue("root.empty-auto-parent"); + Assert.assertTrue("empty-auto-parent is not a ParentQueue", + empty instanceof ParentQueue); + Assert.assertEquals("empty-auto-parent has children", + 0, empty.getChildQueues().size()); + Assert.assertTrue("empty-auto-parent is not eligible " + + "for auto queue creation", + ((ParentQueue)empty).isEligibleForAutoQueueCreation()); + } + private LeafQueue createQueue(String queuePath) throws YarnException { return autoQueueHandler.autoCreateQueue( CSQueueUtils.extractQueuePath(queuePath)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index 4d78d88f785..118a0df6271 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -507,8 +507,6 @@ private static Configuration createWeightConfigInternal(boolean enableAqc) { if (enableAqc) { conf.put("yarn.scheduler.capacity.root.auto-queue-creation-v2.enabled", "true"); - conf.put("yarn.scheduler.capacity.root.default." + - "auto-queue-creation-v2.enabled", "true"); } return createConfiguration(conf); }