From b86a6eb8715e0d722e577fc97ced82c3795ad2d2 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Tue, 25 May 2021 18:27:53 +0200 Subject: [PATCH] YARN-10782. Extend /scheduler endpoint with template properties. Contributed by Andras Gyori --- .../dao/AutoQueueTemplatePropertiesInfo.java | 45 +++++++++++++++++++ .../webapp/dao/CapacitySchedulerInfo.java | 3 ++ .../dao/CapacitySchedulerQueueInfo.java | 6 ++- .../helper/CapacitySchedulerInfoHelper.java | 17 +++++++ .../TestRMWebServicesCapacitySched.java | 6 +-- ...WebServicesCapacitySchedDynamicConfig.java | 43 +++++++++++++++++- .../TestRMWebServicesForCSWithPartitions.java | 2 +- 7 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AutoQueueTemplatePropertiesInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AutoQueueTemplatePropertiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AutoQueueTemplatePropertiesInfo.java new file mode 100644 index 00000000000..43c9ff25685 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AutoQueueTemplatePropertiesInfo.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LeafQueueTemplateInfo.ConfItem; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class AutoQueueTemplatePropertiesInfo { + + private ArrayList property = + new ArrayList<>(); + + public AutoQueueTemplatePropertiesInfo() { + } + + public ArrayList getProperty() { + return property; + } + + public void add(ConfItem confItem) { + property.add(confItem); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 78e2133a90b..cc02c23fd71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -62,6 +62,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected String creationMethod; protected String autoCreationEligibility; protected String defaultNodeLabelExpression; + protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties; @XmlTransient static final float EPSILON = 1e-8f; @@ -109,6 +110,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo { if (parent instanceof ParentQueue) { orderingPolicyInfo = ((ParentQueue) parent).getQueueOrderingPolicy() .getConfigName(); + autoQueueTemplateProperties = CapacitySchedulerInfoHelper + .getAutoCreatedTemplate((ParentQueue) parent); } mode = CapacitySchedulerInfoHelper.getMode(parent); queueType = CapacitySchedulerInfoHelper.getQueueType(parent); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 31857bb94b6..152620d6603 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -19,10 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.Set; -import java.util.regex.Pattern; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -92,6 +90,8 @@ public class CapacitySchedulerQueueInfo { protected String creationMethod; protected String autoCreationEligibility; protected String defaultNodeLabelExpression; + protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties = + new AutoQueueTemplatePropertiesInfo(); CapacitySchedulerQueueInfo() { }; @@ -174,6 +174,8 @@ public class CapacitySchedulerQueueInfo { if (q instanceof ParentQueue) { orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() .getConfigName(); + autoQueueTemplateProperties = CapacitySchedulerInfoHelper + .getAutoCreatedTemplate((ParentQueue) q); } String configuredCapacity = conf.get( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java index 355528d7f1b..f71928b61bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java @@ -23,6 +23,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AutoQueueTemplatePropertiesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LeafQueueTemplateInfo.ConfItem; + +import java.util.Map; /** * Helper class to describe a queue's type, its creation method and its @@ -106,4 +110,17 @@ public class CapacitySchedulerInfoHelper { return AUTO_CREATION_OFF; } } + + public static AutoQueueTemplatePropertiesInfo getAutoCreatedTemplate( + ParentQueue parent) { + AutoQueueTemplatePropertiesInfo propertiesInfo = + new AutoQueueTemplatePropertiesInfo(); + for (Map.Entry e : + parent.getAutoCreatedQueueTemplate().getTemplateProperties() + .entrySet()) { + propertiesInfo.add(new ConfItem(e.getKey(), e.getValue())); + } + + return propertiesInfo; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index ab3294191bf..24744ddd597 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -369,7 +369,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements in: " + info, 19, info.length()); + assertEquals("incorrect number of elements in: " + info, 20, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), @@ -424,10 +424,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 34; + int numExpectedElements = 35; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 52; + numExpectedElements = 53; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index 663bb6763d9..6938519c736 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; @@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -93,6 +95,7 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends private String queueType; private String creationMethod; private String autoCreationEligibility; + private List autoQueueTemplateProperties; public ExpectedQueueWithProperties(String path, float weight, float normalizedWeight, String queueType, String creationMethod, @@ -103,6 +106,20 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends this.queueType = queueType; this.creationMethod = creationMethod; this.autoCreationEligibility = autoCreationEligibility; + this.autoQueueTemplateProperties = new ArrayList<>(); + } + + ExpectedQueueWithProperties( + String path, float weight, float normalizedWeight, String queueType, + String creationMethod, String autoCreationEligibility, + List autoQueueTemplateProperties) { + this.path = path; + this.weight = weight; + this.normalizedWeight = normalizedWeight; + this.queueType = queueType; + this.creationMethod = creationMethod; + this.autoCreationEligibility = autoCreationEligibility; + this.autoQueueTemplateProperties = autoQueueTemplateProperties; } } @@ -260,6 +277,10 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends .createWeightConfigWithAutoQueueCreationEnabled(); config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, YarnConfiguration.MEMORY_CONFIGURATION_STORE); + config.setInt(CapacitySchedulerConfiguration + .getQueuePrefix("root.autoParent1") + + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + + "maximum-applications", 300); initResourceManager(config); initAutoQueueHandler(); @@ -290,6 +311,9 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends new ExpectedQueueWithProperties("root", EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF); + List templateProperties = new ArrayList<>(); + templateProperties.add(new String[] {"maximum-applications", "300"}); + validateSchedulerInfo(json, "weight", expectedRootQ, new ExpectedQueueWithProperties("root.auto1", @@ -307,7 +331,8 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends new ExpectedQueueWithProperties("root.autoParent1", EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE, EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights, - PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE), + PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE, + templateProperties), new ExpectedQueueWithProperties("root.default", 10.0f, 10.0f / sumOfWeights, LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), @@ -474,6 +499,22 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends queuePath, expectedQueue.creationMethod, obj.getString("creationMethod")); + if (!expectedQueue.autoQueueTemplateProperties.isEmpty()) { + JSONArray templates = obj.getJSONObject("autoQueueTemplateProperties") + .getJSONArray("property"); + for (int j = 0; j < templates.length(); j++) { + JSONObject prop = templates.getJSONObject(j); + Assert.assertEquals("Auto creation eligible queue " + + "template key do not match for queue" + queuePath, + expectedQueue.autoQueueTemplateProperties.get(j)[0], + prop.getString("name")); + Assert.assertEquals("Auto creation eligible queue " + + "template value do not match for queue" + queuePath, + expectedQueue.autoQueueTemplateProperties.get(j)[1], + prop.getString("value")); + } + } + Assert.assertEquals("Queue auto creation eligibility does not " + "match for queue " + queuePath, expectedQueue.autoCreationEligibility, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index 86013a0c48f..2e67d698940 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -574,7 +574,7 @@ public class TestRMWebServicesForCSWithPartitions extends JerseyTestBase { JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 19, info.length()); + assertEquals("incorrect number of elements", 20, info.length()); JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES); JSONArray partitionsCapsArray = capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);