Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-10782. Extend /scheduler endpoint with template properties. Contributed by Andras Gyori

This commit is contained in:
Szilard Nemeth 2021-05-25 18:27:53 +02:00
parent 2541efa496
commit b86a6eb871
7 changed files with 115 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AutoQueueTemplatePropertiesInfo.java Normal file
View File

@ -0,0 +1,45 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LeafQueueTemplateInfo.ConfItem;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class AutoQueueTemplatePropertiesInfo {
private ArrayList<ConfItem> property =
new ArrayList<>();
public AutoQueueTemplatePropertiesInfo() {
}
public ArrayList<ConfItem> getProperty() {
return property;
}
public void add(ConfItem confItem) {
property.add(confItem);
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
View File

@ -62,6 +62,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected String creationMethod;
protected String autoCreationEligibility;
protected String defaultNodeLabelExpression;
protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties;
@XmlTransient
static final float EPSILON = 1e-8f;
@ -109,6 +110,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
if (parent instanceof ParentQueue) {
orderingPolicyInfo = ((ParentQueue) parent).getQueueOrderingPolicy()
.getConfigName();
autoQueueTemplateProperties = CapacitySchedulerInfoHelper
.getAutoCreatedTemplate((ParentQueue) parent);
}
mode = CapacitySchedulerInfoHelper.getMode(parent);
queueType = CapacitySchedulerInfoHelper.getQueueType(parent);

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -19,10 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@ -92,6 +90,8 @@ public class CapacitySchedulerQueueInfo {
protected String creationMethod;
protected String autoCreationEligibility;
protected String defaultNodeLabelExpression;
protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties =
new AutoQueueTemplatePropertiesInfo();
CapacitySchedulerQueueInfo() {
};
@ -174,6 +174,8 @@ public class CapacitySchedulerQueueInfo {
if (q instanceof ParentQueue) {
orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy()
.getConfigName();
autoQueueTemplateProperties = CapacitySchedulerInfoHelper
.getAutoCreatedTemplate((ParentQueue) q);
}
String configuredCapacity = conf.get(

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
View File

@ -23,6 +23,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AutoQueueTemplatePropertiesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LeafQueueTemplateInfo.ConfItem;
import java.util.Map;
/**
* Helper class to describe a queue's type, its creation method and its
@ -106,4 +110,17 @@ public class CapacitySchedulerInfoHelper {
return AUTO_CREATION_OFF;
}
}
public static AutoQueueTemplatePropertiesInfo getAutoCreatedTemplate(
ParentQueue parent) {
AutoQueueTemplatePropertiesInfo propertiesInfo =
new AutoQueueTemplatePropertiesInfo();
for (Map.Entry<String, String> e :
parent.getAutoCreatedQueueTemplate().getTemplateProperties()
.entrySet()) {
propertiesInfo.add(new ConfItem(e.getKey(), e.getValue()));
}
return propertiesInfo;
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -369,7 +369,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements in: " + info, 19, info.length());
assertEquals("incorrect number of elements in: " + info, 20, info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
@ -424,10 +424,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
int numExpectedElements = 34;
int numExpectedElements = 35;
boolean isParentQueue = true;
if (!info.has("queues")) {
numExpectedElements = 52;
numExpectedElements = 53;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@ -93,6 +95,7 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
private String queueType;
private String creationMethod;
private String autoCreationEligibility;
private List<String[]> autoQueueTemplateProperties;
public ExpectedQueueWithProperties(String path, float weight,
float normalizedWeight, String queueType, String creationMethod,
@ -103,6 +106,20 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
this.queueType = queueType;
this.creationMethod = creationMethod;
this.autoCreationEligibility = autoCreationEligibility;
this.autoQueueTemplateProperties = new ArrayList<>();
}
ExpectedQueueWithProperties(
String path, float weight, float normalizedWeight, String queueType,
String creationMethod, String autoCreationEligibility,
List<String[]> autoQueueTemplateProperties) {
this.path = path;
this.weight = weight;
this.normalizedWeight = normalizedWeight;
this.queueType = queueType;
this.creationMethod = creationMethod;
this.autoCreationEligibility = autoCreationEligibility;
this.autoQueueTemplateProperties = autoQueueTemplateProperties;
}
}
@ -260,6 +277,10 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
.createWeightConfigWithAutoQueueCreationEnabled();
config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
YarnConfiguration.MEMORY_CONFIGURATION_STORE);
config.setInt(CapacitySchedulerConfiguration
.getQueuePrefix("root.autoParent1") +
AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX +
"maximum-applications", 300);
initResourceManager(config);
initAutoQueueHandler();
@ -290,6 +311,9 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
new ExpectedQueueWithProperties("root",
EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF);
List<String[]> templateProperties = new ArrayList<>();
templateProperties.add(new String[] {"maximum-applications", "300"});
validateSchedulerInfo(json, "weight",
expectedRootQ,
new ExpectedQueueWithProperties("root.auto1",
@ -307,7 +331,8 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
new ExpectedQueueWithProperties("root.autoParent1",
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE),
PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE,
templateProperties),
new ExpectedQueueWithProperties("root.default", 10.0f,
10.0f / sumOfWeights,
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
@ -474,6 +499,22 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
queuePath,
expectedQueue.creationMethod, obj.getString("creationMethod"));
if (!expectedQueue.autoQueueTemplateProperties.isEmpty()) {
JSONArray templates = obj.getJSONObject("autoQueueTemplateProperties")
.getJSONArray("property");
for (int j = 0; j < templates.length(); j++) {
JSONObject prop = templates.getJSONObject(j);
Assert.assertEquals("Auto creation eligible queue " +
"template key do not match for queue" + queuePath,
expectedQueue.autoQueueTemplateProperties.get(j)[0],
prop.getString("name"));
Assert.assertEquals("Auto creation eligible queue " +
"template value do not match for queue" + queuePath,
expectedQueue.autoQueueTemplateProperties.get(j)[1],
prop.getString("value"));
}
}
Assert.assertEquals("Queue auto creation eligibility does not " +
"match for queue " + queuePath,
expectedQueue.autoCreationEligibility,

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
View File

@ -574,7 +574,7 @@ public class TestRMWebServicesForCSWithPartitions extends JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements", 19, info.length());
assertEquals("incorrect number of elements", 20, info.length());
JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
JSONArray partitionsCapsArray =
capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);