YARN-10790. CS Flexible AQC: Add separate parent and leaf template property. Contributed by Andras Gyori

This commit is contained in:
Szilard Nemeth 2021-07-28 16:50:14 +02:00
parent b19dae8db3
commit 1b9efe58c9
7 changed files with 165 additions and 23 deletions

View File

@ -30,6 +30,7 @@ import java.util.Set;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_PREFIX;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix;
/** /**
* A handler for storing and setting auto created queue template settings. * A handler for storing and setting auto created queue template settings.
@ -37,10 +38,17 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
public class AutoCreatedQueueTemplate { public class AutoCreatedQueueTemplate {
public static final String AUTO_QUEUE_TEMPLATE_PREFIX = public static final String AUTO_QUEUE_TEMPLATE_PREFIX =
AUTO_QUEUE_CREATION_V2_PREFIX + "template."; AUTO_QUEUE_CREATION_V2_PREFIX + "template.";
public static final String AUTO_QUEUE_LEAF_TEMPLATE_PREFIX =
AUTO_QUEUE_CREATION_V2_PREFIX + "leaf-template.";
public static final String AUTO_QUEUE_PARENT_TEMPLATE_PREFIX =
AUTO_QUEUE_CREATION_V2_PREFIX + "parent-template.";
private static final String WILDCARD_QUEUE = "*"; private static final String WILDCARD_QUEUE = "*";
private static final int MAX_WILDCARD_LEVEL = 1; private static final int MAX_WILDCARD_LEVEL = 1;
private final Map<String, String> templateProperties = new HashMap<>(); private final Map<String, String> templateProperties = new HashMap<>();
private final Map<String, String> leafOnlyProperties = new HashMap<>();
private final Map<String, String> parentOnlyProperties = new HashMap<>();
public AutoCreatedQueueTemplate(Configuration configuration, public AutoCreatedQueueTemplate(Configuration configuration,
String queuePath) { String queuePath) {
@ -54,7 +62,7 @@ public class AutoCreatedQueueTemplate {
} }
/** /**
* Get the template properties attached to a parent queue. * Get the common template properties specified for a parent queue.
* @return template property names and values * @return template property names and values
*/ */
public Map<String, String> getTemplateProperties() { public Map<String, String> getTemplateProperties() {
@ -62,13 +70,44 @@ public class AutoCreatedQueueTemplate {
} }
/** /**
* Sets the configuration properties of a child queue based on its parent * Get the leaf specific template properties specified for a parent queue.
* template settings. * @return template property names and values
* @param conf configuration to set */
public Map<String, String> getLeafOnlyProperties() {
return leafOnlyProperties;
}
/**
* Get the parent specific template properties specified for a parent queue.
* @return template property names and values
*/
public Map<String, String> getParentOnlyProperties() {
return parentOnlyProperties;
}
/**
* Sets the common template properties and parent specific template
* properties of a child queue based on its parent template settings.
* @param conf configuration to set
* @param childQueuePath child queue path used for prefixing the properties * @param childQueuePath child queue path used for prefixing the properties
*/ */
public void setTemplateEntriesForChild(Configuration conf, public void setTemplateEntriesForChild(Configuration conf,
String childQueuePath) { String childQueuePath) {
setTemplateEntriesForChild(conf, childQueuePath, false);
}
/**
* Sets the common template properties and leaf or parent specific template
* properties of a child queue based on its parent template settings.
* template settings.
* @param conf configuration to set
* @param isLeaf whether to include leaf specific template properties, or
* parent specific template properties
* @param childQueuePath child queue path used for prefixing the properties
*/
public void setTemplateEntriesForChild(Configuration conf,
String childQueuePath,
boolean isLeaf) {
if (childQueuePath.equals(ROOT)) { if (childQueuePath.equals(ROOT)) {
return; return;
} }
@ -77,7 +116,14 @@ public class AutoCreatedQueueTemplate {
Set<String> alreadySetProps = conf.getPropsWithPrefix( Set<String> alreadySetProps = conf.getPropsWithPrefix(
CapacitySchedulerConfiguration.getQueuePrefix(childQueuePath)).keySet(); CapacitySchedulerConfiguration.getQueuePrefix(childQueuePath)).keySet();
for (Map.Entry<String, String> entry : templateProperties.entrySet()) { // Check template properties only set for leaf or parent queues
Map<String, String> queueTypeSpecificTemplates = parentOnlyProperties;
if (isLeaf) {
queueTypeSpecificTemplates = leafOnlyProperties;
}
for (Map.Entry<String, String> entry :
queueTypeSpecificTemplates.entrySet()) {
// Do not overwrite explicitly configured properties // Do not overwrite explicitly configured properties
if (alreadySetProps.contains(entry.getKey())) { if (alreadySetProps.contains(entry.getKey())) {
continue; continue;
@ -85,6 +131,17 @@ public class AutoCreatedQueueTemplate {
conf.set(CapacitySchedulerConfiguration.getQueuePrefix( conf.set(CapacitySchedulerConfiguration.getQueuePrefix(
childQueuePath) + entry.getKey(), entry.getValue()); childQueuePath) + entry.getKey(), entry.getValue());
} }
for (Map.Entry<String, String> entry : templateProperties.entrySet()) {
// Do not overwrite explicitly configured properties or properties set
// by queue type specific templates (parent-template and leaf-template)
if (alreadySetProps.contains(entry.getKey())
|| queueTypeSpecificTemplates.containsKey(entry.getKey())) {
continue;
}
conf.set(CapacitySchedulerConfiguration.getQueuePrefix(
childQueuePath) + entry.getKey(), entry.getValue());
}
} }
/** /**
@ -118,16 +175,15 @@ public class AutoCreatedQueueTemplate {
// Collect all template entries // Collect all template entries
while (wildcardLevel <= supportedWildcardLevel) { while (wildcardLevel <= supportedWildcardLevel) {
// Get all config entries with the specified prefix
String templateQueuePath = String.join(".", queuePathParts); String templateQueuePath = String.join(".", queuePathParts);
// Get all configuration entries with // Get all configuration entries with
// <queuePath>.auto-queue-creation-v2.template prefix // yarn.scheduler.capacity.<queuePath> prefix
Map<String, String> props = configuration.getPropsWithPrefix( Map<String, String> queueProps = configuration.getPropsWithPrefix(
getAutoQueueTemplatePrefix(templateQueuePath)); getQueuePrefix(templateQueuePath));
for (Map.Entry<String, String> entry : props.entrySet()) { // Store template, parent-template and leaf-template properties
// If an entry is already present, it had a higher precedence for (Map.Entry<String, String> entry : queueProps.entrySet()) {
templateProperties.putIfAbsent(entry.getKey(), entry.getValue()); storeConfiguredTemplates(entry.getKey(), entry.getValue());
} }
// Replace a queue part with a wildcard based on the wildcard level // Replace a queue part with a wildcard based on the wildcard level
@ -138,4 +194,28 @@ public class AutoCreatedQueueTemplate {
++wildcardLevel; ++wildcardLevel;
} }
} }
private void storeConfiguredTemplates(
String templateKey, String templateValue) {
String prefix = "";
Map<String, String> properties = templateProperties;
if (templateKey.startsWith(AUTO_QUEUE_TEMPLATE_PREFIX)) {
prefix = AUTO_QUEUE_TEMPLATE_PREFIX;
} else if (templateKey.startsWith(AUTO_QUEUE_LEAF_TEMPLATE_PREFIX)) {
prefix = AUTO_QUEUE_LEAF_TEMPLATE_PREFIX;
properties = leafOnlyProperties;
} else if (templateKey.startsWith(
AUTO_QUEUE_PARENT_TEMPLATE_PREFIX)) {
prefix = AUTO_QUEUE_PARENT_TEMPLATE_PREFIX;
properties = parentOnlyProperties;
}
if (!prefix.isEmpty()) {
// Trim template prefix from key
String key = templateKey.substring(prefix.length());
// If an entry is already present, it had a higher precedence
properties.putIfAbsent(key, templateValue);
}
}
} }

View File

@ -63,6 +63,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected String autoCreationEligibility; protected String autoCreationEligibility;
protected String defaultNodeLabelExpression; protected String defaultNodeLabelExpression;
protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties; protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties;
protected AutoQueueTemplatePropertiesInfo autoQueueParentTemplateProperties;
protected AutoQueueTemplatePropertiesInfo autoQueueLeafTemplateProperties;
@XmlTransient @XmlTransient
static final float EPSILON = 1e-8f; static final float EPSILON = 1e-8f;
@ -108,10 +110,18 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
queuePriority = parent.getPriority().getPriority(); queuePriority = parent.getPriority().getPriority();
if (parent instanceof ParentQueue) { if (parent instanceof ParentQueue) {
orderingPolicyInfo = ((ParentQueue) parent).getQueueOrderingPolicy() ParentQueue queue = (ParentQueue) parent;
orderingPolicyInfo = queue.getQueueOrderingPolicy()
.getConfigName(); .getConfigName();
autoQueueTemplateProperties = CapacitySchedulerInfoHelper autoQueueTemplateProperties = CapacitySchedulerInfoHelper
.getAutoCreatedTemplate((ParentQueue) parent); .getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate()
.getTemplateProperties());
autoQueueParentTemplateProperties = CapacitySchedulerInfoHelper
.getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate()
.getParentOnlyProperties());
autoQueueLeafTemplateProperties = CapacitySchedulerInfoHelper
.getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate()
.getLeafOnlyProperties());
} }
mode = CapacitySchedulerInfoHelper.getMode(parent); mode = CapacitySchedulerInfoHelper.getMode(parent);
queueType = CapacitySchedulerInfoHelper.getQueueType(parent); queueType = CapacitySchedulerInfoHelper.getQueueType(parent);

View File

@ -92,6 +92,10 @@ public class CapacitySchedulerQueueInfo {
protected String defaultNodeLabelExpression; protected String defaultNodeLabelExpression;
protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties = protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties =
new AutoQueueTemplatePropertiesInfo(); new AutoQueueTemplatePropertiesInfo();
protected AutoQueueTemplatePropertiesInfo autoQueueParentTemplateProperties =
new AutoQueueTemplatePropertiesInfo();
protected AutoQueueTemplatePropertiesInfo autoQueueLeafTemplateProperties =
new AutoQueueTemplatePropertiesInfo();
CapacitySchedulerQueueInfo() { CapacitySchedulerQueueInfo() {
}; };
@ -172,10 +176,18 @@ public class CapacitySchedulerQueueInfo {
queuePriority = q.getPriority().getPriority(); queuePriority = q.getPriority().getPriority();
if (q instanceof ParentQueue) { if (q instanceof ParentQueue) {
orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() ParentQueue queue = (ParentQueue) q;
orderingPolicyInfo = queue.getQueueOrderingPolicy()
.getConfigName(); .getConfigName();
autoQueueTemplateProperties = CapacitySchedulerInfoHelper autoQueueTemplateProperties = CapacitySchedulerInfoHelper
.getAutoCreatedTemplate((ParentQueue) q); .getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate()
.getTemplateProperties());
autoQueueParentTemplateProperties = CapacitySchedulerInfoHelper
.getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate()
.getParentOnlyProperties());
autoQueueLeafTemplateProperties = CapacitySchedulerInfoHelper
.getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate()
.getLeafOnlyProperties());
} }
String configuredCapacity = conf.get( String configuredCapacity = conf.get(

View File

@ -112,12 +112,11 @@ public class CapacitySchedulerInfoHelper {
} }
public static AutoQueueTemplatePropertiesInfo getAutoCreatedTemplate( public static AutoQueueTemplatePropertiesInfo getAutoCreatedTemplate(
ParentQueue parent) { Map<String, String> templateProperties) {
AutoQueueTemplatePropertiesInfo propertiesInfo = AutoQueueTemplatePropertiesInfo propertiesInfo =
new AutoQueueTemplatePropertiesInfo(); new AutoQueueTemplatePropertiesInfo();
for (Map.Entry<String, String> e : for (Map.Entry<String, String> e :
parent.getAutoCreatedQueueTemplate().getTemplateProperties() templateProperties.entrySet()) {
.entrySet()) {
propertiesInfo.add(new ConfItem(e.getKey(), e.getValue())); propertiesInfo.add(new ConfItem(e.getKey(), e.getValue()));
} }

View File

@ -22,10 +22,13 @@ import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE;
public class TestAutoCreatedQueueTemplate { public class TestAutoCreatedQueueTemplate {
private static final String TEST_QUEUE_ABC = "root.a.b.c"; private static final String TEST_QUEUE_ABC = "root.a.b.c";
private static final String TEST_QUEUE_AB = "root.a.b"; private static final String TEST_QUEUE_AB = "root.a.b";
private static final String TEST_QUEUE_A = "root.a"; private static final String TEST_QUEUE_A = "root.a";
private static final String TEST_QUEUE_B = "root.b";
private static final String ROOT = "root"; private static final String ROOT = "root";
private CapacitySchedulerConfiguration conf; private CapacitySchedulerConfiguration conf;
@ -120,8 +123,46 @@ public class TestAutoCreatedQueueTemplate {
conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6);
} }
@Test
public void testQueueSpecificTemplates() {
conf.set(getTemplateKey("root", "capacity"), "2w");
conf.set(getLeafTemplateKey("root",
"default-node-label-expression"), "test");
conf.set(getLeafTemplateKey("root", "capacity"), "10w");
conf.setBoolean(getParentTemplateKey(
"root", AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE), false);
AutoCreatedQueueTemplate template =
new AutoCreatedQueueTemplate(conf, ROOT);
template.setTemplateEntriesForChild(conf, TEST_QUEUE_A);
template.setTemplateEntriesForChild(conf, TEST_QUEUE_B, true);
Assert.assertNull("default-node-label-expression is set for parent",
conf.getDefaultNodeLabelExpression(TEST_QUEUE_A));
Assert.assertEquals("default-node-label-expression is not set for leaf",
"test", conf.getDefaultNodeLabelExpression(TEST_QUEUE_B));
Assert.assertFalse("auto queue removal is not disabled for parent",
conf.isAutoExpiredDeletionEnabled(TEST_QUEUE_A));
Assert.assertEquals("weight should not be overridden when set by " +
"queue type specific template",
10f, conf.getNonLabeledQueueWeight(TEST_QUEUE_B), 10e-6);
Assert.assertEquals("weight should be set by common template",
2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6);
}
private String getTemplateKey(String queuePath, String entryKey) { private String getTemplateKey(String queuePath, String entryKey) {
return CapacitySchedulerConfiguration.getQueuePrefix(queuePath) return CapacitySchedulerConfiguration.getQueuePrefix(queuePath)
+ AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + entryKey; + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + entryKey;
} }
private String getParentTemplateKey(String queuePath, String entryKey) {
return CapacitySchedulerConfiguration.getQueuePrefix(queuePath)
+ AutoCreatedQueueTemplate.AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + entryKey;
}
private String getLeafTemplateKey(String queuePath, String entryKey) {
return CapacitySchedulerConfiguration.getQueuePrefix(queuePath)
+ AutoCreatedQueueTemplate.AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + entryKey;
}
} }

View File

@ -369,7 +369,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler"); JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length()); assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo"); info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements in: " + info, 20, info.length()); assertEquals("incorrect number of elements in: " + info, 22, info.length());
verifyClusterSchedulerGeneric(info.getString("type"), verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"), (float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"), (float) info.getDouble("capacity"),
@ -424,10 +424,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifySubQueue(JSONObject info, String q, private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity) float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception { throws JSONException, Exception {
int numExpectedElements = 35; int numExpectedElements = 37;
boolean isParentQueue = true; boolean isParentQueue = true;
if (!info.has("queues")) { if (!info.has("queues")) {
numExpectedElements = 53; numExpectedElements = 55;
isParentQueue = false; isParentQueue = false;
} }
assertEquals("incorrect number of elements", numExpectedElements, info.length()); assertEquals("incorrect number of elements", numExpectedElements, info.length());

View File

@ -574,7 +574,7 @@ public class TestRMWebServicesForCSWithPartitions extends JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler"); JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length()); assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo"); info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements", 20, info.length()); assertEquals("incorrect number of elements", 22, info.length());
JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES); JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
JSONArray partitionsCapsArray = JSONArray partitionsCapsArray =
capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION); capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);