YARN-10872. Replace getPropsWithPrefix calls in AutoCreatedQueueTemplate (#3396)
Co-authored-by: Benjamin Teke <bteke@cloudera.com>
This commit is contained in:
parent
811fd23f23
commit
971f1b8b0a
|
@ -28,11 +28,8 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
/**
|
||||
* A container class for automatically created child leaf queues.
|
||||
|
@ -173,44 +170,22 @@ public abstract class AbstractManagedParentQueue extends ParentQueue {
|
|||
return queueManagementPolicy;
|
||||
}
|
||||
|
||||
protected SortedMap<String, String> getConfigurationsWithPrefix
|
||||
(SortedMap<String, String> sortedConfigs, String prefix) {
|
||||
return sortedConfigs.subMap( prefix, prefix + Character.MAX_VALUE );
|
||||
}
|
||||
|
||||
protected SortedMap<String, String> sortCSConfigurations() {
|
||||
SortedMap<String, String> sortedConfigs = new TreeMap(
|
||||
new Comparator<String>() {
|
||||
public int compare(String s1, String s2) {
|
||||
return s1.compareToIgnoreCase(s2);
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
for (final Iterator<Map.Entry<String, String>> iterator =
|
||||
csContext.getConfiguration().iterator(); iterator.hasNext(); ) {
|
||||
final Map.Entry<String, String> confKeyValuePair = iterator.next();
|
||||
sortedConfigs.put(confKeyValuePair.getKey(), confKeyValuePair.getValue());
|
||||
}
|
||||
return sortedConfigs;
|
||||
}
|
||||
|
||||
protected CapacitySchedulerConfiguration initializeLeafQueueConfigs(String
|
||||
configPrefix) {
|
||||
|
||||
CapacitySchedulerConfiguration leafQueueConfigs = new
|
||||
CapacitySchedulerConfiguration(new Configuration(false), false);
|
||||
|
||||
String prefix = YarnConfiguration.RESOURCE_TYPES + ".";
|
||||
Map<String, String> rtProps = csContext
|
||||
.getConfiguration().getPropsWithPrefix(prefix);
|
||||
.getConfiguration().getConfigurationProperties()
|
||||
.getPropertiesWithPrefix(YarnConfiguration.RESOURCE_TYPES + ".", true);
|
||||
for (Map.Entry<String, String> entry : rtProps.entrySet()) {
|
||||
leafQueueConfigs.set(prefix + entry.getKey(), entry.getValue());
|
||||
leafQueueConfigs.set(entry.getKey(), entry.getValue());
|
||||
}
|
||||
|
||||
SortedMap<String, String> sortedConfigs = sortCSConfigurations();
|
||||
SortedMap<String, String> templateConfigs = getConfigurationsWithPrefix
|
||||
(sortedConfigs, configPrefix);
|
||||
Map<String, String> templateConfigs = csContext
|
||||
.getConfiguration().getConfigurationProperties()
|
||||
.getPropertiesWithPrefix(configPrefix, true);
|
||||
|
||||
for (final Iterator<Map.Entry<String, String>> iterator =
|
||||
templateConfigs.entrySet().iterator(); iterator.hasNext(); ) {
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
@ -50,7 +49,7 @@ public class AutoCreatedQueueTemplate {
|
|||
private final Map<String, String> leafOnlyProperties = new HashMap<>();
|
||||
private final Map<String, String> parentOnlyProperties = new HashMap<>();
|
||||
|
||||
public AutoCreatedQueueTemplate(Configuration configuration,
|
||||
public AutoCreatedQueueTemplate(CapacitySchedulerConfiguration configuration,
|
||||
String queuePath) {
|
||||
setTemplateConfigEntries(configuration, queuePath);
|
||||
}
|
||||
|
@ -91,7 +90,7 @@ public class AutoCreatedQueueTemplate {
|
|||
* @param conf configuration to set
|
||||
* @param childQueuePath child queue path used for prefixing the properties
|
||||
*/
|
||||
public void setTemplateEntriesForChild(Configuration conf,
|
||||
public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
|
||||
String childQueuePath) {
|
||||
setTemplateEntriesForChild(conf, childQueuePath, false);
|
||||
}
|
||||
|
@ -105,16 +104,20 @@ public class AutoCreatedQueueTemplate {
|
|||
* parent specific template properties
|
||||
* @param childQueuePath child queue path used for prefixing the properties
|
||||
*/
|
||||
public void setTemplateEntriesForChild(Configuration conf,
|
||||
public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
|
||||
String childQueuePath,
|
||||
boolean isLeaf) {
|
||||
if (childQueuePath.equals(ROOT)) {
|
||||
return;
|
||||
}
|
||||
|
||||
ConfigurationProperties configurationProperties =
|
||||
conf.getConfigurationProperties();
|
||||
|
||||
// Get all properties that are explicitly set
|
||||
Set<String> alreadySetProps = conf.getPropsWithPrefix(
|
||||
CapacitySchedulerConfiguration.getQueuePrefix(childQueuePath)).keySet();
|
||||
Set<String> alreadySetProps = configurationProperties
|
||||
.getPropertiesWithPrefix(CapacitySchedulerConfiguration
|
||||
.getQueuePrefix(childQueuePath)).keySet();
|
||||
|
||||
// Check template properties only set for leaf or parent queues
|
||||
Map<String, String> queueTypeSpecificTemplates = parentOnlyProperties;
|
||||
|
@ -151,8 +154,11 @@ public class AutoCreatedQueueTemplate {
|
|||
* yarn.scheduler.capacity.root.a.auto-queue-creation-v2.template.capacity
|
||||
* yarn.scheduler.capacity.root.*.auto-queue-creation-v2.template.capacity
|
||||
*/
|
||||
private void setTemplateConfigEntries(Configuration configuration,
|
||||
private void setTemplateConfigEntries(CapacitySchedulerConfiguration configuration,
|
||||
String queuePath) {
|
||||
ConfigurationProperties configurationProperties =
|
||||
configuration.getConfigurationProperties();
|
||||
|
||||
List<String> queuePathParts = new ArrayList<>(Arrays.asList(
|
||||
queuePath.split("\\.")));
|
||||
|
||||
|
@ -178,8 +184,8 @@ public class AutoCreatedQueueTemplate {
|
|||
String templateQueuePath = String.join(".", queuePathParts);
|
||||
// Get all configuration entries with
|
||||
// yarn.scheduler.capacity.<queuePath> prefix
|
||||
Map<String, String> queueProps = configuration.getPropsWithPrefix(
|
||||
getQueuePrefix(templateQueuePath));
|
||||
Map<String, String> queueProps = configurationProperties
|
||||
.getPropertiesWithPrefix(getQueuePrefix(templateQueuePath));
|
||||
|
||||
// Store template, parent-template and leaf-template properties
|
||||
for (Map.Entry<String, String> entry : queueProps.entrySet()) {
|
||||
|
|
|
@ -1607,8 +1607,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
|
|||
*/
|
||||
public Map<String, Set<String>> getConfiguredNodeLabelsByQueue() {
|
||||
Map<String, Set<String>> labelsByQueue = new HashMap<>();
|
||||
Map<String, String> schedulerEntries = getPropsWithPrefix(
|
||||
CapacitySchedulerConfiguration.PREFIX);
|
||||
Map<String, String> schedulerEntries =
|
||||
getConfigurationProperties().getPropertiesWithPrefix(
|
||||
CapacitySchedulerConfiguration.PREFIX);
|
||||
|
||||
for (Map.Entry<String, String> propertyEntry
|
||||
: schedulerEntries.entrySet()) {
|
||||
|
|
|
@ -76,7 +76,17 @@ public class ConfigurationProperties {
|
|||
String prefix, boolean fullyQualifiedKey) {
|
||||
List<String> propertyPrefixParts = splitPropertyByDelimiter(prefix);
|
||||
Map<String, String> properties = new HashMap<>();
|
||||
String trimPrefix = fullyQualifiedKey ? "" : prefix;
|
||||
String trimPrefix;
|
||||
if (fullyQualifiedKey) {
|
||||
trimPrefix = "";
|
||||
} else {
|
||||
// To support the behaviour where the
|
||||
// CapacitySchedulerConfiguration.getQueuePrefix(String queue) method
|
||||
// returned with the queue prefix with a dot appended to it the last dot
|
||||
// should be removed
|
||||
trimPrefix = prefix.endsWith(CapacitySchedulerConfiguration.DOT) ?
|
||||
prefix.substring(0, prefix.length() - 1) : prefix;
|
||||
}
|
||||
|
||||
collectPropertiesRecursively(nodes, properties,
|
||||
propertyPrefixParts.iterator(), trimPrefix);
|
||||
|
|
|
@ -58,6 +58,17 @@ public class TestConfigurationProperties {
|
|||
Assert.assertTrue(props.containsKey("4.5"));
|
||||
Assert.assertEquals("TEST_VALUE_3_2", props.get("4.5"));
|
||||
|
||||
// Test the scenario where the prefix has a dot appended to it
|
||||
// (see CapacitySchedulerConfiguration.getQueuePrefix(String queue)).
|
||||
// The dot is disregarded.
|
||||
props = configurationProperties
|
||||
.getPropertiesWithPrefix("root.1.2.4.");
|
||||
|
||||
Assert.assertEquals(2, props.size());
|
||||
Assert.assertTrue(props.containsKey(""));
|
||||
Assert.assertEquals("TEST_VALUE_3_1", props.get(""));
|
||||
Assert.assertTrue(props.containsKey("5"));
|
||||
Assert.assertEquals("TEST_VALUE_3_2", props.get("5"));
|
||||
|
||||
Map<String, String> propsWithRootPrefix = configurationProperties
|
||||
.getPropertiesWithPrefix("root");
|
||||
|
|
Loading…
Reference in New Issue