YARN-10983. Follow-up changes for YARN-10904. Contributed by Benjamin Teke

This commit is contained in:
9uapaw 2022-03-02 11:16:24 +01:00
parent 9199787215
commit 9e475aede6
3 changed files with 19 additions and 22 deletions

View File

@ -331,7 +331,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// Collect and set the Node label configuration // Collect and set the Node label configuration
this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, parent, this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, parent,
getQueuePath(), queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues()); queuePath, queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues());
// Initialize the queue capacities // Initialize the queue capacities
setupConfigurableCapacities(); setupConfigurableCapacities();
@ -379,7 +379,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// Setup application related limits // Setup application related limits
this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration, this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration,
this, getQueuePath()); this, queuePath);
} finally { } finally {
writeLock.unlock(); writeLock.unlock();
} }

View File

@ -35,9 +35,9 @@ public class QueueAppLifetimeAndLimitSettings {
private int maxParallelApps; private int maxParallelApps;
public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration, public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration,
AbstractCSQueue q, String queuePath) { AbstractCSQueue q, QueuePath queuePath) {
// Store max parallel apps property // Store max parallel apps property
this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath); this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath.getFullPath());
this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration); this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration);
this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration, this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration,
maxApplicationLifetime); maxApplicationLifetime);
@ -48,7 +48,7 @@ public class QueueAppLifetimeAndLimitSettings {
long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath()); long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath());
// If q is the root queue, then get max app lifetime from conf. // If q is the root queue, then get max app lifetime from conf.
if (parentQ == null) { if (q.getQueuePathObject().isRoot()) {
return maxAppLifetime; return maxAppLifetime;
} }
@ -62,16 +62,16 @@ public class QueueAppLifetimeAndLimitSettings {
} }
private long setupInheritedDefaultAppLifetime(CSQueue q, private long setupInheritedDefaultAppLifetime(CSQueue q,
String queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) { QueuePath queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) {
CSQueue parentQ = q.getParent(); CSQueue parentQ = q.getParent();
long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath); long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath.getFullPath());
defaultAppLifetimeWasSpecifiedInConfig = defaultAppLifetimeWasSpecifiedInConfig =
(defaultAppLifetime >= 0 (defaultAppLifetime >= 0
|| (parentQ != null && || (!queuePath.isRoot() &&
parentQ.getDefaultAppLifetimeWasSpecifiedInConfig())); parentQ.getDefaultAppLifetimeWasSpecifiedInConfig()));
// If q is the root queue, then get default app lifetime from conf. // If q is the root queue, then get default app lifetime from conf.
if (parentQ == null) { if (queuePath.isRoot()) {
return defaultAppLifetime; return defaultAppLifetime;
} }

View File

@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.util.Sets; import org.apache.hadoop.util.Sets;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
import java.io.IOException; import java.io.IOException;
import java.util.Set; import java.util.Set;
@ -30,14 +29,14 @@ import java.util.Set;
*/ */
public class QueueNodeLabelsSettings { public class QueueNodeLabelsSettings {
private final CSQueue parent; private final CSQueue parent;
private final String queuePath; private final QueuePath queuePath;
private Set<String> accessibleLabels; private Set<String> accessibleLabels;
private Set<String> configuredNodeLabels; private Set<String> configuredNodeLabels;
private String defaultLabelExpression; private String defaultLabelExpression;
public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration, public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration,
CSQueue parent, CSQueue parent,
String queuePath, QueuePath queuePath,
ConfiguredNodeLabels configuredNodeLabels) throws IOException { ConfiguredNodeLabels configuredNodeLabels) throws IOException {
this.parent = parent; this.parent = parent;
this.queuePath = queuePath; this.queuePath = queuePath;
@ -54,7 +53,7 @@ public class QueueNodeLabelsSettings {
} }
private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) { private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) {
this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath); this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath.getFullPath());
// Inherit labels from parent if not set // Inherit labels from parent if not set
if (this.accessibleLabels == null && parent != null) { if (this.accessibleLabels == null && parent != null) {
this.accessibleLabels = parent.getAccessibleNodeLabels(); this.accessibleLabels = parent.getAccessibleNodeLabels();
@ -62,7 +61,8 @@ public class QueueNodeLabelsSettings {
} }
private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) { private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) {
this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(queuePath); this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(
queuePath.getFullPath());
// If the accessible labels is not null and the queue has a parent with a // If the accessible labels is not null and the queue has a parent with a
// similar set of labels copy the defaultNodeLabelExpression from the parent // similar set of labels copy the defaultNodeLabelExpression from the parent
if (this.accessibleLabels != null && parent != null if (this.accessibleLabels != null && parent != null
@ -75,21 +75,22 @@ public class QueueNodeLabelsSettings {
private void initializeConfiguredNodeLabels(CapacitySchedulerConfiguration configuration, private void initializeConfiguredNodeLabels(CapacitySchedulerConfiguration configuration,
ConfiguredNodeLabels configuredNodeLabelsParam) { ConfiguredNodeLabels configuredNodeLabelsParam) {
if (configuredNodeLabelsParam != null) { if (configuredNodeLabelsParam != null) {
if (queuePath.equals(ROOT)) { if (queuePath.isRoot()) {
this.configuredNodeLabels = configuredNodeLabelsParam.getAllConfiguredLabels(); this.configuredNodeLabels = configuredNodeLabelsParam.getAllConfiguredLabels();
} else { } else {
this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue(queuePath); this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue(
queuePath.getFullPath());
} }
} else { } else {
// Fallback to suboptimal but correct logic // Fallback to suboptimal but correct logic
this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath); this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath.getFullPath());
} }
} }
private void validateNodeLabels() throws IOException { private void validateNodeLabels() throws IOException {
// Check if labels of this queue is a subset of parent queue, only do this // Check if labels of this queue is a subset of parent queue, only do this
// when the queue in question is not root // when the queue in question is not root
if (isNotRoot()) { if (!queuePath.isRoot()) {
if (parent.getAccessibleNodeLabels() != null && !parent if (parent.getAccessibleNodeLabels() != null && !parent
.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) { .getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
// If parent isn't "*", child shouldn't be "*" too // If parent isn't "*", child shouldn't be "*" too
@ -109,10 +110,6 @@ public class QueueNodeLabelsSettings {
} }
} }
private boolean isNotRoot() {
return parent != null && parent.getParent() != null;
}
public boolean isAccessibleToPartition(String nodePartition) { public boolean isAccessibleToPartition(String nodePartition) {
// if queue's label is *, it can access any node // if queue's label is *, it can access any node
if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) { if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) {