YARN-10983. Follow-up changes for YARN-10904. Contributed by Benjamin Teke
This commit is contained in:
parent
9199787215
commit
9e475aede6
|
@ -331,7 +331,7 @@ public abstract class AbstractCSQueue implements CSQueue {
|
||||||
|
|
||||||
// Collect and set the Node label configuration
|
// Collect and set the Node label configuration
|
||||||
this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, parent,
|
this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, parent,
|
||||||
getQueuePath(), queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues());
|
queuePath, queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues());
|
||||||
|
|
||||||
// Initialize the queue capacities
|
// Initialize the queue capacities
|
||||||
setupConfigurableCapacities();
|
setupConfigurableCapacities();
|
||||||
|
@ -379,7 +379,7 @@ public abstract class AbstractCSQueue implements CSQueue {
|
||||||
|
|
||||||
// Setup application related limits
|
// Setup application related limits
|
||||||
this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration,
|
this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration,
|
||||||
this, getQueuePath());
|
this, queuePath);
|
||||||
} finally {
|
} finally {
|
||||||
writeLock.unlock();
|
writeLock.unlock();
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,9 +35,9 @@ public class QueueAppLifetimeAndLimitSettings {
|
||||||
private int maxParallelApps;
|
private int maxParallelApps;
|
||||||
|
|
||||||
public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration,
|
public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration,
|
||||||
AbstractCSQueue q, String queuePath) {
|
AbstractCSQueue q, QueuePath queuePath) {
|
||||||
// Store max parallel apps property
|
// Store max parallel apps property
|
||||||
this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath);
|
this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath.getFullPath());
|
||||||
this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration);
|
this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration);
|
||||||
this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration,
|
this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration,
|
||||||
maxApplicationLifetime);
|
maxApplicationLifetime);
|
||||||
|
@ -48,7 +48,7 @@ public class QueueAppLifetimeAndLimitSettings {
|
||||||
long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath());
|
long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath());
|
||||||
|
|
||||||
// If q is the root queue, then get max app lifetime from conf.
|
// If q is the root queue, then get max app lifetime from conf.
|
||||||
if (parentQ == null) {
|
if (q.getQueuePathObject().isRoot()) {
|
||||||
return maxAppLifetime;
|
return maxAppLifetime;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -62,16 +62,16 @@ public class QueueAppLifetimeAndLimitSettings {
|
||||||
}
|
}
|
||||||
|
|
||||||
private long setupInheritedDefaultAppLifetime(CSQueue q,
|
private long setupInheritedDefaultAppLifetime(CSQueue q,
|
||||||
String queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) {
|
QueuePath queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) {
|
||||||
CSQueue parentQ = q.getParent();
|
CSQueue parentQ = q.getParent();
|
||||||
long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath);
|
long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath.getFullPath());
|
||||||
defaultAppLifetimeWasSpecifiedInConfig =
|
defaultAppLifetimeWasSpecifiedInConfig =
|
||||||
(defaultAppLifetime >= 0
|
(defaultAppLifetime >= 0
|
||||||
|| (parentQ != null &&
|
|| (!queuePath.isRoot() &&
|
||||||
parentQ.getDefaultAppLifetimeWasSpecifiedInConfig()));
|
parentQ.getDefaultAppLifetimeWasSpecifiedInConfig()));
|
||||||
|
|
||||||
// If q is the root queue, then get default app lifetime from conf.
|
// If q is the root queue, then get default app lifetime from conf.
|
||||||
if (parentQ == null) {
|
if (queuePath.isRoot()) {
|
||||||
return defaultAppLifetime;
|
return defaultAppLifetime;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.hadoop.util.Sets;
|
import org.apache.hadoop.util.Sets;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
|
@ -30,14 +29,14 @@ import java.util.Set;
|
||||||
*/
|
*/
|
||||||
public class QueueNodeLabelsSettings {
|
public class QueueNodeLabelsSettings {
|
||||||
private final CSQueue parent;
|
private final CSQueue parent;
|
||||||
private final String queuePath;
|
private final QueuePath queuePath;
|
||||||
private Set<String> accessibleLabels;
|
private Set<String> accessibleLabels;
|
||||||
private Set<String> configuredNodeLabels;
|
private Set<String> configuredNodeLabels;
|
||||||
private String defaultLabelExpression;
|
private String defaultLabelExpression;
|
||||||
|
|
||||||
public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration,
|
public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration,
|
||||||
CSQueue parent,
|
CSQueue parent,
|
||||||
String queuePath,
|
QueuePath queuePath,
|
||||||
ConfiguredNodeLabels configuredNodeLabels) throws IOException {
|
ConfiguredNodeLabels configuredNodeLabels) throws IOException {
|
||||||
this.parent = parent;
|
this.parent = parent;
|
||||||
this.queuePath = queuePath;
|
this.queuePath = queuePath;
|
||||||
|
@ -54,7 +53,7 @@ public class QueueNodeLabelsSettings {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) {
|
private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) {
|
||||||
this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath);
|
this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath.getFullPath());
|
||||||
// Inherit labels from parent if not set
|
// Inherit labels from parent if not set
|
||||||
if (this.accessibleLabels == null && parent != null) {
|
if (this.accessibleLabels == null && parent != null) {
|
||||||
this.accessibleLabels = parent.getAccessibleNodeLabels();
|
this.accessibleLabels = parent.getAccessibleNodeLabels();
|
||||||
|
@ -62,7 +61,8 @@ public class QueueNodeLabelsSettings {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) {
|
private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) {
|
||||||
this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(queuePath);
|
this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(
|
||||||
|
queuePath.getFullPath());
|
||||||
// If the accessible labels is not null and the queue has a parent with a
|
// If the accessible labels is not null and the queue has a parent with a
|
||||||
// similar set of labels copy the defaultNodeLabelExpression from the parent
|
// similar set of labels copy the defaultNodeLabelExpression from the parent
|
||||||
if (this.accessibleLabels != null && parent != null
|
if (this.accessibleLabels != null && parent != null
|
||||||
|
@ -75,21 +75,22 @@ public class QueueNodeLabelsSettings {
|
||||||
private void initializeConfiguredNodeLabels(CapacitySchedulerConfiguration configuration,
|
private void initializeConfiguredNodeLabels(CapacitySchedulerConfiguration configuration,
|
||||||
ConfiguredNodeLabels configuredNodeLabelsParam) {
|
ConfiguredNodeLabels configuredNodeLabelsParam) {
|
||||||
if (configuredNodeLabelsParam != null) {
|
if (configuredNodeLabelsParam != null) {
|
||||||
if (queuePath.equals(ROOT)) {
|
if (queuePath.isRoot()) {
|
||||||
this.configuredNodeLabels = configuredNodeLabelsParam.getAllConfiguredLabels();
|
this.configuredNodeLabels = configuredNodeLabelsParam.getAllConfiguredLabels();
|
||||||
} else {
|
} else {
|
||||||
this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue(queuePath);
|
this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue(
|
||||||
|
queuePath.getFullPath());
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Fallback to suboptimal but correct logic
|
// Fallback to suboptimal but correct logic
|
||||||
this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath);
|
this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath.getFullPath());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void validateNodeLabels() throws IOException {
|
private void validateNodeLabels() throws IOException {
|
||||||
// Check if labels of this queue is a subset of parent queue, only do this
|
// Check if labels of this queue is a subset of parent queue, only do this
|
||||||
// when the queue in question is not root
|
// when the queue in question is not root
|
||||||
if (isNotRoot()) {
|
if (!queuePath.isRoot()) {
|
||||||
if (parent.getAccessibleNodeLabels() != null && !parent
|
if (parent.getAccessibleNodeLabels() != null && !parent
|
||||||
.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
|
.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
|
||||||
// If parent isn't "*", child shouldn't be "*" too
|
// If parent isn't "*", child shouldn't be "*" too
|
||||||
|
@ -109,10 +110,6 @@ public class QueueNodeLabelsSettings {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isNotRoot() {
|
|
||||||
return parent != null && parent.getParent() != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isAccessibleToPartition(String nodePartition) {
|
public boolean isAccessibleToPartition(String nodePartition) {
|
||||||
// if queue's label is *, it can access any node
|
// if queue's label is *, it can access any node
|
||||||
if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) {
|
if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) {
|
||||||
|
|
Loading…
Reference in New Issue