diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index bc19862b632..6c6dbca6755 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -288,6 +288,9 @@ Release 2.8.0 - UNRELEASED YARN-3584. Fixed attempt diagnostics format shown on the UI. (nijel via jianhe) + YARN-2918. RM should not fail on startup if queue's configured labels do + not exist in cluster-node-labels. (Wangda Tan via jianhe) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 7244b17f02c..cd1dacf92fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -504,7 +504,7 @@ public class ApplicationMasterService extends AbstractService implements try { RMServerUtils.normalizeAndValidateRequests(ask, rScheduler.getMaximumResourceCapability(), app.getQueue(), - rScheduler); + rScheduler, rmContext); } catch (InvalidResourceRequestException e) { LOG.warn("Invalid resource ask by application " + appAttemptId, e); throw e; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index ca21f113f52..799042191eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -383,7 +383,7 @@ public class RMAppManager implements EventHandler, try { SchedulerUtils.normalizeAndValidateRequest(amReq, scheduler.getMaximumResourceCapability(), - submissionContext.getQueue(), scheduler, isRecovery); + submissionContext.getQueue(), scheduler, isRecovery, rmContext); } catch (InvalidResourceRequestException e) { LOG.warn("RM app submission failed in validating AM resource request" + " for application " + submissionContext.getApplicationId(), e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 083189a5b9f..4669a28e38d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -91,11 +91,12 @@ public class RMServerUtils { * requested memory/vcore is non-negative and not greater than max */ public static void normalizeAndValidateRequests(List ask, - Resource maximumResource, String queueName, YarnScheduler scheduler) + Resource maximumResource, String queueName, YarnScheduler scheduler, + RMContext rmContext) throws InvalidResourceRequestException { for (ResourceRequest resReq : ask) { SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource, - queueName, scheduler); + queueName, scheduler, rmContext); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index 5e9ea5d921b..0ef5c1e5619 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -210,7 +211,7 @@ public class SchedulerUtils { public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, - boolean isRecovery) + boolean isRecovery, RMContext rmContext) throws InvalidResourceRequestException { QueueInfo queueInfo = null; @@ -222,15 +223,16 @@ public class SchedulerUtils { } SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo); if (!isRecovery) { - validateResourceRequest(resReq, maximumResource, queueInfo); + validateResourceRequest(resReq, maximumResource, queueInfo, rmContext); } } public static void normalizeAndvalidateRequest(ResourceRequest resReq, - Resource maximumResource, String queueName, YarnScheduler scheduler) + Resource maximumResource, String queueName, YarnScheduler scheduler, + RMContext rmContext) throws InvalidResourceRequestException { normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, - false); + false, rmContext); } /** @@ -239,8 +241,8 @@ public class SchedulerUtils { * * @throws InvalidResourceRequestException when there is invalid request */ - public static void validateResourceRequest(ResourceRequest resReq, - Resource maximumResource, QueueInfo queueInfo) + private static void validateResourceRequest(ResourceRequest resReq, + Resource maximumResource, QueueInfo queueInfo, RMContext rmContext) throws InvalidResourceRequestException { if (resReq.getCapability().getMemory() < 0 || resReq.getCapability().getMemory() > maximumResource.getMemory()) { @@ -282,7 +284,7 @@ public class SchedulerUtils { if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) { if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(), - labelExp)) { + labelExp, rmContext)) { throw new InvalidResourceRequestException("Invalid resource request" + ", queue=" + queueInfo.getQueueName() @@ -295,40 +297,37 @@ public class SchedulerUtils { } } } - - public static void checkIfLabelInClusterNodeLabels(RMNodeLabelsManager mgr, - Set labels) throws IOException { - if (mgr == null) { - if (labels != null && !labels.isEmpty()) { - throw new IOException("NodeLabelManager is null, please check"); - } - return; - } - - if (labels != null) { - for (String label : labels) { - if (!label.equals(RMNodeLabelsManager.ANY) - && !mgr.containsNodeLabel(label)) { - throw new IOException("NodeLabelManager doesn't include label = " - + label + ", please check."); - } - } - } - } + /** + * Check queue label expression, check if node label in queue's + * node-label-expression existed in clusterNodeLabels if rmContext != null + */ public static boolean checkQueueLabelExpression(Set queueLabels, - String labelExpression) { - if (queueLabels != null && queueLabels.contains(RMNodeLabelsManager.ANY)) { - return true; - } + String labelExpression, RMContext rmContext) { // if label expression is empty, we can allocate container on any node if (labelExpression == null) { return true; } for (String str : labelExpression.split("&&")) { - if (!str.trim().isEmpty() - && (queueLabels == null || !queueLabels.contains(str.trim()))) { - return false; + str = str.trim(); + if (!str.trim().isEmpty()) { + // check queue label + if (queueLabels == null) { + return false; + } else { + if (!queueLabels.contains(str) + && !queueLabels.contains(RMNodeLabelsManager.ANY)) { + return false; + } + } + + // check node label manager contains this label + if (null != rmContext) { + RMNodeLabelsManager nlm = rmContext.getNodeLabelManager(); + if (nlm != null && !nlm.containsNodeLabel(str)) { + return false; + } + } } } return true; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 47cea19dbde..8b4637a4dd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -118,11 +118,9 @@ public abstract class AbstractCSQueue implements CSQueue { protected void setupConfigurableCapacities() { CSQueueUtils.loadUpdateAndCheckCapacities( getQueuePath(), - accessibleLabels, csContext.getConfiguration(), queueCapacities, - parent == null ? null : parent.getQueueCapacities(), - csContext.getRMContext().getNodeLabelManager()); + parent == null ? null : parent.getQueueCapacities()); } @Override @@ -248,8 +246,6 @@ public abstract class AbstractCSQueue implements CSQueue { if (this.accessibleLabels == null && parent != null) { this.accessibleLabels = parent.getAccessibleNodeLabels(); } - SchedulerUtils.checkIfLabelInClusterNodeLabels(labelManager, - this.accessibleLabels); // inherit from parent if labels not set if (this.defaultLabelExpression == null && parent != null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 0cc75893235..1206026f6a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -116,10 +116,9 @@ class CSQueueUtils { * - Check if capacities/absolute-capacities legal */ public static void loadUpdateAndCheckCapacities(String queuePath, - Set accessibleLabels, CapacitySchedulerConfiguration csConf, - QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities, - RMNodeLabelsManager nlm) { - loadCapacitiesByLabelsFromConf(queuePath, accessibleLabels, nlm, + CapacitySchedulerConfiguration csConf, + QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) { + loadCapacitiesByLabelsFromConf(queuePath, queueCapacities, csConf); updateAbsoluteCapacitiesByNodeLabels(queueCapacities, parentQueueCapacities); @@ -127,28 +126,13 @@ class CSQueueUtils { capacitiesSanityCheck(queuePath, queueCapacities); } - // Considered NO_LABEL, ANY and null cases - private static Set normalizeAccessibleNodeLabels(Set labels, - RMNodeLabelsManager mgr) { - Set accessibleLabels = new HashSet(); - if (labels != null) { - accessibleLabels.addAll(labels); - } - if (accessibleLabels.contains(CommonNodeLabelsManager.ANY)) { - accessibleLabels.addAll(mgr.getClusterNodeLabelNames()); - } - accessibleLabels.add(CommonNodeLabelsManager.NO_LABEL); - - return accessibleLabels; - } - private static void loadCapacitiesByLabelsFromConf(String queuePath, - Set labels, RMNodeLabelsManager mgr, QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) { queueCapacities.clearConfigurableFields(); - labels = normalizeAccessibleNodeLabels(labels, mgr); + Set configuredNodelabels = + csConf.getConfiguredNodeLabels(queuePath); - for (String label : labels) { + for (String label : configuredNodelabels) { if (label.equals(CommonNodeLabelsManager.NO_LABEL)) { queueCapacities.setCapacity(CommonNodeLabelsManager.NO_LABEL, csConf.getNonLabeledQueueCapacity(queuePath) / 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index b00f25c8ef6..55e360df72c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -23,8 +23,10 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.StringTokenizer; @@ -44,13 +46,14 @@ import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.*; - - import com.google.common.collect.ImmutableSet; public class CapacitySchedulerConfiguration extends ReservationSchedulerConfiguration { @@ -909,4 +912,35 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur defaultVal); return preemptionDisabled; } + + /** + * Get configured node labels in a given queuePath + */ + public Set getConfiguredNodeLabels(String queuePath) { + Set configuredNodeLabels = new HashSet(); + Entry e = null; + + Iterator> iter = iterator(); + while (iter.hasNext()) { + e = iter.next(); + String key = e.getKey(); + + if (key.startsWith(getQueuePrefix(queuePath) + ACCESSIBLE_NODE_LABELS + + DOT)) { + // Find in + // .accessible-node-labels..property + int labelStartIdx = + key.indexOf(ACCESSIBLE_NODE_LABELS) + + ACCESSIBLE_NODE_LABELS.length() + 1; + int labelEndIndx = key.indexOf('.', labelStartIdx); + String labelName = key.substring(labelStartIdx, labelEndIndx); + configuredNodeLabels.add(labelName); + } + } + + // always add NO_LABEL + configuredNodeLabels.add(RMNodeLabelsManager.NO_LABEL); + + return configuredNodeLabels; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 56ade8488b2..15d3289d1da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -185,8 +186,8 @@ public class LeafQueue extends AbstractCSQueue { maxAMResourcePerQueuePercent = conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath()); - if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels, - this.defaultLabelExpression)) { + if (!SchedulerUtils.checkQueueLabelExpression( + this.accessibleLabels, this.defaultLabelExpression, null)) { throw new IOException("Invalid default label expression of " + " queue=" + getQueueName() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 8abaeb68eb0..4befe0c120a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; @@ -64,8 +65,10 @@ import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -83,12 +86,15 @@ import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; import org.junit.Test; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; public class TestSchedulerUtils { private static final Log LOG = LogFactory.getLog(TestSchedulerUtils.class); + private RMContext rmContext = getMockRMContext(); + @Test (timeout = 30000) public void testNormalizeRequest() { ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); @@ -206,6 +212,9 @@ public class TestSchedulerUtils { // set queue accessible node labesl to [x, y] queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of(NodeLabel.newInstance("x"), + NodeLabel.newInstance("y"))); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); @@ -213,22 +222,44 @@ public class TestSchedulerUtils { mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); resReq.setNodeLabelExpression("y"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); resReq.setNodeLabelExpression(""); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); resReq.setNodeLabelExpression(" "); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when request labels is a subset of queue labels"); + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y")); + } + + // same as above, but cluster node labels don't contains label being + // requested. should fail + try { + // set queue accessible node labesl to [x, y] + queueAccessibleNodeLabels.clear(); + queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + Resource resource = Resources.createResource( + 0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + resReq.setNodeLabelExpression("x"); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + + fail("Should fail"); + } catch (InvalidResourceRequestException e) { } // queue has labels, failed cases (when ask a label not included by queue) @@ -236,6 +267,9 @@ public class TestSchedulerUtils { // set queue accessible node labesl to [x, y] queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of(NodeLabel.newInstance("x"), + NodeLabel.newInstance("y"))); Resource resource = Resources.createResource( 0, @@ -244,9 +278,12 @@ public class TestSchedulerUtils { mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("z"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y")); } // we don't allow specify more than two node labels in a single expression @@ -255,6 +292,9 @@ public class TestSchedulerUtils { // set queue accessible node labesl to [x, y] queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of(NodeLabel.newInstance("x"), + NodeLabel.newInstance("y"))); Resource resource = Resources.createResource( 0, @@ -263,9 +303,12 @@ public class TestSchedulerUtils { mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x && y"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y")); } // queue doesn't have label, succeed (when request no label) @@ -280,15 +323,15 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); resReq.setNodeLabelExpression(""); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); resReq.setNodeLabelExpression(" "); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when request labels is empty"); @@ -299,6 +342,9 @@ public class TestSchedulerUtils { // set queue accessible node labels to empty queueAccessibleNodeLabels.clear(); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of(NodeLabel.newInstance("x"))); + Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); @@ -306,9 +352,12 @@ public class TestSchedulerUtils { mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x")); } // queue is "*", always succeeded @@ -317,6 +366,40 @@ public class TestSchedulerUtils { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of(NodeLabel.newInstance("x"), + NodeLabel.newInstance("y"), NodeLabel.newInstance("z"))); + + Resource resource = Resources.createResource( + 0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + resReq.setNodeLabelExpression("x"); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + + resReq.setNodeLabelExpression("y"); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + + resReq.setNodeLabelExpression("z"); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + } catch (InvalidResourceRequestException e) { + e.printStackTrace(); + fail("Should be valid when queue can access any labels"); + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y", "z")); + } + + // same as above, but cluster node labels don't contains label, should fail + try { + // set queue accessible node labels to empty + queueAccessibleNodeLabels.clear(); + queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY); + Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); @@ -324,18 +407,9 @@ public class TestSchedulerUtils { mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); - - resReq.setNodeLabelExpression("y"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); - - resReq.setNodeLabelExpression("z"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); + fail("Should fail"); } catch (InvalidResourceRequestException e) { - e.printStackTrace(); - fail("Should be valid when queue can access any labels"); } // we don't allow resource name other than ANY and specify label @@ -343,6 +417,9 @@ public class TestSchedulerUtils { // set queue accessible node labesl to [x, y] queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of(NodeLabel.newInstance("x"), + NodeLabel.newInstance("y"))); Resource resource = Resources.createResource( 0, @@ -351,9 +428,12 @@ public class TestSchedulerUtils { mock(Priority.class), "rack", resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y")); } // we don't allow resource name other than ANY and specify label even if @@ -363,6 +443,8 @@ public class TestSchedulerUtils { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays .asList(CommonNodeLabelsManager.ANY)); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of(NodeLabel.newInstance("x"))); Resource resource = Resources.createResource( 0, @@ -371,9 +453,12 @@ public class TestSchedulerUtils { mock(Priority.class), "rack", resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x")); } } @@ -395,7 +480,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); } catch (InvalidResourceRequestException e) { fail("Zero memory should be accepted"); } @@ -409,7 +494,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); } catch (InvalidResourceRequestException e) { fail("Zero vcores should be accepted"); } @@ -424,7 +509,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); } catch (InvalidResourceRequestException e) { fail("Max memory should be accepted"); } @@ -439,7 +524,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); } catch (InvalidResourceRequestException e) { fail("Max vcores should not be accepted"); } @@ -453,7 +538,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); fail("Negative memory should not be accepted"); } catch (InvalidResourceRequestException e) { // expected @@ -468,7 +553,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); fail("Negative vcores should not be accepted"); } catch (InvalidResourceRequestException e) { // expected @@ -484,7 +569,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); fail("More than max memory should not be accepted"); } catch (InvalidResourceRequestException e) { // expected @@ -501,7 +586,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); fail("More than max vcores should not be accepted"); } catch (InvalidResourceRequestException e) { // expected @@ -632,4 +717,12 @@ public class TestSchedulerUtils { Assert.assertNull(applications.get(appId)); return app; } + + private static RMContext getMockRMContext() { + RMContext rmContext = mock(RMContext.class); + RMNodeLabelsManager nlm = new NullRMNodeLabelsManager(); + nlm.init(new Configuration(false)); + when(rmContext.getNodeLabelManager()).thenReturn(nlm); + return rmContext; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 0206772e087..8d04700690b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -551,7 +551,7 @@ public class TestQueueParsing { ServiceOperations.stopQuietly(capacityScheduler); } - @Test(expected = Exception.class) + @Test public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); @@ -579,7 +579,7 @@ public class TestQueueParsing { ServiceOperations.stopQuietly(nodeLabelsManager); } - @Test(expected = Exception.class) + @Test public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); @@ -607,7 +607,7 @@ public class TestQueueParsing { ServiceOperations.stopQuietly(nodeLabelsManager); } - @Test(expected = Exception.class) + @Test public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); @@ -635,7 +635,7 @@ public class TestQueueParsing { ServiceOperations.stopQuietly(nodeLabelsManager); } - @Test(expected = Exception.class) + @Test public void testQueueParsingWhenLabelsNotExist() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = @@ -769,4 +769,100 @@ public class TestQueueParsing { Assert.assertEquals(0.10 * 0.4, a2.getAbsoluteCapacity(), DELTA); Assert.assertEquals(0.15, a2.getAbsoluteMaximumCapacity(), DELTA); } + + /** + * Test init a queue configuration, children's capacity for a given label + * doesn't equals to 100%. This expect IllegalArgumentException thrown. + */ + @Test(expected = IllegalArgumentException.class) + public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent() + throws IOException { + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet + .of("red", "blue")); + + YarnConfiguration conf = new YarnConfiguration(); + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(conf); + setupQueueConfiguration(csConf); + csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5); + + CapacityScheduler capacityScheduler = new CapacityScheduler(); + RMContextImpl rmContext = + new RMContextImpl(null, null, null, null, null, null, + new RMContainerTokenSecretManager(csConf), + new NMTokenSecretManagerInRM(csConf), + new ClientToAMTokenSecretManagerInRM(), null); + rmContext.setNodeLabelManager(nodeLabelManager); + capacityScheduler.setConf(csConf); + capacityScheduler.setRMContext(rmContext); + capacityScheduler.init(csConf); + capacityScheduler.start(); + ServiceOperations.stopQuietly(capacityScheduler); + } + + /** + * Test init a queue configuration, children's capacity for a given label + * doesn't equals to 100%. This expect IllegalArgumentException thrown. + */ + @Test(expected = IllegalArgumentException.class) + public void testQueueParsingFailWhenSumOfChildrenLabeledCapacityNot100Percent() + throws IOException { + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet + .of("red", "blue")); + + YarnConfiguration conf = new YarnConfiguration(); + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(conf); + setupQueueConfigurationWithLabels(csConf); + csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3", + "red", 24); + + CapacityScheduler capacityScheduler = new CapacityScheduler(); + RMContextImpl rmContext = + new RMContextImpl(null, null, null, null, null, null, + new RMContainerTokenSecretManager(csConf), + new NMTokenSecretManagerInRM(csConf), + new ClientToAMTokenSecretManagerInRM(), null); + rmContext.setNodeLabelManager(nodeLabelManager); + capacityScheduler.setConf(csConf); + capacityScheduler.setRMContext(rmContext); + capacityScheduler.init(csConf); + capacityScheduler.start(); + ServiceOperations.stopQuietly(capacityScheduler); + } + + /** + * Test init a queue configuration, children's capacity for a given label + * doesn't equals to 100%. This expect IllegalArgumentException thrown. + */ + @Test(expected = IllegalArgumentException.class) + public void testQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard() + throws IOException { + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet + .of("red", "blue")); + + YarnConfiguration conf = new YarnConfiguration(); + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(conf); + setupQueueConfigurationWithLabels(csConf); + csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3", + "red", 24); + csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT, + ImmutableSet.of(RMNodeLabelsManager.ANY)); + csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT + ".b", + ImmutableSet.of(RMNodeLabelsManager.ANY)); + + CapacityScheduler capacityScheduler = new CapacityScheduler(); + RMContextImpl rmContext = + new RMContextImpl(null, null, null, null, null, null, + new RMContainerTokenSecretManager(csConf), + new NMTokenSecretManagerInRM(csConf), + new ClientToAMTokenSecretManagerInRM(), null); + rmContext.setNodeLabelManager(nodeLabelManager); + capacityScheduler.setConf(csConf); + capacityScheduler.setRMContext(rmContext); + capacityScheduler.init(csConf); + capacityScheduler.start(); + ServiceOperations.stopQuietly(capacityScheduler); + } }