From 2336264900feb9dc265a96e660c6c5d1dbd17e2d Mon Sep 17 00:00:00 2001 From: Jian He Date: Thu, 7 May 2015 17:35:41 -0700 Subject: [PATCH] YARN-2918. RM should not fail on startup if queue's configured labels do not exist in cluster-node-labels. Contributed by Wangda Tan (cherry picked from commit f489a4ec969f3727d03c8e85d51af1018fc0b2a1) (cherry picked from commit d817fbb34d6e34991c6e512c20d71387750a98f4) --- hadoop-yarn-project/CHANGES.txt | 3 + .../ApplicationMasterService.java | 2 +- .../server/resourcemanager/RMAppManager.java | 2 +- .../server/resourcemanager/RMServerUtils.java | 5 +- .../scheduler/SchedulerUtils.java | 67 ++++---- .../scheduler/capacity/AbstractCSQueue.java | 6 +- .../scheduler/capacity/CSQueueUtils.java | 28 +--- .../CapacitySchedulerConfiguration.java | 33 ++++ .../scheduler/capacity/LeafQueue.java | 4 +- .../scheduler/TestSchedulerUtils.java | 150 ++++++++++++++---- .../scheduler/capacity/TestQueueParsing.java | 104 +++++++++++- 11 files changed, 302 insertions(+), 102 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a6bbdf38146..1ad0199cc8c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -192,6 +192,9 @@ Release 2.6.1 - UNRELEASED YARN-3124. Fixed CS LeafQueue/ParentQueue to use QueueCapacities to track capacities-by-label. (Wangda Tan via jianhe) + YARN-2918. RM should not fail on startup if queue's configured labels do + not exist in cluster-node-labels. (Wangda Tan via jianhe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index c896bf5ae7d..8ed1621003a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -501,7 +501,7 @@ public class ApplicationMasterService extends AbstractService implements try { RMServerUtils.normalizeAndValidateRequests(ask, rScheduler.getMaximumResourceCapability(), app.getQueue(), - rScheduler); + rScheduler, rmContext); } catch (InvalidResourceRequestException e) { LOG.warn("Invalid resource ask by application " + appAttemptId, e); throw e; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index fbae7108caa..e7d6f4c9ac5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -375,7 +375,7 @@ public class RMAppManager implements EventHandler, try { SchedulerUtils.normalizeAndValidateRequest(amReq, scheduler.getMaximumResourceCapability(), - submissionContext.getQueue(), scheduler, isRecovery); + submissionContext.getQueue(), scheduler, isRecovery, rmContext); } catch (InvalidResourceRequestException e) { LOG.warn("RM app submission failed in validating AM resource request" + " for application " + submissionContext.getApplicationId(), e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 800cfec2aec..bbab0a9e09b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -90,11 +90,12 @@ public class RMServerUtils { * requested memory/vcore is non-negative and not greater than max */ public static void normalizeAndValidateRequests(List ask, - Resource maximumResource, String queueName, YarnScheduler scheduler) + Resource maximumResource, String queueName, YarnScheduler scheduler, + RMContext rmContext) throws InvalidResourceRequestException { for (ResourceRequest resReq : ask) { SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource, - queueName, scheduler); + queueName, scheduler, rmContext); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index bdc9da1e893..d13548555f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; @@ -209,7 +210,7 @@ public class SchedulerUtils { public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, - boolean isRecovery) + boolean isRecovery, RMContext rmContext) throws InvalidResourceRequestException { QueueInfo queueInfo = null; @@ -221,15 +222,16 @@ public class SchedulerUtils { } SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo); if (!isRecovery) { - validateResourceRequest(resReq, maximumResource, queueInfo); + validateResourceRequest(resReq, maximumResource, queueInfo, rmContext); } } public static void normalizeAndvalidateRequest(ResourceRequest resReq, - Resource maximumResource, String queueName, YarnScheduler scheduler) + Resource maximumResource, String queueName, YarnScheduler scheduler, + RMContext rmContext) throws InvalidResourceRequestException { normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, - false); + false, rmContext); } /** @@ -239,8 +241,8 @@ public class SchedulerUtils { * @throws InvalidResourceRequestException when there is invalid * request */ - public static void validateResourceRequest(ResourceRequest resReq, - Resource maximumResource, QueueInfo queueInfo) + private static void validateResourceRequest(ResourceRequest resReq, + Resource maximumResource, QueueInfo queueInfo, RMContext rmContext) throws InvalidResourceRequestException { if (resReq.getCapability().getMemory() < 0 || resReq.getCapability().getMemory() > maximumResource.getMemory()) { @@ -283,7 +285,7 @@ public class SchedulerUtils { if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) { if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(), - labelExp)) { + labelExp, rmContext)) { throw new InvalidResourceRequestException("Invalid resource request" + ", queue=" + queueInfo.getQueueName() @@ -316,26 +318,6 @@ public class SchedulerUtils { return false; } - public static void checkIfLabelInClusterNodeLabels(RMNodeLabelsManager mgr, - Set labels) throws IOException { - if (mgr == null) { - if (labels != null && !labels.isEmpty()) { - throw new IOException("NodeLabelManager is null, please check"); - } - return; - } - - if (labels != null) { - for (String label : labels) { - if (!label.equals(RMNodeLabelsManager.ANY) - && !mgr.containsNodeLabel(label)) { - throw new IOException("NodeLabelManager doesn't include label = " - + label + ", please check."); - } - } - } - } - public static boolean checkNodeLabelExpression(Set nodeLabels, String labelExpression) { // empty label expression can only allocate on node with empty labels @@ -356,19 +338,36 @@ public class SchedulerUtils { return true; } + /** + * Check queue label expression, check if node label in queue's + * node-label-expression existed in clusterNodeLabels if rmContext != null + */ public static boolean checkQueueLabelExpression(Set queueLabels, - String labelExpression) { - if (queueLabels != null && queueLabels.contains(RMNodeLabelsManager.ANY)) { - return true; - } + String labelExpression, RMContext rmContext) { // if label expression is empty, we can allocate container on any node if (labelExpression == null) { return true; } for (String str : labelExpression.split("&&")) { - if (!str.trim().isEmpty() - && (queueLabels == null || !queueLabels.contains(str.trim()))) { - return false; + str = str.trim(); + if (!str.trim().isEmpty()) { + // check queue label + if (queueLabels == null) { + return false; + } else { + if (!queueLabels.contains(str) + && !queueLabels.contains(RMNodeLabelsManager.ANY)) { + return false; + } + } + + // check node label manager contains this label + if (null != rmContext) { + RMNodeLabelsManager nlm = rmContext.getNodeLabelManager(); + if (nlm != null && !nlm.containsNodeLabel(str)) { + return false; + } + } } } return true; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 52629b3a1f6..f7d49c804a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -97,11 +97,9 @@ public abstract class AbstractCSQueue implements CSQueue { protected void setupConfigurableCapacities() { CSQueueUtils.loadUpdateAndCheckCapacities( getQueuePath(), - accessibleLabels, csContext.getConfiguration(), queueCapacities, - parent == null ? null : parent.getQueueCapacities(), - csContext.getRMContext().getNodeLabelManager()); + parent == null ? null : parent.getQueueCapacities()); } @Override @@ -232,8 +230,6 @@ public abstract class AbstractCSQueue implements CSQueue { if (this.accessibleLabels == null && parent != null) { this.accessibleLabels = parent.getAccessibleNodeLabels(); } - SchedulerUtils.checkIfLabelInClusterNodeLabels(labelManager, - this.accessibleLabels); // inherit from parent if labels not set if (this.defaultLabelExpression == null && parent != null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 865b0b41979..cbc20b79407 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -117,10 +117,9 @@ class CSQueueUtils { * - Check if capacities/absolute-capacities legal */ public static void loadUpdateAndCheckCapacities(String queuePath, - Set accessibleLabels, CapacitySchedulerConfiguration csConf, - QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities, - RMNodeLabelsManager nlm) { - loadCapacitiesByLabelsFromConf(queuePath, accessibleLabels, nlm, + CapacitySchedulerConfiguration csConf, + QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) { + loadCapacitiesByLabelsFromConf(queuePath, queueCapacities, csConf); updateAbsoluteCapacitiesByNodeLabels(queueCapacities, parentQueueCapacities); @@ -128,28 +127,13 @@ class CSQueueUtils { capacitiesSanityCheck(queuePath, queueCapacities); } - // Considered NO_LABEL, ANY and null cases - private static Set normalizeAccessibleNodeLabels(Set labels, - RMNodeLabelsManager mgr) { - Set accessibleLabels = new HashSet(); - if (labels != null) { - accessibleLabels.addAll(labels); - } - if (accessibleLabels.contains(CommonNodeLabelsManager.ANY)) { - accessibleLabels.addAll(mgr.getClusterNodeLabels()); - } - accessibleLabels.add(CommonNodeLabelsManager.NO_LABEL); - - return accessibleLabels; - } - private static void loadCapacitiesByLabelsFromConf(String queuePath, - Set labels, RMNodeLabelsManager mgr, QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) { queueCapacities.clearConfigurableFields(); - labels = normalizeAccessibleNodeLabels(labels, mgr); + Set configuredNodelabels = + csConf.getConfiguredNodeLabels(queuePath); - for (String label : labels) { + for (String label : configuredNodelabels) { if (label.equals(CommonNodeLabelsManager.NO_LABEL)) { queueCapacities.setCapacity(CommonNodeLabelsManager.NO_LABEL, csConf.getNonLabeledQueueCapacity(queuePath) / 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index f5e86b37e5e..7bee438407c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -23,8 +23,10 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.StringTokenizer; @@ -780,4 +782,35 @@ public class CapacitySchedulerConfiguration extends Configuration { DEFAULT_RESERVATION_ENFORCEMENT_WINDOW); return enforcementWindow; } + + /** + * Get configured node labels in a given queuePath + */ + public Set getConfiguredNodeLabels(String queuePath) { + Set configuredNodeLabels = new HashSet(); + Entry e = null; + + Iterator> iter = iterator(); + while (iter.hasNext()) { + e = iter.next(); + String key = e.getKey(); + + if (key.startsWith(getQueuePrefix(queuePath) + ACCESSIBLE_NODE_LABELS + + DOT)) { + // Find in + // .accessible-node-labels..property + int labelStartIdx = + key.indexOf(ACCESSIBLE_NODE_LABELS) + + ACCESSIBLE_NODE_LABELS.length() + 1; + int labelEndIndx = key.indexOf('.', labelStartIdx); + String labelName = key.substring(labelStartIdx, labelEndIndx); + configuredNodeLabels.add(labelName); + } + } + + // always add NO_LABEL + configuredNodeLabels.add(RMNodeLabelsManager.NO_LABEL); + + return configuredNodeLabels; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index f9f9ca92ae0..f8a6af3ebb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -173,8 +173,8 @@ public class LeafQueue extends AbstractCSQueue { maxAMResourcePerQueuePercent = conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath()); - if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels, - this.defaultLabelExpression)) { + if (!SchedulerUtils.checkQueueLabelExpression( + this.accessibleLabels, this.defaultLabelExpression, null)) { throw new IOException("Invalid default label expression of " + " queue=" + getQueueName() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 8abaeb68eb0..0fd598c6143 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -64,8 +64,10 @@ import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -83,12 +85,15 @@ import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; import org.junit.Test; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; public class TestSchedulerUtils { private static final Log LOG = LogFactory.getLog(TestSchedulerUtils.class); + private RMContext rmContext = getMockRMContext(); + @Test (timeout = 30000) public void testNormalizeRequest() { ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); @@ -206,6 +211,8 @@ public class TestSchedulerUtils { // set queue accessible node labesl to [x, y] queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of("x","y")); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); @@ -213,22 +220,44 @@ public class TestSchedulerUtils { mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); resReq.setNodeLabelExpression("y"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); resReq.setNodeLabelExpression(""); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); resReq.setNodeLabelExpression(" "); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when request labels is a subset of queue labels"); + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y")); + } + + // same as above, but cluster node labels don't contains label being + // requested. should fail + try { + // set queue accessible node labesl to [x, y] + queueAccessibleNodeLabels.clear(); + queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + Resource resource = Resources.createResource( + 0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + resReq.setNodeLabelExpression("x"); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + + fail("Should fail"); + } catch (InvalidResourceRequestException e) { } // queue has labels, failed cases (when ask a label not included by queue) @@ -236,6 +265,8 @@ public class TestSchedulerUtils { // set queue accessible node labesl to [x, y] queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of("x","y")); Resource resource = Resources.createResource( 0, @@ -244,9 +275,12 @@ public class TestSchedulerUtils { mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("z"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y")); } // we don't allow specify more than two node labels in a single expression @@ -255,6 +289,8 @@ public class TestSchedulerUtils { // set queue accessible node labesl to [x, y] queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of("x","y")); Resource resource = Resources.createResource( 0, @@ -263,9 +299,12 @@ public class TestSchedulerUtils { mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x && y"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y")); } // queue doesn't have label, succeed (when request no label) @@ -280,15 +319,15 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); resReq.setNodeLabelExpression(""); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); resReq.setNodeLabelExpression(" "); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when request labels is empty"); @@ -299,6 +338,9 @@ public class TestSchedulerUtils { // set queue accessible node labels to empty queueAccessibleNodeLabels.clear(); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of("x")); + Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); @@ -306,9 +348,12 @@ public class TestSchedulerUtils { mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x")); } // queue is "*", always succeeded @@ -317,6 +362,39 @@ public class TestSchedulerUtils { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of("x","y", "z")); + + Resource resource = Resources.createResource( + 0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + resReq.setNodeLabelExpression("x"); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + + resReq.setNodeLabelExpression("y"); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + + resReq.setNodeLabelExpression("z"); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + } catch (InvalidResourceRequestException e) { + e.printStackTrace(); + fail("Should be valid when queue can access any labels"); + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y", "z")); + } + + // same as above, but cluster node labels don't contains label, should fail + try { + // set queue accessible node labels to empty + queueAccessibleNodeLabels.clear(); + queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY); + Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); @@ -324,18 +402,9 @@ public class TestSchedulerUtils { mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); - - resReq.setNodeLabelExpression("y"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); - - resReq.setNodeLabelExpression("z"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); + fail("Should fail"); } catch (InvalidResourceRequestException e) { - e.printStackTrace(); - fail("Should be valid when queue can access any labels"); } // we don't allow resource name other than ANY and specify label @@ -343,6 +412,9 @@ public class TestSchedulerUtils { // set queue accessible node labesl to [x, y] queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of("x", "y")); + Resource resource = Resources.createResource( 0, @@ -351,9 +423,12 @@ public class TestSchedulerUtils { mock(Priority.class), "rack", resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y")); } // we don't allow resource name other than ANY and specify label even if @@ -363,6 +438,8 @@ public class TestSchedulerUtils { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays .asList(CommonNodeLabelsManager.ANY)); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of("x")); Resource resource = Resources.createResource( 0, @@ -371,9 +448,12 @@ public class TestSchedulerUtils { mock(Priority.class), "rack", resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler); + scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x")); } } @@ -395,7 +475,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); } catch (InvalidResourceRequestException e) { fail("Zero memory should be accepted"); } @@ -409,7 +489,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); } catch (InvalidResourceRequestException e) { fail("Zero vcores should be accepted"); } @@ -424,7 +504,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); } catch (InvalidResourceRequestException e) { fail("Max memory should be accepted"); } @@ -439,7 +519,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); } catch (InvalidResourceRequestException e) { fail("Max vcores should not be accepted"); } @@ -453,7 +533,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); fail("Negative memory should not be accepted"); } catch (InvalidResourceRequestException e) { // expected @@ -468,7 +548,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); fail("Negative vcores should not be accepted"); } catch (InvalidResourceRequestException e) { // expected @@ -484,7 +564,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); fail("More than max memory should not be accepted"); } catch (InvalidResourceRequestException e) { // expected @@ -501,7 +581,7 @@ public class TestSchedulerUtils { BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler); + mockScheduler, rmContext); fail("More than max vcores should not be accepted"); } catch (InvalidResourceRequestException e) { // expected @@ -632,4 +712,12 @@ public class TestSchedulerUtils { Assert.assertNull(applications.get(appId)); return app; } + + private static RMContext getMockRMContext() { + RMContext rmContext = mock(RMContext.class); + RMNodeLabelsManager nlm = new MemoryRMNodeLabelsManager(); + nlm.init(new Configuration(false)); + when(rmContext.getNodeLabelManager()).thenReturn(nlm); + return rmContext; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index fc03581e54a..d0c5d952110 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -501,7 +501,7 @@ public class TestQueueParsing { ServiceOperations.stopQuietly(capacityScheduler); } - @Test(expected = Exception.class) + @Test public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); @@ -529,7 +529,7 @@ public class TestQueueParsing { ServiceOperations.stopQuietly(nodeLabelsManager); } - @Test(expected = Exception.class) + @Test public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); @@ -557,7 +557,7 @@ public class TestQueueParsing { ServiceOperations.stopQuietly(nodeLabelsManager); } - @Test(expected = Exception.class) + @Test public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); @@ -585,7 +585,7 @@ public class TestQueueParsing { ServiceOperations.stopQuietly(nodeLabelsManager); } - @Test(expected = Exception.class) + @Test public void testQueueParsingWhenLabelsNotExist() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = @@ -659,4 +659,100 @@ public class TestQueueParsing { DELTA); capacityScheduler.stop(); } + + /** + * Test init a queue configuration, children's capacity for a given label + * doesn't equals to 100%. This expect IllegalArgumentException thrown. + */ + @Test(expected = IllegalArgumentException.class) + public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent() + throws IOException { + nodeLabelManager.addToCluserNodeLabels(ImmutableSet + .of("red", "blue")); + + YarnConfiguration conf = new YarnConfiguration(); + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(conf); + setupQueueConfiguration(csConf); + csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5); + + CapacityScheduler capacityScheduler = new CapacityScheduler(); + RMContextImpl rmContext = + new RMContextImpl(null, null, null, null, null, null, + new RMContainerTokenSecretManager(csConf), + new NMTokenSecretManagerInRM(csConf), + new ClientToAMTokenSecretManagerInRM(), null); + rmContext.setNodeLabelManager(nodeLabelManager); + capacityScheduler.setConf(csConf); + capacityScheduler.setRMContext(rmContext); + capacityScheduler.init(csConf); + capacityScheduler.start(); + ServiceOperations.stopQuietly(capacityScheduler); + } + + /** + * Test init a queue configuration, children's capacity for a given label + * doesn't equals to 100%. This expect IllegalArgumentException thrown. + */ + @Test(expected = IllegalArgumentException.class) + public void testQueueParsingFailWhenSumOfChildrenLabeledCapacityNot100Percent() + throws IOException { + nodeLabelManager.addToCluserNodeLabels(ImmutableSet + .of("red", "blue")); + + YarnConfiguration conf = new YarnConfiguration(); + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(conf); + setupQueueConfigurationWithLabels(csConf); + csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3", + "red", 24); + + CapacityScheduler capacityScheduler = new CapacityScheduler(); + RMContextImpl rmContext = + new RMContextImpl(null, null, null, null, null, null, + new RMContainerTokenSecretManager(csConf), + new NMTokenSecretManagerInRM(csConf), + new ClientToAMTokenSecretManagerInRM(), null); + rmContext.setNodeLabelManager(nodeLabelManager); + capacityScheduler.setConf(csConf); + capacityScheduler.setRMContext(rmContext); + capacityScheduler.init(csConf); + capacityScheduler.start(); + ServiceOperations.stopQuietly(capacityScheduler); + } + + /** + * Test init a queue configuration, children's capacity for a given label + * doesn't equals to 100%. This expect IllegalArgumentException thrown. + */ + @Test(expected = IllegalArgumentException.class) + public void testQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard() + throws IOException { + nodeLabelManager.addToCluserNodeLabels(ImmutableSet + .of("red", "blue")); + + YarnConfiguration conf = new YarnConfiguration(); + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(conf); + setupQueueConfigurationWithLabels(csConf); + csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3", + "red", 24); + csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT, + ImmutableSet.of(RMNodeLabelsManager.ANY)); + csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT + ".b", + ImmutableSet.of(RMNodeLabelsManager.ANY)); + + CapacityScheduler capacityScheduler = new CapacityScheduler(); + RMContextImpl rmContext = + new RMContextImpl(null, null, null, null, null, null, + new RMContainerTokenSecretManager(csConf), + new NMTokenSecretManagerInRM(csConf), + new ClientToAMTokenSecretManagerInRM(), null); + rmContext.setNodeLabelManager(nodeLabelManager); + capacityScheduler.setConf(csConf); + capacityScheduler.setRMContext(rmContext); + capacityScheduler.init(csConf); + capacityScheduler.start(); + ServiceOperations.stopQuietly(capacityScheduler); + } }