From 7b2940190d26f9fe3787a926c1ea27713b523cc8 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Wed, 23 Nov 2011 01:46:10 +0000 Subject: [PATCH] Merge -c 1205260 from trunk to branch-0.23 to fix MAPREDUCE-3329. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1205261 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../scheduler/capacity/CSQueueUtils.java | 17 +++++++ .../CapacitySchedulerConfiguration.java | 4 ++ .../scheduler/capacity/LeafQueue.java | 18 ++++--- .../scheduler/capacity/ParentQueue.java | 12 +++-- .../scheduler/capacity/TestQueueParsing.java | 47 ++++++++++++++++++- 6 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 056f78ac54c..18bd1b0f77d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -121,6 +121,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3408. yarn-daemon.sh unconditionnaly sets yarn.root.logger (Bruno Mahe via mahadev) + MAPREDUCE-3329. Fixed CapacityScheduler to ensure maximum-capacity cannot + be lesser than capacity for any queue. (acmurthy) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java new file mode 100644 index 00000000000..bd9e7615b6d --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -0,0 +1,17 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +class CSQueueUtils { + + public static void checkMaxCapacity(String queueName, + float capacity, float maximumCapacity) { + if (maximumCapacity != CapacitySchedulerConfiguration.UNDEFINED && + maximumCapacity < capacity) { + throw new IllegalArgumentException( + "Illegal call to setMaxCapacity. " + + "Queue '" + queueName + "' has " + + "capacity (" + capacity + ") greater than " + + "maximumCapacity (" + maximumCapacity + ")" ); + } + } + +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 600706e6eda..dc28c3cb437 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -160,6 +160,10 @@ public class CapacitySchedulerConfiguration extends Configuration { return maxCapacity; } + public void setMaximumCapacity(String queue, int maxCapacity) { + setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity); + } + public int getUserLimit(String queue) { int userLimit = getInt(getQueuePrefix(queue) + USER_LIMIT, DEFAULT_USER_LIMIT); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 40fbefe73fc..c97aeb79a63 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -211,16 +211,19 @@ public class LeafQueue implements CSQueue { private synchronized void setupQueueConfigs( float capacity, float absoluteCapacity, - float maxCapacity, float absoluteMaxCapacity, + float maximumCapacity, float absoluteMaxCapacity, int userLimit, float userLimitFactor, int maxApplications, int maxApplicationsPerUser, int maxActiveApplications, int maxActiveApplicationsPerUser, QueueState state, Map acls) { + // Sanity check + CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); + this.capacity = capacity; this.absoluteCapacity = parent.getAbsoluteCapacity() * capacity; - this.maximumCapacity = maxCapacity; + this.maximumCapacity = maximumCapacity; this.absoluteMaxCapacity = absoluteMaxCapacity; this.userLimit = userLimit; @@ -236,9 +239,9 @@ public class LeafQueue implements CSQueue { this.acls = acls; - this.queueInfo.setCapacity(capacity); - this.queueInfo.setMaximumCapacity(maximumCapacity); - this.queueInfo.setQueueState(state); + this.queueInfo.setCapacity(this.capacity); + this.queueInfo.setMaximumCapacity(this.maximumCapacity); + this.queueInfo.setQueueState(this.state); StringBuilder aclsString = new StringBuilder(); for (Map.Entry e : acls.entrySet()) { @@ -250,7 +253,7 @@ public class LeafQueue implements CSQueue { " [= (float) configuredCapacity / 100 ]" + "\n" + "asboluteCapacity = " + absoluteCapacity + " [= parentAbsoluteCapacity * capacity ]" + "\n" + - "maxCapacity = " + maxCapacity + + "maxCapacity = " + maximumCapacity + " [= configuredMaxCapacity ]" + "\n" + "absoluteMaxCapacity = " + absoluteMaxCapacity + " [= Float.MAX_VALUE if maximumCapacity undefined, " + @@ -394,6 +397,9 @@ public class LeafQueue implements CSQueue { * @param maximumCapacity new max capacity */ synchronized void setMaxCapacity(float maximumCapacity) { + // Sanity check + CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); + this.maximumCapacity = maximumCapacity; this.absoluteMaxCapacity = (maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ? diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index dc363a210f8..38664f0db36 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -153,6 +153,9 @@ public class ParentQueue implements CSQueue { float maximumCapacity, float absoluteMaxCapacity, QueueState state, Map acls ) { + // Sanity check + CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); + this.capacity = capacity; this.absoluteCapacity = absoluteCapacity; this.maximumCapacity = maximumCapacity; @@ -162,9 +165,9 @@ public class ParentQueue implements CSQueue { this.acls = acls; - this.queueInfo.setCapacity(capacity); - this.queueInfo.setMaximumCapacity(maximumCapacity); - this.queueInfo.setQueueState(state); + this.queueInfo.setCapacity(this.capacity); + this.queueInfo.setMaximumCapacity(this.maximumCapacity); + this.queueInfo.setQueueState(this.state); StringBuilder aclsString = new StringBuilder(); for (Map.Entry e : acls.entrySet()) { @@ -484,6 +487,9 @@ public class ParentQueue implements CSQueue { * @param maximumCapacity new max capacity */ synchronized void setMaxCapacity(float maximumCapacity) { + // Sanity check + CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); + this.maximumCapacity = maximumCapacity; float parentAbsoluteCapacity = (rootQueue) ? 100.0f : parent.getAbsoluteCapacity(); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index f353b788a0e..28a0026cec6 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import junit.framework.Assert; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -35,7 +37,6 @@ public class TestQueueParsing { CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.reinitialize(conf, null, null); - //capacityScheduler.g } private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { @@ -104,4 +105,48 @@ public class TestQueueParsing { CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.reinitialize(conf, null, null); } + + public void testMaxCapacity() throws Exception { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + + conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b", "c"}); + conf.setCapacity(CapacityScheduler.ROOT, 100); + + final String A = CapacityScheduler.ROOT + ".a"; + conf.setCapacity(A, 50); + conf.setMaximumCapacity(A, 60); + + final String B = CapacityScheduler.ROOT + ".b"; + conf.setCapacity(B, 50); + conf.setMaximumCapacity(B, 45); // Should throw an exception + + + boolean fail = false; + CapacityScheduler capacityScheduler; + try { + capacityScheduler = new CapacityScheduler(); + capacityScheduler.reinitialize(conf, null, null); + } catch (IllegalArgumentException iae) { + fail = true; + } + Assert.assertTrue("Didn't throw IllegalArgumentException for wrong maxCap", + fail); + + conf.setMaximumCapacity(B, 60); + + // Now this should work + capacityScheduler = new CapacityScheduler(); + capacityScheduler.reinitialize(conf, null, null); + + fail = false; + try { + LeafQueue a = (LeafQueue)capacityScheduler.getQueue(A); + a.setMaxCapacity(45); + } catch (IllegalArgumentException iae) { + fail = true; + } + Assert.assertTrue("Didn't throw IllegalArgumentException for wrong " + + "setMaxCap", fail); + } + }