From e0ced0a835b13759d7c84eb451d327a56df5aad7 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 19 Oct 2012 23:18:00 +0000 Subject: [PATCH] YARN-140. Add capacity-scheduler-default.xml to provide a default set of configurations for the capacity scheduler. (ahmed via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1400337 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 ++ .../main/resources => conf}/capacity-scheduler.xml | 11 ----------- .../hadoop-yarn-server-resourcemanager/pom.xml | 14 ++++++++++++++ .../capacity/CapacitySchedulerConfiguration.java | 7 ++++++- .../scheduler/capacity/TestApplicationLimits.java | 3 +-- .../scheduler/capacity/TestCapacityScheduler.java | 3 +-- .../scheduler/capacity/TestLeafQueue.java | 1 - .../scheduler/capacity/TestParentQueue.java | 2 -- .../scheduler/capacity/TestQueueParsing.java | 4 +--- .../resourcemanager/webapp/TestRMWebApp.java | 1 - .../webapp/TestRMWebServicesCapacitySched.java | 1 - 11 files changed, 25 insertions(+), 24 deletions(-) rename hadoop-yarn-project/hadoop-yarn/{hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources => conf}/capacity-scheduler.xml (86%) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index bc01f157b81..333cd9e4d5b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -69,6 +69,8 @@ Release 2.0.3-alpha - Unreleased YARN-150. Fixes AppRejectedTransition does not unregister a rejected app-attempt from the ApplicationMasterService (Bikas Saha via sseth) + YARN-140. Add capacity-scheduler-default.xml to provide a default set of configurations for the capacity scheduler. (ahmed via tucu) + Release 2.0.2-alpha - 2012-09-07 YARN-9. Rename YARN_HOME to HADOOP_YARN_HOME. (vinodkv via acmurthy) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/conf/capacity-scheduler.xml similarity index 86% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/capacity-scheduler.xml rename to hadoop-yarn-project/hadoop-yarn/conf/capacity-scheduler.xml index 65ef0a2394b..66ea5d6d1bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/capacity-scheduler.xml +++ b/hadoop-yarn-project/hadoop-yarn/conf/capacity-scheduler.xml @@ -26,17 +26,6 @@ - - yarn.scheduler.capacity.root.capacity - 100 - - The total capacity as a percentage out of 100 for this queue. - If it has child queues then this includes their capacity as well. - The child queues capacity should add up to their parent queue's capacity - or less. - - - yarn.scheduler.capacity.root.default.capacity 100 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 3774ca707d7..da914d90509 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -44,6 +44,20 @@ + + + + ${basedir}/src/test/resources + + + ${basedir}/../../conf + + capacity-scheduler.xml + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 540bfcbfd8e..537a689fc05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -172,7 +172,8 @@ public class CapacitySchedulerConfiguration extends Configuration { } public float getCapacity(String queue) { - float capacity = getFloat(getQueuePrefix(queue) + CAPACITY, UNDEFINED); + float capacity = queue.equals("root") ? 100.0f : getFloat( + getQueuePrefix(queue) + CAPACITY, UNDEFINED); if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { throw new IllegalArgumentException("Illegal " + "capacity of " + capacity + " for queue " + queue); @@ -183,6 +184,10 @@ public class CapacitySchedulerConfiguration extends Configuration { } public void setCapacity(String queue, float capacity) { + if (queue.equals("root")) { + throw new IllegalArgumentException( + "Cannot set capacity, root queue has a fixed capacity of 100.0f"); + } setFloat(getQueuePrefix(queue) + CAPACITY, capacity); LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) + ", capacity=" + capacity); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index f2f0e8d770f..123b88f821b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -100,8 +100,7 @@ public class TestApplicationLimits { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - + final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 10); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 3942aee3e1c..8a7b89eaf2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -221,8 +221,7 @@ public class TestCapacityScheduler { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - + conf.setCapacity(A, A_CAPACITY); conf.setCapacity(B, B_CAPACITY); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 93766b93555..f9d6c60479f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -135,7 +135,6 @@ public class TestLeafQueue { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {newRoot}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100); conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 602c09c0d17..998e897eb50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -90,7 +90,6 @@ public class TestParentQueue { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 30); @@ -344,7 +343,6 @@ public class TestParentQueue { // Define top-level queues csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 10); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 1858ddd40a6..e529815cf22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -66,8 +66,7 @@ public class TestQueueParsing { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); @@ -148,7 +147,6 @@ public class TestQueueParsing { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 50); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 924d02cec3b..67b290319f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -215,7 +215,6 @@ public class TestRMWebApp { static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index c0b58619cd7..04b7f01df3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -121,7 +121,6 @@ public class TestRMWebServicesCapacitySched extends JerseyTest { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", "b" }); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10.5f);