From f7a722c38f095c519dd2bb4bbd1e25c8d0a8b6c4 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 23 Jul 2012 19:32:59 +0000 Subject: [PATCH] svn merge -c 1364764 FIXES: MAPREDUCE-3893. allow capacity scheduler configs max-apps and max-am-pct per queue (tgraves via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1364765 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 + .../CapacitySchedulerConfiguration.java | 36 ++++++++- .../scheduler/capacity/LeafQueue.java | 37 ++++++---- .../capacity/TestApplicationLimits.java | 74 ++++++++++++++++++- .../src/site/apt/CapacityScheduler.apt.vm | 17 +++-- 5 files changed, 143 insertions(+), 25 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 486d96b43d8..eba3daabdd4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -618,6 +618,10 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't init. (Jason Lowe via daryn) + MAPREDUCE-3893. allow capacity scheduler configs maximum-applications and + maximum-am-resource-percent configurable on a per queue basis (tgraves via + bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 86e2dd350da..300dda5011a 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -46,13 +46,21 @@ public class CapacitySchedulerConfiguration extends Configuration { @Private public static final String DOT = "."; + @Private + public static final String MAXIMUM_APPLICATIONS_SUFFIX = + "maximum-applications"; + @Private public static final String MAXIMUM_SYSTEM_APPLICATIONS = - PREFIX + "maximum-applications"; + PREFIX + MAXIMUM_APPLICATIONS_SUFFIX; + + @Private + public static final String MAXIMUM_AM_RESOURCE_SUFFIX = + "maximum-am-resource-percent"; @Private public static final String MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT = - PREFIX + "maximum-am-resource-percent"; + PREFIX + MAXIMUM_AM_RESOURCE_SUFFIX; @Private public static final String QUEUES = "queues"; @@ -131,6 +139,30 @@ public float getMaximumApplicationMasterResourcePercent() { return getFloat(MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT); } + + + /** + * Get the maximum applications per queue setting. + * @param queue name of the queue + * @return setting specified or -1 if not set + */ + public int getMaximumApplicationsPerQueue(String queue) { + int maxApplicationsPerQueue = + getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX, + (int)UNDEFINED); + return maxApplicationsPerQueue; + } + + /** + * Get the maximum am resource percent per queue setting. + * @param queue name of the queue + * @return per queue setting or defaults to the global am-resource-percent + * setting if per queue setting not present + */ + public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) { + return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, + getMaximumApplicationMasterResourcePercent()); + } public float getCapacity(String queue) { float capacity = getFloat(getQueuePrefix(queue) + CAPACITY, UNDEFINED); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index ebf594b33c1..cf303cba8f3 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -85,7 +85,7 @@ public class LeafQueue implements CSQueue { private int maxApplications; private int maxApplicationsPerUser; - private float maxAMResourcePercent; + private float maxAMResourcePerQueuePercent; private int maxActiveApplications; // Based on absolute max capacity private int maxActiveAppsUsingAbsCap; // Based on absolute capacity private int maxActiveApplicationsPerUser; @@ -156,21 +156,25 @@ public LeafQueue(CapacitySchedulerContext cs, float userLimitFactor = cs.getConfiguration().getUserLimitFactor(getQueuePath()); - int maxSystemJobs = cs.getConfiguration().getMaximumSystemApplications(); - int maxApplications = (int)(maxSystemJobs * absoluteCapacity); - int maxApplicationsPerUser = + int maxApplications = cs.getConfiguration().getMaximumApplicationsPerQueue(getQueuePath()); + if (maxApplications < 0) { + int maxSystemApps = cs.getConfiguration().getMaximumSystemApplications(); + maxApplications = (int)(maxSystemApps * absoluteCapacity); + } + maxApplicationsPerUser = (int)(maxApplications * (userLimit / 100.0f) * userLimitFactor); - this.maxAMResourcePercent = - cs.getConfiguration().getMaximumApplicationMasterResourcePercent(); + this.maxAMResourcePerQueuePercent = + cs.getConfiguration(). + getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath()); int maxActiveApplications = CSQueueUtils.computeMaxActiveApplications( cs.getClusterResources(), this.minimumAllocation, - maxAMResourcePercent, absoluteMaxCapacity); + maxAMResourcePerQueuePercent, absoluteMaxCapacity); this.maxActiveAppsUsingAbsCap = CSQueueUtils.computeMaxActiveApplications( cs.getClusterResources(), this.minimumAllocation, - maxAMResourcePercent, absoluteCapacity); + maxAMResourcePerQueuePercent, absoluteCapacity); int maxActiveApplicationsPerUser = CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveAppsUsingAbsCap, userLimit, userLimitFactor); @@ -265,15 +269,16 @@ private synchronized void setupQueueConfigs( "userLimitFactor = " + userLimitFactor + " [= configuredUserLimitFactor ]" + "\n" + "maxApplications = " + maxApplications + - " [= (int)(configuredMaximumSystemApplications * absoluteCapacity) ]" + + " [= configuredMaximumSystemApplicationsPerQueue or" + + " (int)(configuredMaximumSystemApplications * absoluteCapacity)]" + "\n" + "maxApplicationsPerUser = " + maxApplicationsPerUser + " [= (int)(maxApplications * (userLimit / 100.0f) * " + "userLimitFactor) ]" + "\n" + "maxActiveApplications = " + maxActiveApplications + " [= max(" + - "(int)ceil((clusterResourceMemory / minimumAllocation) *" + - "maxAMResourcePercent * absoluteMaxCapacity)," + + "(int)ceil((clusterResourceMemory / minimumAllocation) * " + + "maxAMResourcePerQueuePercent * absoluteMaxCapacity)," + "1) ]" + "\n" + "maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap + " [= max(" + @@ -290,7 +295,7 @@ private synchronized void setupQueueConfigs( "(clusterResourceMemory * absoluteCapacity)]" + "\n" + "absoluteUsedCapacity = " + absoluteUsedCapacity + " [= usedResourcesMemory / clusterResourceMemory]" + "\n" + - "maxAMResourcePercent = " + maxAMResourcePercent + + "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent + " [= configuredMaximumAMResourcePercent ]" + "\n" + "minimumAllocationFactor = " + minimumAllocationFactor + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " + @@ -1387,11 +1392,11 @@ public synchronized void updateClusterResource(Resource clusterResource) { maxActiveApplications = CSQueueUtils.computeMaxActiveApplications( clusterResource, minimumAllocation, - maxAMResourcePercent, absoluteMaxCapacity); + maxAMResourcePerQueuePercent, absoluteMaxCapacity); maxActiveAppsUsingAbsCap = - CSQueueUtils.computeMaxActiveApplications( - clusterResource, minimumAllocation, - maxAMResourcePercent, absoluteCapacity); + CSQueueUtils.computeMaxActiveApplications( + clusterResource, minimumAllocation, + maxAMResourcePerQueuePercent, absoluteCapacity); maxActiveApplicationsPerUser = CSQueueUtils.computeMaxActiveApplicationsPerUser( maxActiveAppsUsingAbsCap, userLimit, userLimitFactor); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 8876bd338dd..f2f0e8d770f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -158,7 +158,9 @@ public void testLimitsComputation() throws Exception { int expectedMaxActiveApps = Math.max(1, (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * - csConf.getMaximumApplicationMasterResourcePercent() * + csConf. + getMaximumApplicationMasterResourcePerQueuePercent( + queue.getQueuePath()) * queue.getAbsoluteMaximumCapacity())); assertEquals(expectedMaxActiveApps, queue.getMaximumActiveApplications()); @@ -183,7 +185,9 @@ public void testLimitsComputation() throws Exception { expectedMaxActiveApps = Math.max(1, (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * - csConf.getMaximumApplicationMasterResourcePercent() * + csConf. + getMaximumApplicationMasterResourcePerQueuePercent( + queue.getQueuePath()) * queue.getAbsoluteMaximumCapacity())); assertEquals(expectedMaxActiveApps, queue.getMaximumActiveApplications()); @@ -200,6 +204,72 @@ public void testLimitsComputation() throws Exception { (int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()), queue.getMetrics().getAvailableMB() ); + + // should return -1 if per queue setting not set + assertEquals((int)csConf.UNDEFINED, csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); + int expectedMaxApps = (int)(csConf.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * + queue.getAbsoluteCapacity()); + assertEquals(expectedMaxApps, queue.getMaxApplications()); + + int expectedMaxAppsPerUser = (int)(expectedMaxApps * + (queue.getUserLimit()/100.0f) * queue.getUserLimitFactor()); + assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser()); + + // should default to global setting if per queue setting not set + assertEquals((long) csConf.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, + (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath())); + + // Change the per-queue max AM resources percentage. + csConf.setFloat( + "yarn.scheduler.capacity." + + queue.getQueuePath() + + ".maximum-am-resource-percent", + 0.5f); + // Re-create queues to get new configs. + queues = new HashMap(); + root = + CapacityScheduler.parseQueue(csContext, csConf, null, "root", + queues, queues, + CapacityScheduler.queueComparator, + CapacityScheduler.applicationComparator, + TestUtils.spyHook); + clusterResource = Resources.createResource(100 * 16 * GB); + + queue = (LeafQueue)queues.get(A); + expectedMaxActiveApps = + Math.max(1, + (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * + csConf. + getMaximumApplicationMasterResourcePerQueuePercent( + queue.getQueuePath()) * + queue.getAbsoluteMaximumCapacity())); + + assertEquals((long) 0.5, + (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath())); + assertEquals(expectedMaxActiveApps, + queue.getMaximumActiveApplications()); + + // Change the per-queue max applications. + csConf.setInt( + "yarn.scheduler.capacity." + + queue.getQueuePath() + + ".maximum-applications", 9999); + // Re-create queues to get new configs. + queues = new HashMap(); + root = + CapacityScheduler.parseQueue(csContext, csConf, null, "root", + queues, queues, + CapacityScheduler.queueComparator, + CapacityScheduler.applicationComparator, + TestUtils.spyHook); + + queue = (LeafQueue)queues.get(A); + assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); + assertEquals(9999, queue.getMaxApplications()); + + expectedMaxAppsPerUser = (int)(9999 * + (queue.getUserLimit()/100.0f) * queue.getUserLimitFactor()); + assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser()); } @Test diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm index 01a9f603761..0f57f339080 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm @@ -240,17 +240,24 @@ Hadoop MapReduce Next Generation - Capacity Scheduler *--------------------------------------+--------------------------------------+ || Property || Description | *--------------------------------------+--------------------------------------+ -| <<>> | | +| <<>> / | +| <<.maximum-applications>>> | | | | Maximum number of applications in the system which can be concurrently | | | active both running and pending. Limits on each queue are directly | | | proportional to their queue capacities and user limits. This is a | | hard limit and any applications submitted when this limit is reached will | -| | be rejected. Default is 10000.| +| | be rejected. Default is 10000. This can be set for all queues with | +| | <<>> and can also be overridden on a | +| | per queue basis by setting <<.maximum-applications>>>. | *--------------------------------------+--------------------------------------+ -| yarn.scheduler.capacity.maximum-am-resource-percent | | +| <<>> / | +| <<.maximum-am-resource-percent>>> | | | | Maximum percent of resources in the cluster which can be used to run | -| | application masters - controls number of concurrent running applications. | -| | Specified as a float - ie 0.5 = 50%. Default is 10%. | +| | application masters - controls number of concurrent active applications. Limits on each | +| | queue are directly proportional to their queue capacities and user limits. | +| | Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with | +| | <<>> and can also be overridden on a | +| | per queue basis by setting <<.maximum-am-resource-percent>>> | *--------------------------------------+--------------------------------------+ * Queue Administration & Permissions