From e11ab3382d5d4593aedfd96dd05348aa9dcb35e4 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Sun, 4 Mar 2012 23:47:36 +0000 Subject: [PATCH] Merge -c 1296898 from trunk to branch-0.23 to fix MAPREDUCE-3897. Fixed computation of maxActiveAppsPerUser for queues by using capacity and not max-capacity since we are already scaling it by userLimitFactor. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1296899 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++++ .../scheduler/capacity/LeafQueue.java | 20 ++++++++++++++++--- .../capacity/TestApplicationLimits.java | 14 +++++++++++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 110e682cbff..36140503042 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -173,6 +173,10 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-3960. Fix web-proxy to forward request to AM with configured hostname or IP. (tgraves via acmurthy) + MAPREDUCE-3897. Fixed computation of maxActiveAppsPerUser for queues by + using capacity and not max-capacity since we are already scaling it by + userLimitFactor. (Eric Payne via acmurthy) + Release 0.23.1 - 2012-02-17 NEW FEATURES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 0e3d747523a..df4cebd2ec3 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -88,7 +88,8 @@ public class LeafQueue implements CSQueue { private int maxApplicationsPerUser; private float maxAMResourcePercent; - private int maxActiveApplications; + private int maxActiveApplications; // Based on absolute max capacity + private int maxActiveAppsUsingAbsCap; // Based on absolute capacity private int maxActiveApplicationsPerUser; private Resource usedResources = Resources.createResource(0); @@ -167,8 +168,12 @@ public class LeafQueue implements CSQueue { CSQueueUtils.computeMaxActiveApplications( cs.getClusterResources(), this.minimumAllocation, maxAMResourcePercent, absoluteMaxCapacity); + this.maxActiveAppsUsingAbsCap = + CSQueueUtils.computeMaxActiveApplications( + cs.getClusterResources(), this.minimumAllocation, + maxAMResourcePercent, absoluteCapacity); int maxActiveApplicationsPerUser = - CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveApplications, userLimit, + CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveAppsUsingAbsCap, userLimit, userLimitFactor); this.queueInfo = recordFactory.newRecordInstance(QueueInfo.class); @@ -271,6 +276,11 @@ public class LeafQueue implements CSQueue { "(int)ceil((clusterResourceMemory / minimumAllocation) *" + "maxAMResourcePercent * absoluteMaxCapacity)," + "1) ]" + "\n" + + "maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap + + " [= max(" + + "(int)ceil((clusterResourceMemory / minimumAllocation) *" + + "maxAMResourcePercent * absoluteCapacity)," + + "1) ]" + "\n" + "maxActiveApplicationsPerUser = " + maxActiveApplicationsPerUser + " [= max(" + "(int)(maxActiveApplications * (userLimit / 100.0f) * " + @@ -1376,9 +1386,13 @@ public class LeafQueue implements CSQueue { CSQueueUtils.computeMaxActiveApplications( clusterResource, minimumAllocation, maxAMResourcePercent, absoluteMaxCapacity); + maxActiveAppsUsingAbsCap = + CSQueueUtils.computeMaxActiveApplications( + clusterResource, minimumAllocation, + maxAMResourcePercent, absoluteCapacity); maxActiveApplicationsPerUser = CSQueueUtils.computeMaxActiveApplicationsPerUser( - maxActiveApplications, userLimit, userLimitFactor); + maxActiveAppsUsingAbsCap, userLimit, userLimitFactor); // Update metrics CSQueueUtils.updateQueueStatistics( diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 57e0c69385e..ae3e5f97d19 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -158,9 +158,14 @@ public class TestApplicationLimits { queue.getAbsoluteMaximumCapacity())); assertEquals(expectedMaxActiveApps, queue.getMaximumActiveApplications()); + int expectedMaxActiveAppsUsingAbsCap = + Math.max(1, + (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * + csConf.getMaximumApplicationMasterResourcePercent() * + queue.getAbsoluteCapacity())); assertEquals( (int)Math.ceil( - expectedMaxActiveApps * (queue.getUserLimit() / 100.0f) * + expectedMaxActiveAppsUsingAbsCap * (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()), queue.getMaximumActiveApplicationsPerUser()); assertEquals( @@ -178,8 +183,13 @@ public class TestApplicationLimits { queue.getAbsoluteMaximumCapacity())); assertEquals(expectedMaxActiveApps, queue.getMaximumActiveApplications()); + expectedMaxActiveAppsUsingAbsCap = + Math.max(1, + (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * + csConf.getMaximumApplicationMasterResourcePercent() * + queue.getAbsoluteCapacity())); assertEquals( - (int)Math.ceil(expectedMaxActiveApps * + (int)Math.ceil(expectedMaxActiveAppsUsingAbsCap * (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()), queue.getMaximumActiveApplicationsPerUser()); assertEquals(