Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-3784. Fixed CapacityScheduler so that maxActiveApplications and maxActiveApplicationsPerUser per queue are not too low for small clusters. Contributed by Arun C Murthy. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1239971 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2012-02-03 01:16:32 +00:00
parent 72d34e6594
commit a5c46c9165
4 changed files with 53 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/CHANGES.txt
View File

@ -226,6 +226,10 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3771. Un-deprecated the old mapred apis, port of MAPREDUCE-1735. MAPREDUCE-3771. Un-deprecated the old mapred apis, port of MAPREDUCE-1735.
(acmurthy) (acmurthy)
MAPREDUCE-3784. Fixed CapacityScheduler so that maxActiveApplications and
maxActiveApplicationsPerUser per queue are not too low for small
clusters. (Arun C Murthy via vinodkv)
OPTIMIZATIONS OPTIMIZATIONS
MAPREDUCE-3567. Extraneous JobConf objects in AM heap. (Vinod Kumar MAPREDUCE-3567. Extraneous JobConf objects in AM heap. (Vinod Kumar

14
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
View File

@ -46,17 +46,23 @@ class CSQueueUtils {
} }
public static int computeMaxActiveApplications(Resource clusterResource, public static int computeMaxActiveApplications(Resource clusterResource,
float maxAMResourcePercent, float absoluteCapacity) { Resource minimumAllocation, float maxAMResourcePercent,
float absoluteMaxCapacity) {
return return
Math.max( Math.max(
(int)((clusterResource.getMemory() / (float)LeafQueue.DEFAULT_AM_RESOURCE) * (int)Math.ceil(
maxAMResourcePercent * absoluteCapacity), ((float)clusterResource.getMemory() /
minimumAllocation.getMemory()) *
maxAMResourcePercent * absoluteMaxCapacity),
1); 1);
} }
public static int computeMaxActiveApplicationsPerUser( public static int computeMaxActiveApplicationsPerUser(
int maxActiveApplications, int userLimit, float userLimitFactor) { int maxActiveApplications, int userLimit, float userLimitFactor) {
return (int)(maxActiveApplications * (userLimit / 100.0f) * userLimitFactor); return Math.max(
(int)Math.ceil(
maxActiveApplications * (userLimit / 100.0f) * userLimitFactor),
1);
} }
} }

43
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -125,8 +125,6 @@ public class LeafQueue implements CSQueue {
private final ActiveUsersManager activeUsersManager; private final ActiveUsersManager activeUsersManager;
final static int DEFAULT_AM_RESOURCE = 2 * 1024;
public LeafQueue(CapacitySchedulerContext cs, public LeafQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, String queueName, CSQueue parent,
Comparator<SchedulerApp> applicationComparator, CSQueue old) { Comparator<SchedulerApp> applicationComparator, CSQueue old) {
@ -166,8 +164,9 @@ public class LeafQueue implements CSQueue {
this.maxAMResourcePercent = this.maxAMResourcePercent =
cs.getConfiguration().getMaximumApplicationMasterResourcePercent(); cs.getConfiguration().getMaximumApplicationMasterResourcePercent();
int maxActiveApplications = int maxActiveApplications =
CSQueueUtils.computeMaxActiveApplications(cs.getClusterResources(), CSQueueUtils.computeMaxActiveApplications(
maxAMResourcePercent, absoluteCapacity); cs.getClusterResources(), this.minimumAllocation,
maxAMResourcePercent, absoluteMaxCapacity);
int maxActiveApplicationsPerUser = int maxActiveApplicationsPerUser =
CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveApplications, userLimit, CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveApplications, userLimit,
userLimitFactor); userLimitFactor);
@ -246,30 +245,39 @@ public class LeafQueue implements CSQueue {
" [= configuredMaxCapacity ]" + "\n" + " [= configuredMaxCapacity ]" + "\n" +
"absoluteMaxCapacity = " + absoluteMaxCapacity + "absoluteMaxCapacity = " + absoluteMaxCapacity +
" [= 1.0 maximumCapacity undefined, " + " [= 1.0 maximumCapacity undefined, " +
"(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]" + "\n" + "(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]" +
"\n" +
"userLimit = " + userLimit + "userLimit = " + userLimit +
" [= configuredUserLimit ]" + "\n" + " [= configuredUserLimit ]" + "\n" +
"userLimitFactor = " + userLimitFactor + "userLimitFactor = " + userLimitFactor +
" [= configuredUserLimitFactor ]" + "\n" + " [= configuredUserLimitFactor ]" + "\n" +
"maxApplications = " + maxApplications + "maxApplications = " + maxApplications +
" [= (int)(configuredMaximumSystemApplications * absoluteCapacity) ]" + "\n" + " [= (int)(configuredMaximumSystemApplications * absoluteCapacity) ]" +
"\n" +
"maxApplicationsPerUser = " + maxApplicationsPerUser + "maxApplicationsPerUser = " + maxApplicationsPerUser +
" [= (int)(maxApplications * (userLimit / 100.0f) * userLimitFactor) ]" + "\n" + " [= (int)(maxApplications * (userLimit / 100.0f) * " +
"userLimitFactor) ]" + "\n" +
"maxActiveApplications = " + maxActiveApplications + "maxActiveApplications = " + maxActiveApplications +
" [= max(" + " [= max(" +
"(int)((clusterResourceMemory / (float)DEFAULT_AM_RESOURCE) *" + "(int)ceil((clusterResourceMemory / minimumAllocation) *" +
"maxAMResourcePercent * absoluteCapacity)," + "maxAMResourcePercent * absoluteMaxCapacity)," +
"1) ]" + "\n" + "1) ]" + "\n" +
"maxActiveApplicationsPerUser = " + maxActiveApplicationsPerUser + "maxActiveApplicationsPerUser = " + maxActiveApplicationsPerUser +
" [= (int)(maxActiveApplications * (userLimit / 100.0f) * userLimitFactor) ]" + "\n" + " [= max(" +
"(int)(maxActiveApplications * (userLimit / 100.0f) * " +
"userLimitFactor)," +
"1) ]" + "\n" +
"utilization = " + utilization + "utilization = " + utilization +
" [= usedResourcesMemory / (clusterResourceMemory * absoluteCapacity)]" + "\n" + " [= usedResourcesMemory / " +
"(clusterResourceMemory * absoluteCapacity)]" + "\n" +
"usedCapacity = " + usedCapacity + "usedCapacity = " + usedCapacity +
" [= usedResourcesMemory / (clusterResourceMemory * parent.absoluteCapacity)]" + "\n" + " [= usedResourcesMemory / " +
"(clusterResourceMemory * parent.absoluteCapacity)]" + "\n" +
"maxAMResourcePercent = " + maxAMResourcePercent + "maxAMResourcePercent = " + maxAMResourcePercent +
" [= configuredMaximumAMResourcePercent ]" + "\n" + " [= configuredMaximumAMResourcePercent ]" + "\n" +
"minimumAllocationFactor = " + minimumAllocationFactor + "minimumAllocationFactor = " + minimumAllocationFactor +
" [= (float)(maximumAllocationMemory - minimumAllocationMemory) / maximumAllocationMemory ]" + "\n" + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " +
"maximumAllocationMemory ]" + "\n" +
"numContainers = " + numContainers + "numContainers = " + numContainers +
" [= currentNumContainers ]" + "\n" + " [= currentNumContainers ]" + "\n" +
"state = " + state + "state = " + state +
@ -1359,11 +1367,12 @@ public class LeafQueue implements CSQueue {
public synchronized void updateClusterResource(Resource clusterResource) { public synchronized void updateClusterResource(Resource clusterResource) {
// Update queue properties // Update queue properties
maxActiveApplications = maxActiveApplications =
CSQueueUtils.computeMaxActiveApplications(clusterResource, maxAMResourcePercent, CSQueueUtils.computeMaxActiveApplications(
absoluteCapacity); clusterResource, minimumAllocation,
maxAMResourcePercent, absoluteMaxCapacity);
maxActiveApplicationsPerUser = maxActiveApplicationsPerUser =
CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveApplications, userLimit, CSQueueUtils.computeMaxActiveApplicationsPerUser(
userLimitFactor); maxActiveApplications, userLimit, userLimitFactor);
// Update application properties // Update application properties
for (SchedulerApp application : activeApplications) { for (SchedulerApp application : activeApplications) {

24
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
View File

@ -153,29 +153,31 @@ public class TestApplicationLimits {
queue.getMaximumActiveApplicationsPerUser()); queue.getMaximumActiveApplicationsPerUser());
int expectedMaxActiveApps = int expectedMaxActiveApps =
Math.max(1, Math.max(1,
(int)((clusterResource.getMemory() / LeafQueue.DEFAULT_AM_RESOURCE) * (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
csConf.getMaximumApplicationMasterResourcePercent() * csConf.getMaximumApplicationMasterResourcePercent() *
queue.getAbsoluteCapacity())); queue.getAbsoluteMaximumCapacity()));
assertEquals(expectedMaxActiveApps, assertEquals(expectedMaxActiveApps,
queue.getMaximumActiveApplications()); queue.getMaximumActiveApplications());
assertEquals((int)(expectedMaxActiveApps * (queue.getUserLimit() / 100.0f) * assertEquals(
queue.getUserLimitFactor()), (int)Math.ceil(
queue.getMaximumActiveApplicationsPerUser()); expectedMaxActiveApps * (queue.getUserLimit() / 100.0f) *
queue.getUserLimitFactor()),
queue.getMaximumActiveApplicationsPerUser());
// Add some nodes to the cluster & test new limits // Add some nodes to the cluster & test new limits
clusterResource = Resources.createResource(120 * 16 * GB); clusterResource = Resources.createResource(120 * 16 * GB);
root.updateClusterResource(clusterResource); root.updateClusterResource(clusterResource);
expectedMaxActiveApps = expectedMaxActiveApps =
Math.max(1, Math.max(1,
(int)((clusterResource.getMemory() / LeafQueue.DEFAULT_AM_RESOURCE) * (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
csConf.getMaximumApplicationMasterResourcePercent() * csConf.getMaximumApplicationMasterResourcePercent() *
queue.getAbsoluteCapacity())); queue.getAbsoluteMaximumCapacity()));
assertEquals(expectedMaxActiveApps, assertEquals(expectedMaxActiveApps,
queue.getMaximumActiveApplications()); queue.getMaximumActiveApplications());
assertEquals((int)(expectedMaxActiveApps * (queue.getUserLimit() / 100.0f) * assertEquals(
queue.getUserLimitFactor()), (int)Math.ceil(expectedMaxActiveApps *
queue.getMaximumActiveApplicationsPerUser()); (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()),
queue.getMaximumActiveApplicationsPerUser());
} }
@Test @Test