Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge -c 1235103 from trunk to branch-0.23 to fix MAPREDUCE-3681. Fixed computation of queue's usedCapacity. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1235104 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2012-01-24 01:06:55 +00:00
parent d6ebd72325
commit 439446057d
4 changed files with 83 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-mapreduce-project/CHANGES.txt
View File

@ -493,6 +493,8 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3646. Remove redundant URL info from "mapred job" output.
(Jonathan Eagles via mahadev)
MAPREDUCE-3681. Fixed computation of queue's usedCapacity. (acmurthy)
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES

23
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -272,9 +272,9 @@ public class LeafQueue implements CSQueue {
"maxActiveApplicationsPerUser = " + maxActiveApplicationsPerUser +
" [= (int)(maxActiveApplications * (userLimit / 100.0f) * userLimitFactor) ]" + "\n" +
"utilization = " + utilization +
" [= usedResourcesMemory / queueLimit ]" + "\n" +
" [= usedResourcesMemory / (clusterResourceMemory * absoluteCapacity)]" + "\n" +
"usedCapacity = " + usedCapacity +
" [= usedResourcesMemory / (clusterResourceMemory * capacity) ]" + "\n" +
" [= usedResourcesMemory / (clusterResourceMemory * parent.absoluteCapacity)]" + "\n" +
"maxAMResourcePercent = " + maxAMResourcePercent +
" [= configuredMaximumAMResourcePercent ]" + "\n" +
"minimumAllocationFactor = " + minimumAllocationFactor +
@ -502,9 +502,14 @@ public class LeafQueue implements CSQueue {
}
public String toString() {
return queueName + ":" + capacity + ":" + absoluteCapacity + ":" +
getUsedCapacity() + ":" + getUtilization() + ":" +
getNumApplications() + ":" + getNumContainers();
return queueName + ": " +
"capacity=" + capacity + ", " +
"absoluteCapacity=" + absoluteCapacity + ", " +
"usedResources=" + usedResources.getMemory() + "MB, " +
"usedCapacity=" + getUsedCapacity() + ", " +
"utilization=" + getUtilization() + ", " +
"numApps=" + getNumApplications() + ", " +
"numContainers=" + getNumContainers();
}
private synchronized User getUser(String userName) {
@ -1316,11 +1321,11 @@ public class LeafQueue implements CSQueue {
}
private synchronized void updateResource(Resource clusterResource) {
float queueLimit = clusterResource.getMemory() * absoluteCapacity;
float queueLimit = clusterResource.getMemory() * absoluteCapacity;
setUtilization(usedResources.getMemory() / queueLimit);
setUsedCapacity(
usedResources.getMemory() / (clusterResource.getMemory() * capacity));
setUsedCapacity(usedResources.getMemory()
/ (clusterResource.getMemory() * parent.getAbsoluteCapacity()));
Resource resourceLimit =
Resources.createResource(roundUp((int)queueLimit));
metrics.setAvailableResourcesToQueue(

23
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
View File

@ -333,10 +333,15 @@ public class ParentQueue implements CSQueue {
}
public String toString() {
return queueName + ":" + capacity + ":" + absoluteCapacity + ":" +
getUsedCapacity() + ":" + getUtilization() + ":" +
getNumApplications() + ":" + getNumContainers() + ":" +
childQueues.size() + " child-queues";
return queueName + ": " +
"numChildQueue= " + childQueues.size() + ", " +
"capacity=" + capacity + ", " +
"absoluteCapacity=" + absoluteCapacity + ", " +
"usedResources=" + usedResources.getMemory() + "MB, " +
"usedCapacity=" + getUsedCapacity() + ", " +
"utilization=" + getUtilization() + ", " +
"numApps=" + getNumApplications() + ", " +
"numContainers=" + getNumContainers();
}
@Override
@ -688,11 +693,13 @@ public class ParentQueue implements CSQueue {
}
private synchronized void updateResource(Resource clusterResource) {
float queueLimit = clusterResource.getMemory() * absoluteCapacity;
float queueLimit = clusterResource.getMemory() * absoluteCapacity;
float parentAbsoluteCapacity =
(rootQueue) ? 1.0f : parent.getAbsoluteCapacity();
setUtilization(usedResources.getMemory() / queueLimit);
setUsedCapacity(
usedResources.getMemory() / (clusterResource.getMemory() * capacity));
setUsedCapacity(usedResources.getMemory()
/ (clusterResource.getMemory() * parentAbsoluteCapacity));
Resource resourceLimit =
Resources.createResource((int)queueLimit);
metrics.setAvailableResourcesToQueue(

114
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
View File

@ -138,12 +138,34 @@ public class TestParentQueue {
when(queue).assignContainers(eq(clusterResource), eq(node));
}
private float computeQueueUsedCapacity(CSQueue queue,
int expectedMemory, Resource clusterResource) {
return (
((float)expectedMemory / clusterResource.getMemory()) *
queue.getParent().getAbsoluteCapacity()
);
}
private float computeQueueUtilization(CSQueue queue,
int expectedMemory, Resource clusterResource) {
return (expectedMemory /
(clusterResource.getMemory() * queue.getAbsoluteCapacity()));
}
final static float DELTA = 0.0001f;
private void verifyQueueMetrics(CSQueue queue,
int expectedMemory, Resource clusterResource) {
assertEquals(
computeQueueUtilization(queue, expectedMemory, clusterResource),
queue.getUtilization(),
DELTA);
assertEquals(
computeQueueUsedCapacity(queue, expectedMemory, clusterResource),
queue.getUsedCapacity(),
DELTA);
}
@Test
public void testSingleLevelQueues() throws Exception {
// Setup queue configs
@ -173,15 +195,13 @@ public class TestParentQueue {
// Start testing
LeafQueue a = (LeafQueue)queues.get(A);
LeafQueue b = (LeafQueue)queues.get(B);
final float delta = 0.0001f;
// Simulate B returning a container on node_0
stubQueueAllocation(a, clusterResource, node_0, 0*GB);
stubQueueAllocation(b, clusterResource, node_0, 1*GB);
root.assignContainers(clusterResource, node_0);
assertEquals(0.0f, a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 1*GB, clusterResource),
b.getUtilization(), delta);
verifyQueueMetrics(a, 0*GB, clusterResource);
verifyQueueMetrics(b, 1*GB, clusterResource);
// Now, A should get the scheduling opportunity since A=0G/6G, B=1G/14G
stubQueueAllocation(a, clusterResource, node_1, 2*GB);
@ -192,10 +212,8 @@ public class TestParentQueue {
any(SchedulerNode.class));
allocationOrder.verify(b).assignContainers(eq(clusterResource),
any(SchedulerNode.class));
assertEquals(computeQueueUtilization(a, 2*GB, clusterResource),
a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 2*GB, clusterResource),
b.getUtilization(), delta);
verifyQueueMetrics(a, 2*GB, clusterResource);
verifyQueueMetrics(b, 2*GB, clusterResource);
// Now, B should get the scheduling opportunity
// since A has 2/6G while B has 2/14G
@ -207,10 +225,8 @@ public class TestParentQueue {
any(SchedulerNode.class));
allocationOrder.verify(a).assignContainers(eq(clusterResource),
any(SchedulerNode.class));
assertEquals(computeQueueUtilization(a, 3*GB, clusterResource),
a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 4*GB, clusterResource),
b.getUtilization(), delta);
verifyQueueMetrics(a, 3*GB, clusterResource);
verifyQueueMetrics(b, 4*GB, clusterResource);
// Now, B should still get the scheduling opportunity
// since A has 3/6G while B has 4/14G
@ -222,10 +238,8 @@ public class TestParentQueue {
any(SchedulerNode.class));
allocationOrder.verify(a).assignContainers(eq(clusterResource),
any(SchedulerNode.class));
assertEquals(computeQueueUtilization(a, 3*GB, clusterResource),
a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 8*GB, clusterResource),
b.getUtilization(), delta);
verifyQueueMetrics(a, 3*GB, clusterResource);
verifyQueueMetrics(b, 8*GB, clusterResource);
// Now, A should get the scheduling opportunity
// since A has 3/6G while B has 8/14G
@ -237,10 +251,8 @@ public class TestParentQueue {
any(SchedulerNode.class));
allocationOrder.verify(a).assignContainers(eq(clusterResource),
any(SchedulerNode.class));
assertEquals(computeQueueUtilization(a, 4*GB, clusterResource),
a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 9*GB, clusterResource),
b.getUtilization(), delta);
verifyQueueMetrics(a, 4*GB, clusterResource);
verifyQueueMetrics(b, 9*GB, clusterResource);
}
private static final String C = "c";
@ -323,22 +335,16 @@ public class TestParentQueue {
CSQueue b2 = queues.get(B2);
CSQueue b3 = queues.get(B3);
final float delta = 0.0001f;
// Simulate C returning a container on node_0
stubQueueAllocation(a, clusterResource, node_0, 0*GB);
stubQueueAllocation(b, clusterResource, node_0, 0*GB);
stubQueueAllocation(c, clusterResource, node_0, 1*GB);
stubQueueAllocation(d, clusterResource, node_0, 0*GB);
root.assignContainers(clusterResource, node_0);
assertEquals(computeQueueUtilization(a, 0*GB, clusterResource),
a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 0*GB, clusterResource),
b.getUtilization(), delta);
assertEquals(computeQueueUtilization(c, 1*GB, clusterResource),
c.getUtilization(), delta);
assertEquals(computeQueueUtilization(d, 0*GB, clusterResource),
d.getUtilization(), delta);
verifyQueueMetrics(a, 0*GB, clusterResource);
verifyQueueMetrics(b, 0*GB, clusterResource);
verifyQueueMetrics(c, 1*GB, clusterResource);
verifyQueueMetrics(d, 0*GB, clusterResource);
reset(a); reset(b); reset(c);
// Now get B2 to allocate
@ -347,12 +353,9 @@ public class TestParentQueue {
stubQueueAllocation(b2, clusterResource, node_1, 4*GB);
stubQueueAllocation(c, clusterResource, node_1, 0*GB);
root.assignContainers(clusterResource, node_1);
assertEquals(computeQueueUtilization(a, 0*GB, clusterResource),
a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 4*GB, clusterResource),
b.getUtilization(), delta);
assertEquals(computeQueueUtilization(c, 1*GB, clusterResource),
c.getUtilization(), delta);
verifyQueueMetrics(a, 0*GB, clusterResource);
verifyQueueMetrics(b, 4*GB, clusterResource);
verifyQueueMetrics(c, 1*GB, clusterResource);
reset(a); reset(b); reset(c);
// Now get both A1, C & B3 to allocate in right order
@ -368,12 +371,9 @@ public class TestParentQueue {
any(SchedulerNode.class));
allocationOrder.verify(b).assignContainers(eq(clusterResource),
any(SchedulerNode.class));
assertEquals(computeQueueUtilization(a, 1*GB, clusterResource),
a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 6*GB, clusterResource),
b.getUtilization(), delta);
assertEquals(computeQueueUtilization(c, 3*GB, clusterResource),
c.getUtilization(), delta);
verifyQueueMetrics(a, 1*GB, clusterResource);
verifyQueueMetrics(b, 6*GB, clusterResource);
verifyQueueMetrics(c, 3*GB, clusterResource);
reset(a); reset(b); reset(c);
// Now verify max-capacity
@ -399,16 +399,12 @@ public class TestParentQueue {
any(SchedulerNode.class));
allocationOrder.verify(c).assignContainers(eq(clusterResource),
any(SchedulerNode.class));
assertEquals(computeQueueUtilization(a, 3*GB, clusterResource),
a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 8*GB, clusterResource),
b.getUtilization(), delta);
assertEquals(computeQueueUtilization(c, 4*GB, clusterResource),
c.getUtilization(), delta);
verifyQueueMetrics(a, 3*GB, clusterResource);
verifyQueueMetrics(b, 8*GB, clusterResource);
verifyQueueMetrics(c, 4*GB, clusterResource);
reset(a); reset(b); reset(c);
}
@Test
public void testOffSwitchScheduling() throws Exception {
// Setup queue configs
@ -438,15 +434,13 @@ public class TestParentQueue {
// Start testing
LeafQueue a = (LeafQueue)queues.get(A);
LeafQueue b = (LeafQueue)queues.get(B);
final float delta = 0.0001f;
// Simulate B returning a container on node_0
stubQueueAllocation(a, clusterResource, node_0, 0*GB, NodeType.OFF_SWITCH);
stubQueueAllocation(b, clusterResource, node_0, 1*GB, NodeType.OFF_SWITCH);
root.assignContainers(clusterResource, node_0);
assertEquals(0.0f, a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 1*GB, clusterResource),
b.getUtilization(), delta);
verifyQueueMetrics(a, 0*GB, clusterResource);
verifyQueueMetrics(b, 1*GB, clusterResource);
// Now, A should get the scheduling opportunity since A=0G/6G, B=1G/14G
// also, B gets a scheduling opportunity since A allocates RACK_LOCAL
@ -458,10 +452,8 @@ public class TestParentQueue {
any(SchedulerNode.class));
allocationOrder.verify(b).assignContainers(eq(clusterResource),
any(SchedulerNode.class));
assertEquals(computeQueueUtilization(a, 2*GB, clusterResource),
a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 2*GB, clusterResource),
b.getUtilization(), delta);
verifyQueueMetrics(a, 2*GB, clusterResource);
verifyQueueMetrics(b, 2*GB, clusterResource);
// Now, B should get the scheduling opportunity
// since A has 2/6G while B has 2/14G,
@ -474,10 +466,8 @@ public class TestParentQueue {
any(SchedulerNode.class));
allocationOrder.verify(a).assignContainers(eq(clusterResource),
any(SchedulerNode.class));
assertEquals(computeQueueUtilization(a, 2*GB, clusterResource),
a.getUtilization(), delta);
assertEquals(computeQueueUtilization(b, 4*GB, clusterResource),
b.getUtilization(), delta);
verifyQueueMetrics(a, 2*GB, clusterResource);
verifyQueueMetrics(b, 4*GB, clusterResource);
}