Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-3327. RM web ui scheduler link doesn't show correct max value for queues (Anupam Seth via mahadev) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1212212 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mahadev Konar 2011-12-09 01:38:35 +00:00
parent 7584901685
commit 224fc101fd
6 changed files with 50 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -252,6 +252,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3147. Handle leaf queues with the same name properly. (Ravi Prakash via MAPREDUCE-3147. Handle leaf queues with the same name properly. (Ravi Prakash via
mahadev) mahadev)
MAPREDUCE-3327. RM web ui scheduler link doesn't show correct max value
for queues (Anupam Seth via mahadev)
Release 0.23.0 - 2011-11-01 Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

2
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
View File

@ -21,7 +21,7 @@ class CSQueueUtils {
public static void checkMaxCapacity(String queueName, public static void checkMaxCapacity(String queueName,
float capacity, float maximumCapacity) { float capacity, float maximumCapacity) {
if (maximumCapacity != CapacitySchedulerConfiguration.UNDEFINED && if (Math.round(100 * maximumCapacity) != CapacitySchedulerConfiguration.UNDEFINED &&
maximumCapacity < capacity) { maximumCapacity < capacity) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"Illegal call to setMaxCapacity. " + "Illegal call to setMaxCapacity. " +

2
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -162,6 +162,8 @@ public class CapacitySchedulerConfiguration extends Configuration {
public void setMaximumCapacity(String queue, int maxCapacity) { public void setMaximumCapacity(String queue, int maxCapacity) {
setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity); setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity);
LOG.info("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) +
", maxCapacity=" + maxCapacity);
} }
public int getUserLimit(String queue) { public int getUserLimit(String queue) {

20
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -146,10 +146,10 @@ public class LeafQueue implements CSQueue {
(float)cs.getConfiguration().getCapacity(getQueuePath()) / 100; (float)cs.getConfiguration().getCapacity(getQueuePath()) / 100;
float absoluteCapacity = parent.getAbsoluteCapacity() * capacity; float absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
float maximumCapacity = cs.getConfiguration().getMaximumCapacity(getQueuePath()); float maximumCapacity = (float)cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
float absoluteMaxCapacity = float absoluteMaxCapacity =
(maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ? (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ?
Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity) / 100; Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity);
int userLimit = cs.getConfiguration().getUserLimit(getQueuePath()); int userLimit = cs.getConfiguration().getUserLimit(getQueuePath());
float userLimitFactor = float userLimitFactor =
@ -402,7 +402,7 @@ public class LeafQueue implements CSQueue {
this.maximumCapacity = maximumCapacity; this.maximumCapacity = maximumCapacity;
this.absoluteMaxCapacity = this.absoluteMaxCapacity =
(maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ? (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ?
Float.MAX_VALUE : Float.MAX_VALUE :
(parent.getAbsoluteCapacity() * maximumCapacity); (parent.getAbsoluteCapacity() * maximumCapacity);
} }
@ -829,13 +829,13 @@ public class LeafQueue implements CSQueue {
float potentialNewCapacity = float potentialNewCapacity =
(float)(usedResources.getMemory() + required.getMemory()) / (float)(usedResources.getMemory() + required.getMemory()) /
clusterResource.getMemory(); clusterResource.getMemory();
LOG.info(getQueueName() +
" usedResources: " + usedResources.getMemory() +
" currentCapacity " + ((float)usedResources.getMemory())/clusterResource.getMemory() +
" required " + required.getMemory() +
" potentialNewCapacity: " + potentialNewCapacity + " ( " +
" max-capacity: " + absoluteMaxCapacity + ")");
if (potentialNewCapacity > absoluteMaxCapacity) { if (potentialNewCapacity > absoluteMaxCapacity) {
LOG.info(getQueueName() +
" usedResources: " + usedResources.getMemory() +
" currentCapacity " + ((float)usedResources.getMemory())/clusterResource.getMemory() +
" required " + required.getMemory() +
" potentialNewCapacity: " + potentialNewCapacity + " ( " +
" > max-capacity (" + absoluteMaxCapacity + ")");
return false; return false;
} }
return true; return true;

10
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
View File

@ -123,10 +123,10 @@ public class ParentQueue implements CSQueue {
float absoluteCapacity = parentAbsoluteCapacity * capacity; float absoluteCapacity = parentAbsoluteCapacity * capacity;
float maximumCapacity = float maximumCapacity =
cs.getConfiguration().getMaximumCapacity(getQueuePath()); (float) cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
float absoluteMaxCapacity = float absoluteMaxCapacity =
(maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ? (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ?
1000000000f : (parentAbsoluteCapacity * maximumCapacity) / 100; Float.MAX_VALUE : (parentAbsoluteCapacity * maximumCapacity);
QueueState state = cs.getConfiguration().getState(getQueuePath()); QueueState state = cs.getConfiguration().getState(getQueuePath());
@ -233,12 +233,12 @@ public class ParentQueue implements CSQueue {
@Override @Override
public float getAbsoluteMaximumCapacity() { public float getAbsoluteMaximumCapacity() {
return 0; return absoluteMaxCapacity;
} }
@Override @Override
public float getMaximumCapacity() { public float getMaximumCapacity() {
return 0; return maximumCapacity;
} }
@Override @Override

30
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
View File

@ -107,12 +107,15 @@ public class TestLeafQueue {
// Define top-level queues // Define top-level queues
conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B}); conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B});
conf.setCapacity(CapacityScheduler.ROOT, 100); conf.setCapacity(CapacityScheduler.ROOT, 100);
conf.setMaximumCapacity(CapacityScheduler.ROOT, 100);
final String Q_A = CapacityScheduler.ROOT + "." + A; final String Q_A = CapacityScheduler.ROOT + "." + A;
conf.setCapacity(Q_A, 10); conf.setCapacity(Q_A, 10);
conf.setMaximumCapacity(Q_A, 20);
final String Q_B = CapacityScheduler.ROOT + "." + B; final String Q_B = CapacityScheduler.ROOT + "." + B;
conf.setCapacity(Q_B, 90); conf.setCapacity(Q_B, 90);
conf.setMaximumCapacity(Q_B, 99);
LOG.info("Setup top-level queues a and b"); LOG.info("Setup top-level queues a and b");
} }
@ -158,6 +161,23 @@ public class TestLeafQueue {
return queue; return queue;
} }
@Test
public void testInitializeQueue() throws Exception {
final float epsilon = 1e-5f;
//can add more sturdy test with 3-layer queues
//once MAPREDUCE:3410 is resolved
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
assertEquals(0.1, a.getCapacity(), epsilon);
assertEquals(0.1, a.getAbsoluteCapacity(), epsilon);
assertEquals(0.2, a.getMaximumCapacity(), epsilon);
assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
assertEquals(0.9, b.getCapacity(), epsilon);
assertEquals(0.9, b.getAbsoluteCapacity(), epsilon);
assertEquals(0.99, b.getMaximumCapacity(), epsilon);
assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
}
@Test @Test
public void testSingleQueueOneUserMetrics() throws Exception { public void testSingleQueueOneUserMetrics() throws Exception {
@ -209,6 +229,8 @@ public class TestLeafQueue {
// Manipulate queue 'a' // Manipulate queue 'a'
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
//unset maxCapacity
a.setMaxCapacity(-0.01f);
// Users // Users
final String user_0 = "user_0"; final String user_0 = "user_0";
@ -329,6 +351,8 @@ public class TestLeafQueue {
// Mock the queue // Mock the queue
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
//unset maxCapacity
a.setMaxCapacity(-0.01f);
// Users // Users
final String user_0 = "user_0"; final String user_0 = "user_0";
@ -442,7 +466,7 @@ public class TestLeafQueue {
// Revert max-capacity and user-limit-factor // Revert max-capacity and user-limit-factor
// Now, allocations should goto app_3 since it's under user-limit // Now, allocations should goto app_3 since it's under user-limit
a.setMaxCapacity(-1); a.setMaxCapacity(-0.01f);
a.setUserLimitFactor(1); a.setUserLimitFactor(1);
a.assignContainers(clusterResource, node_0); a.assignContainers(clusterResource, node_0);
assertEquals(7*GB, a.getUsedResources().getMemory()); assertEquals(7*GB, a.getUsedResources().getMemory());
@ -498,6 +522,8 @@ public class TestLeafQueue {
// Manipulate queue 'a' // Manipulate queue 'a'
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
//unset maxCapacity
a.setMaxCapacity(-0.01f);
// Users // Users
final String user_0 = "user_0"; final String user_0 = "user_0";
@ -594,6 +620,8 @@ public class TestLeafQueue {
// Manipulate queue 'a' // Manipulate queue 'a'
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
//unset maxCapacity
a.setMaxCapacity(-0.01f);
a.setUserLimitFactor(10); a.setUserLimitFactor(10);
// Users // Users