MAPREDUCE-3327. RM web ui scheduler link doesn't show correct max value for queues (Anupam Seth via mahadev)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1212212 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
7584901685
commit
224fc101fd
|
@ -252,6 +252,9 @@ Release 0.23.1 - Unreleased
|
|||
MAPREDUCE-3147. Handle leaf queues with the same name properly. (Ravi Prakash via
|
||||
mahadev)
|
||||
|
||||
MAPREDUCE-3327. RM web ui scheduler link doesn't show correct max value
|
||||
for queues (Anupam Seth via mahadev)
|
||||
|
||||
Release 0.23.0 - 2011-11-01
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
@ -21,7 +21,7 @@ class CSQueueUtils {
|
|||
|
||||
public static void checkMaxCapacity(String queueName,
|
||||
float capacity, float maximumCapacity) {
|
||||
if (maximumCapacity != CapacitySchedulerConfiguration.UNDEFINED &&
|
||||
if (Math.round(100 * maximumCapacity) != CapacitySchedulerConfiguration.UNDEFINED &&
|
||||
maximumCapacity < capacity) {
|
||||
throw new IllegalArgumentException(
|
||||
"Illegal call to setMaxCapacity. " +
|
||||
|
|
|
@ -162,6 +162,8 @@ public class CapacitySchedulerConfiguration extends Configuration {
|
|||
|
||||
public void setMaximumCapacity(String queue, int maxCapacity) {
|
||||
setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity);
|
||||
LOG.info("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) +
|
||||
", maxCapacity=" + maxCapacity);
|
||||
}
|
||||
|
||||
public int getUserLimit(String queue) {
|
||||
|
|
|
@ -146,10 +146,10 @@ public class LeafQueue implements CSQueue {
|
|||
(float)cs.getConfiguration().getCapacity(getQueuePath()) / 100;
|
||||
float absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
|
||||
|
||||
float maximumCapacity = cs.getConfiguration().getMaximumCapacity(getQueuePath());
|
||||
float maximumCapacity = (float)cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
|
||||
float absoluteMaxCapacity =
|
||||
(maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ?
|
||||
Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity) / 100;
|
||||
(Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ?
|
||||
Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity);
|
||||
|
||||
int userLimit = cs.getConfiguration().getUserLimit(getQueuePath());
|
||||
float userLimitFactor =
|
||||
|
@ -402,7 +402,7 @@ public class LeafQueue implements CSQueue {
|
|||
|
||||
this.maximumCapacity = maximumCapacity;
|
||||
this.absoluteMaxCapacity =
|
||||
(maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ?
|
||||
(Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ?
|
||||
Float.MAX_VALUE :
|
||||
(parent.getAbsoluteCapacity() * maximumCapacity);
|
||||
}
|
||||
|
@ -829,13 +829,13 @@ public class LeafQueue implements CSQueue {
|
|||
float potentialNewCapacity =
|
||||
(float)(usedResources.getMemory() + required.getMemory()) /
|
||||
clusterResource.getMemory();
|
||||
LOG.info(getQueueName() +
|
||||
" usedResources: " + usedResources.getMemory() +
|
||||
" currentCapacity " + ((float)usedResources.getMemory())/clusterResource.getMemory() +
|
||||
" required " + required.getMemory() +
|
||||
" potentialNewCapacity: " + potentialNewCapacity + " ( " +
|
||||
" max-capacity: " + absoluteMaxCapacity + ")");
|
||||
if (potentialNewCapacity > absoluteMaxCapacity) {
|
||||
LOG.info(getQueueName() +
|
||||
" usedResources: " + usedResources.getMemory() +
|
||||
" currentCapacity " + ((float)usedResources.getMemory())/clusterResource.getMemory() +
|
||||
" required " + required.getMemory() +
|
||||
" potentialNewCapacity: " + potentialNewCapacity + " ( " +
|
||||
" > max-capacity (" + absoluteMaxCapacity + ")");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
|
|
@ -123,10 +123,10 @@ public class ParentQueue implements CSQueue {
|
|||
float absoluteCapacity = parentAbsoluteCapacity * capacity;
|
||||
|
||||
float maximumCapacity =
|
||||
cs.getConfiguration().getMaximumCapacity(getQueuePath());
|
||||
(float) cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
|
||||
float absoluteMaxCapacity =
|
||||
(maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ?
|
||||
1000000000f : (parentAbsoluteCapacity * maximumCapacity) / 100;
|
||||
(Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ?
|
||||
Float.MAX_VALUE : (parentAbsoluteCapacity * maximumCapacity);
|
||||
|
||||
QueueState state = cs.getConfiguration().getState(getQueuePath());
|
||||
|
||||
|
@ -233,12 +233,12 @@ public class ParentQueue implements CSQueue {
|
|||
|
||||
@Override
|
||||
public float getAbsoluteMaximumCapacity() {
|
||||
return 0;
|
||||
return absoluteMaxCapacity;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getMaximumCapacity() {
|
||||
return 0;
|
||||
return maximumCapacity;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -107,12 +107,15 @@ public class TestLeafQueue {
|
|||
// Define top-level queues
|
||||
conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B});
|
||||
conf.setCapacity(CapacityScheduler.ROOT, 100);
|
||||
conf.setMaximumCapacity(CapacityScheduler.ROOT, 100);
|
||||
|
||||
final String Q_A = CapacityScheduler.ROOT + "." + A;
|
||||
conf.setCapacity(Q_A, 10);
|
||||
conf.setMaximumCapacity(Q_A, 20);
|
||||
|
||||
final String Q_B = CapacityScheduler.ROOT + "." + B;
|
||||
conf.setCapacity(Q_B, 90);
|
||||
conf.setMaximumCapacity(Q_B, 99);
|
||||
|
||||
LOG.info("Setup top-level queues a and b");
|
||||
}
|
||||
|
@ -158,6 +161,23 @@ public class TestLeafQueue {
|
|||
return queue;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInitializeQueue() throws Exception {
|
||||
final float epsilon = 1e-5f;
|
||||
//can add more sturdy test with 3-layer queues
|
||||
//once MAPREDUCE:3410 is resolved
|
||||
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
|
||||
assertEquals(0.1, a.getCapacity(), epsilon);
|
||||
assertEquals(0.1, a.getAbsoluteCapacity(), epsilon);
|
||||
assertEquals(0.2, a.getMaximumCapacity(), epsilon);
|
||||
assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
|
||||
|
||||
LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
|
||||
assertEquals(0.9, b.getCapacity(), epsilon);
|
||||
assertEquals(0.9, b.getAbsoluteCapacity(), epsilon);
|
||||
assertEquals(0.99, b.getMaximumCapacity(), epsilon);
|
||||
assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingleQueueOneUserMetrics() throws Exception {
|
||||
|
@ -209,6 +229,8 @@ public class TestLeafQueue {
|
|||
|
||||
// Manipulate queue 'a'
|
||||
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
|
||||
//unset maxCapacity
|
||||
a.setMaxCapacity(-0.01f);
|
||||
|
||||
// Users
|
||||
final String user_0 = "user_0";
|
||||
|
@ -329,6 +351,8 @@ public class TestLeafQueue {
|
|||
|
||||
// Mock the queue
|
||||
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
|
||||
//unset maxCapacity
|
||||
a.setMaxCapacity(-0.01f);
|
||||
|
||||
// Users
|
||||
final String user_0 = "user_0";
|
||||
|
@ -442,7 +466,7 @@ public class TestLeafQueue {
|
|||
|
||||
// Revert max-capacity and user-limit-factor
|
||||
// Now, allocations should goto app_3 since it's under user-limit
|
||||
a.setMaxCapacity(-1);
|
||||
a.setMaxCapacity(-0.01f);
|
||||
a.setUserLimitFactor(1);
|
||||
a.assignContainers(clusterResource, node_0);
|
||||
assertEquals(7*GB, a.getUsedResources().getMemory());
|
||||
|
@ -498,6 +522,8 @@ public class TestLeafQueue {
|
|||
|
||||
// Manipulate queue 'a'
|
||||
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
|
||||
//unset maxCapacity
|
||||
a.setMaxCapacity(-0.01f);
|
||||
|
||||
// Users
|
||||
final String user_0 = "user_0";
|
||||
|
@ -594,6 +620,8 @@ public class TestLeafQueue {
|
|||
|
||||
// Manipulate queue 'a'
|
||||
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
|
||||
//unset maxCapacity
|
||||
a.setMaxCapacity(-0.01f);
|
||||
a.setUserLimitFactor(10);
|
||||
|
||||
// Users
|
||||
|
|
Loading…
Reference in New Issue