Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3969. Allow jobs to be submitted to reservation that is active but does not have any allocations. (subru via curino)

Browse Source 
(cherry picked from commit 0fcb4a8cf2add3f112907ff4e833e2f04947b53e)
(cherry picked from commit 6a71a0a230b337c8d62cd116a9742c54f00728e9)
This commit is contained in:
carlo curino 2015-07-23 19:33:59 -07:00
parent b919c1c044
commit 76339bfb9f
3 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -204,6 +204,9 @@ Release 2.7.1 - 2015-07-06
YARN-3850. NM fails to read files from full disks which can lead to
container logs being lost and other issues (Varun Saxena via jlowe)
YARN-3969. Allow jobs to be submitted to reservation that is active
but does not have any allocations. (subru via curino)
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
View File

@ -39,12 +39,9 @@ public class ReservationQueue extends LeafQueue {
private PlanQueue parent;
private int maxSystemApps;
public ReservationQueue(CapacitySchedulerContext cs, String queueName,
PlanQueue parent) throws IOException {
super(cs, queueName, parent, null);
maxSystemApps = cs.getConfiguration().getMaximumSystemApplications();
// the following parameters are common to all reservation in the plan
updateQuotas(parent.getUserLimitForReservation(),
parent.getUserLimitFactor(),
@ -90,7 +87,6 @@ public synchronized void setEntitlement(QueueEntitlement entitlement)
}
setCapacity(capacity);
setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity());
setMaxApplications((int) (maxSystemApps * getAbsoluteCapacity()));
// note: we currently set maxCapacity to capacity
// this might be revised later
setMaxCapacity(entitlement.getMaxCapacity());

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
@ -39,6 +40,7 @@ public class TestReservationQueue {
CapacitySchedulerConfiguration csConf;
CapacitySchedulerContext csContext;
final static int DEF_MAX_APPS = 10000;
final static int GB = 1024;
private final ResourceCalculator resourceCalculator =
new DefaultResourceCalculator();
@ -66,7 +68,13 @@ public void setup() throws IOException {
// create a queue
PlanQueue pq = new PlanQueue(csContext, "root", null, null);
reservationQueue = new ReservationQueue(csContext, "a", pq);
}
private void validateReservationQueue(double capacity) {
assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
reservationQueue.getCapacity() - capacity < CSQueueUtils.EPSILON);
assertEquals(reservationQueue.maxApplications, DEF_MAX_APPS);
assertEquals(reservationQueue.maxApplicationsPerUser, DEF_MAX_APPS);
}
@Test
@ -74,25 +82,20 @@ public void testAddSubtractCapacity() throws Exception {
// verify that setting, adding, subtracting capacity works
reservationQueue.setCapacity(1.0F);
assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
validateReservationQueue(1);
reservationQueue.setEntitlement(new QueueEntitlement(0.9f, 1f));
assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
reservationQueue.getCapacity() - 0.9 < CSQueueUtils.EPSILON);
validateReservationQueue(0.9);
reservationQueue.setEntitlement(new QueueEntitlement(1f, 1f));
assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
validateReservationQueue(1);
reservationQueue.setEntitlement(new QueueEntitlement(0f, 1f));
assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
reservationQueue.getCapacity() < CSQueueUtils.EPSILON);
validateReservationQueue(0);
try {
reservationQueue.setEntitlement(new QueueEntitlement(1.1f, 1f));
fail();
} catch (SchedulerDynamicEditException iae) {
// expected
assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
validateReservationQueue(1);
}
try {
@ -100,8 +103,7 @@ public void testAddSubtractCapacity() throws Exception {
fail();
} catch (SchedulerDynamicEditException iae) {
// expected
assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
validateReservationQueue(1);
}
}