From 76339bfb9f31a317faa2a8dbcb021a929409098f Mon Sep 17 00:00:00 2001 From: carlo curino Date: Thu, 23 Jul 2015 19:33:59 -0700 Subject: [PATCH] YARN-3969. Allow jobs to be submitted to reservation that is active but does not have any allocations. (subru via curino) (cherry picked from commit 0fcb4a8cf2add3f112907ff4e833e2f04947b53e) (cherry picked from commit 6a71a0a230b337c8d62cd116a9742c54f00728e9) --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../scheduler/capacity/ReservationQueue.java | 4 --- .../capacity/TestReservationQueue.java | 26 ++++++++++--------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f023fd33841..6ce4211957b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -204,6 +204,9 @@ Release 2.7.1 - 2015-07-06 YARN-3850. NM fails to read files from full disks which can lead to container logs being lost and other issues (Varun Saxena via jlowe) + YARN-3969. Allow jobs to be submitted to reservation that is active + but does not have any allocations. (subru via curino) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java index a8d17cfd0d1..19ce9f33123 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java @@ -39,12 +39,9 @@ public class ReservationQueue extends LeafQueue { private PlanQueue parent; - private int maxSystemApps; - public ReservationQueue(CapacitySchedulerContext cs, String queueName, PlanQueue parent) throws IOException { super(cs, queueName, parent, null); - maxSystemApps = cs.getConfiguration().getMaximumSystemApplications(); // the following parameters are common to all reservation in the plan updateQuotas(parent.getUserLimitForReservation(), parent.getUserLimitFactor(), @@ -90,7 +87,6 @@ public class ReservationQueue extends LeafQueue { } setCapacity(capacity); setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity()); - setMaxApplications((int) (maxSystemApps * getAbsoluteCapacity())); // note: we currently set maxCapacity to capacity // this might be revised later setMaxCapacity(entitlement.getMaxCapacity()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java index 4e6c73d56a3..e23e93c99dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -39,6 +40,7 @@ public class TestReservationQueue { CapacitySchedulerConfiguration csConf; CapacitySchedulerContext csContext; + final static int DEF_MAX_APPS = 10000; final static int GB = 1024; private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); @@ -66,7 +68,13 @@ public class TestReservationQueue { // create a queue PlanQueue pq = new PlanQueue(csContext, "root", null, null); reservationQueue = new ReservationQueue(csContext, "a", pq); + } + private void validateReservationQueue(double capacity) { + assertTrue(" actual capacity: " + reservationQueue.getCapacity(), + reservationQueue.getCapacity() - capacity < CSQueueUtils.EPSILON); + assertEquals(reservationQueue.maxApplications, DEF_MAX_APPS); + assertEquals(reservationQueue.maxApplicationsPerUser, DEF_MAX_APPS); } @Test @@ -74,25 +82,20 @@ public class TestReservationQueue { // verify that setting, adding, subtracting capacity works reservationQueue.setCapacity(1.0F); - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON); + validateReservationQueue(1); reservationQueue.setEntitlement(new QueueEntitlement(0.9f, 1f)); - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() - 0.9 < CSQueueUtils.EPSILON); + validateReservationQueue(0.9); reservationQueue.setEntitlement(new QueueEntitlement(1f, 1f)); - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON); + validateReservationQueue(1); reservationQueue.setEntitlement(new QueueEntitlement(0f, 1f)); - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() < CSQueueUtils.EPSILON); + validateReservationQueue(0); try { reservationQueue.setEntitlement(new QueueEntitlement(1.1f, 1f)); fail(); } catch (SchedulerDynamicEditException iae) { // expected - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON); + validateReservationQueue(1); } try { @@ -100,8 +103,7 @@ public class TestReservationQueue { fail(); } catch (SchedulerDynamicEditException iae) { // expected - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON); + validateReservationQueue(1); } }