From 84df660af498cd768e092f072ae6e7ee43df7c67 Mon Sep 17 00:00:00 2001 From: Sandy Ryza Date: Thu, 5 Feb 2015 09:35:47 -0800 Subject: [PATCH] YARN-3101. In Fair Scheduler, fix canceling of reservations for exceeding max share (Anubhav Dhoot via Sandy Ryza) --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/FSAppAttempt.java | 4 + .../scheduler/fair/FairScheduler.java | 13 +- .../scheduler/fair/TestFairScheduler.java | 114 ++++++++++++++---- 4 files changed, 109 insertions(+), 25 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 11f5a6d929b..9e8bb32def4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -448,6 +448,9 @@ Release 2.7.0 - UNRELEASED YARN-3058. Fix error message of tokens' activation delay configuration. (Yi Liu via ozawa) + YARN-3101. In Fair Scheduler, fix canceling of reservations for exceeding + max share (Anubhav Dhoot via Sandy Ryza) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index b23ec3ed30e..2cb0f0bb9ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -530,6 +530,10 @@ public class FSAppAttempt extends SchedulerApplicationAttempt return container.getResource(); } else { + if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) { + return Resources.none(); + } + // The desired container won't fit here, so reserve reserve(request.getPriority(), node, container, reserved); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 1ace6040b25..2b597164baf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -1049,7 +1049,8 @@ public class FairScheduler extends FSQueue queue = reservedAppSchedulable.getQueue(); if (!reservedAppSchedulable.hasContainerForNode(reservedPriority, node) - || !fitInMaxShare(queue)) { + || !fitsInMaxShare(queue, + node.getReservedContainer().getReservedResource())) { // Don't hold the reservation if app can no longer use it LOG.info("Releasing reservation that cannot be satisfied for application " + reservedAppSchedulable.getApplicationAttemptId() @@ -1084,14 +1085,18 @@ public class FairScheduler extends updateRootQueueMetrics(); } - private boolean fitInMaxShare(FSQueue queue) { - if (Resources.fitsIn(queue.getResourceUsage(), queue.getMaxShare())) { + static boolean fitsInMaxShare(FSQueue queue, Resource + additionalResource) { + Resource usagePlusAddition = + Resources.add(queue.getResourceUsage(), additionalResource); + + if (!Resources.fitsIn(usagePlusAddition, queue.getMaxShare())) { return false; } FSQueue parentQueue = queue.getParent(); if (parentQueue != null) { - return fitInMaxShare(parentQueue); + return fitsInMaxShare(parentQueue, additionalResource); } return true; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index d9c53e60a16..f67357abbda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -784,19 +784,18 @@ public class TestFairScheduler extends FairSchedulerTestBase { } - @Test (timeout = 5000) - public void testContainerReservationNotExceedingQueueMax() throws Exception { + @Test (timeout = 500000) + public void testContainerReservationAttemptExceedingQueueMax() + throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); out.println(""); out.println(""); out.println(""); - out.println("1024mb,5vcores"); - out.println("2048mb,10vcores"); + out.println("2048mb,5vcores"); out.println(""); out.println(""); - out.println("1024mb,5vcores"); out.println("2048mb,10vcores"); out.println(""); out.println(""); @@ -806,7 +805,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + // Add a node RMNode node1 = MockNodes @@ -825,7 +824,64 @@ public class TestFairScheduler extends FairSchedulerTestBase { getResourceUsage().getMemory()); // Now queue 2 requests likewise - ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user2", 1); + createSchedulingRequest(1024, "queue2", "user2", 1); + scheduler.update(); + scheduler.handle(updateEvent); + + // Make sure queue 2 is allocated app capacity + assertEquals(1024, scheduler.getQueueManager().getQueue("queue2"). + getResourceUsage().getMemory()); + + ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1", 1); + scheduler.update(); + scheduler.handle(updateEvent); + + // Ensure the reservation does not get created as allocated memory of + // queue1 exceeds max + assertEquals(0, scheduler.getSchedulerApp(attId1). + getCurrentReservation().getMemory()); + } + + @Test (timeout = 500000) + public void testContainerReservationNotExceedingQueueMax() throws Exception { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("3072mb,10vcores"); + out.println(""); + out.println(""); + out.println("2048mb,10vcores"); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + // Add a node + RMNode node1 = + MockNodes + .newNodeInfo(1, Resources.createResource(3072, 5), 1, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + // Queue 1 requests full capacity of the queue + createSchedulingRequest(2048, "queue1", "user1", 1); + scheduler.update(); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); + scheduler.handle(updateEvent); + + // Make sure queue 1 is allocated app capacity + assertEquals(2048, scheduler.getQueueManager().getQueue("queue1"). + getResourceUsage().getMemory()); + + // Now queue 2 requests likewise + createSchedulingRequest(1024, "queue2", "user2", 1); scheduler.update(); scheduler.handle(updateEvent); @@ -841,18 +897,34 @@ public class TestFairScheduler extends FairSchedulerTestBase { assertEquals(1024, scheduler.getSchedulerApp(attId1) .getCurrentReservation().getMemory()); - // Now remove app of queue2 - AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent( - attId, RMAppAttemptState.FINISHED, false); - scheduler.update(); - scheduler.handle(appRemovedEvent1); + // Exercise checks that reservation fits + scheduler.handle(updateEvent); + + // Ensure the reservation still exists as allocated memory of queue1 doesn't + // exceed max + assertEquals(1024, scheduler.getSchedulerApp(attId1). + getCurrentReservation().getMemory()); + + // Now reduce max Resources of queue1 down to 2048 + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("2048mb,10vcores"); + out.println(""); + out.println(""); + out.println("2048mb,10vcores"); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + scheduler.reinitialize(conf, resourceManager.getRMContext()); - // Queue should have no apps - assertEquals(0, scheduler.getQueueManager().getQueue("queue2"). - getResourceUsage().getMemory()); - createSchedulingRequest(1024, "queue2", "user2", 1); scheduler.handle(updateEvent); + // Make sure allocated memory of queue1 doesn't exceed its maximum assertEquals(2048, scheduler.getQueueManager().getQueue("queue1"). getResourceUsage().getMemory()); @@ -2257,10 +2329,9 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); - // Reserved container should will be at higher priority, - // since old reservation cannot be satisfied + // Reserved container should still be at lower priority for (RMContainer container : app.getReservedContainers()) { - assertEquals(1, container.getReservedPriority().getPriority()); + assertEquals(2, container.getReservedPriority().getPriority()); } // Complete container @@ -2273,11 +2344,12 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.update(); scheduler.handle(updateEvent); - // Reserved container (at higher priority) should be run + // Reserved container (at lower priority) should be run Collection liveContainers = app.getLiveContainers(); assertEquals(1, liveContainers.size()); for (RMContainer liveContainer : liveContainers) { - Assert.assertEquals(1, liveContainer.getContainer().getPriority().getPriority()); + Assert.assertEquals(2, liveContainer.getContainer().getPriority() + .getPriority()); } assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());