From 1ab2f5a916a8e942e01b08685caa6c79ad3e1107 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Mon, 24 Feb 2014 22:35:27 +0000 Subject: [PATCH] YARN-1678. Fair scheduler gabs incessantly about reservations (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1571468 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../scheduler/fair/AppSchedulable.java | 53 +++++++++++++++---- .../scheduler/fair/FairScheduler.java | 10 ++-- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index cb2db929fbc..d747a188caa 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -28,6 +28,8 @@ Release 2.5.0 - UNRELEASED YARN-1736. FS: AppSchedulable.assignContainer's priority argument is redundant. (Naren Koneru via kasha) + YARN-1678. Fair scheduler gabs incessantly about reservations (Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java index ac31fb57616..304a2cfa0c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java @@ -201,6 +201,22 @@ public class AppSchedulable extends Schedulable { * Assign a container to this node to facilitate {@code request}. If node does * not have enough memory, create a reservation. This is called once we are * sure the particular request should be facilitated by this node. + * + * @param node + * The node to try placing the container on. + * @param priority + * The requested priority for the container. + * @param request + * The ResourceRequest we're trying to satisfy. + * @param type + * The locality of the assignment. + * @param reserved + * Whether there's already a container reserved for this app on the node. + * @return + * If an assignment was made, returns the resources allocated to the + * container. If a reservation was made, returns + * FairScheduler.CONTAINER_RESERVED. If no assignment or reservation was + * made, returns an empty resource. */ private Resource assignContainer(FSSchedulerNode node, ResourceRequest request, NodeType type, @@ -255,17 +271,6 @@ public class AppSchedulable extends Schedulable { LOG.debug("Node offered to app: " + getName() + " reserved: " + reserved); } - if (reserved) { - RMContainer rmContainer = node.getReservedContainer(); - Priority priority = rmContainer.getReservedPriority(); - - // Make sure the application still needs requests at this priority - if (app.getTotalRequiredResources(priority) == 0) { - unreserve(priority, node); - return Resources.none(); - } - } - Collection prioritiesToTry = (reserved) ? Arrays.asList(node.getReservedContainer().getReservedPriority()) : app.getPriorities(); @@ -338,7 +343,33 @@ public class AppSchedulable extends Schedulable { return Resources.none(); } + /** + * Called when this application already has an existing reservation on the + * given node. Sees whether we can turn the reservation into an allocation. + * Also checks whether the application needs the reservation anymore, and + * releases it if not. + * + * @param node + * Node that the application has an existing reservation on + */ public Resource assignReservedContainer(FSSchedulerNode node) { + RMContainer rmContainer = node.getReservedContainer(); + Priority priority = rmContainer.getReservedPriority(); + + // Make sure the application still needs requests at this priority + if (app.getTotalRequiredResources(priority) == 0) { + unreserve(priority, node); + return Resources.none(); + } + + // Fail early if the reserved container won't fit. + // Note that we have an assumption here that there's only one container size + // per priority. + if (!Resources.fitsIn(node.getReservedContainer().getReservedResource(), + node.getAvailableResource())) { + return Resources.none(); + } + return assignContainer(node, true); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index a852d7b9242..a27a86d3075 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -1046,10 +1046,12 @@ public class FairScheduler extends AbstractYarnScheduler { reservedAppSchedulable = null; } else { // Reservation exists; try to fulfill the reservation - LOG.info("Trying to fulfill reservation for application " - + reservedAppSchedulable.getApp().getApplicationAttemptId() - + " on node: " + node); - + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to fulfill reservation for application " + + reservedAppSchedulable.getApp().getApplicationAttemptId() + + " on node: " + node); + } + node.getReservedAppSchedulable().assignReservedContainer(node); } }