From 9b571e9ebc62db9be8cf7f606a883cf0aebabff5 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Wed, 9 Jan 2013 21:04:39 +0000 Subject: [PATCH] YARN-325. RM CapacityScheduler can deadlock when getQueueInfo() is called and a container is completing (Arun C Murthy via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1431076 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../scheduler/capacity/CSAssignment.java | 22 +++++++++++++++++++ .../scheduler/capacity/CapacityScheduler.java | 15 ++++++++++++- .../scheduler/capacity/LeafQueue.java | 21 +++++------------- .../scheduler/capacity/TestLeafQueue.java | 8 ++++--- 5 files changed, 50 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 56a30da1f7e..c40af7298ef 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -273,6 +273,9 @@ Release 0.23.6 - UNRELEASED YARN-320. RM should always be able to renew its own tokens. (Daryn Sharp via sseth) + YARN-325. RM CapacityScheduler can deadlock when getQueueInfo() is + called and a container is completing (Arun C Murthy via tgraves) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAssignment.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAssignment.java index f994c6d7122..1f1250a2b64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAssignment.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAssignment.java @@ -20,18 +20,32 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; @Private @Unstable public class CSAssignment { final private Resource resource; private NodeType type; + private final RMContainer excessReservation; + private final FiCaSchedulerApp application; public CSAssignment(Resource resource, NodeType type) { this.resource = resource; this.type = type; + this.application = null; + this.excessReservation = null; } + + public CSAssignment(FiCaSchedulerApp application, RMContainer excessReservation) { + this.resource = excessReservation.getContainer().getResource(); + this.type = NodeType.NODE_LOCAL; + this.application = application; + this.excessReservation = excessReservation; + } + public Resource getResource() { return resource; @@ -45,6 +59,14 @@ public class CSAssignment { this.type = type; } + public FiCaSchedulerApp getApplication() { + return application; + } + + public RMContainer getExcessReservation() { + return excessReservation; + } + @Override public String toString() { return resource.getMemory() + ":" + type; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index c6683fe6f13..2ce3a464a86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -604,7 +604,20 @@ implements ResourceScheduler, CapacitySchedulerContext, Configurable { reservedApplication.getApplicationId() + " on node: " + nm); LeafQueue queue = ((LeafQueue)reservedApplication.getQueue()); - queue.assignContainers(clusterResource, node); + CSAssignment assignment = queue.assignContainers(clusterResource, node); + + RMContainer excessReservation = assignment.getExcessReservation(); + if (excessReservation != null) { + Container container = excessReservation.getContainer(); + queue.completedContainer( + clusterResource, assignment.getApplication(), node, + excessReservation, + SchedulerUtils.createAbnormalContainerStatus( + container.getId(), + SchedulerUtils.UNRESERVED_CONTAINER), + RMContainerEventType.RELEASED); + } + } // Try to schedule more if there are no reservations to fulfill diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index b02dda1cb2f..7656ace5b0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -62,7 +62,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -781,11 +780,9 @@ public class LeafQueue implements CSQueue { if (reservedContainer != null) { FiCaSchedulerApp application = getApplication(reservedContainer.getApplicationAttemptId()); - return new CSAssignment( + return assignReservedContainer(application, node, reservedContainer, - clusterResource), - NodeType.NODE_LOCAL); // Don't care about locality constraints - // for reserved containers + clusterResource); } // Try to assign containers to applications in order @@ -873,20 +870,14 @@ public class LeafQueue implements CSQueue { } - private synchronized Resource assignReservedContainer(FiCaSchedulerApp application, + private synchronized CSAssignment + assignReservedContainer(FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer, Resource clusterResource) { // Do we still need this reservation? Priority priority = rmContainer.getReservedPriority(); if (application.getTotalRequiredResources(priority) == 0) { // Release - Container container = rmContainer.getContainer(); - completedContainer(clusterResource, application, node, - rmContainer, - SchedulerUtils.createAbnormalContainerStatus( - container.getId(), - SchedulerUtils.UNRESERVED_CONTAINER), - RMContainerEventType.RELEASED); - return container.getResource(); // Ugh, return resource to force re-sort + return new CSAssignment(application, rmContainer); } // Try to assign if we have sufficient resources @@ -895,7 +886,7 @@ public class LeafQueue implements CSQueue { // Doesn't matter... since it's already charged for at time of reservation // "re-reservation" is *free* - return Resources.none(); + return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL); } private synchronized boolean assignToQueue(Resource clusterResource, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index d0d23af6ba9..ccf2a47c128 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -1181,12 +1181,14 @@ public class TestLeafQueue { // Now finish another container from app_0 and see the reservation cancelled a.completedContainer(clusterResource, app_0, node_0, app_0.getLiveContainers().iterator().next(), null, RMContainerEventType.KILL); - a.assignContainers(clusterResource, node_0); - assertEquals(4*GB, a.getUsedResources().getMemory()); + CSAssignment assignment = a.assignContainers(clusterResource, node_0); + assertEquals(8*GB, a.getUsedResources().getMemory()); assertEquals(0*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(4*GB, app_1.getCurrentConsumption().getMemory()); - assertEquals(0*GB, app_1.getCurrentReservation().getMemory()); + assertEquals(4*GB, app_1.getCurrentReservation().getMemory()); assertEquals(0*GB, node_0.getUsedResource().getMemory()); + assertEquals(4*GB, + assignment.getExcessReservation().getContainer().getResource().getMemory()); }