From b74391ce1495b5fd65216a252d8c4f61955d0018 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Wed, 16 Dec 2015 13:22:37 -0800 Subject: [PATCH] YARN-4416. Deadlock due to synchronised get Methods in AbstractCSQueue. (Naganarasimha G R via wangda) (cherry picked from commit 9b856d9787be5ec88ef34574b9b98755d7b669ea) (cherry picked from commit 5060fd1840ba2962cdfd7d91e2f405d7bdcfd169) --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../scheduler/capacity/AbstractCSQueue.java | 7 +++---- .../scheduler/capacity/LeafQueue.java | 16 ---------------- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 67a9c54c1a1..845a6f2aded 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1083,6 +1083,9 @@ Release 2.8.0 - UNRELEASED YARN-4225. Add preemption status to yarn queue -status for capacity scheduler. (Eric Payne via wangda) + YARN-4416. Deadlock due to synchronised get Methods in AbstractCSQueue. + (Naganarasimha G R via wangda) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 0d70e90e333..acd7ae9641a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -46,7 +46,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; @@ -135,7 +134,7 @@ public abstract class AbstractCSQueue implements CSQueue { } @Override - public synchronized float getAbsoluteUsedCapacity() { + public float getAbsoluteUsedCapacity() { return queueCapacities.getAbsoluteUsedCapacity(); } @@ -154,7 +153,7 @@ public abstract class AbstractCSQueue implements CSQueue { return queueUsage.getUsed(); } - public synchronized int getNumContainers() { + public int getNumContainers() { return numContainers; } @@ -203,7 +202,7 @@ public abstract class AbstractCSQueue implements CSQueue { } @Override - public synchronized void setAbsoluteUsedCapacity(float absUsedCapacity) { + public void setAbsoluteUsedCapacity(float absUsedCapacity) { queueCapacities.setAbsoluteUsedCapacity(absUsedCapacity); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index b74ce45207d..be1ba89ef06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -109,9 +109,6 @@ public class LeafQueue extends AbstractCSQueue { // cache last cluster resource to compute actual capacity private Resource lastClusterResource = Resources.none(); - // absolute capacity as a resource (based on cluster resource) - private Resource absoluteCapacityResource = Resources.none(); - private final QueueResourceLimitsInfo queueResourceLimitsInfo = new QueueResourceLimitsInfo(); @@ -144,7 +141,6 @@ public class LeafQueue extends AbstractCSQueue { super.setupQueueConfigs(clusterResource); this.lastClusterResource = clusterResource; - updateAbsoluteCapacityResource(clusterResource); this.cachedResourceLimitsForHeadroom = new ResourceLimits(clusterResource); @@ -345,10 +341,6 @@ public class LeafQueue extends AbstractCSQueue { public synchronized int getNumActiveApplications(String user) { return getUser(user).getActiveApplications(); } - - public synchronized int getNumContainers() { - return numContainers; - } @Override public synchronized QueueState getState() { @@ -1327,13 +1319,6 @@ public class LeafQueue extends AbstractCSQueue { } } - private void updateAbsoluteCapacityResource(Resource clusterResource) { - absoluteCapacityResource = - Resources.multiplyAndNormalizeUp(resourceCalculator, labelManager - .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource), - queueCapacities.getAbsoluteCapacity(), minimumAllocation); - } - private void updateCurrentResourceLimits( ResourceLimits currentResourceLimits, Resource clusterResource) { // TODO: need consider non-empty node labels when resource limits supports @@ -1359,7 +1344,6 @@ public class LeafQueue extends AbstractCSQueue { ResourceLimits currentResourceLimits) { updateCurrentResourceLimits(currentResourceLimits, clusterResource); lastClusterResource = clusterResource; - updateAbsoluteCapacityResource(clusterResource); // Update headroom info based on new cluster resource value // absoluteMaxCapacity now, will be replaced with absoluteMaxAvailCapacity