Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6029. CapacityScheduler deadlock when ParentQueue#getQueueUserAclInfo is called by one thread and LeafQueue#assignContainers is releasing excessive reserved container is called by another thread. (Tao Yang via wangda)

Browse Source
This commit is contained in:
Wangda Tan 2017-01-03 16:21:46 -08:00
parent a58a1b55bf
commit ebafe075d6
1 changed files with 120 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -895,14 +895,19 @@ private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) {
}
@Override
public synchronized CSAssignment assignContainers(Resource clusterResource,
public CSAssignment assignContainers(Resource clusterResource,
FiCaSchedulerNode node, ResourceLimits currentResourceLimits,
SchedulingMode schedulingMode) {
FiCaSchedulerApp reservedApp = null;
CSAssignment reservedCSAssignment = null;
synchronized (this) {
updateCurrentResourceLimits(currentResourceLimits, clusterResource);
if (LOG.isDebugEnabled()) {
LOG.debug("assignContainers: node=" + node.getNodeName()
+ " #applications=" + orderingPolicy.getNumSchedulableEntities());
LOG.debug(
"assignContainers: node=" + node.getNodeName() + " #applications="
+ orderingPolicy.getNumSchedulableEntities());
}
setPreemptionAllowed(currentResourceLimits, node.getPartition());
@ -910,19 +915,26 @@ public synchronized CSAssignment assignContainers(Resource clusterResource,
// Check for reserved resources
RMContainer reservedContainer = node.getReservedContainer();
if (reservedContainer != null) {
FiCaSchedulerApp application =
getApplication(reservedContainer.getApplicationAttemptId());
synchronized (application) {
CSAssignment assignment =
application.assignContainers(clusterResource, node,
currentResourceLimits, schedulingMode, reservedContainer);
handleExcessReservedContainer(clusterResource, assignment, node,
application);
killToPreemptContainers(clusterResource, node, assignment);
return assignment;
reservedApp = getApplication(
reservedContainer.getApplicationAttemptId());
synchronized (reservedApp) {
reservedCSAssignment = reservedApp.assignContainers(
clusterResource, node, currentResourceLimits, schedulingMode,
reservedContainer);
}
}
}
// Handle possible completedContainer out of synchronized lock to avoid
// deadlock.
if (reservedCSAssignment != null) {
handleExcessReservedContainer(clusterResource, reservedCSAssignment, node,
reservedApp);
killToPreemptContainers(clusterResource, node, reservedCSAssignment);
return reservedCSAssignment;
}
synchronized (this) {
// if our queue cannot access this node, just return
if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
&& !accessibleToPartition(node.getPartition())) {
@ -936,13 +948,15 @@ public synchronized CSAssignment assignContainers(Resource clusterResource,
if (LOG.isDebugEnabled()) {
LOG.debug("Skip this queue=" + getQueuePath()
+ ", because it doesn't need more resource, schedulingMode="
+ schedulingMode.name() + " node-partition=" + node.getPartition());
+ schedulingMode.name() + " node-partition=" + node
.getPartition());
}
return CSAssignment.NULL_ASSIGNMENT;
}
for (Iterator<FiCaSchedulerApp> assignmentIterator =
orderingPolicy.getAssignmentIterator(); assignmentIterator.hasNext();) {
orderingPolicy.getAssignmentIterator(); assignmentIterator
.hasNext(); ) {
FiCaSchedulerApp application = assignmentIterator.next();
// Check queue max-capacity limit
@ -1019,6 +1033,7 @@ public synchronized CSAssignment assignContainers(Resource clusterResource,
return CSAssignment.NULL_ASSIGNMENT;
}
}
protected Resource getHeadroom(User user, Resource queueCurrentLimit,
Resource clusterResource, FiCaSchedulerApp application) {