Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6380. FSAppAttempt keeps redundant copy of the queue

This commit is contained in:
Daniel Templeton 2017-05-11 13:47:02 -07:00
parent 2ada100da7
commit 90cb5b4635
1 changed files with 8 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
View File

@ -41,7 +41,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFini
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
@ -79,7 +78,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
private final ResourceWeights resourceWeights; private final ResourceWeights resourceWeights;
private Resource demand = Resources.createResource(0); private Resource demand = Resources.createResource(0);
private final FairScheduler scheduler; private final FairScheduler scheduler;
private FSQueue fsQueue;
private Resource fairShare = Resources.createResource(0, 0); private Resource fairShare = Resources.createResource(0, 0);
// Preemption related variables // Preemption related variables
@ -118,7 +116,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
super(applicationAttemptId, user, queue, activeUsersManager, rmContext); super(applicationAttemptId, user, queue, activeUsersManager, rmContext);
this.scheduler = scheduler; this.scheduler = scheduler;
this.fsQueue = queue;
this.startTime = scheduler.getClock().getTime(); this.startTime = scheduler.getClock().getTime();
this.lastTimeAtFairShare = this.startTime; this.lastTimeAtFairShare = this.startTime;
this.appPriority = Priority.newInstance(1); this.appPriority = Priority.newInstance(1);
@ -229,11 +226,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
*/ */
@Override @Override
public Resource getHeadroom() { public Resource getHeadroom() {
final FSQueue queue = (FSQueue) this.queue; final FSQueue fsQueue = getQueue();
SchedulingPolicy policy = queue.getPolicy(); SchedulingPolicy policy = fsQueue.getPolicy();
Resource queueFairShare = queue.getFairShare(); Resource queueFairShare = fsQueue.getFairShare();
Resource queueUsage = queue.getResourceUsage(); Resource queueUsage = fsQueue.getResourceUsage();
Resource clusterResource = this.scheduler.getClusterResource(); Resource clusterResource = this.scheduler.getClusterResource();
Resource clusterUsage = this.scheduler.getRootQueueMetrics() Resource clusterUsage = this.scheduler.getRootQueueMetrics()
.getAllocatedResources(); .getAllocatedResources();
@ -243,7 +240,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
subtractResourcesOnBlacklistedNodes(clusterAvailableResources); subtractResourcesOnBlacklistedNodes(clusterAvailableResources);
Resource queueMaxAvailableResources = Resource queueMaxAvailableResources =
Resources.subtract(queue.getMaxShare(), queueUsage); Resources.subtract(fsQueue.getMaxShare(), queueUsage);
Resource maxAvailableResource = Resources.componentwiseMin( Resource maxAvailableResource = Resources.componentwiseMin(
clusterAvailableResources, queueMaxAvailableResources); clusterAvailableResources, queueMaxAvailableResources);
@ -514,8 +511,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
@Override @Override
public FSLeafQueue getQueue() { public FSLeafQueue getQueue() {
Queue queue = super.getQueue();
assert queue instanceof FSLeafQueue;
return (FSLeafQueue) queue; return (FSLeafQueue) queue;
} }
@ -1102,7 +1097,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
Resource fairShareStarvation() { Resource fairShareStarvation() {
long now = scheduler.getClock().getTime(); long now = scheduler.getClock().getTime();
Resource threshold = Resources.multiply( Resource threshold = Resources.multiply(
getFairShare(), fsQueue.getFairSharePreemptionThreshold()); getFairShare(), getQueue().getFairSharePreemptionThreshold());
Resource fairDemand = Resources.componentwiseMin(threshold, demand); Resource fairDemand = Resources.componentwiseMin(threshold, demand);
// Check if the queue is starved for fairshare // Check if the queue is starved for fairshare
@ -1113,7 +1108,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
} }
if (!starved || if (!starved ||
now - lastTimeAtFairShare < fsQueue.getFairSharePreemptionTimeout()) { now - lastTimeAtFairShare < getQueue().getFairSharePreemptionTimeout()) {
fairshareStarvation = Resources.none(); fairshareStarvation = Resources.none();
} else { } else {
// The app has been starved for longer than preemption-timeout. // The app has been starved for longer than preemption-timeout.
@ -1128,7 +1123,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* {@code share}. * {@code share}.
*/ */
private boolean isUsageBelowShare(Resource usage, Resource share) { private boolean isUsageBelowShare(Resource usage, Resource share) {
return fsQueue.getPolicy().getResourceCalculator().compare( return getQueue().getPolicy().getResourceCalculator().compare(
scheduler.getClusterResource(), usage, share, true) < 0; scheduler.getClusterResource(), usage, share, true) < 0;
} }