Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6175. FairScheduler: Negative vcore for resource needed to preempt. (Yufei Gu via kasha)

This commit is contained in:
Karthik Kambatla 2017-02-23 14:29:30 -08:00
parent b1ec790662
commit 1d156c8758
3 changed files with 23 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
View File

@ -172,6 +172,24 @@ public class Resources {
return subtractFrom(clone(lhs), rhs); return subtractFrom(clone(lhs), rhs);
} }
/**
* Subtract <code>rhs</code> from <code>lhs</code> and reset any negative
* values to zero.
* @param lhs {@link Resource} to subtract from
* @param rhs {@link Resource} to subtract
* @return the value of lhs after subtraction
*/
public static Resource subtractFromNonNegative(Resource lhs, Resource rhs) {
subtractFrom(lhs, rhs);
if (lhs.getMemorySize() < 0) {
lhs.setMemorySize(0);
}
if (lhs.getVirtualCores() < 0) {
lhs.setVirtualCores(0);
}
return lhs;
}
public static Resource negate(Resource resource) { public static Resource negate(Resource resource) {
return subtract(NONE, resource); return subtract(NONE, resource);
} }

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
View File

@ -194,16 +194,10 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
for (NodeId nodeId: blacklistNodeIds) { for (NodeId nodeId: blacklistNodeIds) {
SchedulerNode node = scheduler.getSchedulerNode(nodeId); SchedulerNode node = scheduler.getSchedulerNode(nodeId);
if (node != null) { if (node != null) {
Resources.subtractFrom(availableResources, Resources.subtractFromNonNegative(availableResources,
node.getAvailableResource()); node.getAvailableResource());
} }
} }
if (availableResources.getMemory() < 0) {
availableResources.setMemory(0);
}
if (availableResources.getVirtualCores() < 0) {
availableResources.setVirtualCores(0);
}
} }
/** /**

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
View File

@ -531,14 +531,14 @@ public class FairScheduler extends
if (curTime - sched.getLastTimeAtMinShare() > minShareTimeout) { if (curTime - sched.getLastTimeAtMinShare() > minShareTimeout) {
Resource target = Resources.componentwiseMin( Resource target = Resources.componentwiseMin(
sched.getMinShare(), sched.getDemand()); sched.getMinShare(), sched.getDemand());
resDueToMinShare = Resources.max(calc, clusterResource, resDueToMinShare =
Resources.none(), Resources.subtract(target, sched.getResourceUsage())); Resources.subtractFromNonNegative(target, sched.getResourceUsage());
} }
if (curTime - sched.getLastTimeAtFairShareThreshold() > fairShareTimeout) { if (curTime - sched.getLastTimeAtFairShareThreshold() > fairShareTimeout) {
Resource target = Resources.componentwiseMin( Resource target = Resources.componentwiseMin(
sched.getFairShare(), sched.getDemand()); sched.getFairShare(), sched.getDemand());
resDueToFairShare = Resources.max(calc, clusterResource, resDueToFairShare =
Resources.none(), Resources.subtract(target, sched.getResourceUsage())); Resources.subtractFromNonNegative(target, sched.getResourceUsage());
} }
Resource deficit = Resources.max(calc, clusterResource, Resource deficit = Resources.max(calc, clusterResource,
resDueToMinShare, resDueToFairShare); resDueToMinShare, resDueToFairShare);