Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6144. FairScheduler: preempted resources can become negative. (Miklos Szegedi via kasha)

This commit is contained in:
Karthik Kambatla 2017-02-09 14:45:04 -08:00
parent 0c01cf5798
commit 08f93978f3
2 changed files with 25 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
View File

@ -543,18 +543,20 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
void trackContainerForPreemption(RMContainer container) {
containersToPreempt.add(container);
synchronized (preemptedResources) {
Resources.addTo(preemptedResources, container.getAllocatedResource());
if (containersToPreempt.add(container)) {
synchronized (preemptedResources) {
Resources.addTo(preemptedResources, container.getAllocatedResource());
}
}
}
private void untrackContainerForPreemption(RMContainer container) {
synchronized (preemptedResources) {
Resources.subtractFrom(preemptedResources,
container.getAllocatedResource());
if (containersToPreempt.remove(container)) {
synchronized (preemptedResources) {
Resources.subtractFrom(preemptedResources,
container.getAllocatedResource());
}
}
containersToPreempt.remove(container);
}
Set<RMContainer> getPreemptionContainers() {

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
View File

@ -3649,6 +3649,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
assertEquals("Application1's AM should be finished",
0, app1.getLiveContainers().size());
assertEquals("Finished application usage should be none",
Resources.none(), app1.getResourceUsage());
assertEquals("Application3's AM should be running",
1, app3.getLiveContainers().size());
assertEquals("Application3's AM requests 1024 MB memory",
@ -3667,6 +3669,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
0, app4.getAMResource().getMemorySize());
assertEquals("Application4's AM should not be running",
0, app4.getLiveContainers().size());
assertEquals("Finished application usage should be none",
Resources.none(), app4.getResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@ -3681,6 +3685,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
0, app5.getAMResource().getMemorySize());
assertEquals("Application5's AM should not be running",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
Resources.none(), app5.getResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@ -3692,6 +3698,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
assertEquals("Application5's AM should not be running",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
Resources.none(), app5.getResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@ -3706,8 +3714,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
assertEquals("Application2's AM should be finished",
0, app2.getLiveContainers().size());
assertEquals("Finished application usage should be none",
Resources.none(), app2.getResourceUsage());
assertEquals("Application3's AM should be finished",
0, app3.getLiveContainers().size());
assertEquals("Finished application usage should be none",
Resources.none(), app3.getResourceUsage());
assertEquals("Application5's AM should be running",
1, app5.getLiveContainers().size());
assertEquals("Application5's AM requests 2048 MB memory",
@ -3727,6 +3739,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(containerExpired);
assertEquals("Application5's AM should have 0 container",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
Resources.none(), app5.getResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
scheduler.update();
@ -3749,6 +3763,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
assertEquals("Application6's AM should not be running",
0, app6.getLiveContainers().size());
assertEquals("Finished application usage should be none",
Resources.none(), app6.getResourceUsage());
assertEquals("Application6's AM resource shouldn't be updated",
0, app6.getAMResource().getMemorySize());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",