Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4979. FSAppAttempt demand calculation considers demands at multiple locality levels different. (Zhihai Xu via kasha)

This commit is contained in:
Karthik Kambatla 2016-05-23 14:29:28 -07:00
parent eebb39a56f
commit 6d043aa4cf
2 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
View File

@ -929,7 +929,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Add up outstanding resource requests // Add up outstanding resource requests
synchronized (this) { synchronized (this) {
for (Priority p : getPriorities()) { for (Priority p : getPriorities()) {
for (ResourceRequest r : getResourceRequests(p).values()) { ResourceRequest r = getResourceRequest(p, ResourceRequest.ANY);
if (r != null) {
Resources.multiplyAndAddTo(demand, Resources.multiplyAndAddTo(demand,
r.getCapability(), r.getNumContainers()); r.getCapability(), r.getNumContainers());
} }

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
View File

@ -1691,28 +1691,31 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ask1.add(request1); ask1.add(request1);
scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null, null, null, null); scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null, null, null, null);
// Second ask, queue2 requests 1 large + (2 * minReqSize) // Second ask, queue2 requests 1 large.
List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>(); List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
ResourceRequest request2 = createResourceRequest(2 * minReqSize, "foo", 1, 1, ResourceRequest request2 = createResourceRequest(2 * minReqSize, "foo", 1, 1,
false); false);
ResourceRequest request3 = createResourceRequest(minReqSize, "bar", 1, 2, ResourceRequest request3 = createResourceRequest(2 * minReqSize,
false); ResourceRequest.ANY, 1, 1, false);
ask2.add(request2); ask2.add(request2);
ask2.add(request3); ask2.add(request3);
scheduler.allocate(id21, ask2, new ArrayList<ContainerId>(), null, null, null, null); scheduler.allocate(id21, ask2, new ArrayList<ContainerId>(), null, null, null, null);
// Third ask, queue2 requests 1 large // Third ask, queue2 requests 2 small (minReqSize).
List<ResourceRequest> ask3 = new ArrayList<ResourceRequest>(); List<ResourceRequest> ask3 = new ArrayList<ResourceRequest>();
ResourceRequest request4 = ResourceRequest request4 = createResourceRequest(minReqSize, "bar", 2, 2,
createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1, true); true);
ResourceRequest request5 = createResourceRequest(minReqSize,
ResourceRequest.ANY, 2, 2, true);
ask3.add(request4); ask3.add(request4);
ask3.add(request5);
scheduler.allocate(id22, ask3, new ArrayList<ContainerId>(), null, null, null, null); scheduler.allocate(id22, ask3, new ArrayList<ContainerId>(), null, null, null, null);
scheduler.update(); scheduler.update();
assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("root.queue1") assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("root.queue1")
.getDemand().getMemory()); .getDemand().getMemory());
assertEquals(2 * minReqSize + 2 * minReqSize + (2 * minReqSize), scheduler assertEquals(2 * minReqSize + 2 * minReqSize, scheduler
.getQueueManager().getQueue("root.queue2").getDemand() .getQueueManager().getQueue("root.queue2").getDemand()
.getMemory()); .getMemory());
} }