Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-201. Fix CapacityScheduler to be less conservative for starved off-switch requests. Contributed by Jason Lowe. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406834 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2012-11-07 21:56:18 +00:00
parent c44f37a7a5
commit fb5b96dfc3
3 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -202,6 +202,9 @@ Release 0.23.5 - UNRELEASED
YARN-202. Log Aggregation generates a storm of fsync() for namenode YARN-202. Log Aggregation generates a storm of fsync() for namenode
(Kihwal Lee via bobby) (Kihwal Lee via bobby)
YARN-201. Fix CapacityScheduler to be less conservative for starved
off-switch requests. (jlowe via acmurthy)
Release 0.23.4 - UNRELEASED Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -833,8 +833,12 @@ public class LeafQueue implements CSQueue {
// Note: Update headroom to account for current allocation too... // Note: Update headroom to account for current allocation too...
allocateResource(clusterResource, application, assigned); allocateResource(clusterResource, application, assigned);
// Reset scheduling opportunities // Don't reset scheduling opportunities for non-local assignments
application.resetSchedulingOpportunities(priority); // otherwise the app will be delayed for each non-local assignment.
// This helps apps with many off-cluster requests schedule faster.
if (assignment.getType() != NodeType.OFF_SWITCH) {
application.resetSchedulingOpportunities(priority);
}
// Done // Done
return assignment; return assignment;

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
View File

@ -1259,7 +1259,7 @@ public class TestLeafQueue {
assignment = a.assignContainers(clusterResource, node_2); assignment = a.assignContainers(clusterResource, node_2);
verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2), verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2),
any(Priority.class), any(ResourceRequest.class), any(Container.class)); any(Priority.class), any(ResourceRequest.class), any(Container.class));
assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset assertEquals(4, app_0.getSchedulingOpportunities(priority)); // should NOT reset
assertEquals(2, app_0.getTotalRequiredResources(priority)); assertEquals(2, app_0.getTotalRequiredResources(priority));
assertEquals(NodeType.OFF_SWITCH, assignment.getType()); assertEquals(NodeType.OFF_SWITCH, assignment.getType());
@ -1408,11 +1408,11 @@ public class TestLeafQueue {
assertEquals(0, app_0.getSchedulingOpportunities(priority_2)); assertEquals(0, app_0.getSchedulingOpportunities(priority_2));
assertEquals(1, app_0.getTotalRequiredResources(priority_2)); assertEquals(1, app_0.getTotalRequiredResources(priority_2));
// Another off-switch, shouldn allocate OFF_SWITCH P1 // Another off-switch, shouldn't allocate OFF_SWITCH P1
a.assignContainers(clusterResource, node_2); a.assignContainers(clusterResource, node_2);
verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2), verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2),
eq(priority_1), any(ResourceRequest.class), any(Container.class)); eq(priority_1), any(ResourceRequest.class), any(Container.class));
assertEquals(0, app_0.getSchedulingOpportunities(priority_1)); assertEquals(3, app_0.getSchedulingOpportunities(priority_1));
assertEquals(1, app_0.getTotalRequiredResources(priority_1)); assertEquals(1, app_0.getTotalRequiredResources(priority_1));
verify(app_0, never()).allocate(any(NodeType.class), eq(node_2), verify(app_0, never()).allocate(any(NodeType.class), eq(node_2),
eq(priority_2), any(ResourceRequest.class), any(Container.class)); eq(priority_2), any(ResourceRequest.class), any(Container.class));
@ -1438,7 +1438,7 @@ public class TestLeafQueue {
assertEquals(0, app_0.getTotalRequiredResources(priority_1)); assertEquals(0, app_0.getTotalRequiredResources(priority_1));
verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_1), verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_1),
eq(priority_2), any(ResourceRequest.class), any(Container.class)); eq(priority_2), any(ResourceRequest.class), any(Container.class));
assertEquals(0, app_0.getSchedulingOpportunities(priority_2)); assertEquals(1, app_0.getSchedulingOpportunities(priority_2));
assertEquals(0, app_0.getTotalRequiredResources(priority_2)); assertEquals(0, app_0.getTotalRequiredResources(priority_2));
} }