From 05c923a455f1eb36e2fa8cd9d08116ef958f3971 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 11 Dec 2013 20:15:47 +0000 Subject: [PATCH] YARN-408. Change CapacityScheduler to not disable delay-scheduling by default. Contributed by Mayank Bansal. svn merge --ignore-ancestry -c 1550245 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1550246 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../conf/capacity-scheduler.xml | 6 +++--- .../hadoop/yarn/server/resourcemanager/TestRM.java | 6 +++++- .../scheduler/capacity/TestLeafQueue.java | 9 ++++++--- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 06e489a98f4..e746f392004 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -141,6 +141,9 @@ Release 2.4.0 - UNRELEASED YARN-1491. Upgrade JUnit3 TestCase to JUnit 4 (Chen He via jeagles) + YARN-408. Change CapacityScheduler to not disable delay-scheduling by default. + (Mayank Bansal via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml index 80a9fec72c4..2bed464bcc6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml @@ -99,12 +99,12 @@ yarn.scheduler.capacity.node-locality-delay - -1 + 40 Number of missed scheduling opportunities after which the CapacityScheduler attempts to schedule rack-local containers. - Typically this should be set to number of racks in the cluster, this - feature is disabled by default, set to -1. + Typically this should be set to number of nodes in the cluster, By default is setting + approximately number of nodes in one rack which is 40. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index 0908b2fae86..205846a11de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -22,6 +22,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import javax.security.auth.login.Configuration; + import junit.framework.Assert; import org.apache.commons.logging.Log; @@ -88,7 +90,9 @@ public class TestRM { public void testAppOnMultiNode() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); - MockRM rm = new MockRM(); + YarnConfiguration conf = new YarnConfiguration(); + conf.set("yarn.scheduler.capacity.node-locality-delay", "-1"); + MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5120); MockNM nm2 = rm.registerNode("h2:5678", 10240); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 4f4bf2f4708..0ecb230f153 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -1066,6 +1066,9 @@ public class TestLeafQueue { assertEquals(2*GB, a.getMetrics().getAllocatedMB()); // node_1 heartbeats in and gets the DEFAULT_RACK request for app_1 + // We do not need locality delay here + doReturn(-1).when(a).getNodeLocalityDelay(); + a.assignContainers(clusterResource, node_1); assertEquals(10*GB, a.getUsedResources().getMemory()); assertEquals(2*GB, app_0.getCurrentConsumption().getMemory()); @@ -1649,7 +1652,7 @@ public class TestLeafQueue { LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E)); // before reinitialization - assertEquals(0, e.getNodeLocalityDelay()); + assertEquals(40, e.getNodeLocalityDelay()); csConf.setInt(CapacitySchedulerConfiguration .NODE_LOCALITY_DELAY, 60); @@ -1932,10 +1935,10 @@ public class TestLeafQueue { // Now, should allocate since RR(rack_1) = relax: true a.assignContainers(clusterResource, node_1_1); - verify(app_0).allocate(eq(NodeType.RACK_LOCAL), eq(node_1_1), + verify(app_0,never()).allocate(eq(NodeType.RACK_LOCAL), eq(node_1_1), any(Priority.class), any(ResourceRequest.class), any(Container.class)); assertEquals(0, app_0.getSchedulingOpportunities(priority)); - assertEquals(0, app_0.getTotalRequiredResources(priority)); + assertEquals(1, app_0.getTotalRequiredResources(priority)); // Now sanity-check node_local app_0_requests_0.add(