From 39844356617f7ce1dc214d03407ba7bb7087ffdc Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Mon, 22 Sep 2014 16:09:52 -0700 Subject: [PATCH] YARN-2539. FairScheduler: Set the default value for maxAMShare to 0.5. (Wei Yan via kasha) (cherry picked from commit 43efdd30b52f4ee8967a03dac4d548095d6cafdb) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../fair/AllocationConfiguration.java | 2 +- .../fair/AllocationFileLoaderService.java | 2 +- .../scheduler/fair/TestFairScheduler.java | 30 ++++++++++--------- .../src/site/apt/FairScheduler.apt.vm | 4 +-- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 74d88bdf220..73d7f51dc4a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -211,6 +211,9 @@ Release 2.6.0 - UNRELEASED YARN-1372. Ensure all completed containers are reported to the AMs across RM restart. (Anubhav Dhoot via jianhe) + YARN-2539. FairScheduler: Set the default value for maxAMShare to 0.5. + (Wei Yan via kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index de5a999c2dd..70a6496a4a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -130,7 +130,7 @@ public AllocationConfiguration(Configuration conf) { queueMaxAMShares = new HashMap(); userMaxAppsDefault = Integer.MAX_VALUE; queueMaxAppsDefault = Integer.MAX_VALUE; - queueMaxAMShareDefault = -1.0f; + queueMaxAMShareDefault = 0.5f; queueAcls = new HashMap>(); minSharePreemptionTimeouts = new HashMap(); fairSharePreemptionTimeouts = new HashMap(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index c2dfc84a536..2022510c673 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -224,7 +224,7 @@ public synchronized void reloadAllocations() throws IOException, new HashMap>(); int userMaxAppsDefault = Integer.MAX_VALUE; int queueMaxAppsDefault = Integer.MAX_VALUE; - float queueMaxAMShareDefault = -1.0f; + float queueMaxAMShareDefault = 0.5f; long defaultFairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; float defaultFairSharePreemptionThreshold = 0.5f; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 2435a99de80..c71acc8f37b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -3128,7 +3128,7 @@ public void testQueueMaxAMShareDefault() throws Exception { out.println(""); out.println(""); out.println(""); - out.println("1.0"); + out.println("0.4"); out.println(""); out.println(""); out.println(""); @@ -3172,40 +3172,42 @@ public void testQueueMaxAMShareDefault() throws Exception { assertEquals("Queue queue5's fair share should be 0", 0, queue5 .getFairShare().getMemory()); - List queues = Arrays.asList("root.default", "root.queue3", - "root.queue4", "root.queue5"); + List queues = Arrays.asList("root.queue3", "root.queue4", + "root.queue5"); for (String queue : queues) { createSchedulingRequest(1 * 1024, queue, "user1"); scheduler.update(); scheduler.handle(updateEvent); } - Resource amResource1 = Resource.newInstance(2048, 1); + Resource amResource1 = Resource.newInstance(1024, 1); int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority(); - // Exceeds queue limit, but default maxAMShare is -1.0 so it doesn't matter + // The fair share is 2048 MB, and the default maxAMShare is 0.5f, + // so the AM is accepted. ApplicationAttemptId attId1 = createAppAttemptId(1, 1); createApplicationWithAMResource(attId1, "queue1", "test1", amResource1); - createSchedulingRequestExistingApplication(2048, 1, amPriority, attId1); + createSchedulingRequestExistingApplication(1024, 1, amPriority, attId1); FSAppAttempt app1 = scheduler.getSchedulerApp(attId1); scheduler.update(); scheduler.handle(updateEvent); - assertEquals("Application1's AM requests 2048 MB memory", - 2048, app1.getAMResource().getMemory()); + assertEquals("Application1's AM requests 1024 MB memory", + 1024, app1.getAMResource().getMemory()); assertEquals("Application1's AM should be running", 1, app1.getLiveContainers().size()); - assertEquals("Queue1's AM resource usage should be 2048 MB memory", - 2048, queue1.getAmResourceUsage().getMemory()); + assertEquals("Queue1's AM resource usage should be 1024 MB memory", + 1024, queue1.getAmResourceUsage().getMemory()); - // Exceeds queue limit, and maxAMShare is 1.0 + // Now the fair share is 1639 MB, and the maxAMShare is 0.4f, + // so the AM is not accepted. ApplicationAttemptId attId2 = createAppAttemptId(2, 1); createApplicationWithAMResource(attId2, "queue2", "test1", amResource1); - createSchedulingRequestExistingApplication(2048, 1, amPriority, attId2); + createSchedulingRequestExistingApplication(1024, 1, amPriority, attId2); FSAppAttempt app2 = scheduler.getSchedulerApp(attId2); scheduler.update(); scheduler.handle(updateEvent); - assertEquals("Application2's AM requests 2048 MB memory", - 2048, app2.getAMResource().getMemory()); + assertEquals("Application2's AM requests 1024 MB memory", + 1024, app2.getAMResource().getMemory()); assertEquals("Application2's AM should not be running", 0, app2.getLiveContainers().size()); assertEquals("Queue2's AM resource usage should be 0 MB memory", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm index d1ce36a1f59..7755c23d264 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm @@ -246,8 +246,8 @@ Allocation file format * maxAMShare: limit the fraction of the queue's fair share that can be used to run application masters. This property can only be used for leaf queues. For example, if set to 1.0f, then AMs in the leaf queue can take up to 100% - of both the memory and CPU fair share. The default value is -1.0f, which - means that this check is disabled. + of both the memory and CPU fair share. The value of -1.0f will disable + this feature and the amShare will not be checked. The default value is 0.5f. * weight: to share the cluster non-proportionally with other queues. Weights default to 1, and a queue with weight 2 should receive approximately twice