From e75ee47fc9358d6088eaa4de98b3c85ec2ca9667 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 12 Jun 2013 00:25:07 +0000 Subject: [PATCH] YARN-795. Fair scheduler queue metrics should subtract allocated vCores from available vCores. (ywskycn via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1492021 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../scheduler/QueueMetrics.java | 2 +- .../scheduler/fair/TestFairScheduler.java | 36 ++++++++++++++----- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8e16f2584f7..40c29f35a8f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -544,6 +544,9 @@ Release 2.1.0-beta - UNRELEASED YARN-493. Fixed some shell related flaws in YARN on Windows. (Chris Nauroth via vinodkv) + YARN-795. Fair scheduler queue metrics should subtract allocated vCores from + available vCores. (ywskycn via tucu) + Release 2.0.5-alpha - 06/06/2013 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 58d9d4d9766..a9cf2fdeadf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -452,7 +452,7 @@ public class QueueMetrics implements MetricsSource { } public Resource getAllocatedResources() { - return BuilderUtils.newResource(allocatedMB.value(), 0); + return BuilderUtils.newResource(allocatedMB.value(), allocatedVCores.value()); } public int getAllocatedMB() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 40c6be0e448..03b3a7fd3cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -219,6 +219,13 @@ public class TestFairScheduler { createSchedulingRequestExistingApplication(request, attId); } + private void createSchedulingRequestExistingApplication(int memory, int vcores, + int priority, ApplicationAttemptId attId) { + ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, + priority, 1, true); + createSchedulingRequestExistingApplication(request, attId); + } + private void createSchedulingRequestExistingApplication(ResourceRequest request, ApplicationAttemptId attId) { List ask = new ArrayList(); @@ -352,17 +359,17 @@ public class TestFairScheduler { // Add a node RMNode node1 = MockNodes - .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); + .newNodeInfo(1, Resources.createResource(1024, 4), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); // Add another node RMNode node2 = - MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2"); + MockNodes.newNodeInfo(1, Resources.createResource(512, 2), 2, "127.0.0.2"); NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); scheduler.handle(nodeEvent2); - createSchedulingRequest(512, "queue1", "user1", 2); + createSchedulingRequest(512, 2, "queue1", "user1", 2); scheduler.update(); @@ -379,13 +386,18 @@ public class TestFairScheduler { assertEquals(1024, scheduler.getQueueManager().getQueue("queue1"). getResourceUsage().getMemory()); + assertEquals(2, scheduler.getQueueManager().getQueue("queue1"). + getResourceUsage().getVirtualCores()); // verify metrics QueueMetrics queue1Metrics = scheduler.getQueueManager().getQueue("queue1") .getMetrics(); assertEquals(1024, queue1Metrics.getAllocatedMB()); + assertEquals(2, queue1Metrics.getAllocatedVirtualCores()); assertEquals(1024, scheduler.getRootQueueMetrics().getAllocatedMB()); + assertEquals(2, scheduler.getRootQueueMetrics().getAllocatedVirtualCores()); assertEquals(512, scheduler.getRootQueueMetrics().getAvailableMB()); + assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); } @Test (timeout = 5000) @@ -1334,11 +1346,11 @@ public class TestFairScheduler { // Add a node RMNode node1 = MockNodes - .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); + .newNodeInfo(1, Resources.createResource(1024, 4), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); - ApplicationAttemptId attId = createSchedulingRequest(1024, "queue1", + ApplicationAttemptId attId = createSchedulingRequest(1024, 4, "queue1", "user1", 1, 2); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); @@ -1351,15 +1363,16 @@ public class TestFairScheduler { .getLiveContainers().iterator().next().getContainerId(); // Cause reservation to be created - createSchedulingRequestExistingApplication(1024, 2, attId); + createSchedulingRequestExistingApplication(1024, 4, 2, attId); scheduler.update(); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); + assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); // Create request at higher priority - createSchedulingRequestExistingApplication(1024, 1, attId); + createSchedulingRequestExistingApplication(1024, 4, 1, attId); scheduler.update(); scheduler.handle(updateEvent); @@ -1373,6 +1386,7 @@ public class TestFairScheduler { scheduler.allocate(attId, new ArrayList(), Arrays.asList(containerId), null, null); assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); + assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); // Schedule at opening scheduler.update(); @@ -1385,6 +1399,7 @@ public class TestFairScheduler { Assert.assertEquals(2, liveContainer.getContainer().getPriority().getPriority()); } assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); + assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); } @Test @@ -1708,22 +1723,27 @@ public class TestFairScheduler { @Test public void testRemoveNodeUpdatesRootQueueMetrics() { assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); + assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); - RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024, 4), 1, "127.0.0.1"); NodeAddedSchedulerEvent addEvent = new NodeAddedSchedulerEvent(node1); scheduler.handle(addEvent); assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); + assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); scheduler.update(); // update shouldn't change things assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); + assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); NodeRemovedSchedulerEvent removeEvent = new NodeRemovedSchedulerEvent(node1); scheduler.handle(removeEvent); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); + assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); scheduler.update(); // update shouldn't change things assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); + assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); } @Test