From 452079af8bc56195945e28b8cf76620f0aca01c3 Mon Sep 17 00:00:00 2001 From: Jian He Date: Wed, 16 Sep 2015 14:58:32 +0800 Subject: [PATCH] YARN-4078. Add getPendingResourceRequestForAttempt in YarnScheduler interface. Contributed by Naganarasimha G R --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../scheduler/AbstractYarnScheduler.java | 1 + .../scheduler/YarnScheduler.java | 6 ++++++ .../resourcemanager/webapp/dao/AppInfo.java | 6 ++---- .../TestWorkPreservingRMRestart.java | 21 ++++++------------- .../scheduler/TestAbstractYarnScheduler.java | 14 ++++++------- .../capacity/TestCapacityScheduler.java | 10 ++++----- 7 files changed, 29 insertions(+), 32 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b044078c425..c5143ff629f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -842,6 +842,9 @@ Release 2.8.0 - UNRELEASED YARN-4151. Fix findbugs errors in hadoop-yarn-server-common module. (Meng Ding via wangda) + YARN-4078. Add getPendingResourceRequestForAttempt in YarnScheduler interface. + (Naganarasimha G R via jianhe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 0b6b8efdf5f..4c34c47a314 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -687,6 +687,7 @@ public abstract class AbstractYarnScheduler } } + @Override public List getPendingResourceRequestsForAttempt( ApplicationAttemptId attemptId) { SchedulerApplicationAttempt attempt = getApplicationAttempt(attemptId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index f2753e60cb5..e3c79f7acdc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -337,4 +337,10 @@ public interface YarnScheduler extends EventHandler { * @throws YarnException */ void setClusterMaxPriority(Configuration conf) throws YarnException; + + /** + * @param attemptId + */ + List getPendingResourceRequestsForAttempt( + ApplicationAttemptId attemptId); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 7f8031529a1..a035aebd2b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -177,9 +176,8 @@ public class AppInfo { allocatedVCores = usedResources.getVirtualCores(); runningContainers = resourceReport.getNumUsedContainers(); } - resourceRequests = - ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) - .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId()); + resourceRequests = rm.getRMContext().getScheduler() + .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 77e98383ce7..a9a88daeabc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -23,14 +23,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; @@ -62,10 +58,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueNotFoundException; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; @@ -73,9 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQu import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.Records; @@ -616,8 +610,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase // Wait for RM to settle down on recovering containers; Thread.sleep(3000); - AbstractYarnScheduler scheduler = - (AbstractYarnScheduler) rm2.getResourceScheduler(); + YarnScheduler scheduler = rm2.getResourceScheduler(); // Previous AM failed, The failed AM should once again release the // just-recovered containers. assertNull(scheduler.getRMContainer(runningContainer.getContainerId())); @@ -669,8 +662,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase // Wait for RM to settle down on recovering containers; Thread.sleep(3000); - AbstractYarnScheduler scheduler = - (AbstractYarnScheduler) rm2.getResourceScheduler(); + YarnScheduler scheduler = rm2.getResourceScheduler(); // scheduler should not recover containers for finished apps. assertNull(scheduler.getRMContainer(runningContainer.getContainerId())); @@ -724,9 +716,8 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1); RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt(); - AbstractYarnScheduler scheduler = - ((AbstractYarnScheduler) rm1.getResourceScheduler()); - + YarnScheduler scheduler = rm1.getResourceScheduler(); + Assert.assertTrue(scheduler.getRMContainer( attempt0.getMasterContainer().getId()).isAMContainer()); @@ -742,7 +733,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase // Wait for RM to settle down on recovering containers; waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId()); - scheduler = ((AbstractYarnScheduler) rm2.getResourceScheduler()); + scheduler = rm2.getResourceScheduler(); Assert.assertTrue(scheduler.getRMContainer( attempt0.getMasterContainer().getId()).isAMContainer()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java index ffd1c1f40d3..7c33f78358a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java @@ -79,7 +79,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase { try { rm.start(); testMaximumAllocationMemoryHelper( - (AbstractYarnScheduler) rm.getResourceScheduler(), + rm.getResourceScheduler(), node1MaxMemory, node2MaxMemory, node3MaxMemory, configuredMaxMemory, configuredMaxMemory, configuredMaxMemory, configuredMaxMemory, configuredMaxMemory, configuredMaxMemory); @@ -94,7 +94,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase { try { rm.start(); testMaximumAllocationMemoryHelper( - (AbstractYarnScheduler) rm.getResourceScheduler(), + rm.getResourceScheduler(), node1MaxMemory, node2MaxMemory, node3MaxMemory, configuredMaxMemory, configuredMaxMemory, configuredMaxMemory, node2MaxMemory, node3MaxMemory, node2MaxMemory); @@ -104,7 +104,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase { } private void testMaximumAllocationMemoryHelper( - AbstractYarnScheduler scheduler, + YarnScheduler scheduler, final int node1MaxMemory, final int node2MaxMemory, final int node3MaxMemory, final int... expectedMaxMemory) throws Exception { @@ -166,7 +166,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase { try { rm.start(); testMaximumAllocationVCoresHelper( - (AbstractYarnScheduler) rm.getResourceScheduler(), + rm.getResourceScheduler(), node1MaxVCores, node2MaxVCores, node3MaxVCores, configuredMaxVCores, configuredMaxVCores, configuredMaxVCores, configuredMaxVCores, configuredMaxVCores, configuredMaxVCores); @@ -181,7 +181,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase { try { rm.start(); testMaximumAllocationVCoresHelper( - (AbstractYarnScheduler) rm.getResourceScheduler(), + rm.getResourceScheduler(), node1MaxVCores, node2MaxVCores, node3MaxVCores, configuredMaxVCores, configuredMaxVCores, configuredMaxVCores, node2MaxVCores, node3MaxVCores, node2MaxVCores); @@ -191,7 +191,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase { } private void testMaximumAllocationVCoresHelper( - AbstractYarnScheduler scheduler, + YarnScheduler scheduler, final int node1MaxVCores, final int node2MaxVCores, final int node3MaxVCores, final int... expectedMaxVCores) throws Exception { @@ -494,7 +494,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase { } private void verifyMaximumResourceCapability( - Resource expectedMaximumResource, AbstractYarnScheduler scheduler) { + Resource expectedMaximumResource, YarnScheduler scheduler) { final Resource schedulerMaximumResourceCapability = scheduler .getMaximumResourceCapability(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 76a1351e56d..fb7fce400b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -113,6 +113,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicat import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; @@ -1950,8 +1951,7 @@ public class TestCapacityScheduler { @Test public void testMoveAllAppsInvalidDestination() throws Exception { MockRM rm = setUpMove(); - AbstractYarnScheduler scheduler = - (AbstractYarnScheduler) rm.getResourceScheduler(); + YarnScheduler scheduler = rm.getResourceScheduler(); // submit an app RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); @@ -2009,8 +2009,7 @@ public class TestCapacityScheduler { @Test public void testMoveAllAppsInvalidSource() throws Exception { MockRM rm = setUpMove(); - AbstractYarnScheduler scheduler = - (AbstractYarnScheduler) rm.getResourceScheduler(); + YarnScheduler scheduler = rm.getResourceScheduler(); // submit an app RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); @@ -2113,8 +2112,7 @@ public class TestCapacityScheduler { @Test public void testKillAllAppsInvalidSource() throws Exception { MockRM rm = setUpMove(); - AbstractYarnScheduler scheduler = - (AbstractYarnScheduler) rm.getResourceScheduler(); + YarnScheduler scheduler = rm.getResourceScheduler(); // submit an app RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1");