YARN-4078. Add getPendingResourceRequestForAttempt in YarnScheduler interface. Contributed by Naganarasimha G R

(cherry picked from commit 452079af8b)

hadoop-yarn-project/CHANGES.txt

@@ -790,6 +790,9 @@ Release 2.8.0 - UNRELEASED
     YARN-4151. Fix findbugs errors in hadoop-yarn-server-common module.
     (Meng Ding via wangda)
 
+    YARN-4078. Add getPendingResourceRequestForAttempt in YarnScheduler interface.
+    (Naganarasimha G R via jianhe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java

@@ -687,6 +687,7 @@ public abstract class AbstractYarnScheduler
     }
   }
 
+  @Override
   public List<ResourceRequest> getPendingResourceRequestsForAttempt(
       ApplicationAttemptId attemptId) {
     SchedulerApplicationAttempt attempt = getApplicationAttempt(attemptId);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java

@@ -337,4 +337,10 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
    * @throws YarnException
    */
   void setClusterMaxPriority(Configuration conf) throws YarnException;
+
+  /**
+   * @param attemptId
+   */
+  List<ResourceRequest> getPendingResourceRequestsForAttempt(
+      ApplicationAttemptId attemptId);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java

@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Times;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -177,9 +176,8 @@ public class AppInfo {
             allocatedVCores = usedResources.getVirtualCores();
             runningContainers = resourceReport.getNumUsedContainers();
           }
-          resourceRequests =
+          resourceRequests = rm.getRMContext().getScheduler()
-              ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
+              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
-                .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
         }
       }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java

@@ -23,14 +23,10 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
-import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
-import java.io.PrintWriter;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -62,10 +58,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueNotFoundException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
@@ -73,9 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQu
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
 import org.apache.hadoop.yarn.util.ControlledClock;
 import org.apache.hadoop.yarn.util.Records;
@@ -616,8 +610,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
     // Wait for RM to settle down on recovering containers;
     Thread.sleep(3000);
 
-    AbstractYarnScheduler scheduler =
+    YarnScheduler scheduler = rm2.getResourceScheduler();
-        (AbstractYarnScheduler) rm2.getResourceScheduler();
     // Previous AM failed, The failed AM should once again release the
     // just-recovered containers.
     assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
@@ -669,8 +662,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
     // Wait for RM to settle down on recovering containers;
     Thread.sleep(3000);
 
-    AbstractYarnScheduler scheduler =
+    YarnScheduler scheduler = rm2.getResourceScheduler();
-        (AbstractYarnScheduler) rm2.getResourceScheduler();
 
     // scheduler should not recover containers for finished apps.
     assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
@@ -724,9 +716,8 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
     MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1);
     
     RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt();
-    AbstractYarnScheduler scheduler =
+    YarnScheduler scheduler = rm1.getResourceScheduler();
-        ((AbstractYarnScheduler) rm1.getResourceScheduler());
+
-    
     Assert.assertTrue(scheduler.getRMContainer(
         attempt0.getMasterContainer().getId()).isAMContainer());
 
@@ -742,7 +733,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
     // Wait for RM to settle down on recovering containers;
     waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId());
 
-    scheduler = ((AbstractYarnScheduler) rm2.getResourceScheduler());
+    scheduler = rm2.getResourceScheduler();
     Assert.assertTrue(scheduler.getRMContainer(
         attempt0.getMasterContainer().getId()).isAMContainer());
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java

@@ -79,7 +79,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
     try {
       rm.start();
       testMaximumAllocationMemoryHelper(
-          (AbstractYarnScheduler) rm.getResourceScheduler(),
+          rm.getResourceScheduler(),
           node1MaxMemory, node2MaxMemory, node3MaxMemory,
           configuredMaxMemory, configuredMaxMemory, configuredMaxMemory,
           configuredMaxMemory, configuredMaxMemory, configuredMaxMemory);
@@ -94,7 +94,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
     try {
       rm.start();
       testMaximumAllocationMemoryHelper(
-          (AbstractYarnScheduler) rm.getResourceScheduler(),
+          rm.getResourceScheduler(),
           node1MaxMemory, node2MaxMemory, node3MaxMemory,
           configuredMaxMemory, configuredMaxMemory, configuredMaxMemory,
           node2MaxMemory, node3MaxMemory, node2MaxMemory);
@@ -104,7 +104,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
   }
 
   private void testMaximumAllocationMemoryHelper(
-       AbstractYarnScheduler scheduler,
+       YarnScheduler scheduler,
        final int node1MaxMemory, final int node2MaxMemory,
        final int node3MaxMemory, final int... expectedMaxMemory)
        throws Exception {
@@ -166,7 +166,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
     try {
       rm.start();
       testMaximumAllocationVCoresHelper(
-          (AbstractYarnScheduler) rm.getResourceScheduler(),
+          rm.getResourceScheduler(),
           node1MaxVCores, node2MaxVCores, node3MaxVCores,
           configuredMaxVCores, configuredMaxVCores, configuredMaxVCores,
           configuredMaxVCores, configuredMaxVCores, configuredMaxVCores);
@@ -181,7 +181,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
     try {
       rm.start();
       testMaximumAllocationVCoresHelper(
-          (AbstractYarnScheduler) rm.getResourceScheduler(),
+          rm.getResourceScheduler(),
           node1MaxVCores, node2MaxVCores, node3MaxVCores,
           configuredMaxVCores, configuredMaxVCores, configuredMaxVCores,
           node2MaxVCores, node3MaxVCores, node2MaxVCores);
@@ -191,7 +191,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
   }
 
   private void testMaximumAllocationVCoresHelper(
-      AbstractYarnScheduler scheduler,
+      YarnScheduler scheduler,
       final int node1MaxVCores, final int node2MaxVCores,
       final int node3MaxVCores, final int... expectedMaxVCores)
       throws Exception {
@@ -494,7 +494,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
   }
 
   private void verifyMaximumResourceCapability(
-      Resource expectedMaximumResource, AbstractYarnScheduler scheduler) {
+      Resource expectedMaximumResource, YarnScheduler scheduler) {
 
     final Resource schedulerMaximumResourceCapability = scheduler
         .getMaximumResourceCapability();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java

@@ -113,6 +113,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicat
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
@@ -1950,8 +1951,7 @@ public class TestCapacityScheduler {
   @Test
   public void testMoveAllAppsInvalidDestination() throws Exception {
     MockRM rm = setUpMove();
-    AbstractYarnScheduler scheduler =
+    YarnScheduler scheduler = rm.getResourceScheduler();
-        (AbstractYarnScheduler) rm.getResourceScheduler();
 
     // submit an app
     RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1");
@@ -2009,8 +2009,7 @@ public class TestCapacityScheduler {
   @Test
   public void testMoveAllAppsInvalidSource() throws Exception {
     MockRM rm = setUpMove();
-    AbstractYarnScheduler scheduler =
+    YarnScheduler scheduler = rm.getResourceScheduler();
-        (AbstractYarnScheduler) rm.getResourceScheduler();
 
     // submit an app
     RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1");
@@ -2113,8 +2112,7 @@ public class TestCapacityScheduler {
   @Test
   public void testKillAllAppsInvalidSource() throws Exception {
     MockRM rm = setUpMove();
-    AbstractYarnScheduler scheduler =
+    YarnScheduler scheduler = rm.getResourceScheduler();
-        (AbstractYarnScheduler) rm.getResourceScheduler();
 
     // submit an app
     RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1");