YARN-3785. Support for Resource as an argument during submitApp call in

MockRM test class. Contributed by Sunil G

(cherry picked from commit 5583f88bf7)

hadoop-yarn-project/CHANGES.txt

@@ -255,6 +255,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3787. Allowed generic history service to load a number of applications whose
     started time is within the given range. (Xuan Gong via zjshen)
 
+    YARN-3785. Support for Resource as an argument during submitApp call in MockRM
+    test class. (Sunil G via xgong)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java

@@ -322,6 +322,14 @@ public class MockRM extends ResourceManager {
         YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null);
   }
   
+  public RMApp submitApp(Resource resource, String name, String user,
+      Map<ApplicationAccessType, String> acls, String queue) throws Exception {
+    return submitApp(resource, name, user, acls, false, queue,
+        super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+          YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null,
+          true, false, false, null, 0, null, true);
+  }
+
   public RMApp submitApp(int masterMemory, String name, String user,
       Map<ApplicationAccessType, String> acls, String queue, 
       boolean waitForAccepted) throws Exception {
@@ -358,14 +366,18 @@ public class MockRM extends ResourceManager {
       Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
       int maxAppAttempts, Credentials ts, String appType,
       boolean waitForAccepted, boolean keepContainers) throws Exception {
-    return submitApp(masterMemory, name, user, acls, unmanaged, queue,
+    Resource resource = Records.newRecord(Resource.class);
+    resource.setMemory(masterMemory);
+    return submitApp(resource, name, user, acls, unmanaged, queue,
         maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
         false, null, 0, null, true);
   }
 
   public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval)
       throws Exception {
-    return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser()
+    Resource resource = Records.newRecord(Resource.class);
+    resource.setMemory(masterMemory);
+    return submitApp(resource, "", UserGroupInformation.getCurrentUser()
       .getShortUserName(), null, false, null,
       super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
       YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false,
@@ -377,21 +389,25 @@ public class MockRM extends ResourceManager {
       int maxAppAttempts, Credentials ts, String appType,
       boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
       ApplicationId applicationId) throws Exception {
-    return submitApp(masterMemory, name, user, acls, unmanaged, queue,
+    Resource resource = Records.newRecord(Resource.class);
+    resource.setMemory(masterMemory);
+    return submitApp(resource, name, user, acls, unmanaged, queue,
       maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
       isAppIdProvided, applicationId, 0, null, true);
   }
 
   public RMApp submitApp(int masterMemory,
       LogAggregationContext logAggregationContext) throws Exception {
-    return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser()
+    Resource resource = Records.newRecord(Resource.class);
+    resource.setMemory(masterMemory);
+    return submitApp(resource, "", UserGroupInformation.getCurrentUser()
       .getShortUserName(), null, false, null,
       super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
       YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false,
       false, null, 0, logAggregationContext, true);
    }
 
-  public RMApp submitApp(int masterMemory, String name, String user,
+  public RMApp submitApp(Resource capability, String name, String user,
       Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
       int maxAppAttempts, Credentials ts, String appType,
       boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
@@ -422,8 +438,6 @@ public class MockRM extends ResourceManager {
     sub.setApplicationType(appType);
     ContainerLaunchContext clc = Records
         .newRecord(ContainerLaunchContext.class);
-    final Resource capability = Records.newRecord(Resource.class);
-    capability.setMemory(masterMemory);
     sub.setResource(capability);
     clc.setApplicationACLs(acls);
     if (ts != null && UserGroupInformation.isSecurityEnabled()) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java

@@ -78,6 +78,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.Dom
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
 import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.util.SystemClock;
 import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -1050,9 +1051,12 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
     nm1.registerNode();
 
     // submit app with keepContainersAcrossApplicationAttempts true
-    RMApp app0 = rm1.submitApp(200, "", UserGroupInformation.getCurrentUser()
+    Resource resource = Records.newRecord(Resource.class);
-        .getShortUserName(), null, false, null, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, 
+    resource.setMemory(200);
-        null, null, true, true, false, null, 0, null, true);
+    RMApp app0 = rm1.submitApp(resource, "", UserGroupInformation
+        .getCurrentUser().getShortUserName(), null, false, null,
+        YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true,
+        false, null, 0, null, true);
     MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);
 
     am0.allocate("127.0.0.1", 1000, 2, new ArrayList<ContainerId>());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java

@@ -2986,13 +2986,16 @@ public class TestCapacityScheduler {
         (LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName);
     Resource amResourceLimit = queueA.getAMResourceLimit();
 
-    Resource amResource =
+    Resource amResource1 =
-        Resource.newInstance(amResourceLimit.getMemory() + 1,
+        Resource.newInstance(amResourceLimit.getMemory() + 1024,
+            amResourceLimit.getVirtualCores() + 1);
+    Resource amResource2 =
+        Resource.newInstance(amResourceLimit.getMemory() + 2048,
             amResourceLimit.getVirtualCores() + 1);
 
-    rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
+    rm.submitApp(amResource1, "app-1", userName, null, queueName);
 
-    rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
+    rm.submitApp(amResource2, "app-2", userName, null, queueName);
 
     // When AM limit is exceeded, 1 applications will be activated.Rest all
     // applications will be in pending

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java

@@ -91,6 +91,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer.DelegationTokenToRenew;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.Records;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -1044,16 +1045,16 @@ public class TestDelegationTokenRenewer {
     credentials.addToken(userText1, token1);
 
     // submit app1 with a token, set cancelTokenWhenComplete to false;
-    RMApp app1 =
+    Resource resource = Records.newRecord(Resource.class);
-        rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
+    resource.setMemory(200);
-          null, true, false, false, null, 0, null, false);
+    RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2,
+        credentials, null, true, false, false, null, 0, null, false);
     MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
     rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
 
     // submit app2 with the same token, set cancelTokenWhenComplete to true;
-    RMApp app2 =
+    RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2,
-        rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
+        credentials, null, true, false, false, null, 0, null, true);
-          null, true, false, false, null, 0, null, true);
     MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
     rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
     MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2);
@@ -1109,8 +1110,10 @@ public class TestDelegationTokenRenewer {
     Assert.assertTrue(renewer.getAllTokens().isEmpty());
     Assert.assertFalse(Renewer.cancelled);
 
+    Resource resource = Records.newRecord(Resource.class);
+    resource.setMemory(200);
     RMApp app1 =
-        rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
+        rm.submitApp(resource, "name", "user", null, false, null, 2, credentials,
           null, true, false, false, null, 0, null, true);
     MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
     rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
@@ -1118,9 +1121,8 @@ public class TestDelegationTokenRenewer {
     DelegationTokenToRenew dttr = renewer.getAllTokens().get(token1);
     Assert.assertNotNull(dttr);
     Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
-    RMApp app2 =
+    RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2,
-        rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
+        credentials, null, true, false, false, null, 0, null, true);
-          null, true, false, false, null, 0, null, true);
     MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
     rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
     Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
@@ -1136,9 +1138,8 @@ public class TestDelegationTokenRenewer {
     Assert.assertFalse(dttr.isTimerCancelled());
     Assert.assertFalse(Renewer.cancelled);
 
-    RMApp app3 =
+    RMApp app3 = rm.submitApp(resource, "name", "user", null, false, null, 2,
-        rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
+        credentials, null, true, false, false, null, 0, null, true);
-          null, true, false, false, null, 0, null, true);
     MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm1);
     rm.waitForState(app3.getApplicationId(), RMAppState.RUNNING);
     Assert.assertTrue(renewer.getAllTokens().containsKey(token1));