From 8a6c7d99733630416efde5ede7ff8657ff13903c Mon Sep 17 00:00:00 2001 From: Xuan Date: Wed, 10 Jun 2015 21:40:48 -0700 Subject: [PATCH] YARN-3785. Support for Resource as an argument during submitApp call in MockRM test class. Contributed by Sunil G (cherry picked from commit 5583f88bf7f1852dc0907ce55d0755e4fb22107a) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../yarn/server/resourcemanager/MockRM.java | 28 ++++++++++++++----- .../TestWorkPreservingRMRestart.java | 10 +++++-- .../capacity/TestCapacityScheduler.java | 11 +++++--- .../security/TestDelegationTokenRenewer.java | 27 +++++++++--------- 5 files changed, 52 insertions(+), 27 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7a4d686e555..4a83ed508df 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -255,6 +255,9 @@ Release 2.8.0 - UNRELEASED YARN-3787. Allowed generic history service to load a number of applications whose started time is within the given range. (Xuan Gong via zjshen) + YARN-3785. Support for Resource as an argument during submitApp call in MockRM + test class. (Sunil G via xgong) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 672ce13c18d..d068a94f16b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -322,6 +322,14 @@ public class MockRM extends ResourceManager { YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); } + public RMApp submitApp(Resource resource, String name, String user, + Map acls, String queue) throws Exception { + return submitApp(resource, name, user, acls, false, queue, + super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, + true, false, false, null, 0, null, true); + } + public RMApp submitApp(int masterMemory, String name, String user, Map acls, String queue, boolean waitForAccepted) throws Exception { @@ -358,14 +366,18 @@ public class MockRM extends ResourceManager { Map acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers) throws Exception { - return submitApp(masterMemory, name, user, acls, unmanaged, queue, + Resource resource = Records.newRecord(Resource.class); + resource.setMemory(masterMemory); + return submitApp(resource, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, false, null, 0, null, true); } public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval) throws Exception { - return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser() + Resource resource = Records.newRecord(Resource.class); + resource.setMemory(masterMemory); + return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, @@ -377,21 +389,25 @@ public class MockRM extends ResourceManager { int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId) throws Exception { - return submitApp(masterMemory, name, user, acls, unmanaged, queue, + Resource resource = Records.newRecord(Resource.class); + resource.setMemory(masterMemory); + return submitApp(resource, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, isAppIdProvided, applicationId, 0, null, true); } public RMApp submitApp(int masterMemory, LogAggregationContext logAggregationContext) throws Exception { - return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser() + Resource resource = Records.newRecord(Resource.class); + resource.setMemory(masterMemory); + return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, logAggregationContext, true); } - public RMApp submitApp(int masterMemory, String name, String user, + public RMApp submitApp(Resource capability, String name, String user, Map acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, @@ -422,8 +438,6 @@ public class MockRM extends ResourceManager { sub.setApplicationType(appType); ContainerLaunchContext clc = Records .newRecord(ContainerLaunchContext.class); - final Resource capability = Records.newRecord(Resource.class); - capability.setMemory(masterMemory); sub.setResource(capability); clc.setApplicationACLs(acls); if (ts != null && UserGroupInformation.isSecurityEnabled()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index c6fe37128a9..32743c97bad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -78,6 +78,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.Dom import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.util.ControlledClock; +import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -1050,9 +1051,12 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase nm1.registerNode(); // submit app with keepContainersAcrossApplicationAttempts true - RMApp app0 = rm1.submitApp(200, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, - null, null, true, true, false, null, 0, null, true); + Resource resource = Records.newRecord(Resource.class); + resource.setMemory(200); + RMApp app0 = rm1.submitApp(resource, "", UserGroupInformation + .getCurrentUser().getShortUserName(), null, false, null, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true, + false, null, 0, null, true); MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); am0.allocate("127.0.0.1", 1000, 2, new ArrayList()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 3827f8537b9..e8afab2ae9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -2986,13 +2986,16 @@ public class TestCapacityScheduler { (LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName); Resource amResourceLimit = queueA.getAMResourceLimit(); - Resource amResource = - Resource.newInstance(amResourceLimit.getMemory() + 1, + Resource amResource1 = + Resource.newInstance(amResourceLimit.getMemory() + 1024, + amResourceLimit.getVirtualCores() + 1); + Resource amResource2 = + Resource.newInstance(amResourceLimit.getMemory() + 2048, amResourceLimit.getVirtualCores() + 1); - rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName); + rm.submitApp(amResource1, "app-1", userName, null, queueName); - rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName); + rm.submitApp(amResource2, "app-2", userName, null, queueName); // When AM limit is exceeded, 1 applications will be activated.Rest all // applications will be in pending diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 10e840bee04..49c7bf96ac6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -91,6 +91,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer.DelegationTokenToRenew; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.Records; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -1044,16 +1045,16 @@ public class TestDelegationTokenRenewer { credentials.addToken(userText1, token1); // submit app1 with a token, set cancelTokenWhenComplete to false; - RMApp app1 = - rm.submitApp(200, "name", "user", null, false, null, 2, credentials, - null, true, false, false, null, 0, null, false); + Resource resource = Records.newRecord(Resource.class); + resource.setMemory(200); + RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, + credentials, null, true, false, false, null, 0, null, false); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING); // submit app2 with the same token, set cancelTokenWhenComplete to true; - RMApp app2 = - rm.submitApp(200, "name", "user", null, false, null, 2, credentials, - null, true, false, false, null, 0, null, true); + RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2, + credentials, null, true, false, false, null, 0, null, true); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING); MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2); @@ -1109,8 +1110,10 @@ public class TestDelegationTokenRenewer { Assert.assertTrue(renewer.getAllTokens().isEmpty()); Assert.assertFalse(Renewer.cancelled); + Resource resource = Records.newRecord(Resource.class); + resource.setMemory(200); RMApp app1 = - rm.submitApp(200, "name", "user", null, false, null, 2, credentials, + rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, true); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING); @@ -1118,9 +1121,8 @@ public class TestDelegationTokenRenewer { DelegationTokenToRenew dttr = renewer.getAllTokens().get(token1); Assert.assertNotNull(dttr); Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId())); - RMApp app2 = - rm.submitApp(200, "name", "user", null, false, null, 2, credentials, - null, true, false, false, null, 0, null, true); + RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2, + credentials, null, true, false, false, null, 0, null, true); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING); Assert.assertTrue(renewer.getAllTokens().containsKey(token1)); @@ -1136,9 +1138,8 @@ public class TestDelegationTokenRenewer { Assert.assertFalse(dttr.isTimerCancelled()); Assert.assertFalse(Renewer.cancelled); - RMApp app3 = - rm.submitApp(200, "name", "user", null, false, null, 2, credentials, - null, true, false, false, null, 0, null, true); + RMApp app3 = rm.submitApp(resource, "name", "user", null, false, null, 2, + credentials, null, true, false, false, null, 0, null, true); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm1); rm.waitForState(app3.getApplicationId(), RMAppState.RUNNING); Assert.assertTrue(renewer.getAllTokens().containsKey(token1));