YARN-3785. Support for Resource as an argument during submitApp call in
MockRM test class. Contributed by Sunil G
(cherry picked from commit 5583f88bf7
)
This commit is contained in:
parent
9523e2c098
commit
8a6c7d9973
|
@ -255,6 +255,9 @@ Release 2.8.0 - UNRELEASED
|
|||
YARN-3787. Allowed generic history service to load a number of applications whose
|
||||
started time is within the given range. (Xuan Gong via zjshen)
|
||||
|
||||
YARN-3785. Support for Resource as an argument during submitApp call in MockRM
|
||||
test class. (Sunil G via xgong)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
YARN-3339. TestDockerContainerExecutor should pull a single image and not
|
||||
|
|
|
@ -322,6 +322,14 @@ public class MockRM extends ResourceManager {
|
|||
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null);
|
||||
}
|
||||
|
||||
public RMApp submitApp(Resource resource, String name, String user,
|
||||
Map<ApplicationAccessType, String> acls, String queue) throws Exception {
|
||||
return submitApp(resource, name, user, acls, false, queue,
|
||||
super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
|
||||
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null,
|
||||
true, false, false, null, 0, null, true);
|
||||
}
|
||||
|
||||
public RMApp submitApp(int masterMemory, String name, String user,
|
||||
Map<ApplicationAccessType, String> acls, String queue,
|
||||
boolean waitForAccepted) throws Exception {
|
||||
|
@ -358,14 +366,18 @@ public class MockRM extends ResourceManager {
|
|||
Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
|
||||
int maxAppAttempts, Credentials ts, String appType,
|
||||
boolean waitForAccepted, boolean keepContainers) throws Exception {
|
||||
return submitApp(masterMemory, name, user, acls, unmanaged, queue,
|
||||
Resource resource = Records.newRecord(Resource.class);
|
||||
resource.setMemory(masterMemory);
|
||||
return submitApp(resource, name, user, acls, unmanaged, queue,
|
||||
maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
|
||||
false, null, 0, null, true);
|
||||
}
|
||||
|
||||
public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval)
|
||||
throws Exception {
|
||||
return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser()
|
||||
Resource resource = Records.newRecord(Resource.class);
|
||||
resource.setMemory(masterMemory);
|
||||
return submitApp(resource, "", UserGroupInformation.getCurrentUser()
|
||||
.getShortUserName(), null, false, null,
|
||||
super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
|
||||
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false,
|
||||
|
@ -377,21 +389,25 @@ public class MockRM extends ResourceManager {
|
|||
int maxAppAttempts, Credentials ts, String appType,
|
||||
boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
|
||||
ApplicationId applicationId) throws Exception {
|
||||
return submitApp(masterMemory, name, user, acls, unmanaged, queue,
|
||||
Resource resource = Records.newRecord(Resource.class);
|
||||
resource.setMemory(masterMemory);
|
||||
return submitApp(resource, name, user, acls, unmanaged, queue,
|
||||
maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
|
||||
isAppIdProvided, applicationId, 0, null, true);
|
||||
}
|
||||
|
||||
public RMApp submitApp(int masterMemory,
|
||||
LogAggregationContext logAggregationContext) throws Exception {
|
||||
return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser()
|
||||
Resource resource = Records.newRecord(Resource.class);
|
||||
resource.setMemory(masterMemory);
|
||||
return submitApp(resource, "", UserGroupInformation.getCurrentUser()
|
||||
.getShortUserName(), null, false, null,
|
||||
super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
|
||||
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false,
|
||||
false, null, 0, logAggregationContext, true);
|
||||
}
|
||||
|
||||
public RMApp submitApp(int masterMemory, String name, String user,
|
||||
public RMApp submitApp(Resource capability, String name, String user,
|
||||
Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
|
||||
int maxAppAttempts, Credentials ts, String appType,
|
||||
boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
|
||||
|
@ -422,8 +438,6 @@ public class MockRM extends ResourceManager {
|
|||
sub.setApplicationType(appType);
|
||||
ContainerLaunchContext clc = Records
|
||||
.newRecord(ContainerLaunchContext.class);
|
||||
final Resource capability = Records.newRecord(Resource.class);
|
||||
capability.setMemory(masterMemory);
|
||||
sub.setResource(capability);
|
||||
clc.setApplicationACLs(acls);
|
||||
if (ts != null && UserGroupInformation.isSecurityEnabled()) {
|
||||
|
|
|
@ -78,6 +78,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.Dom
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
|
||||
import org.apache.hadoop.yarn.util.ControlledClock;
|
||||
import org.apache.hadoop.yarn.util.Records;
|
||||
import org.apache.hadoop.yarn.util.SystemClock;
|
||||
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
|
||||
|
@ -1050,9 +1051,12 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
|
|||
nm1.registerNode();
|
||||
|
||||
// submit app with keepContainersAcrossApplicationAttempts true
|
||||
RMApp app0 = rm1.submitApp(200, "", UserGroupInformation.getCurrentUser()
|
||||
.getShortUserName(), null, false, null, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS,
|
||||
null, null, true, true, false, null, 0, null, true);
|
||||
Resource resource = Records.newRecord(Resource.class);
|
||||
resource.setMemory(200);
|
||||
RMApp app0 = rm1.submitApp(resource, "", UserGroupInformation
|
||||
.getCurrentUser().getShortUserName(), null, false, null,
|
||||
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true,
|
||||
false, null, 0, null, true);
|
||||
MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);
|
||||
|
||||
am0.allocate("127.0.0.1", 1000, 2, new ArrayList<ContainerId>());
|
||||
|
|
|
@ -2986,13 +2986,16 @@ public class TestCapacityScheduler {
|
|||
(LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName);
|
||||
Resource amResourceLimit = queueA.getAMResourceLimit();
|
||||
|
||||
Resource amResource =
|
||||
Resource.newInstance(amResourceLimit.getMemory() + 1,
|
||||
Resource amResource1 =
|
||||
Resource.newInstance(amResourceLimit.getMemory() + 1024,
|
||||
amResourceLimit.getVirtualCores() + 1);
|
||||
Resource amResource2 =
|
||||
Resource.newInstance(amResourceLimit.getMemory() + 2048,
|
||||
amResourceLimit.getVirtualCores() + 1);
|
||||
|
||||
rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
|
||||
rm.submitApp(amResource1, "app-1", userName, null, queueName);
|
||||
|
||||
rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
|
||||
rm.submitApp(amResource2, "app-2", userName, null, queueName);
|
||||
|
||||
// When AM limit is exceeded, 1 applications will be activated.Rest all
|
||||
// applications will be in pending
|
||||
|
|
|
@ -91,6 +91,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer.DelegationTokenToRenew;
|
||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||
import org.apache.hadoop.yarn.util.Records;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
|
@ -1044,16 +1045,16 @@ public class TestDelegationTokenRenewer {
|
|||
credentials.addToken(userText1, token1);
|
||||
|
||||
// submit app1 with a token, set cancelTokenWhenComplete to false;
|
||||
RMApp app1 =
|
||||
rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
|
||||
null, true, false, false, null, 0, null, false);
|
||||
Resource resource = Records.newRecord(Resource.class);
|
||||
resource.setMemory(200);
|
||||
RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2,
|
||||
credentials, null, true, false, false, null, 0, null, false);
|
||||
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
|
||||
rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
|
||||
|
||||
// submit app2 with the same token, set cancelTokenWhenComplete to true;
|
||||
RMApp app2 =
|
||||
rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
|
||||
null, true, false, false, null, 0, null, true);
|
||||
RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2,
|
||||
credentials, null, true, false, false, null, 0, null, true);
|
||||
MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
|
||||
rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
|
||||
MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2);
|
||||
|
@ -1109,8 +1110,10 @@ public class TestDelegationTokenRenewer {
|
|||
Assert.assertTrue(renewer.getAllTokens().isEmpty());
|
||||
Assert.assertFalse(Renewer.cancelled);
|
||||
|
||||
Resource resource = Records.newRecord(Resource.class);
|
||||
resource.setMemory(200);
|
||||
RMApp app1 =
|
||||
rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
|
||||
rm.submitApp(resource, "name", "user", null, false, null, 2, credentials,
|
||||
null, true, false, false, null, 0, null, true);
|
||||
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
|
||||
rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
|
||||
|
@ -1118,9 +1121,8 @@ public class TestDelegationTokenRenewer {
|
|||
DelegationTokenToRenew dttr = renewer.getAllTokens().get(token1);
|
||||
Assert.assertNotNull(dttr);
|
||||
Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
|
||||
RMApp app2 =
|
||||
rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
|
||||
null, true, false, false, null, 0, null, true);
|
||||
RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2,
|
||||
credentials, null, true, false, false, null, 0, null, true);
|
||||
MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
|
||||
rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
|
||||
Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
|
||||
|
@ -1136,9 +1138,8 @@ public class TestDelegationTokenRenewer {
|
|||
Assert.assertFalse(dttr.isTimerCancelled());
|
||||
Assert.assertFalse(Renewer.cancelled);
|
||||
|
||||
RMApp app3 =
|
||||
rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
|
||||
null, true, false, false, null, 0, null, true);
|
||||
RMApp app3 = rm.submitApp(resource, "name", "user", null, false, null, 2,
|
||||
credentials, null, true, false, false, null, 0, null, true);
|
||||
MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm1);
|
||||
rm.waitForState(app3.getApplicationId(), RMAppState.RUNNING);
|
||||
Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
|
||||
|
|
Loading…
Reference in New Issue