YARN-8244. TestContainerSchedulerQueuing.testStartMultipleContainers failed. Contributed by Jim Brennan

(cherry picked from commit dc912994a1)
2018-05-11 14:07:32 -05:00 · 2018-05-11 14:07:32 -05:00 · 5e3369ccd2
parent d88d9f2874
commit 5e3369ccd2
2 changed files with 42 additions and 78 deletions
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@ -1487,8 +1487,6 @@ public class TestContainerManager extends BaseContainerManagerTest {
    containerManager.start();

    List<StartContainerRequest> list = new ArrayList<>();
-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
    for (int i = 0; i < 10; i++) {
      ContainerId cId = createContainerId(i);
      long identifier = 0;
@ -1501,8 +1499,9 @@ public class TestContainerManager extends BaseContainerManagerTest {
          createContainerToken(cId, identifier, context.getNodeId(), user,
            context.getContainerTokenSecretManager());
      StartContainerRequest request =
-          StartContainerRequest.newInstance(containerLaunchContext,
-            containerToken);
+          StartContainerRequest.newInstance(
+              recordFactory.newRecordInstance(ContainerLaunchContext.class),
+              containerToken);
      list.add(request);
    }
    StartContainersRequest requestList =
@ -1532,9 +1531,6 @@ public class TestContainerManager extends BaseContainerManagerTest {
  public void testMultipleContainersStopAndGetStatus() throws Exception {
    containerManager.start();
    List<StartContainerRequest> startRequest = new ArrayList<>();
-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
    List<ContainerId> containerIds = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
      ContainerId cId;
@ -1548,8 +1544,9 @@ public class TestContainerManager extends BaseContainerManagerTest {
          createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
            user, context.getContainerTokenSecretManager());
      StartContainerRequest request =
-          StartContainerRequest.newInstance(containerLaunchContext,
-            containerToken);
+          StartContainerRequest.newInstance(
+              recordFactory.newRecordInstance(ContainerLaunchContext.class),
+              containerToken);
      startRequest.add(request);
      containerIds.add(cId);
    }
@ -1789,15 +1786,14 @@ public class TestContainerManager extends BaseContainerManagerTest {
    containerManager.start();
    // Start 4 containers 0..4 with default resource (1024, 1)
    List<StartContainerRequest> list = new ArrayList<>();
-    ContainerLaunchContext containerLaunchContext = recordFactory
-        .newRecordInstance(ContainerLaunchContext.class);
    for (int i = 0; i < 4; i++) {
      ContainerId cId = createContainerId(i);
      long identifier = DUMMY_RM_IDENTIFIER;
      Token containerToken = createContainerToken(cId, identifier,
          context.getNodeId(), user, context.getContainerTokenSecretManager());
      StartContainerRequest request = StartContainerRequest.newInstance(
-          containerLaunchContext, containerToken);
+          recordFactory.newRecordInstance(ContainerLaunchContext.class),
+          containerToken);
      list.add(request);
    }
    StartContainersRequest requestList = StartContainersRequest
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@ -221,19 +221,16 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
  public void testStartMultipleContainers() throws Exception {
    containerManager.start();

-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(1024, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.GUARANTEED)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(1024, 1),
@ -277,19 +274,16 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
  public void testQueueMultipleContainers() throws Exception {
    containerManager.start();

-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(3072, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.GUARANTEED)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(3072, 1),
@ -335,26 +329,23 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
  public void testStartAndQueueMultipleContainers() throws Exception {
    containerManager.start();

-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(2048, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(1024, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(1024, 1),
@ -407,12 +398,9 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
  public void testStartOpportunistcsWhenOppQueueIsFull() throws Exception {
    containerManager.start();

-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(2048, 1),
@ -424,7 +412,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
        YarnConfiguration.DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH);
    for (int i = 1; i < maxOppQueueLength + 2; i++) {
      list.add(StartContainerRequest.newInstance(
-          containerLaunchContext,
+          recordFactory.newRecordInstance(ContainerLaunchContext.class),
          createContainerToken(createContainerId(i), DUMMY_RM_IDENTIFIER,
              context.getNodeId(),
              user, BuilderUtils.newResource(2048, 1),
@ -492,26 +480,23 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
  public void testKillOpportunisticForGuaranteedContainer() throws Exception {
    containerManager.start();

-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(2048, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(2048, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(2048, 1),
@ -581,12 +566,10 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
    Listener listener = new Listener();
    ((NodeManager.DefaultContainerStateListener)containerManager.getContext().
        getContainerStateTransitionListener()).addListener(listener);
-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);

    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(2048, 1),
@ -602,7 +585,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {

    list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(2048, 1),
@ -710,42 +693,42 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {

    list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(3), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(4), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(5), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(6), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
@ -832,14 +815,14 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {

    list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
@ -879,26 +862,23 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
  public void testKillMultipleOpportunisticContainers() throws Exception {
    containerManager.start();

-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
@ -911,7 +891,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {

    list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(3), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(1500, 1),
@ -959,14 +939,11 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
  public void testKillOnlyRequiredOpportunisticContainers() throws Exception {
    containerManager.start();

-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
    List<StartContainerRequest> list = new ArrayList<>();
    // Fill NM with Opportunistic containers
    for (int i = 0; i < 4; i++) {
      list.add(StartContainerRequest.newInstance(
-          containerLaunchContext,
+          recordFactory.newRecordInstance(ContainerLaunchContext.class),
          createContainerToken(createContainerId(i), DUMMY_RM_IDENTIFIER,
              context.getNodeId(),
              user, BuilderUtils.newResource(512, 1),
@ -982,7 +959,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
    // Now ask for two Guaranteed containers
    for (int i = 4; i < 6; i++) {
      list.add(StartContainerRequest.newInstance(
-          containerLaunchContext,
+          recordFactory.newRecordInstance(ContainerLaunchContext.class),
          createContainerToken(createContainerId(i), DUMMY_RM_IDENTIFIER,
              context.getNodeId(),
              user, BuilderUtils.newResource(512, 1),
@ -1028,26 +1005,23 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
  public void testStopQueuedContainer() throws Exception {
    containerManager.start();

-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(2048, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.GUARANTEED)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(512, 1),
@ -1134,19 +1108,16 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
    ((NodeManager.DefaultContainerStateListener)containerManager.getContext().
        getContainerStateTransitionListener()).addListener(listener);

-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(2048, 1),
            context.getContainerTokenSecretManager(), null,
            ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(
-        containerLaunchContext,
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
        createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
            context.getNodeId(),
            user, BuilderUtils.newResource(1024, 1),
@ -1252,12 +1223,9 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
    containerManager.start();
    // Construct the Container-id
    ContainerId cId = createContainerId(0);
-    ContainerLaunchContext containerLaunchContext =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
    StartContainerRequest scRequest =
        StartContainerRequest.newInstance(
-            containerLaunchContext,
+            recordFactory.newRecordInstance(ContainerLaunchContext.class),
            createContainerToken(cId, DUMMY_RM_IDENTIFIER,
                context.getNodeId(), user, BuilderUtils.newResource(512, 1),
                context.getContainerTokenSecretManager(), null));