From f6d6a2ab048cd696338e3be70a73963f8b165ab6 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Fri, 11 May 2018 14:07:32 -0500 Subject: [PATCH] YARN-8244. TestContainerSchedulerQueuing.testStartMultipleContainers failed. Contributed by Jim Brennan (cherry picked from commit dc912994a1bcb511dfda32a0649cef0c9bdc47d3) --- .../TestContainerManager.java | 20 ++-- .../TestContainerSchedulerQueuing.java | 100 ++++++------------ 2 files changed, 42 insertions(+), 78 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index 6d198a444b2..ee5259f8a81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -1486,8 +1486,6 @@ public class TestContainerManager extends BaseContainerManagerTest { containerManager.start(); List list = new ArrayList<>(); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); for (int i = 0; i < 10; i++) { ContainerId cId = createContainerId(i); long identifier = 0; @@ -1500,8 +1498,9 @@ public class TestContainerManager extends BaseContainerManagerTest { createContainerToken(cId, identifier, context.getNodeId(), user, context.getContainerTokenSecretManager()); StartContainerRequest request = - StartContainerRequest.newInstance(containerLaunchContext, - containerToken); + StartContainerRequest.newInstance( + recordFactory.newRecordInstance(ContainerLaunchContext.class), + containerToken); list.add(request); } StartContainersRequest requestList = @@ -1531,9 +1530,6 @@ public class TestContainerManager extends BaseContainerManagerTest { public void testMultipleContainersStopAndGetStatus() throws Exception { containerManager.start(); List startRequest = new ArrayList<>(); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - List containerIds = new ArrayList<>(); for (int i = 0; i < 10; i++) { ContainerId cId; @@ -1547,8 +1543,9 @@ public class TestContainerManager extends BaseContainerManagerTest { createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()); StartContainerRequest request = - StartContainerRequest.newInstance(containerLaunchContext, - containerToken); + StartContainerRequest.newInstance( + recordFactory.newRecordInstance(ContainerLaunchContext.class), + containerToken); startRequest.add(request); containerIds.add(cId); } @@ -1788,15 +1785,14 @@ public class TestContainerManager extends BaseContainerManagerTest { containerManager.start(); // Start 4 containers 0..4 with default resource (1024, 1) List list = new ArrayList<>(); - ContainerLaunchContext containerLaunchContext = recordFactory - .newRecordInstance(ContainerLaunchContext.class); for (int i = 0; i < 4; i++) { ContainerId cId = createContainerId(i); long identifier = DUMMY_RM_IDENTIFIER; Token containerToken = createContainerToken(cId, identifier, context.getNodeId(), user, context.getContainerTokenSecretManager()); StartContainerRequest request = StartContainerRequest.newInstance( - containerLaunchContext, containerToken); + recordFactory.newRecordInstance(ContainerLaunchContext.class), + containerToken); list.add(request); } StartContainersRequest requestList = StartContainersRequest diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java index 5c72e7e8b4f..d4b2eff3e62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java @@ -224,19 +224,16 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { public void testStartMultipleContainers() throws Exception { containerManager.start(); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - List list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), @@ -280,19 +277,16 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { public void testQueueMultipleContainers() throws Exception { containerManager.start(); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - List list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(3072, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(3072, 1), @@ -338,26 +332,23 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { public void testStartAndQueueMultipleContainers() throws Exception { containerManager.start(); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - List list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), @@ -410,12 +401,9 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { public void testStartOpportunistcsWhenOppQueueIsFull() throws Exception { containerManager.start(); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - List list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), @@ -427,7 +415,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { YarnConfiguration.DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH); for (int i = 1; i < maxOppQueueLength + 2; i++) { list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(i), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), @@ -495,26 +483,23 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { public void testKillOpportunisticForGuaranteedContainer() throws Exception { containerManager.start(); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - List list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), @@ -584,12 +569,10 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { Listener listener = new Listener(); ((NodeManager.DefaultContainerStateListener)containerManager.getContext(). getContainerStateTransitionListener()).addListener(listener); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); List list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), @@ -605,7 +588,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), @@ -713,42 +696,42 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(3), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(4), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(5), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(6), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), @@ -835,14 +818,14 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), @@ -882,26 +865,23 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { public void testKillMultipleOpportunisticContainers() throws Exception { containerManager.start(); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - List list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), @@ -914,7 +894,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(3), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1500, 1), @@ -962,14 +942,11 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { public void testKillOnlyRequiredOpportunisticContainers() throws Exception { containerManager.start(); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - List list = new ArrayList<>(); // Fill NM with Opportunistic containers for (int i = 0; i < 4; i++) { list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(i), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), @@ -985,7 +962,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { // Now ask for two Guaranteed containers for (int i = 4; i < 6; i++) { list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(i), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), @@ -1031,26 +1008,23 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { public void testStopQueuedContainer() throws Exception { containerManager.start(); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - List list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), @@ -1137,19 +1111,16 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { ((NodeManager.DefaultContainerStateListener)containerManager.getContext(). getContainerStateTransitionListener()).addListener(listener); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - List list = new ArrayList<>(); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC))); list.add(StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), @@ -1255,12 +1226,9 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest { containerManager.start(); // Construct the Container-id ContainerId cId = createContainerId(0); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - StartContainerRequest scRequest = StartContainerRequest.newInstance( - containerLaunchContext, + recordFactory.newRecordInstance(ContainerLaunchContext.class), createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null));