From f3f7a11336d225488576ecdf22f9a719a9745180 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Fri, 31 May 2013 04:17:33 +0000 Subject: [PATCH] MAPREDUCE-5286. Change MapReduce to use ContainerTokenIdentifier instead of the entire Container in the startContainer call - YARN-684. Contributed by Vinod Kumar Vavilapalli. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1488087 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++++ .../v2/app/launcher/ContainerLauncherImpl.java | 2 +- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 18 ++++++++++++++---- .../v2/app/launcher/TestContainerLauncher.java | 14 +++++++------- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 86ddbc7e7ba..e3e30079ce8 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -447,6 +447,10 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-5282. Updating MR App to use immutable ApplicationID after YARN-716. (Siddharth Seth via vinodkv) + MAPREDUCE-5286. Change MapReduce to use ContainerTokenIdentifier instead + of the entire Container in the startContainer call - YARN-684. + (Vinod Kumar Vavilapalli via sseth) + BREAKDOWN OF HADOOP-8562 SUBTASKS MAPREDUCE-4739. Some MapReduce tests fail to find winutils. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java index b48348610ef..3e64ad4bbf4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java @@ -154,7 +154,7 @@ public class ContainerLauncherImpl extends AbstractService implements StartContainerRequest startRequest = Records .newRecord(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainer(event.getAllocatedContainer()); + startRequest.setContainerToken(event.getContainerToken()); StartContainerResponse response = proxy.startContainer(startRequest); ByteBuffer portInfo = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index 2eb5344ee94..ee6e3b4957e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -90,14 +90,17 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.service.Service; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.BuilderUtils; /** @@ -512,12 +515,19 @@ public class MRApp extends MRAppMaster { @Override public void handle(ContainerAllocatorEvent event) { - ContainerId cId = recordFactory.newRecordInstance(ContainerId.class); - cId.setApplicationAttemptId(getContext().getApplicationAttemptId()); - cId.setId(containerCount++); + ContainerId cId = + ContainerId.newInstance(getContext().getApplicationAttemptId(), + containerCount++); NodeId nodeId = NodeId.newInstance(NM_HOST, NM_PORT); + Resource resource = Resource.newInstance(1234, 2); + ContainerTokenIdentifier containerTokenIdentifier = + new ContainerTokenIdentifier(cId, nodeId.toString(), "user", + resource, System.currentTimeMillis() + 10000, 42, 42); + ContainerToken containerToken = + BuilderUtils.newContainerToken(nodeId, "password".getBytes(), + containerTokenIdentifier); Container container = Container.newInstance(cId, nodeId, - NM_HOST + ":" + NM_HTTP_PORT, null, null, null); + NM_HOST + ":" + NM_HTTP_PORT, resource, null, containerToken); JobID id = TypeConverter.fromYarn(applicationId); JobId jobId = TypeConverter.toYarn(id); getContext().getEventHandler().handle(new JobHistoryEvent(jobId, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java index 02c8e6b861d..2291a75c5c3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.Test; @@ -376,13 +377,12 @@ public class TestContainerLauncher { public StartContainerResponse startContainer(StartContainerRequest request) throws IOException { + ContainerTokenIdentifier containerTokenIdentifier = + BuilderUtils.newContainerTokenIdentifier(request.getContainerToken()); + // Validate that the container is what RM is giving. - Assert.assertEquals(MRApp.NM_HOST, request.getContainer().getNodeId() - .getHost()); - Assert.assertEquals(MRApp.NM_PORT, request.getContainer().getNodeId() - .getPort()); - Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_HTTP_PORT, request - .getContainer().getNodeHttpAddress()); + Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT, + containerTokenIdentifier.getNmHostAddress()); StartContainerResponse response = recordFactory .newRecordInstance(StartContainerResponse.class); @@ -395,7 +395,7 @@ public class TestContainerLauncher { throw new UndeclaredThrowableException(e); } status.setState(ContainerState.RUNNING); - status.setContainerId(request.getContainer().getId()); + status.setContainerId(containerTokenIdentifier.getContainerID()); status.setExitStatus(0); return response; }