From 6c5055fe92d3000e2d3960545440f38b5a43a8f0 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 24 May 2013 23:11:25 +0000 Subject: [PATCH] YARN-571. Remove user from ContainerLaunchContext. Contributed by Omkar Vinit Joshi. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1486251 13f79535-47bb-0310-9956-ffa450edef68 --- .../v2/app/job/impl/TaskAttemptImpl.java | 4 +- .../org/apache/hadoop/mapred/YARNRunner.java | 3 +- hadoop-yarn-project/CHANGES.txt | 3 + .../api/records/ContainerLaunchContext.java | 19 +---- .../impl/pb/ContainerLaunchContextPBImpl.java | 19 ----- .../src/main/proto/yarn_protos.proto | 13 ++- ...erver_resourcemanager_service_protos.proto | 1 + .../distributedshell/ApplicationMaster.java | 5 -- .../apache/hadoop/yarn/util/BuilderUtils.java | 28 ++++++- .../hadoop/yarn/TestContainerLaunchRPC.java | 1 - .../java/org/apache/hadoop/yarn/TestRPC.java | 1 - .../ContainerManagerImpl.java | 33 +++----- .../application/ApplicationImpl.java | 2 +- .../container/ContainerImpl.java | 17 ++-- .../launcher/ContainerLaunch.java | 6 +- .../server/nodemanager/TestEventFlow.java | 14 +++- .../nodemanager/TestNodeManagerReboot.java | 2 - .../nodemanager/TestNodeManagerShutdown.java | 13 +-- .../nodemanager/TestNodeStatusUpdater.java | 8 +- .../BaseContainerManagerTest.java | 10 --- .../TestContainerManager.java | 83 +++++++++++-------- .../container/TestContainer.java | 27 ++++-- .../launcher/TestContainerLaunch.java | 44 ++++++---- .../TestLogAggregationService.java | 12 +-- .../monitor/TestContainersMonitor.java | 16 ++-- .../nodemanager/webapp/MockContainer.java | 15 +++- .../nodemanager/webapp/TestNMWebServer.java | 11 ++- .../webapp/TestNMWebServicesApps.java | 4 +- .../webapp/TestNMWebServicesContainers.java | 4 +- .../resourcemanager/ClientRMService.java | 3 +- .../server/resourcemanager/RMAppManager.java | 7 +- .../amlauncher/AMLauncher.java | 1 - .../recovery/FileSystemRMStateStore.java | 3 +- .../recovery/MemoryRMStateStore.java | 4 +- .../recovery/RMStateStore.java | 14 +++- .../records/ApplicationStateData.java | 11 +++ .../impl/pb/ApplicationStateDataPBImpl.java | 17 ++++ .../resourcemanager/rmapp/RMAppImpl.java | 2 +- .../rmapp/attempt/RMAppAttemptImpl.java | 9 +- .../resourcemanager/webapp/AppBlock.java | 2 +- .../resourcemanager/webapp/RMWebServices.java | 2 +- .../webapp/dao/AppAttemptInfo.java | 5 +- .../server/resourcemanager/Application.java | 2 - .../yarn/server/resourcemanager/MockRM.java | 1 - .../resourcemanager/TestAppManager.java | 12 +-- .../resourcemanager/TestClientRMService.java | 3 - .../resourcemanager/TestClientRMTokens.java | 2 +- .../recovery/TestRMStateStore.java | 1 + .../attempt/TestRMAppAttemptTransitions.java | 4 +- .../scheduler/fair/TestFairScheduler.java | 2 +- .../webapp/TestRMWebServicesApps.java | 21 +++-- .../src/test/resources/core-site.xml | 25 ++++++ .../server/TestContainerManagerSecurity.java | 15 ++-- 53 files changed, 333 insertions(+), 253 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/core-site.xml diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 480f8a32030..d374ef9c3a9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -763,8 +763,7 @@ private static ContainerLaunchContext createCommonContainerLaunchContext( // The null fields are per-container and will be constructed for each // container separately. ContainerLaunchContext container = BuilderUtils - .newContainerLaunchContext(conf - .get(MRJobConfig.USER_NAME), localResources, + .newContainerLaunchContext(localResources, environment, null, serviceData, taskCredentialsBuffer, applicationACLs); @@ -808,7 +807,6 @@ static ContainerLaunchContext createContainerLaunchContext( // Construct the actual Container ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext( - commonContainerSpec.getUser(), commonContainerSpec.getLocalResources(), myEnv, commands, myServiceData, commonContainerSpec.getTokens().duplicate(), applicationACLs); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index b1d4d10e718..1f5b0a06f1c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -476,8 +476,7 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( // Setup ContainerLaunchContext for AM container ContainerLaunchContext amContainer = BuilderUtils - .newContainerLaunchContext(UserGroupInformation - .getCurrentUser().getShortUserName(), localResources, + .newContainerLaunchContext(localResources, environment, vargsFinal, null, securityTokens, acls); // Set up the ApplicationSubmissionContext diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a16aabc8d72..c711bf34da5 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -49,6 +49,9 @@ Release 2.0.5-beta - UNRELEASED YARN-615. Rename ContainerLaunchContext.containerTokens to tokens. (Vinod Kumar Vavilapalli via sseth) + YARN-571. Remove user from ContainerLaunchContext. (Omkar Vinit Joshi via + vinodkv) + NEW FEATURES YARN-482. FS: Extend SchedulingMode to intermediate queues. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java index ae360913813..59e7614bd1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java @@ -56,13 +56,12 @@ public abstract class ContainerLaunchContext { @Public @Stable public static ContainerLaunchContext newInstance( - String user, Map localResources, + Map localResources, Map environment, List commands, Map serviceData, ByteBuffer tokens, Map acls) { ContainerLaunchContext container = Records.newRecord(ContainerLaunchContext.class); - container.setUser(user); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); @@ -72,22 +71,6 @@ public static ContainerLaunchContext newInstance( return container; } - /** - * Get the user to whom the container has been allocated. - * @return the user to whom the container has been allocated - */ - @Public - @Stable - public abstract String getUser(); - - /** - * Set the user to whom the container has been allocated - * @param user user to whom the container has been allocated - */ - @Public - @Stable - public abstract void setUser(String user); - /** * Get all the tokens needed by this container. It may include file-system * tokens, ApplicationMaster related tokens if this container is an diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java index bd74b923725..667e09bdd2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java @@ -169,25 +169,6 @@ private void addCommandsToProto() { builder.addAllCommand(this.commands); } - @Override - public String getUser() { - ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasUser()) { - return null; - } - return (p.getUser()); - } - - @Override - public void setUser(String user) { - maybeInitBuilder(); - if (user == null) { - builder.clearUser(); - return; - } - builder.setUser((user)); - } - @Override public Map getLocalResources() { initLocalResources(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 6f8682d260a..92cd9f980ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -267,13 +267,12 @@ message QueueUserACLInfoProto { //////////////////////////////////////////////////////////////////////// message ContainerLaunchContextProto { - optional string user = 1; - repeated StringLocalResourceMapProto localResources = 2; - optional bytes tokens = 3; - repeated StringBytesMapProto service_data = 4; - repeated StringStringMapProto environment = 5; - repeated string command = 6; - repeated ApplicationACLMapProto application_ACLs = 7; + repeated StringLocalResourceMapProto localResources = 1; + optional bytes tokens = 2; + repeated StringBytesMapProto service_data = 3; + repeated StringStringMapProto environment = 4; + repeated string command = 5; + repeated ApplicationACLMapProto application_ACLs = 6; } message ContainerStatusProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_server_resourcemanager_service_protos.proto index 142dc451bbb..4e4efcc586c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_server_resourcemanager_service_protos.proto @@ -67,6 +67,7 @@ message GetGroupsForUserResponseProto { message ApplicationStateDataProto { optional int64 submit_time = 1; optional ApplicationSubmissionContextProto application_submission_context = 2; + optional string user = 3; } message ApplicationAttemptStateDataProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index e92f45d9deb..9f1437e6061 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -698,11 +698,6 @@ public void run() { ContainerLaunchContext ctx = Records .newRecord(ContainerLaunchContext.class); - String jobUserName = System.getenv(ApplicationConstants.Environment.USER - .key()); - ctx.setUser(jobUserName); - LOG.info("Setting user in ContainerLaunchContext to: " + jobUserName); - // Set the environment ctx.setEnvironment(shellEnv); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java index 9ec00b8f3ee..c13d65d03dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.util; +import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; import java.nio.ByteBuffer; @@ -25,6 +26,7 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; @@ -188,6 +190,16 @@ public static ContainerId newContainerId(int appId, int appAttemptId, return cId; } + public static ContainerToken newContainerToken(ContainerId cId, String host, + int port, String user, Resource r, long expiryTime, int masterKeyId, + byte[] password) throws IOException { + ContainerTokenIdentifier identifier = + new ContainerTokenIdentifier(cId, host, user, r, expiryTime, + masterKeyId); + return newContainerToken(BuilderUtils.newNodeId(host, port), password, + identifier); + } + public static ContainerId newContainerId(RecordFactory recordFactory, ApplicationId appId, ApplicationAttemptId appAttemptId, int containerId) { @@ -286,14 +298,24 @@ public static ContainerToken newContainerToken(NodeId nodeId, return containerToken; } + public static ContainerTokenIdentifier newContainerTokenIdentifier( + ContainerToken containerToken) throws IOException { + org.apache.hadoop.security.token.Token token = + new org.apache.hadoop.security.token.Token( + containerToken.getIdentifier() + .array(), containerToken.getPassword().array(), new Text( + containerToken.getKind()), + new Text(containerToken.getService())); + return token.decodeIdentifier(); + } + public static ContainerLaunchContext newContainerLaunchContext( - String user, Map localResources, + Map localResources, Map environment, List commands, - Map serviceData, ByteBuffer tokens, + Map serviceData, ByteBuffer tokens, Map acls) { ContainerLaunchContext container = recordFactory .newRecordInstance(ContainerLaunchContext.class); - container.setUser(user); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java index cf0a441b17a..05b153dcc77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java @@ -89,7 +89,6 @@ private void testRPCTimeout(String rpcClass) throws Exception { server.getListenerAddress(), conf); ContainerLaunchContext containerLaunchContext = recordFactory .newRecordInstance(ContainerLaunchContext.class); - containerLaunchContext.setUser("dummy-user"); ContainerId containerId = recordFactory .newRecordInstance(ContainerId.class); ApplicationId applicationId = recordFactory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java index 5485927dcae..43f9dc33af9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java @@ -111,7 +111,6 @@ private void test(String rpcClass) throws Exception { NetUtils.getConnectAddress(server), conf); ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - containerLaunchContext.setUser("dummy-user"); ContainerId containerId = recordFactory.newRecordInstance(ContainerId.class); ApplicationId applicationId = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index ce2a9cefca2..67529c706b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -20,8 +20,6 @@ import static org.apache.hadoop.yarn.service.Service.STATE.STARTED; -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; @@ -105,9 +103,9 @@ import org.apache.hadoop.yarn.service.CompositeService; import org.apache.hadoop.yarn.service.Service; import org.apache.hadoop.yarn.service.ServiceStateChangeListener; +import org.apache.hadoop.yarn.util.BuilderUtils; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.RpcUtil; public class ContainerManagerImpl extends CompositeService implements ServiceStateChangeListener, ContainerManager, @@ -321,13 +319,9 @@ protected ContainerTokenIdentifier getContainerTokenIdentifier( // Get the tokenId from the remote user ugi return selectContainerTokenIdentifier(remoteUgi); } else { - ContainerToken containerToken = container.getContainerToken(); - Token token = - new Token(containerToken.getIdentifier() - .array(), containerToken.getPassword().array(), new Text( - containerToken.getKind()), new Text(containerToken.getService())); try { - return token.decodeIdentifier(); + return BuilderUtils.newContainerTokenIdentifier(container + .getContainerToken()); } catch (IOException e) { throw RPCUtil.getRemoteException(e); } @@ -370,15 +364,6 @@ protected void authorizeRequest(String containerIDStr, messageBuilder .append("\nNo ContainerToken found for " + containerIDStr); } else { - - // Is the container coming in with correct user-name? - if (!launchContext.getUser().equals(tokenId.getApplicationSubmitter())) { - unauthorized = true; - messageBuilder.append("\n Expected user-name " - + tokenId.getApplicationSubmitter() + " but found " - + launchContext.getUser()); - } - // Is the container being relaunched? Or RPC layer let startCall with // tokens generated off old-secret through? @@ -451,7 +436,7 @@ public StartContainerResponse startContainer(StartContainerRequest request) } LOG.info("Start request for " + containerIDStr + " by user " - + launchContext.getUser()); + + tokenId.getApplicationSubmitter()); // //////////// Parse credentials ByteBuffer tokens = launchContext.getTokens(); @@ -473,13 +458,14 @@ public StartContainerResponse startContainer(StartContainerRequest request) } } // //////////// End of parsing credentials + String user = tokenId.getApplicationSubmitter(); Container container = new ContainerImpl(getConfig(), this.dispatcher, - launchContext, lauchContainer, credentials, metrics); + launchContext, lauchContainer, credentials, metrics, tokenId); ApplicationId applicationID = containerID.getApplicationAttemptId().getApplicationId(); if (context.getContainers().putIfAbsent(containerID, container) != null) { - NMAuditLogger.logFailure(launchContext.getUser(), + NMAuditLogger.logFailure(user, AuditConstants.START_CONTAINER, "ContainerManagerImpl", "Container already running on this node!", applicationID, containerID); @@ -490,7 +476,8 @@ public StartContainerResponse startContainer(StartContainerRequest request) // Create the application Application application = new ApplicationImpl(dispatcher, this.aclsManager, - launchContext.getUser(), applicationID, credentials, context); + user, applicationID, credentials, + context); if (null == context.getApplications().putIfAbsent(applicationID, application)) { LOG.info("Creating a new application reference for app " @@ -506,7 +493,7 @@ public StartContainerResponse startContainer(StartContainerRequest request) this.context.getContainerTokenSecretManager().startContainerSuccessful( tokenId); - NMAuditLogger.logSuccess(launchContext.getUser(), + NMAuditLogger.logSuccess(user, AuditConstants.START_CONTAINER, "ContainerManageImpl", applicationID, containerID); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index e9dda5d9b7b..dc065a51654 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -77,7 +77,7 @@ public ApplicationImpl(Dispatcher dispatcher, ApplicationACLsManager aclsManager, String user, ApplicationId appId, Credentials credentials, Context context) { this.dispatcher = dispatcher; - this.user = user.toString(); + this.user = user; this.appId = appId; this.credentials = credentials; this.aclsManager = aclsManager; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 3344e8bbcec..8a95043febc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.container; +import java.io.IOException; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -44,6 +45,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger; import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent; @@ -75,6 +77,7 @@ public class ContainerImpl implements Container { private final NodeManagerMetrics metrics; private final ContainerLaunchContext launchContext; private final org.apache.hadoop.yarn.api.records.Container container; + private final String user; private int exitCode = ContainerExitStatus.INVALID; private final StringBuilder diagnostics; @@ -93,10 +96,11 @@ public class ContainerImpl implements Container { private final List appRsrcs = new ArrayList(); - public ContainerImpl(Configuration conf, - Dispatcher dispatcher, ContainerLaunchContext launchContext, + public ContainerImpl(Configuration conf, Dispatcher dispatcher, + ContainerLaunchContext launchContext, org.apache.hadoop.yarn.api.records.Container container, - Credentials creds, NodeManagerMetrics metrics) { + Credentials creds, NodeManagerMetrics metrics, + ContainerTokenIdentifier identifier) throws IOException { this.daemonConf = conf; this.dispatcher = dispatcher; this.launchContext = launchContext; @@ -104,7 +108,7 @@ public ContainerImpl(Configuration conf, this.diagnostics = new StringBuilder(); this.credentials = creds; this.metrics = metrics; - + user = identifier.getApplicationSubmitter(); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); @@ -311,7 +315,7 @@ private org.apache.hadoop.yarn.api.records.ContainerState getCurrentState() { public String getUser() { this.readLock.lock(); try { - return this.launchContext.getUser(); + return this.user; } finally { this.readLock.unlock(); } @@ -382,7 +386,6 @@ public org.apache.hadoop.yarn.api.records.Container getContainer() { @SuppressWarnings({"fallthrough", "unchecked"}) private void finished() { ContainerId containerID = this.container.getId(); - String user = this.launchContext.getUser(); switch (getContainerState()) { case EXITED_WITH_SUCCESS: metrics.endRunningContainer(); @@ -486,7 +489,7 @@ public ContainerState transition(ContainerImpl container, for (Map.Entry service : csd.entrySet()) { container.dispatcher.getEventHandler().handle( new AuxServicesEvent(AuxServicesEventType.APPLICATION_INIT, - ctxt.getUser(), container.container.getId() + container.user, container.container.getId() .getApplicationAttemptId().getApplicationId(), service.getKey().toString(), service.getValue())); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 59a4c43fefa..d5142a1ebfb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.DelayedProcessKiller; @@ -67,6 +68,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader; import org.apache.hadoop.yarn.util.Apps; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; public class ContainerLaunch implements Callable { @@ -120,11 +122,11 @@ public Integer call() { container.getLocalizedResources(); ContainerId containerID = container.getContainer().getId(); String containerIdStr = ConverterUtils.toString(containerID); - final String user = launchContext.getUser(); final List command = launchContext.getCommands(); int ret = -1; try { + final String user = container.getUser(); // /////////////////////////// Variable expansion // Before the container script gets written out. List newCmds = new ArrayList(command.size()); @@ -334,7 +336,7 @@ public void cleanupContainer() throws IOException { // kill process if (processId != null) { - String user = container.getLaunchContext().getUser(); + String user = container.getUser(); LOG.debug("Sending signal to pid " + processId + " as user " + user + " for container " + containerIdStr); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java index 5064f548248..1e61d4e65c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -47,6 +48,7 @@ import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.Test; @@ -135,10 +137,16 @@ public long getRMIdentifier() { cID.setApplicationAttemptId(applicationAttemptId); Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(cID); - when(mockContainer.getResource()).thenReturn(recordFactory - .newRecordInstance(Resource.class)); + Resource r = BuilderUtils.newResource(1024, 1); + when(mockContainer.getResource()).thenReturn(r); when(mockContainer.getRMIdentifer()).thenReturn(SIMULATED_RM_IDENTIFIER); - launchContext.setUser("testing"); + String user = "testing"; + String host = "127.0.0.1"; + int port = 1234; + ContainerToken containerToken = + BuilderUtils.newContainerToken(cID, host, port, user, r, + System.currentTimeMillis() + 10000L, 123, "password".getBytes()); + when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class); request.setContainerLaunchContext(launchContext); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java index 896132ac6db..302aa95a47d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java @@ -115,8 +115,6 @@ public void testClearLocalDirWhenNodeReboot() throws IOException, Records.newRecord(org.apache.hadoop.yarn.api.records.Container.class); mockContainer.setId(cId); - containerLaunchContext.setUser(user); - URL localResourceUri = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(localResourceDir.getAbsolutePath()))); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index c61c74efbbb..5c63343ec24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -21,12 +21,10 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; -import java.io.FileWriter; import java.io.IOException; +import java.io.PrintWriter; import java.net.InetSocketAddress; import java.security.PrivilegedAction; -import java.io.PrintWriter; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -40,8 +38,8 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.ContainerManager; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.ContainerManager; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -66,9 +64,6 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.api.records.MasterKey; -import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; -import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.After; @@ -169,8 +164,7 @@ public static void startContainer(NodeManager nm, ContainerId cId, NodeId nodeId = BuilderUtils.newNodeId("localhost", 1234); mockContainer.setNodeId(nodeId); mockContainer.setNodeHttpAddress("localhost:12345"); - containerLaunchContext.setUser(cId.toString()); - + URL localResourceUri = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(scriptFile.getAbsolutePath()))); @@ -186,7 +180,6 @@ public static void startContainer(NodeManager nm, ContainerId cId, new HashMap(); localResources.put(destinationFile, localResource); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); Resource resource = BuilderUtils.newResource(1024, 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index 74c938c4f66..6c2f8c9731c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -206,7 +206,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) when(mockContainer.getResource()).thenReturn(resource); Container container = new ContainerImpl(conf, mockDispatcher, launchContext, - mockContainer, null, mockMetrics); + mockContainer, null, mockMetrics, null); this.context.getContainers().put(firstContainerID, container); } else if (heartBeatID == 2) { // Checks on the RM end @@ -232,7 +232,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) when(mockContainer.getResource()).thenReturn(resource); Container container = new ContainerImpl(conf, mockDispatcher, launchContext, - mockContainer, null, mockMetrics); + mockContainer, null, mockMetrics, null); this.context.getContainers().put(secondContainerID, container); } else if (heartBeatID == 3) { // Checks on the RM end @@ -665,8 +665,8 @@ public void run() { } waitCount = 0; - while (heartBeatID <= 3 && waitCount++ != 20) { - Thread.sleep(500); + while (heartBeatID <= 3 && waitCount++ != 200) { + Thread.sleep(1000); } Assert.assertFalse(heartBeatID <= 3); Assert.assertEquals("Number of registered NMs is wrong!!", 1, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index b7a6d0b800f..2e607844676 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -182,16 +182,6 @@ protected void authorizeRequest(String containerIDStr, throws YarnRemoteException { // do nothing } - - @Override - protected ContainerTokenIdentifier getContainerTokenIdentifier( - UserGroupInformation remoteUgi, - org.apache.hadoop.yarn.api.records.Container container) - throws YarnRemoteException { - return new ContainerTokenIdentifier(container.getId(), - container.getNodeHttpAddress(), remoteUgi.getUserName(), - container.getResource(), System.currentTimeMillis(), 123); - } }; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index 29f8b1b8a3b..b48f851e61c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -26,7 +26,6 @@ import java.io.FileReader; import java.io.IOException; import java.io.PrintWriter; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -49,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.server.api.ResourceManagerConstants; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; -import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; @@ -130,8 +129,6 @@ public void testContainerSetup() throws IOException, InterruptedException, // ////// Construct the Container-id ContainerId cId = createContainerId(); - container.setUser(user); - // ////// Construct the container-spec. ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); @@ -149,17 +146,21 @@ public void testContainerSetup() throws IOException, InterruptedException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(container.getUser()); Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(cId); - when(mockContainer.getResource()).thenReturn( - BuilderUtils.newResource(512, 1)); + Resource r = BuilderUtils.newResource(512, 1); + when(mockContainer.getResource()).thenReturn(r); when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + int port = 12345; when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":12345"); + context.getNodeId().getHost() + ":" + port); when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); - + ContainerToken containerToken = + BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), + port, user, r, System.currentTimeMillis() + 10000L, 123, + "password".getBytes()); + when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); @@ -230,8 +231,6 @@ public void testContainerLaunchAndStop() throws IOException, ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - containerLaunchContext.setUser(user); - URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(scriptFile.getAbsolutePath()))); @@ -247,17 +246,22 @@ public void testContainerLaunchAndStop() throws IOException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(cId); - when(mockContainer.getResource()).thenReturn( - BuilderUtils.newResource(100, 1)); // MB + Resource r = BuilderUtils.newResource(100, 1); + when(mockContainer.getResource()).thenReturn(r); // MB when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + int port = 12345; when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":12345"); + context.getNodeId().getHost() + ":" + port); when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); + ContainerToken containerToken = + BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), + port, user, r, System.currentTimeMillis() + 10000L, 123, + "password".getBytes()); + when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); @@ -343,8 +347,6 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException, ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - containerLaunchContext.setUser(user); - URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(scriptFile.getAbsolutePath()))); @@ -360,19 +362,22 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(cId); - when(mockContainer.getResource()).thenReturn( - BuilderUtils.newResource(100, 1)); // MB - + Resource r = BuilderUtils.newResource(100, 1); + when(mockContainer.getResource()).thenReturn(r); // MB + int port = 12345; when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":12345"); + context.getNodeId().getHost() + ":" + port); when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); - + ContainerToken containerToken = + BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), + port, user, r, System.currentTimeMillis() + 10000L, 123, + "password".getBytes()); + when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); startRequest.setContainer(mockContainer); @@ -439,8 +444,6 @@ public void testLocalFilesCleanup() throws InterruptedException, ContainerId cId = createContainerId(); ApplicationId appId = cId.getApplicationAttemptId().getApplicationId(); - container.setUser(user); - // ////// Construct the container-spec. ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); // containerLaunchContext.resources = @@ -459,18 +462,22 @@ public void testLocalFilesCleanup() throws InterruptedException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(container.getUser()); Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(cId); - when(mockContainer.getResource()).thenReturn( - BuilderUtils.newResource(100, 1)); + Resource r = BuilderUtils.newResource(100, 1); + when(mockContainer.getResource()).thenReturn(r); when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + int port = 12345; when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":12345"); + context.getNodeId().getHost() + ":" + port); when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); // containerLaunchContext.command = new ArrayList(); - + ContainerToken containerToken = + BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), + port, user, r, System.currentTimeMillis() + 10000L, 123, + "password".getBytes()); + when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class); request.setContainerLaunchContext(containerLaunchContext); request.setContainer(mockContainer); @@ -544,13 +551,13 @@ public void testContainerLaunchFromPreviousRM() throws IOException, ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); + String host = "127.0.0.1"; + int port = 1234; ContainerId cId1 = createContainerId(); ContainerId cId2 = createContainerId(); - containerLaunchContext.setUser(user); containerLaunchContext .setLocalResources(new HashMap()); - containerLaunchContext.setUser(containerLaunchContext.getUser()); - Resource mockResource = mock(Resource.class); + Resource mockResource = BuilderUtils.newResource(1024, 1); Container mockContainer1 = mock(Container.class); when(mockContainer1.getId()).thenReturn(cId1); @@ -560,6 +567,11 @@ public void testContainerLaunchFromPreviousRM() throws IOException, StartContainerRequest startRequest1 = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest1.setContainerLaunchContext(containerLaunchContext); + + ContainerToken containerToken1 = + BuilderUtils.newContainerToken(cId1, host, port, user, mockResource, + System.currentTimeMillis() + 10000, 123, "password".getBytes()); + when(mockContainer1.getContainerToken()).thenReturn(containerToken1); startRequest1.setContainer(mockContainer1); boolean catchException = false; try { @@ -586,6 +598,11 @@ public void testContainerLaunchFromPreviousRM() throws IOException, StartContainerRequest startRequest2 = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest2.setContainerLaunchContext(containerLaunchContext); + ContainerToken containerToken2 = + BuilderUtils.newContainerToken(cId1, host, port, user, mockResource, + System.currentTimeMillis() + 10000, 123, "password".getBytes()); + when(mockContainer2.getContainerToken()).thenReturn(containerToken2); + startRequest2.setContainer(mockContainer2); boolean noException = true; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index 9c5918dc7c9..7d8a3b04c1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.AbstractMap.SimpleEntry; @@ -44,6 +45,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -53,6 +55,7 @@ import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType; @@ -524,8 +527,10 @@ private static Map createServiceData(Random r) { } private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx, - org.apache.hadoop.yarn.api.records.Container container) { - return new ContainerImpl(conf, disp, ctx, container, null, metrics); + org.apache.hadoop.yarn.api.records.Container container, + ContainerTokenIdentifier identifier) throws IOException { + return new ContainerImpl(conf, disp, ctx, container, null, metrics, + identifier); } @SuppressWarnings("unchecked") @@ -545,12 +550,13 @@ private class WrappedContainer { final Map serviceData; final String user; - WrappedContainer(int appId, long timestamp, int id, String user) { + WrappedContainer(int appId, long timestamp, int id, String user) + throws IOException { this(appId, timestamp, id, user, true, false); } WrappedContainer(int appId, long timestamp, int id, String user, - boolean withLocalRes, boolean withServiceData) { + boolean withLocalRes, boolean withServiceData) throws IOException { dispatcher = new DrainDispatcher(); dispatcher.init(new Configuration()); @@ -572,12 +578,19 @@ private class WrappedContainer { org.apache.hadoop.yarn.api.records.Container mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class); cId = BuilderUtils.newContainerId(appId, 1, timestamp, id); - when(ctxt.getUser()).thenReturn(this.user); when(mockContainer.getId()).thenReturn(cId); Resource resource = BuilderUtils.newResource(1024, 1); when(mockContainer.getResource()).thenReturn(resource); - + String host = "127.0.0.1"; + int port = 1234; + ContainerTokenIdentifier identifier = + new ContainerTokenIdentifier(cId, "127.0.0.1", user, resource, + System.currentTimeMillis() + 10000L, 123); + ContainerToken token = + BuilderUtils.newContainerToken(BuilderUtils.newNodeId(host, port), + "password".getBytes(), identifier); + when(mockContainer.getContainerToken()).thenReturn(token); if (withLocalRes) { Random r = new Random(); long seed = r.nextLong(); @@ -600,7 +613,7 @@ private class WrappedContainer { } when(ctxt.getServiceData()).thenReturn(serviceData); - c = newContainer(dispatcher, ctxt, mockContainer); + c = newContainer(dispatcher, ctxt, mockContainer, identifier); dispatcher.start(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 22248dd4feb..16be2ea17ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.BufferedReader; import java.io.File; @@ -26,14 +28,14 @@ import java.io.FileReader; import java.io.IOException; import java.io.PrintWriter; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import junit.framework.Assert; + import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; @@ -50,9 +52,11 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; @@ -67,9 +71,6 @@ import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import org.junit.Before; import org.junit.Test; -import static org.mockito.Mockito.*; - -import junit.framework.Assert; public class TestContainerLaunch extends BaseContainerManagerTest { @@ -157,7 +158,7 @@ public void testSpecialCharSymlinks() throws IOException { * See if environment variable is forwarded using sanitizeEnv. * @throws Exception */ - @Test (timeout = 5000) + @Test (timeout = 60000) public void testContainerEnvVariables() throws Exception { containerManager.start(); @@ -175,12 +176,13 @@ public void testContainerEnvVariables() throws Exception { appAttemptId.setAttemptId(1); ContainerId cId = recordFactory.newRecordInstance(ContainerId.class); + int port = 12345; cId.setApplicationAttemptId(appAttemptId); when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":12345"); + context.getNodeId().getHost() + ":" + port); when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); Map userSetEnv = new HashMap(); @@ -189,7 +191,6 @@ public void testContainerEnvVariables() throws Exception { userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT"); userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT"); userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR"); - containerLaunchContext.setUser(user); containerLaunchContext.setEnvironment(userSetEnv); File scriptFile = new File(tmpDir, "scriptFile.sh"); @@ -243,13 +244,17 @@ public void testContainerEnvVariables() throws Exception { containerLaunchContext.setLocalResources(localResources); // set up the rest of the container - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - when(mockContainer.getResource()).thenReturn( - BuilderUtils.newResource(1024, 1)); + Resource r = BuilderUtils.newResource(1024, 1); + when(mockContainer.getResource()).thenReturn(r); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); + ContainerToken containerToken = + BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), + port, user, r, System.currentTimeMillis() + 10000L, 1234, + "password".getBytes()); + when(mockContainer.getContainerToken()).thenReturn(containerToken); startRequest.setContainer(mockContainer); containerManager.startContainer(startRequest); @@ -376,12 +381,11 @@ public void testDelayedKill() throws Exception { recordFactory.newRecordInstance(ContainerLaunchContext.class); when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + int port = 12345; when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":12345"); + context.getNodeId().getHost() + ":" + port); when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); - containerLaunchContext.setUser(user); - // upload the script file so that the container can run it URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS @@ -400,11 +404,15 @@ public void testDelayedKill() throws Exception { containerLaunchContext.setLocalResources(localResources); // set up the rest of the container - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - when(mockContainer.getResource()).thenReturn( - BuilderUtils.newResource(1024, 1)); + Resource r = BuilderUtils.newResource(1024, 1); + when(mockContainer.getResource()).thenReturn(r); + ContainerToken containerToken = + BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), + port, user, r, System.currentTimeMillis() + 10000L, 123, + "password".getBytes()); + when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); startRequest.setContainer(mockContainer); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 3c97f4367f4..8228a88fbb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.DrainDispatcher; @@ -75,6 +76,7 @@ import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; @@ -694,8 +696,6 @@ public void testLogAggregationForRealContainerLaunch() throws IOException, when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); - containerLaunchContext.setUser(this.user); - URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(scriptFile.getAbsolutePath()))); @@ -711,13 +711,15 @@ public void testLogAggregationForRealContainerLaunch() throws IOException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = new ArrayList(); commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); - when(mockContainer.getResource()).thenReturn( - BuilderUtils.newResource(100 * 1024 * 1024, 1)); + Resource r = BuilderUtils.newResource(100 * 1024 * 1024, 1); + when(mockContainer.getResource()).thenReturn(r); + when(mockContainer.getContainerToken()).thenReturn( + BuilderUtils.newContainerToken(cId, "127.0.0.1", 1234, user, r, + System.currentTimeMillis() + 10000L, 123, "password".getBytes())); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index 75f7272c2b5..222d954b2e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -49,9 +49,11 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -215,11 +217,11 @@ public void testContainerKillOnMemoryOverflow() throws IOException, when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + int port = 12345; when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":12345"); + context.getNodeId().getHost() + ":" + port); when(mockContainer.getRMIdentifer()).thenReturn( super.DUMMY_RM_IDENTIFIER); - containerLaunchContext.setUser(user); URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS @@ -236,16 +238,20 @@ public void testContainerKillOnMemoryOverflow() throws IOException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = new ArrayList(); commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); - when(mockContainer.getResource()).thenReturn( - BuilderUtils.newResource(8 * 1024 * 1024, 1)); + Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1); + when(mockContainer.getResource()).thenReturn(r); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); + ContainerToken containerToken = + BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), + port, user, r, System.currentTimeMillis() + 10000L, 123, + "password".getBytes()); + when(mockContainer.getContainerToken()).thenReturn(containerToken); startRequest.setContainer(mockContainer); containerManager.startContainer(startRequest); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index f2331737411..6973227af39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -18,6 +18,10 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -38,7 +42,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.util.BuilderUtils; -import static org.mockito.Mockito.*; public class MockContainer implements Container { @@ -53,7 +56,7 @@ public class MockContainer implements Container { public MockContainer(ApplicationAttemptId appAttemptId, Dispatcher dispatcher, Configuration conf, String user, - ApplicationId appId, int uniqId) { + ApplicationId appId, int uniqId) throws IOException{ this.user = user; this.recordFactory = RecordFactoryProvider.getRecordFactory(conf); @@ -61,10 +64,14 @@ public MockContainer(ApplicationAttemptId appAttemptId, uniqId); this.launchContext = recordFactory .newRecordInstance(ContainerLaunchContext.class); - launchContext.setUser(user); + ContainerToken containerToken = + BuilderUtils.newContainerToken(id, "127.0.0.1", 1234, user, + BuilderUtils.newResource(1024, 1), + System.currentTimeMillis() + 10000, 123, "password".getBytes()); this.state = ContainerState.NEW; mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class); + when(mockContainer.getContainerToken()).thenReturn(containerToken); when(mockContainer.getId()).thenReturn(id); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java index 48abd9e7b19..75943edcc3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; @@ -181,11 +183,16 @@ public boolean isPmemCheckEnabled() { recordFactory.newRecordInstance(ContainerLaunchContext.class); org.apache.hadoop.yarn.api.records.Container mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class); + ContainerToken containerToken = + BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user, + BuilderUtils.newResource(1024, 1), + System.currentTimeMillis() + 10000L, 123, "password".getBytes()); + when(mockContainer.getContainerToken()).thenReturn(containerToken); when(mockContainer.getId()).thenReturn(containerId); - launchContext.setUser(user); Container container = new ContainerImpl(conf, dispatcher, launchContext, mockContainer, - null, metrics) { + null, metrics, + BuilderUtils.newContainerTokenIdentifier(containerToken)) { @Override public ContainerState getContainerState() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java index bde9feeee5c..5333a613e91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import java.io.File; +import java.io.IOException; import java.io.StringReader; import java.util.HashMap; @@ -177,7 +178,8 @@ public void testNodeAppsNone() throws JSONException, Exception { assertEquals("apps isn't NULL", JSONObject.NULL, json.get("apps")); } - private HashMap addAppContainers(Application app) { + private HashMap addAppContainers(Application app) + throws IOException { Dispatcher dispatcher = new AsyncDispatcher(); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( app.getAppId(), 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java index 0afa735196e..5112c6c96b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -24,6 +24,7 @@ import static org.junit.Assert.fail; import java.io.File; +import java.io.IOException; import java.io.StringReader; import java.util.HashMap; @@ -178,7 +179,8 @@ public void testNodeContainersNone() throws JSONException, Exception { assertEquals("apps isn't NULL", JSONObject.NULL, json.get("containers")); } - private HashMap addAppContainers(Application app) { + private HashMap addAppContainers(Application app) + throws IOException { Dispatcher dispatcher = new AsyncDispatcher(); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( app.getAppId(), 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 22df580012a..1166c5f9c58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -270,7 +270,6 @@ public SubmitApplicationResponse submitApplication( try { // Safety user = UserGroupInformation.getCurrentUser().getShortUserName(); - submissionContext.getAMContainerSpec().setUser(user); } catch (IOException ie) { LOG.warn("Unable to get the current user.", ie); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, @@ -312,7 +311,7 @@ public SubmitApplicationResponse submitApplication( try { // call RMAppManager to submit application directly rmAppManager.submitApplication(submissionContext, - System.currentTimeMillis(), false); + System.currentTimeMillis(), false, user); LOG.info("Application with id " + applicationId.getId() + " submitted by user " + user); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index a8ac80d519a..fdf45b0bcb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -239,7 +239,7 @@ protected synchronized void checkAppNumCompletedLimit() { @SuppressWarnings("unchecked") protected void submitApplication( ApplicationSubmissionContext submissionContext, long submitTime, - boolean isRecovered) throws YarnRemoteException { + boolean isRecovered, String user) throws YarnRemoteException { ApplicationId applicationId = submissionContext.getApplicationId(); // Validation of the ApplicationSubmissionContext needs to be completed @@ -265,8 +265,7 @@ protected void submitApplication( // Create RMApp RMApp application = new RMAppImpl(applicationId, rmContext, this.conf, - submissionContext.getApplicationName(), - submissionContext.getAMContainerSpec().getUser(), + submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, submitTime, submissionContext.getApplicationType()); @@ -370,7 +369,7 @@ public void recover(RMState state) throws Exception { if(shouldRecover) { LOG.info("Recovering application " + appState.getAppId()); submitApplication(appState.getApplicationSubmissionContext(), - appState.getSubmitTime(), true); + appState.getSubmitTime(), true, appState.getUser()); // re-populate attempt information in application RMAppImpl appImpl = (RMAppImpl) rmContext.getRMApps().get( appState.getAppId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index b5af79b162b..7c8f1d0e54a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -165,7 +165,6 @@ private ContainerLaunchContext createAMContainerLaunchContext( new String[0]))); // Finalize the container - container.setUser(applicationMasterContext.getAMContainerSpec().getUser()); setupTokensAndEnv(container, containerID); return container; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index 03154b61b80..db85c535b37 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -103,7 +103,8 @@ public synchronized RMState loadState() throws Exception { ApplicationStateDataProto.parseFrom(childData)); ApplicationState appState = new ApplicationState( appStateData.getSubmitTime(), - appStateData.getApplicationSubmissionContext()); + appStateData.getApplicationSubmissionContext(), + appStateData.getUser()); // assert child node name is same as actual applicationId assert appId.equals(appState.context.getApplicationId()); state.appState.put(appId, appState); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index dd6fab50731..f9b3c2f36d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -65,8 +65,8 @@ public void storeApplicationState(String appId, ApplicationStateDataPBImpl appStateData) throws Exception { ApplicationState appState = new ApplicationState( - appStateData.getSubmitTime(), - appStateData.getApplicationSubmissionContext()); + appStateData.getSubmitTime(), + appStateData.getApplicationSubmissionContext(), appStateData.getUser()); if (state.appState.containsKey(appState.getAppId())) { Exception e = new IOException("App: " + appId + " is already stored."); LOG.info("Error storing info for app: " + appId, e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index e1400f46a1f..cb4b5099b0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -91,12 +91,15 @@ public Credentials getAppAttemptTokens() { public static class ApplicationState { final ApplicationSubmissionContext context; final long submitTime; + final String user; Map attempts = new HashMap(); - ApplicationState(long submitTime, ApplicationSubmissionContext context) { + ApplicationState(long submitTime, ApplicationSubmissionContext context, + String user) { this.submitTime = submitTime; this.context = context; + this.user = user; } public ApplicationId getAppId() { @@ -114,6 +117,9 @@ public ApplicationSubmissionContext getApplicationSubmissionContext() { public ApplicationAttemptState getAttempt(ApplicationAttemptId attemptId) { return attempts.get(attemptId); } + public String getUser() { + return user; + } } /** @@ -190,7 +196,7 @@ public synchronized void storeApplication(RMApp app) { .getApplicationSubmissionContext(); assert context instanceof ApplicationSubmissionContextPBImpl; ApplicationState appState = new ApplicationState( - app.getSubmitTime(), context); + app.getSubmitTime(), context, app.getUser()); dispatcher.getEventHandler().handle(new RMStateStoreAppEvent(appState)); } @@ -240,7 +246,8 @@ protected abstract void storeApplicationAttemptState(String attemptId, */ public synchronized void removeApplication(RMApp app) { ApplicationState appState = new ApplicationState( - app.getSubmitTime(), app.getApplicationSubmissionContext()); + app.getSubmitTime(), app.getApplicationSubmissionContext(), + app.getUser()); for(RMAppAttempt appAttempt : app.getAppAttempts().values()) { Credentials credentials = getTokensFromAppAttempt(appAttempt); ApplicationAttemptState attemptState = @@ -295,6 +302,7 @@ private synchronized void handleStoreEvent(RMStateStoreEvent event) { appStateData.setSubmitTime(apptState.getSubmitTime()); appStateData.setApplicationSubmissionContext( apptState.getApplicationSubmissionContext()); + appStateData.setUser(apptState.getUser()); ApplicationId appId = apptState.getApplicationSubmissionContext().getApplicationId(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java index feffca9b2c4..35b12e56dc4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java @@ -43,6 +43,17 @@ public interface ApplicationStateData { @Unstable public void setSubmitTime(long submitTime); + /** + * The application submitter + */ + @Public + @Unstable + public void setUser(String user); + + @Public + @Unstable + public String getUser(); + /** * The {@link ApplicationSubmissionContext} for the application * {@link ApplicationId} can be obtained from the this diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java index c95d6e13344..75c50f7a047 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProtoOrBuilder; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; +import org.mortbay.log.Log; public class ApplicationStateDataPBImpl extends ProtoBase @@ -91,6 +92,22 @@ public void setSubmitTime(long submitTime) { builder.setSubmitTime(submitTime); } + @Override + public String getUser() { + ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasUser()) { + return null; + } + return (p.getUser()); + + } + + @Override + public void setUser(String user) { + maybeInitBuilder(); + builder.setUser(user); + } + @Override public ApplicationSubmissionContext getApplicationSubmissionContext() { ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 2e14fe87a6d..db68c29d121 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -586,7 +586,7 @@ private void createNewAttempt(boolean startAttempt) { RMAppAttempt attempt = new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService, - submissionContext, conf); + submissionContext, conf, user); attempts.put(appAttemptId, attempt); currentAttempt = attempt; if(startAttempt) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 49db01bb009..13bbe42957c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -153,7 +153,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { private final StringBuilder diagnostics = new StringBuilder(); private Configuration conf; - + private String user; + private static final ExpiredTransition EXPIRED_TRANSITION = new ExpiredTransition(); @@ -364,7 +365,7 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, RMContext rmContext, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationSubmissionContext submissionContext, - Configuration conf) { + Configuration conf, String user) { this.conf = conf; this.applicationAttemptId = appAttemptId; this.rmContext = rmContext; @@ -380,6 +381,7 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, this.proxiedTrackingUrl = generateProxyUriWithoutScheme(); this.stateMachine = stateMachineFactory.make(this); + this.user = user; } @Override @@ -746,8 +748,7 @@ public void transition(RMAppAttemptImpl appAttempt, // Add the application to the scheduler appAttempt.eventHandler.handle( new AppAddedSchedulerEvent(appAttempt.applicationAttemptId, - appAttempt.submissionContext.getQueue(), - appAttempt.submissionContext.getAMContainerSpec().getUser())); + appAttempt.submissionContext.getQueue(), appAttempt.user)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index e9dc06a3bc9..edc59707d16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -135,7 +135,7 @@ protected void render(Block html) { boolean odd = false; for (RMAppAttempt attempt : attempts) { - AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt); + AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser()); table.tr((odd = !odd) ? _ODD : _EVEN). td(String.valueOf(attemptInfo.getAttemptId())). td(Times.format(attemptInfo.getStartTime())). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 334d3a8d331..e2efdd7454e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -409,7 +409,7 @@ public AppAttemptsInfo getAppAttempts(@PathParam("appid") String appId) { AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo(); for (RMAppAttempt attempt : app.getAppAttempts().values()) { - AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt); + AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser()); appAttemptsInfo.add(attemptInfo); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java index 875212f5558..6c903387fe9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java @@ -42,7 +42,7 @@ public class AppAttemptInfo { public AppAttemptInfo() { } - public AppAttemptInfo(RMAppAttempt attempt) { + public AppAttemptInfo(RMAppAttempt attempt, String user) { this.startTime = 0; this.containerId = ""; this.nodeHttpAddress = ""; @@ -59,8 +59,7 @@ public AppAttemptInfo(RMAppAttempt attempt) { this.logsLink = join(HttpConfig.getSchemePrefix(), masterContainer.getNodeHttpAddress(), "/node", "/containerlogs/", - ConverterUtils.toString(masterContainer.getId()), "/", - attempt.getSubmissionContext().getAMContainerSpec().getUser()); + ConverterUtils.toString(masterContainer.getId()), "/", user); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index c9306e6673c..841bc28e44f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -131,7 +131,6 @@ public Resource getUsedResources() { public synchronized void submit() throws IOException, YarnRemoteException { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(this.applicationId); - context.getAMContainerSpec().setUser(this.user); context.setQueue(this.queue); SubmitApplicationRequest request = recordFactory .newRecordInstance(SubmitApplicationRequest.class); @@ -401,7 +400,6 @@ private void updateResourceRequest(ResourceRequest request) { private ContainerLaunchContext createCLC() { ContainerLaunchContext clc = recordFactory.newRecordInstance(ContainerLaunchContext.class); - clc.setUser(this.user); return clc; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 9328d2e94e1..977c7689044 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -188,7 +188,6 @@ public RMApp submitApp(int masterMemory, String name, String user, capability.setMemory(masterMemory); sub.setResource(capability); clc.setApplicationACLs(acls); - clc.setUser(user); if (ts != null && UserGroupInformation.isSecurityEnabled()) { DataOutputBuffer dob = new DataOutputBuffer(); ts.writeTokenStorageToStream(dob); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 6d446611a6a..afd951324c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -169,10 +169,10 @@ public void setCompletedAppsMax(int max) { super.setCompletedAppsMax(max); } public void submitApplication( - ApplicationSubmissionContext submissionContext) + ApplicationSubmissionContext submissionContext, String user) throws YarnRemoteException { super.submitApplication(submissionContext, System.currentTimeMillis(), - false); + false, user); } } @@ -375,7 +375,7 @@ protected void setupDispatcher(RMContext rmContext, Configuration conf) { @Test public void testRMAppSubmit() throws Exception { - appMonitor.submitApplication(asContext); + appMonitor.submitApplication(asContext, "test"); RMApp app = rmContext.getRMApps().get(appId); Assert.assertNotNull("app is null", app); Assert.assertEquals("app id doesn't match", appId, app.getApplicationId()); @@ -416,7 +416,7 @@ public void testRMAppSubmitMaxAppAttempts() throws Exception { if (individualMaxAppAttempts[i][j] != 0) { asContext.setMaxAppAttempts(individualMaxAppAttempts[i][j]); } - appMonitor.submitApplication(asContext); + appMonitor.submitApplication(asContext, "test"); RMApp app = rmContext.getRMApps().get(appID); Assert.assertEquals("max application attempts doesn't match", expectedNums[i][j], app.getMaxAppAttempts()); @@ -441,7 +441,7 @@ public void testRMAppSubmitDuplicateApplicationId() throws Exception { // our testApp1 should be rejected and original app with same id should be left in place try { - appMonitor.submitApplication(asContext); + appMonitor.submitApplication(asContext, "test"); Assert.fail("Exception is expected when applicationId is duplicate."); } catch (YarnRemoteException e) { Assert.assertTrue("The thrown exception is not the expectd one.", @@ -462,7 +462,7 @@ public void testRMAppSubmitInvalidResourceRequest() throws Exception { // submit an app try { - appMonitor.submitApplication(asContext); + appMonitor.submitApplication(asContext, "test"); Assert.fail("Application submission should fail because resource" + " request is invalid."); } catch (YarnRemoteException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 0141968782e..e0ee009ae64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -381,8 +381,6 @@ public void run() { private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue) { - String user = MockApps.newUserName(); - ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( @@ -391,7 +389,6 @@ private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); submissionContext.setAMContainerSpec(amContainerSpec); - submissionContext.getAMContainerSpec().setUser(user); submissionContext.setApplicationName(name); submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java index 88f68e4fba0..0d079f89f8d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java @@ -234,7 +234,7 @@ public void testDelegationToken() throws IOException, InterruptedException { public void testShortCircuitRenewCancel() throws IOException, InterruptedException { InetSocketAddress addr = - new InetSocketAddress(InetAddress.getLocalHost(), 123); + new InetSocketAddress(InetAddress.getLocalHost(), 123); checkShortCircuitRenewCancel(addr, addr, true); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java index a245e54692c..67109d65cdb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java @@ -194,6 +194,7 @@ void storeApp(RMStateStore store, ApplicationId appId, long time) when(mockApp.getApplicationId()).thenReturn(appId); when(mockApp.getSubmitTime()).thenReturn(time); when(mockApp.getApplicationSubmissionContext()).thenReturn(context); + when(mockApp.getUser()).thenReturn("test"); store.storeApplication(mockApp); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index f736edf2038..3f82db1a405 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -203,7 +203,7 @@ null, new ApplicationTokenSecretManager(conf), when(submissionContext.getQueue()).thenReturn(queue); Resource resource = BuilderUtils.newResource(1536, 1); ContainerLaunchContext amContainerSpec = - BuilderUtils.newContainerLaunchContext(user, null, null, + BuilderUtils.newContainerLaunchContext(null, null, null, null, null, null); when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec); when(submissionContext.getResource()).thenReturn(resource); @@ -213,7 +213,7 @@ null, new ApplicationTokenSecretManager(conf), application = mock(RMApp.class); applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, rmContext, scheduler, - masterService, submissionContext, new Configuration()); + masterService, submissionContext, new Configuration(), user); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index abf1d412844..9f21031c89b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -1612,7 +1612,7 @@ public void testNotAllowSubmitApplication() throws Exception { new ApplicationMasterService(resourceManager.getRMContext(), scheduler); ApplicationSubmissionContext submissionContext = new ApplicationSubmissionContextPBImpl(); ContainerLaunchContext clc = - BuilderUtils.newContainerLaunchContext(user, null, null, null, null, + BuilderUtils.newContainerLaunchContext(null, null, null, null, null, null); submissionContext.setApplicationId(applicationId); submissionContext.setAMContainerSpec(clc); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 74fed119042..deb53eb7a66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1009,7 +1009,7 @@ public void testAppAttemptsHelper(String path, RMApp app, String media) // Verify these parallel arrays are the same int i = 0; for (RMAppAttempt attempt : attempts) { - verifyAppAttemptsInfo(jsonArray.getJSONObject(i), attempt); + verifyAppAttemptsInfo(jsonArray.getJSONObject(i), attempt, app.getUser()); ++i; } } @@ -1017,8 +1017,9 @@ public void testAppAttemptsHelper(String path, RMApp app, String media) @Test public void testAppAttemptsXML() throws JSONException, Exception { rm.start(); + String user = "user1"; MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + RMApp app1 = rm.submitApp(1024, "testwordcount", user); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -1037,11 +1038,12 @@ public void testAppAttemptsXML() throws JSONException, Exception { assertEquals("incorrect number of elements", 1, nodes.getLength()); NodeList attempt = dom.getElementsByTagName("appAttempt"); assertEquals("incorrect number of elements", 1, attempt.getLength()); - verifyAppAttemptsXML(attempt, app1.getCurrentAppAttempt()); + verifyAppAttemptsXML(attempt, app1.getCurrentAppAttempt(), user); rm.stop(); } - public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt) + public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt, + String user) throws JSONException, Exception { for (int i = 0; i < nodes.getLength(); i++) { @@ -1053,11 +1055,12 @@ public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt) WebServicesTestUtils.getXmlString(element, "containerId"), WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"), WebServicesTestUtils.getXmlString(element, "nodeId"), - WebServicesTestUtils.getXmlString(element, "logsLink")); + WebServicesTestUtils.getXmlString(element, "logsLink"), user); } } - public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt) + public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt, + String user) throws JSONException, Exception { assertEquals("incorrect number of elements", 6, info.length()); @@ -1065,12 +1068,12 @@ public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt) verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"), info.getLong("startTime"), info.getString("containerId"), info.getString("nodeHttpAddress"), info.getString("nodeId"), - info.getString("logsLink")); + info.getString("logsLink"), user); } public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id, long startTime, String containerId, String nodeHttpAddress, String nodeId, - String logsLink) + String logsLink, String user) throws JSONException, Exception { assertEquals("id doesn't match", appAttempt.getAppAttemptId() @@ -1087,7 +1090,7 @@ public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id, logsLink.startsWith("http://")); assertTrue( "logsLink doesn't contain user info", logsLink.endsWith("/" - + appAttempt.getSubmissionContext().getAMContainerSpec().getUser())); + + user)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/core-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/core-site.xml new file mode 100644 index 00000000000..f0d3085ef85 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/core-site.xml @@ -0,0 +1,25 @@ + + + + + + + + + hadoop.security.token.service.use_ip + false + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index cb2a05cd59e..9ff5d36569f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -321,7 +321,8 @@ public ContainerManager run() { callWithIllegalContainerID(client, tokenId, allocatedContainer); callWithIllegalResource(client, tokenId, allocatedContainer); - callWithIllegalUserName(client, tokenId, allocatedContainer); + // UserName is no longer sent using containerLaunchContext. +// callWithIllegalUserName(client, tokenId, allocatedContainer); return client; } @@ -411,9 +412,9 @@ private AMRMProtocol submitAndRegisterApplication( Arrays.asList("ping", "-n", "100", "127.0.0.1", ">nul") : Arrays.asList("sleep", "100"); - ContainerLaunchContext amContainer = BuilderUtils - .newContainerLaunchContext("testUser", - Collections.emptyMap(), + ContainerLaunchContext amContainer = + BuilderUtils.newContainerLaunchContext( + Collections. emptyMap(), new HashMap(), cmd, new HashMap(), null, new HashMap()); @@ -422,7 +423,6 @@ private AMRMProtocol submitAndRegisterApplication( .newRecordInstance(ApplicationSubmissionContext.class); appSubmissionContext.setApplicationId(appID); appSubmissionContext.setAMContainerSpec(amContainer); - appSubmissionContext.getAMContainerSpec().setUser("testUser"); appSubmissionContext.setResource(BuilderUtils.newResource(1024, 1)); SubmitApplicationRequest submitRequest = recordFactory @@ -590,7 +590,7 @@ void callWithIllegalUserName(ContainerManager client, // Authenticated but unauthorized, due to wrong resource ContainerLaunchContext context = createContainerLaunchContextForTest(tokenId); - context.setUser("Saruman"); // Set a different user-name. + String user = "invalidUser"; request.setContainerLaunchContext(context); request.setContainer(container); try { @@ -603,7 +603,7 @@ void callWithIllegalUserName(ContainerManager client, "Unauthorized request to start container. ")); Assert.assertTrue(e.getMessage().contains( "Expected user-name " + tokenId.getApplicationSubmitter() - + " but found " + context.getUser())); + + " but found " + user)); } catch (IOException e) { LOG.info("Got IOException: ",e); fail("IOException is not expected."); @@ -614,7 +614,6 @@ private ContainerLaunchContext createContainerLaunchContextForTest( ContainerTokenIdentifier tokenId) { ContainerLaunchContext context = BuilderUtils.newContainerLaunchContext( - tokenId.getApplicationSubmitter(), new HashMap(), new HashMap(), new ArrayList(), new HashMap(), null,