Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-571. Remove user from ContainerLaunchContext. Contributed by Omkar Vinit Joshi.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1486251 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-05-24 23:11:25 +00:00
parent 599b00e087
commit 6c5055fe92
53 changed files with 333 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
View File

@ -763,8 +763,7 @@ private static ContainerLaunchContext createCommonContainerLaunchContext(
// The null fields are per-container and will be constructed for each
// container separately.
ContainerLaunchContext container = BuilderUtils
.newContainerLaunchContext(conf
.get(MRJobConfig.USER_NAME), localResources,
.newContainerLaunchContext(localResources,
environment, null, serviceData, taskCredentialsBuffer,
applicationACLs);
@ -808,7 +807,6 @@ static ContainerLaunchContext createContainerLaunchContext(
// Construct the actual Container
ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext(
commonContainerSpec.getUser(),
commonContainerSpec.getLocalResources(), myEnv, commands,
myServiceData, commonContainerSpec.getTokens().duplicate(),
applicationACLs);

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
View File

@ -476,8 +476,7 @@ public ApplicationSubmissionContext createApplicationSubmissionContext(
// Setup ContainerLaunchContext for AM container
ContainerLaunchContext amContainer = BuilderUtils
.newContainerLaunchContext(UserGroupInformation
.getCurrentUser().getShortUserName(), localResources,
.newContainerLaunchContext(localResources,
environment, vargsFinal, null, securityTokens, acls);
// Set up the ApplicationSubmissionContext

3
hadoop-yarn-project/CHANGES.txt
View File

@ -49,6 +49,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-615. Rename ContainerLaunchContext.containerTokens to tokens.
(Vinod Kumar Vavilapalli via sseth)
YARN-571. Remove user from ContainerLaunchContext. (Omkar Vinit Joshi via
vinodkv)
NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues.

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
View File

@ -56,13 +56,12 @@ public abstract class ContainerLaunchContext {
@Public
@Stable
public static ContainerLaunchContext newInstance(
String user, Map<String, LocalResource> localResources,
Map<String, LocalResource> localResources,
Map<String, String> environment, List<String> commands,
Map<String, ByteBuffer> serviceData, ByteBuffer tokens,
Map<ApplicationAccessType, String> acls) {
ContainerLaunchContext container =
Records.newRecord(ContainerLaunchContext.class);
container.setUser(user);
container.setLocalResources(localResources);
container.setEnvironment(environment);
container.setCommands(commands);
@ -72,22 +71,6 @@ public static ContainerLaunchContext newInstance(
return container;
}
/**
* Get the <em>user</em> to whom the container has been allocated.
* @return the <em>user</em> to whom the container has been allocated
*/
@Public
@Stable
public abstract String getUser();
/**
* Set the <em>user</em> to whom the container has been allocated
* @param user <em>user</em> to whom the container has been allocated
*/
@Public
@Stable
public abstract void setUser(String user);
/**
* Get all the tokens needed by this container. It may include file-system
* tokens, ApplicationMaster related tokens if this container is an

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
View File

@ -169,25 +169,6 @@ private void addCommandsToProto() {
builder.addAllCommand(this.commands);
}
@Override
public String getUser() {
ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasUser()) {
return null;
}
return (p.getUser());
}
@Override
public void setUser(String user) {
maybeInitBuilder();
if (user == null) {
builder.clearUser();
return;
}
builder.setUser((user));
}
@Override
public Map<String, LocalResource> getLocalResources() {
initLocalResources();

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -267,13 +267,12 @@ message QueueUserACLInfoProto {
////////////////////////////////////////////////////////////////////////
message ContainerLaunchContextProto {
optional string user = 1;
repeated StringLocalResourceMapProto localResources = 2;
optional bytes tokens = 3;
repeated StringBytesMapProto service_data = 4;
repeated StringStringMapProto environment = 5;
repeated string command = 6;
repeated ApplicationACLMapProto application_ACLs = 7;
repeated StringLocalResourceMapProto localResources = 1;
optional bytes tokens = 2;
repeated StringBytesMapProto service_data = 3;
repeated StringStringMapProto environment = 4;
repeated string command = 5;
repeated ApplicationACLMapProto application_ACLs = 6;
}
message ContainerStatusProto {

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_server_resourcemanager_service_protos.proto
View File

@ -67,6 +67,7 @@ message GetGroupsForUserResponseProto {
message ApplicationStateDataProto {
optional int64 submit_time = 1;
optional ApplicationSubmissionContextProto application_submission_context = 2;
optional string user = 3;
}
message ApplicationAttemptStateDataProto {

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
View File

@ -698,11 +698,6 @@ public void run() {
ContainerLaunchContext ctx = Records
.newRecord(ContainerLaunchContext.class);
String jobUserName = System.getenv(ApplicationConstants.Environment.USER
.key());
ctx.setUser(jobUserName);
LOG.info("Setting user in ContainerLaunchContext to: " + jobUserName);
// Set the environment
ctx.setEnvironment(shellEnv);

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.util;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.nio.ByteBuffer;
@ -25,6 +26,7 @@
import java.util.List;
import java.util.Map;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@ -188,6 +190,16 @@ public static ContainerId newContainerId(int appId, int appAttemptId,
return cId;
}
public static ContainerToken newContainerToken(ContainerId cId, String host,
int port, String user, Resource r, long expiryTime, int masterKeyId,
byte[] password) throws IOException {
ContainerTokenIdentifier identifier =
new ContainerTokenIdentifier(cId, host, user, r, expiryTime,
masterKeyId);
return newContainerToken(BuilderUtils.newNodeId(host, port), password,
identifier);
}
public static ContainerId newContainerId(RecordFactory recordFactory,
ApplicationId appId, ApplicationAttemptId appAttemptId,
int containerId) {
@ -286,14 +298,24 @@ public static ContainerToken newContainerToken(NodeId nodeId,
return containerToken;
}
public static ContainerTokenIdentifier newContainerTokenIdentifier(
ContainerToken containerToken) throws IOException {
org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token =
new org.apache.hadoop.security.token.Token<ContainerTokenIdentifier>(
containerToken.getIdentifier()
.array(), containerToken.getPassword().array(), new Text(
containerToken.getKind()),
new Text(containerToken.getService()));
return token.decodeIdentifier();
}
public static ContainerLaunchContext newContainerLaunchContext(
String user, Map<String, LocalResource> localResources,
Map<String, LocalResource> localResources,
Map<String, String> environment, List<String> commands,
Map<String, ByteBuffer> serviceData, ByteBuffer tokens,
Map<String, ByteBuffer> serviceData, ByteBuffer tokens,
Map<ApplicationAccessType, String> acls) {
ContainerLaunchContext container = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
container.setUser(user);
container.setLocalResources(localResources);
container.setEnvironment(environment);
container.setCommands(commands);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
View File

@ -89,7 +89,6 @@ private void testRPCTimeout(String rpcClass) throws Exception {
server.getListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
containerLaunchContext.setUser("dummy-user");
ContainerId containerId = recordFactory
.newRecordInstance(ContainerId.class);
ApplicationId applicationId = recordFactory

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
View File

@ -111,7 +111,6 @@ private void test(String rpcClass) throws Exception {
NetUtils.getConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
containerLaunchContext.setUser("dummy-user");
ContainerId containerId =
recordFactory.newRecordInstance(ContainerId.class);
ApplicationId applicationId =

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
View File

@ -20,8 +20,6 @@
import static org.apache.hadoop.yarn.service.Service.STATE.STARTED;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
@ -105,9 +103,9 @@
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.ServiceStateChangeListener;
import org.apache.hadoop.yarn.util.BuilderUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.RpcUtil;
public class ContainerManagerImpl extends CompositeService implements
ServiceStateChangeListener, ContainerManager,
@ -321,13 +319,9 @@ protected ContainerTokenIdentifier getContainerTokenIdentifier(
// Get the tokenId from the remote user ugi
return selectContainerTokenIdentifier(remoteUgi);
} else {
ContainerToken containerToken = container.getContainerToken();
Token<ContainerTokenIdentifier> token =
new Token<ContainerTokenIdentifier>(containerToken.getIdentifier()
.array(), containerToken.getPassword().array(), new Text(
containerToken.getKind()), new Text(containerToken.getService()));
try {
return token.decodeIdentifier();
return BuilderUtils.newContainerTokenIdentifier(container
.getContainerToken());
} catch (IOException e) {
throw RPCUtil.getRemoteException(e);
}
@ -371,15 +365,6 @@ protected void authorizeRequest(String containerIDStr,
.append("\nNo ContainerToken found for " + containerIDStr);
} else {
// Is the container coming in with correct user-name?
if (!launchContext.getUser().equals(tokenId.getApplicationSubmitter())) {
unauthorized = true;
messageBuilder.append("\n Expected user-name "
+ tokenId.getApplicationSubmitter() + " but found "
+ launchContext.getUser());
}
// Is the container being relaunched? Or RPC layer let startCall with
// tokens generated off old-secret through?
if (!this.context.getContainerTokenSecretManager()
@ -451,7 +436,7 @@ public StartContainerResponse startContainer(StartContainerRequest request)
}
LOG.info("Start request for " + containerIDStr + " by user "
+ launchContext.getUser());
+ tokenId.getApplicationSubmitter());
// //////////// Parse credentials
ByteBuffer tokens = launchContext.getTokens();
@ -473,13 +458,14 @@ public StartContainerResponse startContainer(StartContainerRequest request)
}
}
// //////////// End of parsing credentials
String user = tokenId.getApplicationSubmitter();
Container container = new ContainerImpl(getConfig(), this.dispatcher,
launchContext, lauchContainer, credentials, metrics);
launchContext, lauchContainer, credentials, metrics, tokenId);
ApplicationId applicationID =
containerID.getApplicationAttemptId().getApplicationId();
if (context.getContainers().putIfAbsent(containerID, container) != null) {
NMAuditLogger.logFailure(launchContext.getUser(),
NMAuditLogger.logFailure(user,
AuditConstants.START_CONTAINER, "ContainerManagerImpl",
"Container already running on this node!",
applicationID, containerID);
@ -490,7 +476,8 @@ public StartContainerResponse startContainer(StartContainerRequest request)
// Create the application
Application application =
new ApplicationImpl(dispatcher, this.aclsManager,
launchContext.getUser(), applicationID, credentials, context);
user, applicationID, credentials,
context);
if (null ==
context.getApplications().putIfAbsent(applicationID, application)) {
LOG.info("Creating a new application reference for app "
@ -506,7 +493,7 @@ public StartContainerResponse startContainer(StartContainerRequest request)
this.context.getContainerTokenSecretManager().startContainerSuccessful(
tokenId);
NMAuditLogger.logSuccess(launchContext.getUser(),
NMAuditLogger.logSuccess(user,
AuditConstants.START_CONTAINER, "ContainerManageImpl",
applicationID, containerID);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
View File

@ -77,7 +77,7 @@ public ApplicationImpl(Dispatcher dispatcher,
ApplicationACLsManager aclsManager, String user, ApplicationId appId,
Credentials credentials, Context context) {
this.dispatcher = dispatcher;
this.user = user.toString();
this.user = user;
this.appId = appId;
this.credentials = credentials;
this.aclsManager = aclsManager;

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@ -44,6 +45,7 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
@ -75,6 +77,7 @@ public class ContainerImpl implements Container {
private final NodeManagerMetrics metrics;
private final ContainerLaunchContext launchContext;
private final org.apache.hadoop.yarn.api.records.Container container;
private final String user;
private int exitCode = ContainerExitStatus.INVALID;
private final StringBuilder diagnostics;
@ -93,10 +96,11 @@ public class ContainerImpl implements Container {
private final List<LocalResourceRequest> appRsrcs =
new ArrayList<LocalResourceRequest>();
public ContainerImpl(Configuration conf,
Dispatcher dispatcher, ContainerLaunchContext launchContext,
public ContainerImpl(Configuration conf, Dispatcher dispatcher,
ContainerLaunchContext launchContext,
org.apache.hadoop.yarn.api.records.Container container,
Credentials creds, NodeManagerMetrics metrics) {
Credentials creds, NodeManagerMetrics metrics,
ContainerTokenIdentifier identifier) throws IOException {
this.daemonConf = conf;
this.dispatcher = dispatcher;
this.launchContext = launchContext;
@ -104,7 +108,7 @@ public ContainerImpl(Configuration conf,
this.diagnostics = new StringBuilder();
this.credentials = creds;
this.metrics = metrics;
user = identifier.getApplicationSubmitter();
ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
this.readLock = readWriteLock.readLock();
this.writeLock = readWriteLock.writeLock();
@ -311,7 +315,7 @@ private org.apache.hadoop.yarn.api.records.ContainerState getCurrentState() {
public String getUser() {
this.readLock.lock();
try {
return this.launchContext.getUser();
return this.user;
} finally {
this.readLock.unlock();
}
@ -382,7 +386,6 @@ public org.apache.hadoop.yarn.api.records.Container getContainer() {
@SuppressWarnings({"fallthrough", "unchecked"})
private void finished() {
ContainerId containerID = this.container.getId();
String user = this.launchContext.getUser();
switch (getContainerState()) {
case EXITED_WITH_SUCCESS:
metrics.endRunningContainer();
@ -486,7 +489,7 @@ public ContainerState transition(ContainerImpl container,
for (Map.Entry<String,ByteBuffer> service : csd.entrySet()) {
container.dispatcher.getEventHandler().handle(
new AuxServicesEvent(AuxServicesEventType.APPLICATION_INIT,
ctxt.getUser(), container.container.getId()
container.user, container.container.getId()
.getApplicationAttemptId().getApplicationId(),
service.getKey().toString(), service.getValue()));
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
View File

@ -53,6 +53,7 @@
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.DelayedProcessKiller;
@ -67,6 +68,7 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
public class ContainerLaunch implements Callable<Integer> {
@ -120,11 +122,11 @@ public Integer call() {
container.getLocalizedResources();
ContainerId containerID = container.getContainer().getId();
String containerIdStr = ConverterUtils.toString(containerID);
final String user = launchContext.getUser();
final List<String> command = launchContext.getCommands();
int ret = -1;
try {
final String user = container.getUser();
// /////////////////////////// Variable expansion
// Before the container script gets written out.
List<String> newCmds = new ArrayList<String>(command.size());
@ -334,7 +336,7 @@ public void cleanupContainer() throws IOException {
// kill process
if (processId != null) {
String user = container.getLaunchContext().getUser();
String user = container.getUser();
LOG.debug("Sending signal to pid " + processId
+ " as user " + user
+ " for container " + containerIdStr);

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
View File

@ -34,6 +34,7 @@
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@ -47,6 +48,7 @@
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
@ -135,10 +137,16 @@ public long getRMIdentifier() {
cID.setApplicationAttemptId(applicationAttemptId);
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cID);
when(mockContainer.getResource()).thenReturn(recordFactory
.newRecordInstance(Resource.class));
Resource r = BuilderUtils.newResource(1024, 1);
when(mockContainer.getResource()).thenReturn(r);
when(mockContainer.getRMIdentifer()).thenReturn(SIMULATED_RM_IDENTIFIER);
launchContext.setUser("testing");
String user = "testing";
String host = "127.0.0.1";
int port = 1234;
ContainerToken containerToken =
BuilderUtils.newContainerToken(cID, host, port, user, r,
System.currentTimeMillis() + 10000L, 123, "password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest request =
recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
View File

@ -115,8 +115,6 @@ public void testClearLocalDirWhenNodeReboot() throws IOException,
Records.newRecord(org.apache.hadoop.yarn.api.records.Container.class);
mockContainer.setId(cId);
containerLaunchContext.setUser(user);
URL localResourceUri =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(localResourceDir.getAbsolutePath())));

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
View File

@ -21,12 +21,10 @@
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@ -40,8 +38,8 @@
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@ -66,9 +64,6 @@
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
@ -169,7 +164,6 @@ public static void startContainer(NodeManager nm, ContainerId cId,
NodeId nodeId = BuilderUtils.newNodeId("localhost", 1234);
mockContainer.setNodeId(nodeId);
mockContainer.setNodeHttpAddress("localhost:12345");
containerLaunchContext.setUser(cId.toString());
URL localResourceUri =
ConverterUtils.getYarnUrlFromPath(localFS
@ -186,7 +180,6 @@ public static void startContainer(NodeManager nm, ContainerId cId,
new HashMap<String, LocalResource>();
localResources.put(destinationFile, localResource);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Resource resource = BuilderUtils.newResource(1024, 1);

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
View File

@ -206,7 +206,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
when(mockContainer.getResource()).thenReturn(resource);
Container container =
new ContainerImpl(conf, mockDispatcher, launchContext,
mockContainer, null, mockMetrics);
mockContainer, null, mockMetrics, null);
this.context.getContainers().put(firstContainerID, container);
} else if (heartBeatID == 2) {
// Checks on the RM end
@ -232,7 +232,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
when(mockContainer.getResource()).thenReturn(resource);
Container container =
new ContainerImpl(conf, mockDispatcher, launchContext,
mockContainer, null, mockMetrics);
mockContainer, null, mockMetrics, null);
this.context.getContainers().put(secondContainerID, container);
} else if (heartBeatID == 3) {
// Checks on the RM end
@ -665,8 +665,8 @@ public void run() {
}
waitCount = 0;
while (heartBeatID <= 3 && waitCount++ != 20) {
Thread.sleep(500);
while (heartBeatID <= 3 && waitCount++ != 200) {
Thread.sleep(1000);
}
Assert.assertFalse(heartBeatID <= 3);
Assert.assertEquals("Number of registered NMs is wrong!!", 1,

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
View File

@ -182,16 +182,6 @@ protected void authorizeRequest(String containerIDStr,
throws YarnRemoteException {
// do nothing
}
@Override
protected ContainerTokenIdentifier getContainerTokenIdentifier(
UserGroupInformation remoteUgi,
org.apache.hadoop.yarn.api.records.Container container)
throws YarnRemoteException {
return new ContainerTokenIdentifier(container.getId(),
container.getNodeHttpAddress(), remoteUgi.getUserName(),
container.getResource(), System.currentTimeMillis(), 123);
}
};
}

83
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
View File

@ -26,7 +26,6 @@
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@ -49,6 +48,7 @@
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@ -58,7 +58,6 @@
import org.apache.hadoop.yarn.server.api.ResourceManagerConstants;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
@ -130,8 +129,6 @@ public void testContainerSetup() throws IOException, InterruptedException,
// ////// Construct the Container-id
ContainerId cId = createContainerId();
container.setUser(user);
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
@ -149,17 +146,21 @@ public void testContainerSetup() throws IOException, InterruptedException,
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(container.getUser());
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(512, 1));
Resource r = BuilderUtils.newResource(512, 1);
when(mockContainer.getResource()).thenReturn(r);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
@ -230,8 +231,6 @@ public void testContainerLaunchAndStop() throws IOException,
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
containerLaunchContext.setUser(user);
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@ -247,17 +246,22 @@ public void testContainerLaunchAndStop() throws IOException,
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(100, 1)); // MB
Resource r = BuilderUtils.newResource(100, 1);
when(mockContainer.getResource()).thenReturn(r); // MB
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
@ -343,8 +347,6 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException,
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
containerLaunchContext.setUser(user);
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@ -360,19 +362,22 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException,
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(100, 1)); // MB
Resource r = BuilderUtils.newResource(100, 1);
when(mockContainer.getResource()).thenReturn(r); // MB
int port = 12345;
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainer(mockContainer);
@ -439,8 +444,6 @@ public void testLocalFilesCleanup() throws InterruptedException,
ContainerId cId = createContainerId();
ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
container.setUser(user);
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
// containerLaunchContext.resources =
@ -459,18 +462,22 @@ public void testLocalFilesCleanup() throws InterruptedException,
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(container.getUser());
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(100, 1));
Resource r = BuilderUtils.newResource(100, 1);
when(mockContainer.getResource()).thenReturn(r);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
// containerLaunchContext.command = new ArrayList<CharSequence>();
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(containerLaunchContext);
request.setContainer(mockContainer);
@ -544,13 +551,13 @@ public void testContainerLaunchFromPreviousRM() throws IOException,
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
String host = "127.0.0.1";
int port = 1234;
ContainerId cId1 = createContainerId();
ContainerId cId2 = createContainerId();
containerLaunchContext.setUser(user);
containerLaunchContext
.setLocalResources(new HashMap<String, LocalResource>());
containerLaunchContext.setUser(containerLaunchContext.getUser());
Resource mockResource = mock(Resource.class);
Resource mockResource = BuilderUtils.newResource(1024, 1);
Container mockContainer1 = mock(Container.class);
when(mockContainer1.getId()).thenReturn(cId1);
@ -560,6 +567,11 @@ public void testContainerLaunchFromPreviousRM() throws IOException,
StartContainerRequest startRequest1 =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest1.setContainerLaunchContext(containerLaunchContext);
ContainerToken containerToken1 =
BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
System.currentTimeMillis() + 10000, 123, "password".getBytes());
when(mockContainer1.getContainerToken()).thenReturn(containerToken1);
startRequest1.setContainer(mockContainer1);
boolean catchException = false;
try {
@ -586,6 +598,11 @@ public void testContainerLaunchFromPreviousRM() throws IOException,
StartContainerRequest startRequest2 =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest2.setContainerLaunchContext(containerLaunchContext);
ContainerToken containerToken2 =
BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
System.currentTimeMillis() + 10000, 123, "password".getBytes());
when(mockContainer2.getContainerToken()).thenReturn(containerToken2);
startRequest2.setContainer(mockContainer2);
boolean noException = true;
try {

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
View File

@ -25,6 +25,7 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.AbstractMap.SimpleEntry;
@ -44,6 +45,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@ -53,6 +55,7 @@
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType;
@ -524,8 +527,10 @@ private static Map<String,ByteBuffer> createServiceData(Random r) {
}
private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx,
org.apache.hadoop.yarn.api.records.Container container) {
return new ContainerImpl(conf, disp, ctx, container, null, metrics);
org.apache.hadoop.yarn.api.records.Container container,
ContainerTokenIdentifier identifier) throws IOException {
return new ContainerImpl(conf, disp, ctx, container, null, metrics,
identifier);
}
@SuppressWarnings("unchecked")
@ -545,12 +550,13 @@ private class WrappedContainer {
final Map<String, ByteBuffer> serviceData;
final String user;
WrappedContainer(int appId, long timestamp, int id, String user) {
WrappedContainer(int appId, long timestamp, int id, String user)
throws IOException {
this(appId, timestamp, id, user, true, false);
}
WrappedContainer(int appId, long timestamp, int id, String user,
boolean withLocalRes, boolean withServiceData) {
boolean withLocalRes, boolean withServiceData) throws IOException {
dispatcher = new DrainDispatcher();
dispatcher.init(new Configuration());
@ -572,12 +578,19 @@ private class WrappedContainer {
org.apache.hadoop.yarn.api.records.Container mockContainer =
mock(org.apache.hadoop.yarn.api.records.Container.class);
cId = BuilderUtils.newContainerId(appId, 1, timestamp, id);
when(ctxt.getUser()).thenReturn(this.user);
when(mockContainer.getId()).thenReturn(cId);
Resource resource = BuilderUtils.newResource(1024, 1);
when(mockContainer.getResource()).thenReturn(resource);
String host = "127.0.0.1";
int port = 1234;
ContainerTokenIdentifier identifier =
new ContainerTokenIdentifier(cId, "127.0.0.1", user, resource,
System.currentTimeMillis() + 10000L, 123);
ContainerToken token =
BuilderUtils.newContainerToken(BuilderUtils.newNodeId(host, port),
"password".getBytes(), identifier);
when(mockContainer.getContainerToken()).thenReturn(token);
if (withLocalRes) {
Random r = new Random();
long seed = r.nextLong();
@ -600,7 +613,7 @@ private class WrappedContainer {
}
when(ctxt.getServiceData()).thenReturn(serviceData);
c = newContainer(dispatcher, ctxt, mockContainer);
c = newContainer(dispatcher, ctxt, mockContainer, identifier);
dispatcher.start();
}

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
View File

@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.BufferedReader;
import java.io.File;
@ -26,14 +28,14 @@
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import junit.framework.Assert;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@ -50,9 +52,11 @@
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
@ -67,9 +71,6 @@
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.junit.Before;
import org.junit.Test;
import static org.mockito.Mockito.*;
import junit.framework.Assert;
public class TestContainerLaunch extends BaseContainerManagerTest {
@ -157,7 +158,7 @@ public void testSpecialCharSymlinks() throws IOException {
* See if environment variable is forwarded using sanitizeEnv.
* @throws Exception
*/
@Test (timeout = 5000)
@Test (timeout = 60000)
public void testContainerEnvVariables() throws Exception {
containerManager.start();
@ -175,12 +176,13 @@ public void testContainerEnvVariables() throws Exception {
appAttemptId.setAttemptId(1);
ContainerId cId =
recordFactory.newRecordInstance(ContainerId.class);
int port = 12345;
cId.setApplicationAttemptId(appAttemptId);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
Map<String, String> userSetEnv = new HashMap<String, String>();
@ -189,7 +191,6 @@ public void testContainerEnvVariables() throws Exception {
userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT");
userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT");
userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR");
containerLaunchContext.setUser(user);
containerLaunchContext.setEnvironment(userSetEnv);
File scriptFile = new File(tmpDir, "scriptFile.sh");
@ -243,13 +244,17 @@ public void testContainerEnvVariables() throws Exception {
containerLaunchContext.setLocalResources(localResources);
// set up the rest of the container
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(1024, 1));
Resource r = BuilderUtils.newResource(1024, 1);
when(mockContainer.getResource()).thenReturn(r);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 1234,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
@ -376,12 +381,11 @@ public void testDelayedKill() throws Exception {
recordFactory.newRecordInstance(ContainerLaunchContext.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
containerLaunchContext.setUser(user);
// upload the script file so that the container can run it
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
@ -400,11 +404,15 @@ public void testDelayedKill() throws Exception {
containerLaunchContext.setLocalResources(localResources);
// set up the rest of the container
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(1024, 1));
Resource r = BuilderUtils.newResource(1024, 1);
when(mockContainer.getResource()).thenReturn(r);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainer(mockContainer);

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
View File

@ -63,6 +63,7 @@
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@ -75,6 +76,7 @@
import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
@ -694,8 +696,6 @@ public void testLogAggregationForRealContainerLaunch() throws IOException,
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
containerLaunchContext.setUser(this.user);
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@ -711,13 +711,15 @@ public void testLogAggregationForRealContainerLaunch() throws IOException,
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = new ArrayList<String>();
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(100 * 1024 * 1024, 1));
Resource r = BuilderUtils.newResource(100 * 1024 * 1024, 1);
when(mockContainer.getResource()).thenReturn(r);
when(mockContainer.getContainerToken()).thenReturn(
BuilderUtils.newContainerToken(cId, "127.0.0.1", 1234, user, r,
System.currentTimeMillis() + 10000L, 123, "password".getBytes()));
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
View File

@ -49,9 +49,11 @@
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@ -215,11 +217,11 @@ public void testContainerKillOnMemoryOverflow() throws IOException,
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(
super.DUMMY_RM_IDENTIFIER);
containerLaunchContext.setUser(user);
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
@ -236,16 +238,20 @@ public void testContainerKillOnMemoryOverflow() throws IOException,
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = new ArrayList<String>();
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(8 * 1024 * 1024, 1));
Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1);
when(mockContainer.getResource()).thenReturn(r);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
View File

@ -18,6 +18,10 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -30,7 +34,7 @@
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@ -38,7 +42,6 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.util.BuilderUtils;
import static org.mockito.Mockito.*;
public class MockContainer implements Container {
@ -53,7 +56,7 @@ public class MockContainer implements Container {
public MockContainer(ApplicationAttemptId appAttemptId,
Dispatcher dispatcher, Configuration conf, String user,
ApplicationId appId, int uniqId) {
ApplicationId appId, int uniqId) throws IOException{
this.user = user;
this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
@ -61,10 +64,14 @@ public MockContainer(ApplicationAttemptId appAttemptId,
uniqId);
this.launchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
launchContext.setUser(user);
ContainerToken containerToken =
BuilderUtils.newContainerToken(id, "127.0.0.1", 1234, user,
BuilderUtils.newResource(1024, 1),
System.currentTimeMillis() + 10000, 123, "password".getBytes());
this.state = ContainerState.NEW;
mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class);
when(mockContainer.getContainerToken()).thenReturn(containerToken);
when(mockContainer.getId()).thenReturn(id);
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
View File

@ -33,6 +33,8 @@
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@ -181,11 +183,16 @@ public boolean isPmemCheckEnabled() {
recordFactory.newRecordInstance(ContainerLaunchContext.class);
org.apache.hadoop.yarn.api.records.Container mockContainer =
mock(org.apache.hadoop.yarn.api.records.Container.class);
ContainerToken containerToken =
BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user,
BuilderUtils.newResource(1024, 1),
System.currentTimeMillis() + 10000L, 123, "password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
when(mockContainer.getId()).thenReturn(containerId);
launchContext.setUser(user);
Container container =
new ContainerImpl(conf, dispatcher, launchContext, mockContainer,
null, metrics) {
null, metrics,
BuilderUtils.newContainerTokenIdentifier(containerToken)) {
@Override
public ContainerState getContainerState() {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
View File

@ -23,6 +23,7 @@
import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.HashMap;
@ -177,7 +178,8 @@ public void testNodeAppsNone() throws JSONException, Exception {
assertEquals("apps isn't NULL", JSONObject.NULL, json.get("apps"));
}
private HashMap<String, String> addAppContainers(Application app) {
private HashMap<String, String> addAppContainers(Application app)
throws IOException {
Dispatcher dispatcher = new AsyncDispatcher();
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
app.getAppId(), 1);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
View File

@ -24,6 +24,7 @@
import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.HashMap;
@ -178,7 +179,8 @@ public void testNodeContainersNone() throws JSONException, Exception {
assertEquals("apps isn't NULL", JSONObject.NULL, json.get("containers"));
}
private HashMap<String, String> addAppContainers(Application app) {
private HashMap<String, String> addAppContainers(Application app)
throws IOException {
Dispatcher dispatcher = new AsyncDispatcher();
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
app.getAppId(), 1);

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -270,7 +270,6 @@ public SubmitApplicationResponse submitApplication(
try {
// Safety
user = UserGroupInformation.getCurrentUser().getShortUserName();
submissionContext.getAMContainerSpec().setUser(user);
} catch (IOException ie) {
LOG.warn("Unable to get the current user.", ie);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
@ -312,7 +311,7 @@ public SubmitApplicationResponse submitApplication(
try {
// call RMAppManager to submit application directly
rmAppManager.submitApplication(submissionContext,
System.currentTimeMillis(), false);
System.currentTimeMillis(), false, user);
LOG.info("Application with id " + applicationId.getId() +
" submitted by user " + user);

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -239,7 +239,7 @@ protected synchronized void checkAppNumCompletedLimit() {
@SuppressWarnings("unchecked")
protected void submitApplication(
ApplicationSubmissionContext submissionContext, long submitTime,
boolean isRecovered) throws YarnRemoteException {
boolean isRecovered, String user) throws YarnRemoteException {
ApplicationId applicationId = submissionContext.getApplicationId();
// Validation of the ApplicationSubmissionContext needs to be completed
@ -265,8 +265,7 @@ protected void submitApplication(
// Create RMApp
RMApp application =
new RMAppImpl(applicationId, rmContext, this.conf,
submissionContext.getApplicationName(),
submissionContext.getAMContainerSpec().getUser(),
submissionContext.getApplicationName(), user,
submissionContext.getQueue(),
submissionContext, this.scheduler, this.masterService,
submitTime, submissionContext.getApplicationType());
@ -370,7 +369,7 @@ public void recover(RMState state) throws Exception {
if(shouldRecover) {
LOG.info("Recovering application " + appState.getAppId());
submitApplication(appState.getApplicationSubmissionContext(),
appState.getSubmitTime(), true);
appState.getSubmitTime(), true, appState.getUser());
// re-populate attempt information in application
RMAppImpl appImpl = (RMAppImpl) rmContext.getRMApps().get(
appState.getAppId());

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
View File

@ -165,7 +165,6 @@ private ContainerLaunchContext createAMContainerLaunchContext(
new String[0])));
// Finalize the container
container.setUser(applicationMasterContext.getAMContainerSpec().getUser());
setupTokensAndEnv(container, containerID);
return container;

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
View File

@ -103,7 +103,8 @@ public synchronized RMState loadState() throws Exception {
ApplicationStateDataProto.parseFrom(childData));
ApplicationState appState = new ApplicationState(
appStateData.getSubmitTime(),
appStateData.getApplicationSubmissionContext());
appStateData.getApplicationSubmissionContext(),
appStateData.getUser());
// assert child node name is same as actual applicationId
assert appId.equals(appState.context.getApplicationId());
state.appState.put(appId, appState);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
View File

@ -65,8 +65,8 @@ public void storeApplicationState(String appId,
ApplicationStateDataPBImpl appStateData)
throws Exception {
ApplicationState appState = new ApplicationState(
appStateData.getSubmitTime(),
appStateData.getApplicationSubmissionContext());
appStateData.getSubmitTime(),
appStateData.getApplicationSubmissionContext(), appStateData.getUser());
if (state.appState.containsKey(appState.getAppId())) {
Exception e = new IOException("App: " + appId + " is already stored.");
LOG.info("Error storing info for app: " + appId, e);

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
View File

@ -91,12 +91,15 @@ public Credentials getAppAttemptTokens() {
public static class ApplicationState {
final ApplicationSubmissionContext context;
final long submitTime;
final String user;
Map<ApplicationAttemptId, ApplicationAttemptState> attempts =
new HashMap<ApplicationAttemptId, ApplicationAttemptState>();
ApplicationState(long submitTime, ApplicationSubmissionContext context) {
ApplicationState(long submitTime, ApplicationSubmissionContext context,
String user) {
this.submitTime = submitTime;
this.context = context;
this.user = user;
}
public ApplicationId getAppId() {
@ -114,6 +117,9 @@ public ApplicationSubmissionContext getApplicationSubmissionContext() {
public ApplicationAttemptState getAttempt(ApplicationAttemptId attemptId) {
return attempts.get(attemptId);
}
public String getUser() {
return user;
}
}
/**
@ -190,7 +196,7 @@ public synchronized void storeApplication(RMApp app) {
.getApplicationSubmissionContext();
assert context instanceof ApplicationSubmissionContextPBImpl;
ApplicationState appState = new ApplicationState(
app.getSubmitTime(), context);
app.getSubmitTime(), context, app.getUser());
dispatcher.getEventHandler().handle(new RMStateStoreAppEvent(appState));
}
@ -240,7 +246,8 @@ protected abstract void storeApplicationAttemptState(String attemptId,
*/
public synchronized void removeApplication(RMApp app) {
ApplicationState appState = new ApplicationState(
app.getSubmitTime(), app.getApplicationSubmissionContext());
app.getSubmitTime(), app.getApplicationSubmissionContext(),
app.getUser());
for(RMAppAttempt appAttempt : app.getAppAttempts().values()) {
Credentials credentials = getTokensFromAppAttempt(appAttempt);
ApplicationAttemptState attemptState =
@ -295,6 +302,7 @@ private synchronized void handleStoreEvent(RMStateStoreEvent event) {
appStateData.setSubmitTime(apptState.getSubmitTime());
appStateData.setApplicationSubmissionContext(
apptState.getApplicationSubmissionContext());
appStateData.setUser(apptState.getUser());
ApplicationId appId =
apptState.getApplicationSubmissionContext().getApplicationId();

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
View File

@ -43,6 +43,17 @@ public interface ApplicationStateData {
@Unstable
public void setSubmitTime(long submitTime);
/**
* The application submitter
*/
@Public
@Unstable
public void setUser(String user);
@Public
@Unstable
public String getUser();
/**
* The {@link ApplicationSubmissionContext} for the application
* {@link ApplicationId} can be obtained from the this

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
View File

@ -24,6 +24,7 @@
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProtoOrBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.mortbay.log.Log;
public class ApplicationStateDataPBImpl
extends ProtoBase<ApplicationStateDataProto>
@ -91,6 +92,22 @@ public void setSubmitTime(long submitTime) {
builder.setSubmitTime(submitTime);
}
@Override
public String getUser() {
ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasUser()) {
return null;
}
return (p.getUser());
}
@Override
public void setUser(String user) {
maybeInitBuilder();
builder.setUser(user);
}
@Override
public ApplicationSubmissionContext getApplicationSubmissionContext() {
ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
View File

@ -586,7 +586,7 @@ private void createNewAttempt(boolean startAttempt) {
RMAppAttempt attempt =
new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService,
submissionContext, conf);
submissionContext, conf, user);
attempts.put(appAttemptId, attempt);
currentAttempt = attempt;
if(startAttempt) {

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -153,6 +153,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
private final StringBuilder diagnostics = new StringBuilder();
private Configuration conf;
private String user;
private static final ExpiredTransition EXPIRED_TRANSITION =
new ExpiredTransition();
@ -364,7 +365,7 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
ApplicationSubmissionContext submissionContext,
Configuration conf) {
Configuration conf, String user) {
this.conf = conf;
this.applicationAttemptId = appAttemptId;
this.rmContext = rmContext;
@ -380,6 +381,7 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
this.proxiedTrackingUrl = generateProxyUriWithoutScheme();
this.stateMachine = stateMachineFactory.make(this);
this.user = user;
}
@Override
@ -746,8 +748,7 @@ public void transition(RMAppAttemptImpl appAttempt,
// Add the application to the scheduler
appAttempt.eventHandler.handle(
new AppAddedSchedulerEvent(appAttempt.applicationAttemptId,
appAttempt.submissionContext.getQueue(),
appAttempt.submissionContext.getAMContainerSpec().getUser()));
appAttempt.submissionContext.getQueue(), appAttempt.user));
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
View File

@ -135,7 +135,7 @@ protected void render(Block html) {
boolean odd = false;
for (RMAppAttempt attempt : attempts) {
AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt);
AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser());
table.tr((odd = !odd) ? _ODD : _EVEN).
td(String.valueOf(attemptInfo.getAttemptId())).
td(Times.format(attemptInfo.getStartTime())).

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -409,7 +409,7 @@ public AppAttemptsInfo getAppAttempts(@PathParam("appid") String appId) {
AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo();
for (RMAppAttempt attempt : app.getAppAttempts().values()) {
AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt);
AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser());
appAttemptsInfo.add(attemptInfo);
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
View File

@ -42,7 +42,7 @@ public class AppAttemptInfo {
public AppAttemptInfo() {
}
public AppAttemptInfo(RMAppAttempt attempt) {
public AppAttemptInfo(RMAppAttempt attempt, String user) {
this.startTime = 0;
this.containerId = "";
this.nodeHttpAddress = "";
@ -59,8 +59,7 @@ public AppAttemptInfo(RMAppAttempt attempt) {
this.logsLink = join(HttpConfig.getSchemePrefix(),
masterContainer.getNodeHttpAddress(),
"/node", "/containerlogs/",
ConverterUtils.toString(masterContainer.getId()), "/",
attempt.getSubmissionContext().getAMContainerSpec().getUser());
ConverterUtils.toString(masterContainer.getId()), "/", user);
}
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
View File

@ -131,7 +131,6 @@ public Resource getUsedResources() {
public synchronized void submit() throws IOException, YarnRemoteException {
ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
context.setApplicationId(this.applicationId);
context.getAMContainerSpec().setUser(this.user);
context.setQueue(this.queue);
SubmitApplicationRequest request = recordFactory
.newRecordInstance(SubmitApplicationRequest.class);
@ -401,7 +400,6 @@ private void updateResourceRequest(ResourceRequest request) {
private ContainerLaunchContext createCLC() {
ContainerLaunchContext clc = recordFactory.newRecordInstance(ContainerLaunchContext.class);
clc.setUser(this.user);
return clc;
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
View File

@ -188,7 +188,6 @@ public RMApp submitApp(int masterMemory, String name, String user,
capability.setMemory(masterMemory);
sub.setResource(capability);
clc.setApplicationACLs(acls);
clc.setUser(user);
if (ts != null && UserGroupInformation.isSecurityEnabled()) {
DataOutputBuffer dob = new DataOutputBuffer();
ts.writeTokenStorageToStream(dob);

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -169,10 +169,10 @@ public void setCompletedAppsMax(int max) {
super.setCompletedAppsMax(max);
}
public void submitApplication(
ApplicationSubmissionContext submissionContext)
ApplicationSubmissionContext submissionContext, String user)
throws YarnRemoteException {
super.submitApplication(submissionContext, System.currentTimeMillis(),
false);
false, user);
}
}
@ -375,7 +375,7 @@ protected void setupDispatcher(RMContext rmContext, Configuration conf) {
@Test
public void testRMAppSubmit() throws Exception {
appMonitor.submitApplication(asContext);
appMonitor.submitApplication(asContext, "test");
RMApp app = rmContext.getRMApps().get(appId);
Assert.assertNotNull("app is null", app);
Assert.assertEquals("app id doesn't match", appId, app.getApplicationId());
@ -416,7 +416,7 @@ public void testRMAppSubmitMaxAppAttempts() throws Exception {
if (individualMaxAppAttempts[i][j] != 0) {
asContext.setMaxAppAttempts(individualMaxAppAttempts[i][j]);
}
appMonitor.submitApplication(asContext);
appMonitor.submitApplication(asContext, "test");
RMApp app = rmContext.getRMApps().get(appID);
Assert.assertEquals("max application attempts doesn't match",
expectedNums[i][j], app.getMaxAppAttempts());
@ -441,7 +441,7 @@ public void testRMAppSubmitDuplicateApplicationId() throws Exception {
// our testApp1 should be rejected and original app with same id should be left in place
try {
appMonitor.submitApplication(asContext);
appMonitor.submitApplication(asContext, "test");
Assert.fail("Exception is expected when applicationId is duplicate.");
} catch (YarnRemoteException e) {
Assert.assertTrue("The thrown exception is not the expectd one.",
@ -462,7 +462,7 @@ public void testRMAppSubmitInvalidResourceRequest() throws Exception {
// submit an app
try {
appMonitor.submitApplication(asContext);
appMonitor.submitApplication(asContext, "test");
Assert.fail("Application submission should fail because resource" +
" request is invalid.");
} catch (YarnRemoteException e) {

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -381,8 +381,6 @@ public void run() {
private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId,
String name, String queue) {
String user = MockApps.newUserName();
ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);
Resource resource = Resources.createResource(
@ -391,7 +389,6 @@ private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId,
ApplicationSubmissionContext submissionContext =
recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
submissionContext.setAMContainerSpec(amContainerSpec);
submissionContext.getAMContainerSpec().setUser(user);
submissionContext.setApplicationName(name);
submissionContext.setQueue(queue);
submissionContext.setApplicationId(appId);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
View File

@ -194,6 +194,7 @@ void storeApp(RMStateStore store, ApplicationId appId, long time)
when(mockApp.getApplicationId()).thenReturn(appId);
when(mockApp.getSubmitTime()).thenReturn(time);
when(mockApp.getApplicationSubmissionContext()).thenReturn(context);
when(mockApp.getUser()).thenReturn("test");
store.storeApplication(mockApp);
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
View File

@ -203,7 +203,7 @@ null, new ApplicationTokenSecretManager(conf),
when(submissionContext.getQueue()).thenReturn(queue);
Resource resource = BuilderUtils.newResource(1536, 1);
ContainerLaunchContext amContainerSpec =
BuilderUtils.newContainerLaunchContext(user, null, null,
BuilderUtils.newContainerLaunchContext(null, null,
null, null, null, null);
when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
when(submissionContext.getResource()).thenReturn(resource);
@ -213,7 +213,7 @@ null, new ApplicationTokenSecretManager(conf),
application = mock(RMApp.class);
applicationAttempt =
new RMAppAttemptImpl(applicationAttemptId, rmContext, scheduler,
masterService, submissionContext, new Configuration());
masterService, submissionContext, new Configuration(), user);
when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
when(application.getApplicationId()).thenReturn(applicationId);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
View File

@ -1612,7 +1612,7 @@ public void testNotAllowSubmitApplication() throws Exception {
new ApplicationMasterService(resourceManager.getRMContext(), scheduler);
ApplicationSubmissionContext submissionContext = new ApplicationSubmissionContextPBImpl();
ContainerLaunchContext clc =
BuilderUtils.newContainerLaunchContext(user, null, null, null, null,
BuilderUtils.newContainerLaunchContext(null, null, null, null,
null, null);
submissionContext.setApplicationId(applicationId);
submissionContext.setAMContainerSpec(clc);

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -1009,7 +1009,7 @@ public void testAppAttemptsHelper(String path, RMApp app, String media)
// Verify these parallel arrays are the same
int i = 0;
for (RMAppAttempt attempt : attempts) {
verifyAppAttemptsInfo(jsonArray.getJSONObject(i), attempt);
verifyAppAttemptsInfo(jsonArray.getJSONObject(i), attempt, app.getUser());
++i;
}
}
@ -1017,8 +1017,9 @@ public void testAppAttemptsHelper(String path, RMApp app, String media)
@Test
public void testAppAttemptsXML() throws JSONException, Exception {
rm.start();
String user = "user1";
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
RMApp app1 = rm.submitApp(1024, "testwordcount", "user1");
RMApp app1 = rm.submitApp(1024, "testwordcount", user);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
@ -1037,11 +1038,12 @@ public void testAppAttemptsXML() throws JSONException, Exception {
assertEquals("incorrect number of elements", 1, nodes.getLength());
NodeList attempt = dom.getElementsByTagName("appAttempt");
assertEquals("incorrect number of elements", 1, attempt.getLength());
verifyAppAttemptsXML(attempt, app1.getCurrentAppAttempt());
verifyAppAttemptsXML(attempt, app1.getCurrentAppAttempt(), user);
rm.stop();
}
public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt)
public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt,
String user)
throws JSONException, Exception {
for (int i = 0; i < nodes.getLength(); i++) {
@ -1053,11 +1055,12 @@ public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt)
WebServicesTestUtils.getXmlString(element, "containerId"),
WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
WebServicesTestUtils.getXmlString(element, "nodeId"),
WebServicesTestUtils.getXmlString(element, "logsLink"));
WebServicesTestUtils.getXmlString(element, "logsLink"), user);
}
}
public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt)
public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt,
String user)
throws JSONException, Exception {
assertEquals("incorrect number of elements", 6, info.length());
@ -1065,12 +1068,12 @@ public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt)
verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"),
info.getLong("startTime"), info.getString("containerId"),
info.getString("nodeHttpAddress"), info.getString("nodeId"),
info.getString("logsLink"));
info.getString("logsLink"), user);
}
public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id,
long startTime, String containerId, String nodeHttpAddress, String nodeId,
String logsLink)
String logsLink, String user)
throws JSONException, Exception {
assertEquals("id doesn't match", appAttempt.getAppAttemptId()
@ -1087,7 +1090,7 @@ public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id,
logsLink.startsWith("http://"));
assertTrue(
"logsLink doesn't contain user info", logsLink.endsWith("/"
+ appAttempt.getSubmissionContext().getAMContainerSpec().getUser()));
+ user));
}
}

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/core-site.xml Normal file
View File

@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>hadoop.security.token.service.use_ip</name>
<value>false</value>
</property>
</configuration>

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
View File

@ -321,7 +321,8 @@ public ContainerManager run() {
callWithIllegalContainerID(client, tokenId, allocatedContainer);
callWithIllegalResource(client, tokenId, allocatedContainer);
callWithIllegalUserName(client, tokenId, allocatedContainer);
// UserName is no longer sent using containerLaunchContext.
// callWithIllegalUserName(client, tokenId, allocatedContainer);
return client;
}
@ -411,9 +412,9 @@ private AMRMProtocol submitAndRegisterApplication(
Arrays.asList("ping", "-n", "100", "127.0.0.1", ">nul") :
Arrays.asList("sleep", "100");
ContainerLaunchContext amContainer = BuilderUtils
.newContainerLaunchContext("testUser",
Collections.<String, LocalResource>emptyMap(),
ContainerLaunchContext amContainer =
BuilderUtils.newContainerLaunchContext(
Collections.<String, LocalResource> emptyMap(),
new HashMap<String, String>(), cmd,
new HashMap<String, ByteBuffer>(), null,
new HashMap<ApplicationAccessType, String>());
@ -422,7 +423,6 @@ private AMRMProtocol submitAndRegisterApplication(
.newRecordInstance(ApplicationSubmissionContext.class);
appSubmissionContext.setApplicationId(appID);
appSubmissionContext.setAMContainerSpec(amContainer);
appSubmissionContext.getAMContainerSpec().setUser("testUser");
appSubmissionContext.setResource(BuilderUtils.newResource(1024, 1));
SubmitApplicationRequest submitRequest = recordFactory
@ -590,7 +590,7 @@ void callWithIllegalUserName(ContainerManager client,
// Authenticated but unauthorized, due to wrong resource
ContainerLaunchContext context =
createContainerLaunchContextForTest(tokenId);
context.setUser("Saruman"); // Set a different user-name.
String user = "invalidUser";
request.setContainerLaunchContext(context);
request.setContainer(container);
try {
@ -603,7 +603,7 @@ void callWithIllegalUserName(ContainerManager client,
"Unauthorized request to start container. "));
Assert.assertTrue(e.getMessage().contains(
"Expected user-name " + tokenId.getApplicationSubmitter()
+ " but found " + context.getUser()));
+ " but found " + user));
} catch (IOException e) {
LOG.info("Got IOException: ",e);
fail("IOException is not expected.");
@ -614,7 +614,6 @@ private ContainerLaunchContext createContainerLaunchContextForTest(
ContainerTokenIdentifier tokenId) {
ContainerLaunchContext context =
BuilderUtils.newContainerLaunchContext(
tokenId.getApplicationSubmitter(),
new HashMap<String, LocalResource>(),
new HashMap<String, String>(), new ArrayList<String>(),
new HashMap<String, ByteBuffer>(), null,