Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-571. Remove user from ContainerLaunchContext. Contributed by Omkar Vinit Joshi. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1485928 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-05-24 03:22:54 +00:00
parent a791527dd1
commit 259edf8dca
53 changed files with 332 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
View File

@ -763,8 +763,7 @@ public abstract class TaskAttemptImpl implements
// The null fields are per-container and will be constructed for each
// container separately.
ContainerLaunchContext container = BuilderUtils
.newContainerLaunchContext(conf
.get(MRJobConfig.USER_NAME), localResources,
.newContainerLaunchContext(localResources,
environment, null, serviceData, taskCredentialsBuffer,
applicationACLs);
@ -808,7 +807,6 @@ public abstract class TaskAttemptImpl implements
// Construct the actual Container
ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext(
commonContainerSpec.getUser(),
commonContainerSpec.getLocalResources(), myEnv, commands,
myServiceData, commonContainerSpec.getTokens().duplicate(),
applicationACLs);

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
View File

@ -476,8 +476,7 @@ public class YARNRunner implements ClientProtocol {
// Setup ContainerLaunchContext for AM container
ContainerLaunchContext amContainer = BuilderUtils
.newContainerLaunchContext(UserGroupInformation
.getCurrentUser().getShortUserName(), localResources,
.newContainerLaunchContext(localResources,
environment, vargsFinal, null, securityTokens, acls);
// Set up the ApplicationSubmissionContext

3
hadoop-yarn-project/CHANGES.txt
View File

@ -127,6 +127,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-615. Rename ContainerLaunchContext.containerTokens to tokens.
(Vinod Kumar Vavilapalli via sseth)
YARN-571. Remove user from ContainerLaunchContext. (Omkar Vinit Joshi via
vinodkv)
NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues.

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
View File

@ -56,13 +56,12 @@ public abstract class ContainerLaunchContext {
@Public
@Stable
public static ContainerLaunchContext newInstance(
String user, Map<String, LocalResource> localResources,
Map<String, LocalResource> localResources,
Map<String, String> environment, List<String> commands,
Map<String, ByteBuffer> serviceData, ByteBuffer tokens,
Map<ApplicationAccessType, String> acls) {
ContainerLaunchContext container =
Records.newRecord(ContainerLaunchContext.class);
container.setUser(user);
container.setLocalResources(localResources);
container.setEnvironment(environment);
container.setCommands(commands);
@ -72,22 +71,6 @@ public abstract class ContainerLaunchContext {
return container;
}
/**
* Get the <em>user</em> to whom the container has been allocated.
* @return the <em>user</em> to whom the container has been allocated
*/
@Public
@Stable
public abstract String getUser();
/**
* Set the <em>user</em> to whom the container has been allocated
* @param user <em>user</em> to whom the container has been allocated
*/
@Public
@Stable
public abstract void setUser(String user);
/**
* Get all the tokens needed by this container. It may include file-system
* tokens, ApplicationMaster related tokens if this container is an

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
View File

@ -169,25 +169,6 @@ extends ContainerLaunchContext {
builder.addAllCommand(this.commands);
}
@Override
public String getUser() {
ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasUser()) {
return null;
}
return (p.getUser());
}
@Override
public void setUser(String user) {
maybeInitBuilder();
if (user == null) {
builder.clearUser();
return;
}
builder.setUser((user));
}
@Override
public Map<String, LocalResource> getLocalResources() {
initLocalResources();

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -267,13 +267,12 @@ message QueueUserACLInfoProto {
////////////////////////////////////////////////////////////////////////
message ContainerLaunchContextProto {
optional string user = 1;
repeated StringLocalResourceMapProto localResources = 2;
optional bytes tokens = 3;
repeated StringBytesMapProto service_data = 4;
repeated StringStringMapProto environment = 5;
repeated string command = 6;
repeated ApplicationACLMapProto application_ACLs = 7;
repeated StringLocalResourceMapProto localResources = 1;
optional bytes tokens = 2;
repeated StringBytesMapProto service_data = 3;
repeated StringStringMapProto environment = 4;
repeated string command = 5;
repeated ApplicationACLMapProto application_ACLs = 6;
}
message ContainerStatusProto {

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_server_resourcemanager_service_protos.proto
View File

@ -67,6 +67,7 @@ message GetGroupsForUserResponseProto {
message ApplicationStateDataProto {
optional int64 submit_time = 1;
optional ApplicationSubmissionContextProto application_submission_context = 2;
optional string user = 3;
}
message ApplicationAttemptStateDataProto {

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
View File

@ -698,11 +698,6 @@ public class ApplicationMaster {
ContainerLaunchContext ctx = Records
.newRecord(ContainerLaunchContext.class);
String jobUserName = System.getenv(ApplicationConstants.Environment.USER
.key());
ctx.setUser(jobUserName);
LOG.info("Setting user in ContainerLaunchContext to: " + jobUserName);
// Set the environment
ctx.setEnvironment(shellEnv);

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.util;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.nio.ByteBuffer;
@ -25,6 +26,7 @@ import java.util.Comparator;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@ -188,6 +190,16 @@ public class BuilderUtils {
return cId;
}
public static ContainerToken newContainerToken(ContainerId cId, String host,
int port, String user, Resource r, long expiryTime, int masterKeyId,
byte[] password) throws IOException {
ContainerTokenIdentifier identifier =
new ContainerTokenIdentifier(cId, host, user, r, expiryTime,
masterKeyId);
return newContainerToken(BuilderUtils.newNodeId(host, port), password,
identifier);
}
public static ContainerId newContainerId(RecordFactory recordFactory,
ApplicationId appId, ApplicationAttemptId appAttemptId,
int containerId) {
@ -286,14 +298,24 @@ public class BuilderUtils {
return containerToken;
}
public static ContainerTokenIdentifier newContainerTokenIdentifier(
ContainerToken containerToken) throws IOException {
org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token =
new org.apache.hadoop.security.token.Token<ContainerTokenIdentifier>(
containerToken.getIdentifier()
.array(), containerToken.getPassword().array(), new Text(
containerToken.getKind()),
new Text(containerToken.getService()));
return token.decodeIdentifier();
}
public static ContainerLaunchContext newContainerLaunchContext(
String user, Map<String, LocalResource> localResources,
Map<String, LocalResource> localResources,
Map<String, String> environment, List<String> commands,
Map<String, ByteBuffer> serviceData, ByteBuffer tokens,
Map<String, ByteBuffer> serviceData, ByteBuffer tokens,
Map<ApplicationAccessType, String> acls) {
ContainerLaunchContext container = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
container.setUser(user);
container.setLocalResources(localResources);
container.setEnvironment(environment);
container.setCommands(commands);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
View File

@ -89,7 +89,6 @@ public class TestContainerLaunchRPC {
server.getListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
containerLaunchContext.setUser("dummy-user");
ContainerId containerId = recordFactory
.newRecordInstance(ContainerId.class);
ApplicationId applicationId = recordFactory

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
View File

@ -111,7 +111,6 @@ public class TestRPC {
NetUtils.getConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
containerLaunchContext.setUser("dummy-user");
ContainerId containerId =
recordFactory.newRecordInstance(ContainerId.class);
ApplicationId applicationId =

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
View File

@ -20,8 +20,6 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager;
import static org.apache.hadoop.yarn.service.Service.STATE.STARTED;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
@ -105,9 +103,9 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.ServiceStateChangeListener;
import org.apache.hadoop.yarn.util.BuilderUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.RpcUtil;
public class ContainerManagerImpl extends CompositeService implements
ServiceStateChangeListener, ContainerManager,
@ -321,13 +319,9 @@ public class ContainerManagerImpl extends CompositeService implements
// Get the tokenId from the remote user ugi
return selectContainerTokenIdentifier(remoteUgi);
} else {
ContainerToken containerToken = container.getContainerToken();
Token<ContainerTokenIdentifier> token =
new Token<ContainerTokenIdentifier>(containerToken.getIdentifier()
.array(), containerToken.getPassword().array(), new Text(
containerToken.getKind()), new Text(containerToken.getService()));
try {
return token.decodeIdentifier();
return BuilderUtils.newContainerTokenIdentifier(container
.getContainerToken());
} catch (IOException e) {
throw RPCUtil.getRemoteException(e);
}
@ -370,15 +364,6 @@ public class ContainerManagerImpl extends CompositeService implements
messageBuilder
.append("\nNo ContainerToken found for " + containerIDStr);
} else {
// Is the container coming in with correct user-name?
if (!launchContext.getUser().equals(tokenId.getApplicationSubmitter())) {
unauthorized = true;
messageBuilder.append("\n Expected user-name "
+ tokenId.getApplicationSubmitter() + " but found "
+ launchContext.getUser());
}
// Is the container being relaunched? Or RPC layer let startCall with
// tokens generated off old-secret through?
@ -451,7 +436,7 @@ public class ContainerManagerImpl extends CompositeService implements
}
LOG.info("Start request for " + containerIDStr + " by user "
+ launchContext.getUser());
+ tokenId.getApplicationSubmitter());
// //////////// Parse credentials
ByteBuffer tokens = launchContext.getTokens();
@ -473,13 +458,14 @@ public class ContainerManagerImpl extends CompositeService implements
}
}
// //////////// End of parsing credentials
String user = tokenId.getApplicationSubmitter();
Container container = new ContainerImpl(getConfig(), this.dispatcher,
launchContext, lauchContainer, credentials, metrics);
launchContext, lauchContainer, credentials, metrics, tokenId);
ApplicationId applicationID =
containerID.getApplicationAttemptId().getApplicationId();
if (context.getContainers().putIfAbsent(containerID, container) != null) {
NMAuditLogger.logFailure(launchContext.getUser(),
NMAuditLogger.logFailure(user,
AuditConstants.START_CONTAINER, "ContainerManagerImpl",
"Container already running on this node!",
applicationID, containerID);
@ -490,7 +476,8 @@ public class ContainerManagerImpl extends CompositeService implements
// Create the application
Application application =
new ApplicationImpl(dispatcher, this.aclsManager,
launchContext.getUser(), applicationID, credentials, context);
user, applicationID, credentials,
context);
if (null ==
context.getApplications().putIfAbsent(applicationID, application)) {
LOG.info("Creating a new application reference for app "
@ -506,7 +493,7 @@ public class ContainerManagerImpl extends CompositeService implements
this.context.getContainerTokenSecretManager().startContainerSuccessful(
tokenId);
NMAuditLogger.logSuccess(launchContext.getUser(),
NMAuditLogger.logSuccess(user,
AuditConstants.START_CONTAINER, "ContainerManageImpl",
applicationID, containerID);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
View File

@ -77,7 +77,7 @@ public class ApplicationImpl implements Application {
ApplicationACLsManager aclsManager, String user, ApplicationId appId,
Credentials credentials, Context context) {
this.dispatcher = dispatcher;
this.user = user.toString();
this.user = user;
this.appId = appId;
this.credentials = credentials;
this.aclsManager = aclsManager;

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@ -44,6 +45,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
@ -75,6 +77,7 @@ public class ContainerImpl implements Container {
private final NodeManagerMetrics metrics;
private final ContainerLaunchContext launchContext;
private final org.apache.hadoop.yarn.api.records.Container container;
private final String user;
private int exitCode = ContainerExitStatus.INVALID;
private final StringBuilder diagnostics;
@ -93,10 +96,11 @@ public class ContainerImpl implements Container {
private final List<LocalResourceRequest> appRsrcs =
new ArrayList<LocalResourceRequest>();
public ContainerImpl(Configuration conf,
Dispatcher dispatcher, ContainerLaunchContext launchContext,
public ContainerImpl(Configuration conf, Dispatcher dispatcher,
ContainerLaunchContext launchContext,
org.apache.hadoop.yarn.api.records.Container container,
Credentials creds, NodeManagerMetrics metrics) {
Credentials creds, NodeManagerMetrics metrics,
ContainerTokenIdentifier identifier) throws IOException {
this.daemonConf = conf;
this.dispatcher = dispatcher;
this.launchContext = launchContext;
@ -104,7 +108,7 @@ public class ContainerImpl implements Container {
this.diagnostics = new StringBuilder();
this.credentials = creds;
this.metrics = metrics;
user = identifier.getApplicationSubmitter();
ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
this.readLock = readWriteLock.readLock();
this.writeLock = readWriteLock.writeLock();
@ -311,7 +315,7 @@ public class ContainerImpl implements Container {
public String getUser() {
this.readLock.lock();
try {
return this.launchContext.getUser();
return this.user;
} finally {
this.readLock.unlock();
}
@ -382,7 +386,6 @@ public class ContainerImpl implements Container {
@SuppressWarnings({"fallthrough", "unchecked"})
private void finished() {
ContainerId containerID = this.container.getId();
String user = this.launchContext.getUser();
switch (getContainerState()) {
case EXITED_WITH_SUCCESS:
metrics.endRunningContainer();
@ -486,7 +489,7 @@ public class ContainerImpl implements Container {
for (Map.Entry<String,ByteBuffer> service : csd.entrySet()) {
container.dispatcher.getEventHandler().handle(
new AuxServicesEvent(AuxServicesEventType.APPLICATION_INIT,
ctxt.getUser(), container.container.getId()
container.user, container.container.getId()
.getApplicationAttemptId().getApplicationId(),
service.getKey().toString(), service.getValue()));
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
View File

@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.DelayedProcessKiller;
@ -67,6 +68,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
public class ContainerLaunch implements Callable<Integer> {
@ -120,11 +122,11 @@ public class ContainerLaunch implements Callable<Integer> {
container.getLocalizedResources();
ContainerId containerID = container.getContainer().getId();
String containerIdStr = ConverterUtils.toString(containerID);
final String user = launchContext.getUser();
final List<String> command = launchContext.getCommands();
int ret = -1;
try {
final String user = container.getUser();
// /////////////////////////// Variable expansion
// Before the container script gets written out.
List<String> newCmds = new ArrayList<String>(command.size());
@ -334,7 +336,7 @@ public class ContainerLaunch implements Callable<Integer> {
// kill process
if (processId != null) {
String user = container.getLaunchContext().getUser();
String user = container.getUser();
LOG.debug("Sending signal to pid " + processId
+ " as user " + user
+ " for container " + containerIdStr);

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
View File

@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@ -47,6 +48,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerM
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
@ -135,10 +137,16 @@ public class TestEventFlow {
cID.setApplicationAttemptId(applicationAttemptId);
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cID);
when(mockContainer.getResource()).thenReturn(recordFactory
.newRecordInstance(Resource.class));
Resource r = BuilderUtils.newResource(1024, 1);
when(mockContainer.getResource()).thenReturn(r);
when(mockContainer.getRMIdentifer()).thenReturn(SIMULATED_RM_IDENTIFIER);
launchContext.setUser("testing");
String user = "testing";
String host = "127.0.0.1";
int port = 1234;
ContainerToken containerToken =
BuilderUtils.newContainerToken(cID, host, port, user, r,
System.currentTimeMillis() + 10000L, 123, "password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest request =
recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
View File

@ -115,8 +115,6 @@ public class TestNodeManagerReboot {
Records.newRecord(org.apache.hadoop.yarn.api.records.Container.class);
mockContainer.setId(cId);
containerLaunchContext.setUser(user);
URL localResourceUri =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(localResourceDir.getAbsolutePath())));

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
View File

@ -166,8 +166,6 @@ public class TestNodeManagerShutdown {
mockContainer.setNodeId(nodeId);
mockContainer.setNodeHttpAddress("localhost:12345");
containerLaunchContext.setUser(cId.toString());
URL localResourceUri =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@ -183,7 +181,6 @@ public class TestNodeManagerShutdown {
new HashMap<String, LocalResource>();
localResources.put(destinationFile, localResource);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Resource resource = BuilderUtils.newResource(1024, 1);

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
View File

@ -206,7 +206,7 @@ public class TestNodeStatusUpdater {
when(mockContainer.getResource()).thenReturn(resource);
Container container =
new ContainerImpl(conf, mockDispatcher, launchContext,
mockContainer, null, mockMetrics);
mockContainer, null, mockMetrics, null);
this.context.getContainers().put(firstContainerID, container);
} else if (heartBeatID == 2) {
// Checks on the RM end
@ -232,7 +232,7 @@ public class TestNodeStatusUpdater {
when(mockContainer.getResource()).thenReturn(resource);
Container container =
new ContainerImpl(conf, mockDispatcher, launchContext,
mockContainer, null, mockMetrics);
mockContainer, null, mockMetrics, null);
this.context.getContainers().put(secondContainerID, container);
} else if (heartBeatID == 3) {
// Checks on the RM end
@ -665,8 +665,8 @@ public class TestNodeStatusUpdater {
}
waitCount = 0;
while (heartBeatID <= 3 && waitCount++ != 20) {
Thread.sleep(500);
while (heartBeatID <= 3 && waitCount++ != 200) {
Thread.sleep(1000);
}
Assert.assertFalse(heartBeatID <= 3);
Assert.assertEquals("Number of registered NMs is wrong!!", 1,

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
View File

@ -182,16 +182,6 @@ public abstract class BaseContainerManagerTest {
throws YarnRemoteException {
// do nothing
}
@Override
protected ContainerTokenIdentifier getContainerTokenIdentifier(
UserGroupInformation remoteUgi,
org.apache.hadoop.yarn.api.records.Container container)
throws YarnRemoteException {
return new ContainerTokenIdentifier(container.getId(),
container.getNodeHttpAddress(), remoteUgi.getUserName(),
container.getResource(), System.currentTimeMillis(), 123);
}
};
}

80
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
View File

@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@ -128,8 +129,6 @@ public class TestContainerManager extends BaseContainerManagerTest {
// ////// Construct the Container-id
ContainerId cId = createContainerId();
container.setUser(user);
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
@ -147,17 +146,21 @@ public class TestContainerManager extends BaseContainerManagerTest {
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(container.getUser());
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(512, 1));
Resource r = BuilderUtils.newResource(512, 1);
when(mockContainer.getResource()).thenReturn(r);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
@ -228,8 +231,6 @@ public class TestContainerManager extends BaseContainerManagerTest {
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
containerLaunchContext.setUser(user);
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@ -245,17 +246,22 @@ public class TestContainerManager extends BaseContainerManagerTest {
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(100, 1)); // MB
Resource r = BuilderUtils.newResource(100, 1);
when(mockContainer.getResource()).thenReturn(r); // MB
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
@ -341,8 +347,6 @@ public class TestContainerManager extends BaseContainerManagerTest {
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
containerLaunchContext.setUser(user);
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@ -358,18 +362,22 @@ public class TestContainerManager extends BaseContainerManagerTest {
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(100, 1)); // MB
Resource r = BuilderUtils.newResource(100, 1);
when(mockContainer.getResource()).thenReturn(r); // MB
int port = 12345;
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainer(mockContainer);
@ -436,8 +444,6 @@ public class TestContainerManager extends BaseContainerManagerTest {
ContainerId cId = createContainerId();
ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
container.setUser(user);
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
// containerLaunchContext.resources =
@ -456,18 +462,22 @@ public class TestContainerManager extends BaseContainerManagerTest {
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(container.getUser());
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(100, 1));
Resource r = BuilderUtils.newResource(100, 1);
when(mockContainer.getResource()).thenReturn(r);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
// containerLaunchContext.command = new ArrayList<CharSequence>();
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(containerLaunchContext);
request.setContainer(mockContainer);
@ -541,13 +551,13 @@ public class TestContainerManager extends BaseContainerManagerTest {
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
String host = "127.0.0.1";
int port = 1234;
ContainerId cId1 = createContainerId();
ContainerId cId2 = createContainerId();
containerLaunchContext.setUser(user);
containerLaunchContext
.setLocalResources(new HashMap<String, LocalResource>());
containerLaunchContext.setUser(containerLaunchContext.getUser());
Resource mockResource = mock(Resource.class);
Resource mockResource = BuilderUtils.newResource(1024, 1);
Container mockContainer1 = mock(Container.class);
when(mockContainer1.getId()).thenReturn(cId1);
@ -557,6 +567,11 @@ public class TestContainerManager extends BaseContainerManagerTest {
StartContainerRequest startRequest1 =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest1.setContainerLaunchContext(containerLaunchContext);
ContainerToken containerToken1 =
BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
System.currentTimeMillis() + 10000, 123, "password".getBytes());
when(mockContainer1.getContainerToken()).thenReturn(containerToken1);
startRequest1.setContainer(mockContainer1);
boolean catchException = false;
try {
@ -583,6 +598,11 @@ public class TestContainerManager extends BaseContainerManagerTest {
StartContainerRequest startRequest2 =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest2.setContainerLaunchContext(containerLaunchContext);
ContainerToken containerToken2 =
BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
System.currentTimeMillis() + 10000, 123, "password".getBytes());
when(mockContainer2.getContainerToken()).thenReturn(containerToken2);
startRequest2.setContainer(mockContainer2);
boolean noException = true;
try {

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
View File

@ -25,6 +25,7 @@ import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.AbstractMap.SimpleEntry;
@ -44,6 +45,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@ -53,6 +55,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType;
@ -524,8 +527,10 @@ public class TestContainer {
}
private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx,
org.apache.hadoop.yarn.api.records.Container container) {
return new ContainerImpl(conf, disp, ctx, container, null, metrics);
org.apache.hadoop.yarn.api.records.Container container,
ContainerTokenIdentifier identifier) throws IOException {
return new ContainerImpl(conf, disp, ctx, container, null, metrics,
identifier);
}
@SuppressWarnings("unchecked")
@ -545,12 +550,13 @@ public class TestContainer {
final Map<String, ByteBuffer> serviceData;
final String user;
WrappedContainer(int appId, long timestamp, int id, String user) {
WrappedContainer(int appId, long timestamp, int id, String user)
throws IOException {
this(appId, timestamp, id, user, true, false);
}
WrappedContainer(int appId, long timestamp, int id, String user,
boolean withLocalRes, boolean withServiceData) {
boolean withLocalRes, boolean withServiceData) throws IOException {
dispatcher = new DrainDispatcher();
dispatcher.init(new Configuration());
@ -572,12 +578,19 @@ public class TestContainer {
org.apache.hadoop.yarn.api.records.Container mockContainer =
mock(org.apache.hadoop.yarn.api.records.Container.class);
cId = BuilderUtils.newContainerId(appId, 1, timestamp, id);
when(ctxt.getUser()).thenReturn(this.user);
when(mockContainer.getId()).thenReturn(cId);
Resource resource = BuilderUtils.newResource(1024, 1);
when(mockContainer.getResource()).thenReturn(resource);
String host = "127.0.0.1";
int port = 1234;
ContainerTokenIdentifier identifier =
new ContainerTokenIdentifier(cId, "127.0.0.1", user, resource,
System.currentTimeMillis() + 10000L, 123);
ContainerToken token =
BuilderUtils.newContainerToken(BuilderUtils.newNodeId(host, port),
"password".getBytes(), identifier);
when(mockContainer.getContainerToken()).thenReturn(token);
if (withLocalRes) {
Random r = new Random();
long seed = r.nextLong();
@ -600,7 +613,7 @@ public class TestContainer {
}
when(ctxt.getServiceData()).thenReturn(serviceData);
c = newContainer(dispatcher, ctxt, mockContainer);
c = newContainer(dispatcher, ctxt, mockContainer, identifier);
dispatcher.start();
}

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
View File

@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.BufferedReader;
import java.io.File;
@ -26,14 +28,14 @@ import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import junit.framework.Assert;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@ -50,15 +52,16 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
@ -66,9 +69,6 @@ import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.junit.Before;
import org.junit.Test;
import static org.mockito.Mockito.*;
import junit.framework.Assert;
public class TestContainerLaunch extends BaseContainerManagerTest {
@ -156,7 +156,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
* See if environment variable is forwarded using sanitizeEnv.
* @throws Exception
*/
@Test (timeout = 5000)
@Test (timeout = 60000)
public void testContainerEnvVariables() throws Exception {
containerManager.start();
@ -174,12 +174,13 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
appAttemptId.setAttemptId(1);
ContainerId cId =
recordFactory.newRecordInstance(ContainerId.class);
int port = 12345;
cId.setApplicationAttemptId(appAttemptId);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
Map<String, String> userSetEnv = new HashMap<String, String>();
@ -188,7 +189,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT");
userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT");
userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR");
containerLaunchContext.setUser(user);
containerLaunchContext.setEnvironment(userSetEnv);
File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
@ -242,13 +242,17 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
containerLaunchContext.setLocalResources(localResources);
// set up the rest of the container
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(1024, 1));
Resource r = BuilderUtils.newResource(1024, 1);
when(mockContainer.getResource()).thenReturn(r);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 1234,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
@ -375,12 +379,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
recordFactory.newRecordInstance(ContainerLaunchContext.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
containerLaunchContext.setUser(user);
// upload the script file so that the container can run it
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
@ -399,11 +402,15 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
containerLaunchContext.setLocalResources(localResources);
// set up the rest of the container
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(1024, 1));
Resource r = BuilderUtils.newResource(1024, 1);
when(mockContainer.getResource()).thenReturn(r);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainer(mockContainer);

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
View File

@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@ -75,6 +76,7 @@ import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
@ -694,8 +696,6 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
containerLaunchContext.setUser(this.user);
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@ -711,13 +711,15 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = new ArrayList<String>();
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(100 * 1024 * 1024, 1));
Resource r = BuilderUtils.newResource(100 * 1024 * 1024, 1);
when(mockContainer.getResource()).thenReturn(r);
when(mockContainer.getContainerToken()).thenReturn(
BuilderUtils.newContainerToken(cId, "127.0.0.1", 1234, user, r,
System.currentTimeMillis() + 10000L, 123, "password".getBytes()));
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
View File

@ -49,9 +49,11 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@ -215,11 +217,11 @@ public class TestContainersMonitor extends BaseContainerManagerTest {
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
context.getNodeId().getHost() + ":12345");
context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(
super.DUMMY_RM_IDENTIFIER);
containerLaunchContext.setUser(user);
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
@ -236,16 +238,20 @@ public class TestContainersMonitor extends BaseContainerManagerTest {
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = new ArrayList<String>();
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(8 * 1024 * 1024, 1));
Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1);
when(mockContainer.getResource()).thenReturn(r);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
View File

@ -18,6 +18,10 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -30,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@ -38,7 +42,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.util.BuilderUtils;
import static org.mockito.Mockito.*;
public class MockContainer implements Container {
@ -53,7 +56,7 @@ public class MockContainer implements Container {
public MockContainer(ApplicationAttemptId appAttemptId,
Dispatcher dispatcher, Configuration conf, String user,
ApplicationId appId, int uniqId) {
ApplicationId appId, int uniqId) throws IOException{
this.user = user;
this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
@ -61,10 +64,14 @@ public class MockContainer implements Container {
uniqId);
this.launchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
launchContext.setUser(user);
ContainerToken containerToken =
BuilderUtils.newContainerToken(id, "127.0.0.1", 1234, user,
BuilderUtils.newResource(1024, 1),
System.currentTimeMillis() + 10000, 123, "password".getBytes());
this.state = ContainerState.NEW;
mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class);
when(mockContainer.getContainerToken()).thenReturn(containerToken);
when(mockContainer.getId()).thenReturn(id);
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
View File

@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@ -181,11 +183,16 @@ public class TestNMWebServer {
recordFactory.newRecordInstance(ContainerLaunchContext.class);
org.apache.hadoop.yarn.api.records.Container mockContainer =
mock(org.apache.hadoop.yarn.api.records.Container.class);
ContainerToken containerToken =
BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user,
BuilderUtils.newResource(1024, 1),
System.currentTimeMillis() + 10000L, 123, "password".getBytes());
when(mockContainer.getContainerToken()).thenReturn(containerToken);
when(mockContainer.getId()).thenReturn(containerId);
launchContext.setUser(user);
Container container =
new ContainerImpl(conf, dispatcher, launchContext, mockContainer,
null, metrics) {
null, metrics,
BuilderUtils.newContainerTokenIdentifier(containerToken)) {
@Override
public ContainerState getContainerState() {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
View File

@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.HashMap;
@ -177,7 +178,8 @@ public class TestNMWebServicesApps extends JerseyTest {
assertEquals("apps isn't NULL", JSONObject.NULL, json.get("apps"));
}
private HashMap<String, String> addAppContainers(Application app) {
private HashMap<String, String> addAppContainers(Application app)
throws IOException {
Dispatcher dispatcher = new AsyncDispatcher();
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
app.getAppId(), 1);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
View File

@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.HashMap;
@ -178,7 +179,8 @@ public class TestNMWebServicesContainers extends JerseyTest {
assertEquals("apps isn't NULL", JSONObject.NULL, json.get("containers"));
}
private HashMap<String, String> addAppContainers(Application app) {
private HashMap<String, String> addAppContainers(Application app)
throws IOException {
Dispatcher dispatcher = new AsyncDispatcher();
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
app.getAppId(), 1);

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -270,7 +270,6 @@ public class ClientRMService extends AbstractService implements
try {
// Safety
user = UserGroupInformation.getCurrentUser().getShortUserName();
submissionContext.getAMContainerSpec().setUser(user);
} catch (IOException ie) {
LOG.warn("Unable to get the current user.", ie);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
@ -312,7 +311,7 @@ public class ClientRMService extends AbstractService implements
try {
// call RMAppManager to submit application directly
rmAppManager.submitApplication(submissionContext,
System.currentTimeMillis(), false);
System.currentTimeMillis(), false, user);
LOG.info("Application with id " + applicationId.getId() +
" submitted by user " + user);

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -239,7 +239,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
@SuppressWarnings("unchecked")
protected void submitApplication(
ApplicationSubmissionContext submissionContext, long submitTime,
boolean isRecovered) throws YarnRemoteException {
boolean isRecovered, String user) throws YarnRemoteException {
ApplicationId applicationId = submissionContext.getApplicationId();
// Validation of the ApplicationSubmissionContext needs to be completed
@ -265,8 +265,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
// Create RMApp
RMApp application =
new RMAppImpl(applicationId, rmContext, this.conf,
submissionContext.getApplicationName(),
submissionContext.getAMContainerSpec().getUser(),
submissionContext.getApplicationName(), user,
submissionContext.getQueue(),
submissionContext, this.scheduler, this.masterService,
submitTime, submissionContext.getApplicationType());
@ -370,7 +369,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
if(shouldRecover) {
LOG.info("Recovering application " + appState.getAppId());
submitApplication(appState.getApplicationSubmissionContext(),
appState.getSubmitTime(), true);
appState.getSubmitTime(), true, appState.getUser());
// re-populate attempt information in application
RMAppImpl appImpl = (RMAppImpl) rmContext.getRMApps().get(
appState.getAppId());

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
View File

@ -165,7 +165,6 @@ public class AMLauncher implements Runnable {
new String[0])));
// Finalize the container
container.setUser(applicationMasterContext.getAMContainerSpec().getUser());
setupTokensAndEnv(container, containerID);
return container;

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
View File

@ -103,7 +103,8 @@ public class FileSystemRMStateStore extends RMStateStore {
ApplicationStateDataProto.parseFrom(childData));
ApplicationState appState = new ApplicationState(
appStateData.getSubmitTime(),
appStateData.getApplicationSubmissionContext());
appStateData.getApplicationSubmissionContext(),
appStateData.getUser());
// assert child node name is same as actual applicationId
assert appId.equals(appState.context.getApplicationId());
state.appState.put(appId, appState);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
View File

@ -65,8 +65,8 @@ public class MemoryRMStateStore extends RMStateStore {
ApplicationStateDataPBImpl appStateData)
throws Exception {
ApplicationState appState = new ApplicationState(
appStateData.getSubmitTime(),
appStateData.getApplicationSubmissionContext());
appStateData.getSubmitTime(),
appStateData.getApplicationSubmissionContext(), appStateData.getUser());
if (state.appState.containsKey(appState.getAppId())) {
Exception e = new IOException("App: " + appId + " is already stored.");
LOG.info("Error storing info for app: " + appId, e);

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
View File

@ -91,12 +91,15 @@ public abstract class RMStateStore {
public static class ApplicationState {
final ApplicationSubmissionContext context;
final long submitTime;
final String user;
Map<ApplicationAttemptId, ApplicationAttemptState> attempts =
new HashMap<ApplicationAttemptId, ApplicationAttemptState>();
ApplicationState(long submitTime, ApplicationSubmissionContext context) {
ApplicationState(long submitTime, ApplicationSubmissionContext context,
String user) {
this.submitTime = submitTime;
this.context = context;
this.user = user;
}
public ApplicationId getAppId() {
@ -114,6 +117,9 @@ public abstract class RMStateStore {
public ApplicationAttemptState getAttempt(ApplicationAttemptId attemptId) {
return attempts.get(attemptId);
}
public String getUser() {
return user;
}
}
/**
@ -190,7 +196,7 @@ public abstract class RMStateStore {
.getApplicationSubmissionContext();
assert context instanceof ApplicationSubmissionContextPBImpl;
ApplicationState appState = new ApplicationState(
app.getSubmitTime(), context);
app.getSubmitTime(), context, app.getUser());
dispatcher.getEventHandler().handle(new RMStateStoreAppEvent(appState));
}
@ -240,7 +246,8 @@ public abstract class RMStateStore {
*/
public synchronized void removeApplication(RMApp app) {
ApplicationState appState = new ApplicationState(
app.getSubmitTime(), app.getApplicationSubmissionContext());
app.getSubmitTime(), app.getApplicationSubmissionContext(),
app.getUser());
for(RMAppAttempt appAttempt : app.getAppAttempts().values()) {
Credentials credentials = getTokensFromAppAttempt(appAttempt);
ApplicationAttemptState attemptState =
@ -295,6 +302,7 @@ public abstract class RMStateStore {
appStateData.setSubmitTime(apptState.getSubmitTime());
appStateData.setApplicationSubmissionContext(
apptState.getApplicationSubmissionContext());
appStateData.setUser(apptState.getUser());
ApplicationId appId =
apptState.getApplicationSubmissionContext().getApplicationId();

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
View File

@ -43,6 +43,17 @@ public interface ApplicationStateData {
@Unstable
public void setSubmitTime(long submitTime);
/**
* The application submitter
*/
@Public
@Unstable
public void setUser(String user);
@Public
@Unstable
public String getUser();
/**
* The {@link ApplicationSubmissionContext} for the application
* {@link ApplicationId} can be obtained from the this

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
View File

@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPB
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProtoOrBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.mortbay.log.Log;
public class ApplicationStateDataPBImpl
extends ProtoBase<ApplicationStateDataProto>
@ -91,6 +92,22 @@ implements ApplicationStateData {
builder.setSubmitTime(submitTime);
}
@Override
public String getUser() {
ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasUser()) {
return null;
}
return (p.getUser());
}
@Override
public void setUser(String user) {
maybeInitBuilder();
builder.setUser(user);
}
@Override
public ApplicationSubmissionContext getApplicationSubmissionContext() {
ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
View File

@ -586,7 +586,7 @@ public class RMAppImpl implements RMApp, Recoverable {
RMAppAttempt attempt =
new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService,
submissionContext, conf);
submissionContext, conf, user);
attempts.put(appAttemptId, attempt);
currentAttempt = attempt;
if(startAttempt) {

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -153,7 +153,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
private final StringBuilder diagnostics = new StringBuilder();
private Configuration conf;
private String user;
private static final ExpiredTransition EXPIRED_TRANSITION =
new ExpiredTransition();
@ -364,7 +365,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
ApplicationSubmissionContext submissionContext,
Configuration conf) {
Configuration conf, String user) {
this.conf = conf;
this.applicationAttemptId = appAttemptId;
this.rmContext = rmContext;
@ -380,6 +381,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.proxiedTrackingUrl = generateProxyUriWithoutScheme();
this.stateMachine = stateMachineFactory.make(this);
this.user = user;
}
@Override
@ -746,8 +748,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
// Add the application to the scheduler
appAttempt.eventHandler.handle(
new AppAddedSchedulerEvent(appAttempt.applicationAttemptId,
appAttempt.submissionContext.getQueue(),
appAttempt.submissionContext.getAMContainerSpec().getUser()));
appAttempt.submissionContext.getQueue(), appAttempt.user));
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
View File

@ -135,7 +135,7 @@ public class AppBlock extends HtmlBlock {
boolean odd = false;
for (RMAppAttempt attempt : attempts) {
AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt);
AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser());
table.tr((odd = !odd) ? _ODD : _EVEN).
td(String.valueOf(attemptInfo.getAttemptId())).
td(Times.format(attemptInfo.getStartTime())).

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -409,7 +409,7 @@ public class RMWebServices {
AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo();
for (RMAppAttempt attempt : app.getAppAttempts().values()) {
AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt);
AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser());
appAttemptsInfo.add(attemptInfo);
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
View File

@ -42,7 +42,7 @@ public class AppAttemptInfo {
public AppAttemptInfo() {
}
public AppAttemptInfo(RMAppAttempt attempt) {
public AppAttemptInfo(RMAppAttempt attempt, String user) {
this.startTime = 0;
this.containerId = "";
this.nodeHttpAddress = "";
@ -59,8 +59,7 @@ public class AppAttemptInfo {
this.logsLink = join(HttpConfig.getSchemePrefix(),
masterContainer.getNodeHttpAddress(),
"/node", "/containerlogs/",
ConverterUtils.toString(masterContainer.getId()), "/",
attempt.getSubmissionContext().getAMContainerSpec().getUser());
ConverterUtils.toString(masterContainer.getId()), "/", user);
}
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
View File

@ -131,7 +131,6 @@ public class Application {
public synchronized void submit() throws IOException, YarnRemoteException {
ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
context.setApplicationId(this.applicationId);
context.getAMContainerSpec().setUser(this.user);
context.setQueue(this.queue);
SubmitApplicationRequest request = recordFactory
.newRecordInstance(SubmitApplicationRequest.class);
@ -401,7 +400,6 @@ public class Application {
private ContainerLaunchContext createCLC() {
ContainerLaunchContext clc = recordFactory.newRecordInstance(ContainerLaunchContext.class);
clc.setUser(this.user);
return clc;
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
View File

@ -188,7 +188,6 @@ public class MockRM extends ResourceManager {
capability.setMemory(masterMemory);
sub.setResource(capability);
clc.setApplicationACLs(acls);
clc.setUser(user);
if (ts != null && UserGroupInformation.isSecurityEnabled()) {
DataOutputBuffer dob = new DataOutputBuffer();
ts.writeTokenStorageToStream(dob);

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -169,10 +169,10 @@ public class TestAppManager{
super.setCompletedAppsMax(max);
}
public void submitApplication(
ApplicationSubmissionContext submissionContext)
ApplicationSubmissionContext submissionContext, String user)
throws YarnRemoteException {
super.submitApplication(submissionContext, System.currentTimeMillis(),
false);
false, user);
}
}
@ -375,7 +375,7 @@ public class TestAppManager{
@Test
public void testRMAppSubmit() throws Exception {
appMonitor.submitApplication(asContext);
appMonitor.submitApplication(asContext, "test");
RMApp app = rmContext.getRMApps().get(appId);
Assert.assertNotNull("app is null", app);
Assert.assertEquals("app id doesn't match", appId, app.getApplicationId());
@ -416,7 +416,7 @@ public class TestAppManager{
if (individualMaxAppAttempts[i][j] != 0) {
asContext.setMaxAppAttempts(individualMaxAppAttempts[i][j]);
}
appMonitor.submitApplication(asContext);
appMonitor.submitApplication(asContext, "test");
RMApp app = rmContext.getRMApps().get(appID);
Assert.assertEquals("max application attempts doesn't match",
expectedNums[i][j], app.getMaxAppAttempts());
@ -441,7 +441,7 @@ public class TestAppManager{
// our testApp1 should be rejected and original app with same id should be left in place
try {
appMonitor.submitApplication(asContext);
appMonitor.submitApplication(asContext, "test");
Assert.fail("Exception is expected when applicationId is duplicate.");
} catch (YarnRemoteException e) {
Assert.assertTrue("The thrown exception is not the expectd one.",
@ -462,7 +462,7 @@ public class TestAppManager{
// submit an app
try {
appMonitor.submitApplication(asContext);
appMonitor.submitApplication(asContext, "test");
Assert.fail("Application submission should fail because resource" +
" request is invalid.");
} catch (YarnRemoteException e) {

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -381,8 +381,6 @@ public class TestClientRMService {
private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId,
String name, String queue) {
String user = MockApps.newUserName();
ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);
Resource resource = Resources.createResource(
@ -391,7 +389,6 @@ public class TestClientRMService {
ApplicationSubmissionContext submissionContext =
recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
submissionContext.setAMContainerSpec(amContainerSpec);
submissionContext.getAMContainerSpec().setUser(user);
submissionContext.setApplicationName(name);
submissionContext.setQueue(queue);
submissionContext.setApplicationId(appId);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java
View File

@ -234,7 +234,7 @@ public class TestClientRMTokens {
public void testShortCircuitRenewCancel()
throws IOException, InterruptedException {
InetSocketAddress addr =
new InetSocketAddress(InetAddress.getLocalHost(), 123);
new InetSocketAddress(InetAddress.getLocalHost(), 123);
checkShortCircuitRenewCancel(addr, addr, true);
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
View File

@ -194,6 +194,7 @@ public class TestRMStateStore {
when(mockApp.getApplicationId()).thenReturn(appId);
when(mockApp.getSubmitTime()).thenReturn(time);
when(mockApp.getApplicationSubmissionContext()).thenReturn(context);
when(mockApp.getUser()).thenReturn("test");
store.storeApplication(mockApp);
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
View File

@ -203,7 +203,7 @@ public class TestRMAppAttemptTransitions {
when(submissionContext.getQueue()).thenReturn(queue);
Resource resource = BuilderUtils.newResource(1536, 1);
ContainerLaunchContext amContainerSpec =
BuilderUtils.newContainerLaunchContext(user, null, null,
BuilderUtils.newContainerLaunchContext(null, null,
null, null, null, null);
when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
when(submissionContext.getResource()).thenReturn(resource);
@ -213,7 +213,7 @@ public class TestRMAppAttemptTransitions {
application = mock(RMApp.class);
applicationAttempt =
new RMAppAttemptImpl(applicationAttemptId, rmContext, scheduler,
masterService, submissionContext, new Configuration());
masterService, submissionContext, new Configuration(), user);
when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
when(application.getApplicationId()).thenReturn(applicationId);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
View File

@ -1612,7 +1612,7 @@ public class TestFairScheduler {
new ApplicationMasterService(resourceManager.getRMContext(), scheduler);
ApplicationSubmissionContext submissionContext = new ApplicationSubmissionContextPBImpl();
ContainerLaunchContext clc =
BuilderUtils.newContainerLaunchContext(user, null, null, null, null,
BuilderUtils.newContainerLaunchContext(null, null, null, null,
null, null);
submissionContext.setApplicationId(applicationId);
submissionContext.setAMContainerSpec(clc);

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -1009,7 +1009,7 @@ public class TestRMWebServicesApps extends JerseyTest {
// Verify these parallel arrays are the same
int i = 0;
for (RMAppAttempt attempt : attempts) {
verifyAppAttemptsInfo(jsonArray.getJSONObject(i), attempt);
verifyAppAttemptsInfo(jsonArray.getJSONObject(i), attempt, app.getUser());
++i;
}
}
@ -1017,8 +1017,9 @@ public class TestRMWebServicesApps extends JerseyTest {
@Test
public void testAppAttemptsXML() throws JSONException, Exception {
rm.start();
String user = "user1";
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
RMApp app1 = rm.submitApp(1024, "testwordcount", "user1");
RMApp app1 = rm.submitApp(1024, "testwordcount", user);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
@ -1037,11 +1038,12 @@ public class TestRMWebServicesApps extends JerseyTest {
assertEquals("incorrect number of elements", 1, nodes.getLength());
NodeList attempt = dom.getElementsByTagName("appAttempt");
assertEquals("incorrect number of elements", 1, attempt.getLength());
verifyAppAttemptsXML(attempt, app1.getCurrentAppAttempt());
verifyAppAttemptsXML(attempt, app1.getCurrentAppAttempt(), user);
rm.stop();
}
public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt)
public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt,
String user)
throws JSONException, Exception {
for (int i = 0; i < nodes.getLength(); i++) {
@ -1053,11 +1055,12 @@ public class TestRMWebServicesApps extends JerseyTest {
WebServicesTestUtils.getXmlString(element, "containerId"),
WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
WebServicesTestUtils.getXmlString(element, "nodeId"),
WebServicesTestUtils.getXmlString(element, "logsLink"));
WebServicesTestUtils.getXmlString(element, "logsLink"), user);
}
}
public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt)
public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt,
String user)
throws JSONException, Exception {
assertEquals("incorrect number of elements", 6, info.length());
@ -1065,12 +1068,12 @@ public class TestRMWebServicesApps extends JerseyTest {
verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"),
info.getLong("startTime"), info.getString("containerId"),
info.getString("nodeHttpAddress"), info.getString("nodeId"),
info.getString("logsLink"));
info.getString("logsLink"), user);
}
public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id,
long startTime, String containerId, String nodeHttpAddress, String nodeId,
String logsLink)
String logsLink, String user)
throws JSONException, Exception {
assertEquals("id doesn't match", appAttempt.getAppAttemptId()
@ -1087,7 +1090,7 @@ public class TestRMWebServicesApps extends JerseyTest {
logsLink.startsWith("http://"));
assertTrue(
"logsLink doesn't contain user info", logsLink.endsWith("/"
+ appAttempt.getSubmissionContext().getAMContainerSpec().getUser()));
+ user));
}
}

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/core-site.xml Normal file
View File

@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>hadoop.security.token.service.use_ip</name>
<value>false</value>
</property>
</configuration>

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
View File

@ -321,7 +321,8 @@ public class TestContainerManagerSecurity {
callWithIllegalContainerID(client, tokenId, allocatedContainer);
callWithIllegalResource(client, tokenId, allocatedContainer);
callWithIllegalUserName(client, tokenId, allocatedContainer);
// UserName is no longer sent using containerLaunchContext.
// callWithIllegalUserName(client, tokenId, allocatedContainer);
return client;
}
@ -412,17 +413,16 @@ public class TestContainerManagerSecurity {
Arrays.asList("sleep", "100");
ContainerLaunchContext amContainer =
BuilderUtils.newContainerLaunchContext("testUser",
Collections.<String, LocalResource> emptyMap(),
new HashMap<String, String>(), cmd,
new HashMap<String, ByteBuffer>(), null,
new HashMap<ApplicationAccessType, String>());
BuilderUtils.newContainerLaunchContext(
Collections.<String, LocalResource> emptyMap(),
new HashMap<String, String>(), cmd,
new HashMap<String, ByteBuffer>(), null,
new HashMap<ApplicationAccessType, String>());
ApplicationSubmissionContext appSubmissionContext = recordFactory
.newRecordInstance(ApplicationSubmissionContext.class);
appSubmissionContext.setApplicationId(appID);
appSubmissionContext.setAMContainerSpec(amContainer);
appSubmissionContext.getAMContainerSpec().setUser("testUser");
appSubmissionContext.setResource(BuilderUtils.newResource(1024, 1));
SubmitApplicationRequest submitRequest = recordFactory
@ -590,7 +590,7 @@ public class TestContainerManagerSecurity {
// Authenticated but unauthorized, due to wrong resource
ContainerLaunchContext context =
createContainerLaunchContextForTest(tokenId);
context.setUser("Saruman"); // Set a different user-name.
String user = "invalidUser";
request.setContainerLaunchContext(context);
request.setContainer(container);
try {
@ -603,7 +603,7 @@ public class TestContainerManagerSecurity {
"Unauthorized request to start container. "));
Assert.assertTrue(e.getMessage().contains(
"Expected user-name " + tokenId.getApplicationSubmitter()
+ " but found " + context.getUser()));
+ " but found " + user));
} catch (IOException e) {
LOG.info("Got IOException: ",e);
fail("IOException is not expected.");
@ -614,7 +614,6 @@ public class TestContainerManagerSecurity {
ContainerTokenIdentifier tokenId) {
ContainerLaunchContext context =
BuilderUtils.newContainerLaunchContext(
tokenId.getApplicationSubmitter(),
new HashMap<String, LocalResource>(),
new HashMap<String, String>(), new ArrayList<String>(),
new HashMap<String, ByteBuffer>(), null,