Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-822. Renamed ApplicationToken to be AMRMToken, and similarly the corresponding TokenSelector and SecretManager. Contributed by Omkar Vinit Joshi. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1493889 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-06-17 19:11:29 +00:00
parent eff5e6da1d
commit 7d0c43d7cf
28 changed files with 133 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
View File

@ -127,7 +127,7 @@ import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.SystemClock;
@ -1380,7 +1380,7 @@ public class MRAppMaster extends CompositeService {
Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
while (iter.hasNext()) { while (iter.hasNext()) {
Token<?> token = iter.next(); Token<?> token = iter.next();
if (token.getKind().equals(ApplicationTokenIdentifier.KIND_NAME)) { if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
iter.remove(); iter.remove();
} }
} }

8
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
View File

@ -62,7 +62,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.apache.log4j.LogManager; import org.apache.log4j.LogManager;
@ -346,9 +346,9 @@ public class TestMRAppMaster {
credentials.addToken(tokenAlias, myToken); credentials.addToken(tokenAlias, myToken);
Text appTokenService = new Text("localhost:0"); Text appTokenService = new Text("localhost:0");
Token<ApplicationTokenIdentifier> appToken = Token<AMRMTokenIdentifier> appToken =
new Token<ApplicationTokenIdentifier>(identifier, password, new Token<AMRMTokenIdentifier>(identifier, password,
ApplicationTokenIdentifier.KIND_NAME, appTokenService); AMRMTokenIdentifier.KIND_NAME, appTokenService);
credentials.addToken(appTokenService, appToken); credentials.addToken(appTokenService, appToken);
Text keyAlias = new Text("mySecretKeyAlias"); Text keyAlias = new Text("mySecretKeyAlias");

8
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
View File

@ -61,7 +61,7 @@ import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@ -104,7 +104,7 @@ public class TestPipeApplication {
conf.set(MRJobConfig.CACHE_LOCALFILES, fCommand.getAbsolutePath()); conf.set(MRJobConfig.CACHE_LOCALFILES, fCommand.getAbsolutePath());
// token for authorization // token for authorization
Token<ApplicationTokenIdentifier> token = new Token<ApplicationTokenIdentifier>( Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>(
"user".getBytes(), "password".getBytes(), new Text("kind"), new Text( "user".getBytes(), "password".getBytes(), new Text("kind"), new Text(
"service")); "service"));
TokenCache.setJobToken(token, conf.getCredentials()); TokenCache.setJobToken(token, conf.getCredentials());
@ -168,7 +168,7 @@ public class TestPipeApplication {
conf.set(MRJobConfig.CACHE_LOCALFILES, fCommand.getAbsolutePath()); conf.set(MRJobConfig.CACHE_LOCALFILES, fCommand.getAbsolutePath());
// token for authorization // token for authorization
Token<ApplicationTokenIdentifier> token = new Token<ApplicationTokenIdentifier>( Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>(
"user".getBytes(), "password".getBytes(), new Text("kind"), new Text( "user".getBytes(), "password".getBytes(), new Text("kind"), new Text(
"service")); "service"));
@ -389,7 +389,7 @@ public class TestPipeApplication {
File[] psw = cleanTokenPasswordFile(); File[] psw = cleanTokenPasswordFile();
JobConf conf = new JobConf(); JobConf conf = new JobConf();
try { try {
Token<ApplicationTokenIdentifier> token = new Token<ApplicationTokenIdentifier>( Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>(
"user".getBytes(), "password".getBytes(), new Text("kind"), new Text( "user".getBytes(), "password".getBytes(), new Text("kind"), new Text(
"service")); "service"));
TokenCache.setJobToken(token, conf.getCredentials()); TokenCache.setJobToken(token, conf.getCredentials());

3
hadoop-yarn-project/CHANGES.txt
View File

@ -180,6 +180,9 @@ Release 2.1.0-beta - UNRELEASED
YARN-837. Moved yarn.ClusterInfo into MapReduce project as it doesn't belong YARN-837. Moved yarn.ClusterInfo into MapReduce project as it doesn't belong
to YARN. (Zhijie Shen via vinodkv) to YARN. (Zhijie Shen via vinodkv)
YARN-822. Renamed ApplicationToken to be AMRMToken, and similarly the
corresponding TokenSelector and SecretManager. (Omkar Vinit Joshi via vinodkv)
NEW FEATURES NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues. YARN-482. FS: Extend SchedulingMode to intermediate queues.

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -274,10 +274,10 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS =
"60,300,1440"; "60,300,1440";
public static final String RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX public static final String RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX
+ "application-tokens.master-key-rolling-interval-secs"; + "am-rm-tokens.master-key-rolling-interval-secs";
public static final long DEFAULT_RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = public static final long DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
24 * 60 * 60; 24 * 60 * 60;
public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java
View File

@ -34,21 +34,21 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
/** /**
* ApplicationTokenIdentifier is the TokenIdentifier to be used by * AMRMTokenIdentifier is the TokenIdentifier to be used by
* ApplicationMasters to authenticate to the ResourceManager. * ApplicationMasters to authenticate to the ResourceManager.
*/ */
@Public @Public
@Evolving @Evolving
public class ApplicationTokenIdentifier extends TokenIdentifier { public class AMRMTokenIdentifier extends TokenIdentifier {
public static final Text KIND_NAME = new Text("YARN_APPLICATION_TOKEN"); public static final Text KIND_NAME = new Text("YARN_AM_RM_TOKEN");
private ApplicationAttemptId applicationAttemptId; private ApplicationAttemptId applicationAttemptId;
public ApplicationTokenIdentifier() { public AMRMTokenIdentifier() {
} }
public ApplicationTokenIdentifier(ApplicationAttemptId appAttemptId) { public AMRMTokenIdentifier(ApplicationAttemptId appAttemptId) {
this(); this();
this.applicationAttemptId = appAttemptId; this.applicationAttemptId = appAttemptId;
} }

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenSelector.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenSelector.java
View File

@ -31,14 +31,14 @@ import org.apache.hadoop.security.token.TokenSelector;
@Public @Public
@Evolving @Evolving
public class ApplicationTokenSelector implements public class AMRMTokenSelector implements
TokenSelector<ApplicationTokenIdentifier> { TokenSelector<AMRMTokenIdentifier> {
private static final Log LOG = LogFactory private static final Log LOG = LogFactory
.getLog(ApplicationTokenSelector.class); .getLog(AMRMTokenSelector.class);
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Token<ApplicationTokenIdentifier> selectToken(Text service, public Token<AMRMTokenIdentifier> selectToken(Text service,
Collection<Token<? extends TokenIdentifier>> tokens) { Collection<Token<? extends TokenIdentifier>> tokens) {
if (service == null) { if (service == null) {
return null; return null;
@ -47,9 +47,9 @@ public class ApplicationTokenSelector implements
for (Token<? extends TokenIdentifier> token : tokens) { for (Token<? extends TokenIdentifier> token : tokens) {
LOG.debug("Token kind is " + token.getKind().toString() LOG.debug("Token kind is " + token.getKind().toString()
+ " and the token's service name is " + token.getService()); + " and the token's service name is " + token.getService());
if (ApplicationTokenIdentifier.KIND_NAME.equals(token.getKind()) if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())
&& service.equals(token.getService())) { && service.equals(token.getService())) {
return (Token<ApplicationTokenIdentifier>) token; return (Token<AMRMTokenIdentifier>) token;
} }
} }
return null; return null;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java
View File

@ -54,7 +54,7 @@ public class SchedulerSecurityInfo extends SecurityInfo {
@Override @Override
public Class<? extends TokenSelector<? extends TokenIdentifier>> public Class<? extends TokenSelector<? extends TokenIdentifier>>
value() { value() {
return ApplicationTokenSelector.class; return AMRMTokenSelector.class;
} }
}; };
} }

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
View File

@ -12,6 +12,6 @@
# limitations under the License. # limitations under the License.
# #
org.apache.hadoop.yarn.security.ContainerTokenIdentifier org.apache.hadoop.yarn.security.ContainerTokenIdentifier
org.apache.hadoop.yarn.security.ApplicationTokenIdentifier org.apache.hadoop.yarn.security.AMRMTokenIdentifier
org.apache.hadoop.yarn.security.client.ClientTokenIdentifier org.apache.hadoop.yarn.security.client.ClientTokenIdentifier
org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
View File

@ -11,6 +11,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
org.apache.hadoop.yarn.security.ApplicationTokenIdentifier$Renewer org.apache.hadoop.yarn.security.AMRMTokenIdentifier$Renewer
org.apache.hadoop.yarn.security.ContainerTokenIdentifier$Renewer org.apache.hadoop.yarn.security.ContainerTokenIdentifier$Renewer
org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier$Renewer org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier$Renewer

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
View File

@ -118,7 +118,7 @@ public class ApplicationMasterService extends AbstractService implements
this.server = this.server =
rpc.getServer(ApplicationMasterProtocol.class, this, masterServiceAddress, rpc.getServer(ApplicationMasterProtocol.class, this, masterServiceAddress,
conf, this.rmContext.getApplicationTokenSecretManager(), conf, this.rmContext.getAMRMTokenSecretManager(),
conf.getInt(YarnConfiguration.RM_SCHEDULER_CLIENT_THREAD_COUNT, conf.getInt(YarnConfiguration.RM_SCHEDULER_CLIENT_THREAD_COUNT,
YarnConfiguration.DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT)); YarnConfiguration.DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT));

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
View File

@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
@ -57,7 +57,7 @@ public interface RMContext {
DelegationTokenRenewer getDelegationTokenRenewer(); DelegationTokenRenewer getDelegationTokenRenewer();
ApplicationTokenSecretManager getApplicationTokenSecretManager(); AMRMTokenSecretManager getAMRMTokenSecretManager();
RMContainerTokenSecretManager getContainerTokenSecretManager(); RMContainerTokenSecretManager getContainerTokenSecretManager();

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
View File

@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
@ -57,7 +57,7 @@ public class RMContextImpl implements RMContext {
private RMStateStore stateStore = null; private RMStateStore stateStore = null;
private ContainerAllocationExpirer containerAllocationExpirer; private ContainerAllocationExpirer containerAllocationExpirer;
private final DelegationTokenRenewer tokenRenewer; private final DelegationTokenRenewer tokenRenewer;
private final ApplicationTokenSecretManager appTokenSecretManager; private final AMRMTokenSecretManager appTokenSecretManager;
private final RMContainerTokenSecretManager containerTokenSecretManager; private final RMContainerTokenSecretManager containerTokenSecretManager;
private final NMTokenSecretManagerInRM nmTokenSecretManager; private final NMTokenSecretManagerInRM nmTokenSecretManager;
private final ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager; private final ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager;
@ -68,7 +68,7 @@ public class RMContextImpl implements RMContext {
AMLivelinessMonitor amLivelinessMonitor, AMLivelinessMonitor amLivelinessMonitor,
AMLivelinessMonitor amFinishingMonitor, AMLivelinessMonitor amFinishingMonitor,
DelegationTokenRenewer tokenRenewer, DelegationTokenRenewer tokenRenewer,
ApplicationTokenSecretManager appTokenSecretManager, AMRMTokenSecretManager appTokenSecretManager,
RMContainerTokenSecretManager containerTokenSecretManager, RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager, NMTokenSecretManagerInRM nmTokenSecretManager,
ClientToAMTokenSecretManagerInRM clientTokenSecretManager) { ClientToAMTokenSecretManagerInRM clientTokenSecretManager) {
@ -91,7 +91,7 @@ public class RMContextImpl implements RMContext {
AMLivelinessMonitor amLivelinessMonitor, AMLivelinessMonitor amLivelinessMonitor,
AMLivelinessMonitor amFinishingMonitor, AMLivelinessMonitor amFinishingMonitor,
DelegationTokenRenewer tokenRenewer, DelegationTokenRenewer tokenRenewer,
ApplicationTokenSecretManager appTokenSecretManager, AMRMTokenSecretManager appTokenSecretManager,
RMContainerTokenSecretManager containerTokenSecretManager, RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager, NMTokenSecretManagerInRM nmTokenSecretManager,
ClientToAMTokenSecretManagerInRM clientTokenSecretManager) { ClientToAMTokenSecretManagerInRM clientTokenSecretManager) {
@ -155,7 +155,7 @@ public class RMContextImpl implements RMContext {
} }
@Override @Override
public ApplicationTokenSecretManager getApplicationTokenSecretManager() { public AMRMTokenSecretManager getAMRMTokenSecretManager() {
return this.appTokenSecretManager; return this.appTokenSecretManager;
} }

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
View File

@ -67,7 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
@ -87,7 +87,7 @@ import com.google.common.annotations.VisibleForTesting;
/** /**
* The ResourceManager is the main class that is a set of components. * The ResourceManager is the main class that is a set of components.
* "I am the ResourceManager. All your resources are belong to us..." * "I am the ResourceManager. All your resources belong to us..."
* *
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@ -107,7 +107,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
protected RMContainerTokenSecretManager containerTokenSecretManager; protected RMContainerTokenSecretManager containerTokenSecretManager;
protected NMTokenSecretManagerInRM nmTokenSecretManager; protected NMTokenSecretManagerInRM nmTokenSecretManager;
protected ApplicationTokenSecretManager appTokenSecretManager; protected AMRMTokenSecretManager amRmTokenSecretManager;
private Dispatcher rmDispatcher; private Dispatcher rmDispatcher;
@ -150,7 +150,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
this.rmDispatcher = createDispatcher(); this.rmDispatcher = createDispatcher();
addIfService(this.rmDispatcher); addIfService(this.rmDispatcher);
this.appTokenSecretManager = createApplicationTokenSecretManager(conf); this.amRmTokenSecretManager = createAMRMTokenSecretManager(conf);
this.containerAllocationExpirer = new ContainerAllocationExpirer( this.containerAllocationExpirer = new ContainerAllocationExpirer(
this.rmDispatcher); this.rmDispatcher);
@ -193,7 +193,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
this.rmContext = this.rmContext =
new RMContextImpl(this.rmDispatcher, rmStore, new RMContextImpl(this.rmDispatcher, rmStore,
this.containerAllocationExpirer, amLivelinessMonitor, this.containerAllocationExpirer, amLivelinessMonitor,
amFinishingMonitor, tokenRenewer, this.appTokenSecretManager, amFinishingMonitor, tokenRenewer, this.amRmTokenSecretManager,
this.containerTokenSecretManager, this.nmTokenSecretManager, this.containerTokenSecretManager, this.nmTokenSecretManager,
this.clientToAMSecretManager); this.clientToAMSecretManager);
@ -294,9 +294,9 @@ public class ResourceManager extends CompositeService implements Recoverable {
} }
} }
protected ApplicationTokenSecretManager createApplicationTokenSecretManager( protected AMRMTokenSecretManager createAMRMTokenSecretManager(
Configuration conf) { Configuration conf) {
return new ApplicationTokenSecretManager(conf); return new AMRMTokenSecretManager(conf);
} }
protected ResourceScheduler createScheduler() { protected ResourceScheduler createScheduler() {
@ -557,7 +557,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
throw new YarnRuntimeException("Failed to login", ie); throw new YarnRuntimeException("Failed to login", ie);
} }
this.appTokenSecretManager.start(); this.amRmTokenSecretManager.start();
this.containerTokenSecretManager.start(); this.containerTokenSecretManager.start();
this.nmTokenSecretManager.start(); this.nmTokenSecretManager.start();
@ -617,8 +617,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
rmDTSecretManager.stopThreads(); rmDTSecretManager.stopThreads();
} }
if (appTokenSecretManager != null) { if (amRmTokenSecretManager != null) {
this.appTokenSecretManager.stop(); this.amRmTokenSecretManager.stop();
} }
if (containerTokenSecretManager != null) { if (containerTokenSecretManager != null) {
this.containerTokenSecretManager.stop(); this.containerTokenSecretManager.stop();
@ -731,8 +731,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
} }
@Private @Private
public ApplicationTokenSecretManager getApplicationTokenSecretManager(){ public AMRMTokenSecretManager getAMRMTokenSecretManager(){
return this.appTokenSecretManager; return this.amRmTokenSecretManager;
} }
@Override @Override

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
View File

@ -52,7 +52,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@ -201,10 +201,10 @@ public class AMLauncher implements Runnable {
} }
// Add application token // Add application token
Token<ApplicationTokenIdentifier> applicationToken = Token<AMRMTokenIdentifier> amrmToken =
application.getApplicationToken(); application.getAMRMToken();
if(applicationToken != null) { if(amrmToken != null) {
credentials.addToken(applicationToken.getService(), applicationToken); credentials.addToken(amrmToken.getService(), amrmToken);
} }
DataOutputBuffer dob = new DataOutputBuffer(); DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob); credentials.writeTokenStorageToStream(dob);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
View File

@ -41,7 +41,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPB
import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
@ -378,7 +378,7 @@ public abstract class RMStateStore {
private Credentials getTokensFromAppAttempt(RMAppAttempt appAttempt) { private Credentials getTokensFromAppAttempt(RMAppAttempt appAttempt) {
Credentials credentials = new Credentials(); Credentials credentials = new Credentials();
Token<ApplicationTokenIdentifier> appToken = appAttempt.getApplicationToken(); Token<AMRMTokenIdentifier> appToken = appAttempt.getAMRMToken();
if(appToken != null){ if(appToken != null){
credentials.addToken(appToken.getService(), appToken); credentials.addToken(appToken.getService(), appToken);
} }

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
View File

@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@ -149,10 +149,10 @@ public interface RMAppAttempt extends EventHandler<RMAppAttemptEvent> {
ApplicationSubmissionContext getSubmissionContext(); ApplicationSubmissionContext getSubmissionContext();
/** /**
* The application token belonging to this app attempt * The AMRMToken belonging to this app attempt
* @return The application token belonging to this app attempt * @return The AMRMToken belonging to this app attempt
*/ */
Token<ApplicationTokenIdentifier> getApplicationToken(); Token<AMRMTokenIdentifier> getAMRMToken();
/** /**
* Get application container and resource usage information. * Get application container and resource usage information.

38
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -61,8 +61,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.ApplicationTokenSelector; import org.apache.hadoop.yarn.security.AMRMTokenSelector;
import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientTokenSelector; import org.apache.hadoop.yarn.security.client.ClientTokenSelector;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
@ -131,7 +131,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
private final ApplicationAttemptId applicationAttemptId; private final ApplicationAttemptId applicationAttemptId;
private Token<ClientTokenIdentifier> clientToken; private Token<ClientTokenIdentifier> clientToken;
private final ApplicationSubmissionContext submissionContext; private final ApplicationSubmissionContext submissionContext;
private Token<ApplicationTokenIdentifier> applicationToken = null; private Token<AMRMTokenIdentifier> amrmToken = null;
//nodes on while this attempt's containers ran //nodes on while this attempt's containers ran
private final Set<NodeId> ranNodes = private final Set<NodeId> ranNodes =
@ -503,8 +503,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
} }
@Override @Override
public Token<ApplicationTokenIdentifier> getApplicationToken() { public Token<AMRMTokenIdentifier> getAMRMToken() {
return this.applicationToken; return this.amrmToken;
} }
@Override @Override
@ -682,14 +682,14 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
ApplicationTokenSelector appTokenSelector = new ApplicationTokenSelector(); AMRMTokenSelector appTokenSelector = new AMRMTokenSelector();
this.applicationToken = this.amrmToken =
appTokenSelector.selectToken( appTokenSelector.selectToken(
SecurityUtil.buildTokenService(serviceAddr), SecurityUtil.buildTokenService(serviceAddr),
appAttemptTokens.getAllTokens()); appAttemptTokens.getAllTokens());
// For now, no need to populate tokens back to // For now, no need to populate tokens back to
// ApplicationTokenSecretManager, because running attempts are rebooted // AMRMTokenSecretManager, because running attempts are rebooted
// Later in work-preserve restart, we'll create NEW->RUNNING transition // Later in work-preserve restart, we'll create NEW->RUNNING transition
// in which the restored tokens will be added to the secret manager // in which the restored tokens will be added to the secret manager
} }
@ -727,11 +727,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
appAttempt.rmContext.getClientToAMTokenSecretManager()); appAttempt.rmContext.getClientToAMTokenSecretManager());
// create application token // create application token
ApplicationTokenIdentifier id = AMRMTokenIdentifier id =
new ApplicationTokenIdentifier(appAttempt.applicationAttemptId); new AMRMTokenIdentifier(appAttempt.applicationAttemptId);
Token<ApplicationTokenIdentifier> applicationToken = Token<AMRMTokenIdentifier> amRmToken =
new Token<ApplicationTokenIdentifier>(id, new Token<AMRMTokenIdentifier>(id,
appAttempt.rmContext.getApplicationTokenSecretManager()); appAttempt.rmContext.getAMRMTokenSecretManager());
InetSocketAddress serviceAddr = InetSocketAddress serviceAddr =
appAttempt.conf.getSocketAddr( appAttempt.conf.getSocketAddr(
YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.RM_SCHEDULER_ADDRESS,
@ -739,9 +739,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
// normally the client should set the service after acquiring the // normally the client should set the service after acquiring the
// token, but this token is directly provided to the AMs // token, but this token is directly provided to the AMs
SecurityUtil.setTokenService(applicationToken, serviceAddr); SecurityUtil.setTokenService(amRmToken, serviceAddr);
appAttempt.applicationToken = applicationToken; appAttempt.amrmToken = amRmToken;
} }
@ -902,8 +902,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
appAttempt.eventHandler.handle(new AppRemovedSchedulerEvent(appAttemptId, appAttempt.eventHandler.handle(new AppRemovedSchedulerEvent(appAttemptId,
finalAttemptState)); finalAttemptState));
// Remove the AppAttempt from the ApplicationTokenSecretManager // Remove the AppAttempt from the AMRMTokenSecretManager
appAttempt.rmContext.getApplicationTokenSecretManager() appAttempt.rmContext.getAMRMTokenSecretManager()
.applicationMasterFinished(appAttemptId); .applicationMasterFinished(appAttemptId);
} }
} }
@ -1125,8 +1125,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
appAttempt.rmContext.getAMLivelinessMonitor().unregister(appAttemptId); appAttempt.rmContext.getAMLivelinessMonitor().unregister(appAttemptId);
// Remove the AppAttempt from the ApplicationTokenSecretManager // Remove the AppAttempt from the AMRMTokenSecretManager
appAttempt.rmContext.getApplicationTokenSecretManager() appAttempt.rmContext.getAMRMTokenSecretManager()
.applicationMasterFinished(appAttemptId); .applicationMasterFinished(appAttemptId);
appAttempt.progress = 1.0f; appAttempt.progress = 1.0f;

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ApplicationTokenSecretManager.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
View File

@ -32,20 +32,20 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
/** /**
* Application-tokens are per ApplicationAttempt. If users redistribute their * AMRM-tokens are per ApplicationAttempt. If users redistribute their
* tokens, it is their headache, god save them. I mean you are not supposed to * tokens, it is their headache, god save them. I mean you are not supposed to
* distribute keys to your vault, right? Anyways, ResourceManager saves each * distribute keys to your vault, right? Anyways, ResourceManager saves each
* token locally in memory till application finishes and to a store for restart, * token locally in memory till application finishes and to a store for restart,
* so no need to remember master-keys even after rolling them. * so no need to remember master-keys even after rolling them.
*/ */
public class ApplicationTokenSecretManager extends public class AMRMTokenSecretManager extends
SecretManager<ApplicationTokenIdentifier> { SecretManager<AMRMTokenIdentifier> {
private static final Log LOG = LogFactory private static final Log LOG = LogFactory
.getLog(ApplicationTokenSecretManager.class); .getLog(AMRMTokenSecretManager.class);
private SecretKey masterKey; private SecretKey masterKey;
private final Timer timer; private final Timer timer;
@ -55,16 +55,16 @@ public class ApplicationTokenSecretManager extends
new HashMap<ApplicationAttemptId, byte[]>(); new HashMap<ApplicationAttemptId, byte[]>();
/** /**
* Create an {@link ApplicationTokenSecretManager} * Create an {@link AMRMTokenSecretManager}
*/ */
public ApplicationTokenSecretManager(Configuration conf) { public AMRMTokenSecretManager(Configuration conf) {
rollMasterKey(); rollMasterKey();
this.timer = new Timer(); this.timer = new Timer();
this.rollingInterval = this.rollingInterval =
conf conf
.getLong( .getLong(
YarnConfiguration.RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
YarnConfiguration.DEFAULT_RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS) * 1000; YarnConfiguration.DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS) * 1000;
} }
public void start() { public void start() {
@ -102,17 +102,17 @@ public class ApplicationTokenSecretManager extends
@Private @Private
synchronized void rollMasterKey() { synchronized void rollMasterKey() {
LOG.info("Rolling master-key for application-tokens"); LOG.info("Rolling master-key for amrm-tokens");
this.masterKey = generateSecret(); this.masterKey = generateSecret();
} }
/** /**
* Create a password for a given {@link ApplicationTokenIdentifier}. Used to * Create a password for a given {@link AMRMTokenIdentifier}. Used to
* send to the AppicationAttempt which can give it back during authentication. * send to the AppicationAttempt which can give it back during authentication.
*/ */
@Override @Override
public synchronized byte[] createPassword( public synchronized byte[] createPassword(
ApplicationTokenIdentifier identifier) { AMRMTokenIdentifier identifier) {
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId applicationAttemptId =
identifier.getApplicationAttemptId(); identifier.getApplicationAttemptId();
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
@ -124,12 +124,12 @@ public class ApplicationTokenSecretManager extends
} }
/** /**
* Retrieve the password for the given {@link ApplicationTokenIdentifier}. * Retrieve the password for the given {@link AMRMTokenIdentifier}.
* Used by RPC layer to validate a remote {@link ApplicationTokenIdentifier}. * Used by RPC layer to validate a remote {@link AMRMTokenIdentifier}.
*/ */
@Override @Override
public synchronized byte[] retrievePassword( public synchronized byte[] retrievePassword(
ApplicationTokenIdentifier identifier) throws InvalidToken { AMRMTokenIdentifier identifier) throws InvalidToken {
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId applicationAttemptId =
identifier.getApplicationAttemptId(); identifier.getApplicationAttemptId();
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
@ -145,11 +145,11 @@ public class ApplicationTokenSecretManager extends
/** /**
* Creates an empty TokenId to be used for de-serializing an * Creates an empty TokenId to be used for de-serializing an
* {@link ApplicationTokenIdentifier} by the RPC layer. * {@link AMRMTokenIdentifier} by the RPC layer.
*/ */
@Override @Override
public ApplicationTokenIdentifier createIdentifier() { public AMRMTokenIdentifier createIdentifier() {
return new ApplicationTokenIdentifier(); return new AMRMTokenIdentifier();
} }
} }

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
View File

@ -536,7 +536,7 @@ public class TestRMRestart {
// the appToken and clientToken that are generated when RMAppAttempt is created, // the appToken and clientToken that are generated when RMAppAttempt is created,
HashSet<Token<?>> tokenSet = new HashSet<Token<?>>(); HashSet<Token<?>> tokenSet = new HashSet<Token<?>>();
tokenSet.add(attempt1.getApplicationToken()); tokenSet.add(attempt1.getAMRMToken());
tokenSet.add(attempt1.getClientToken()); tokenSet.add(attempt1.getClientToken());
// assert application Token is saved // assert application Token is saved
@ -555,7 +555,7 @@ public class TestRMRestart {
// assert loaded attempt recovered attempt tokens // assert loaded attempt recovered attempt tokens
Assert.assertNotNull(loadedAttempt1); Assert.assertNotNull(loadedAttempt1);
savedTokens.clear(); savedTokens.clear();
savedTokens.add(loadedAttempt1.getApplicationToken()); savedTokens.add(loadedAttempt1.getAMRMToken());
savedTokens.add(loadedAttempt1.getClientToken()); savedTokens.add(loadedAttempt1.getClientToken());
Assert.assertEquals(tokenSet, savedTokens); Assert.assertEquals(tokenSet, savedTokens);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
View File

@ -51,7 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventT
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
View File

@ -57,7 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
View File

@ -54,7 +54,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState;
@ -64,7 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMSta
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStoredEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStoredEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Test; import org.junit.Test;
@ -205,7 +205,7 @@ public class TestRMStateStore {
} }
ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId, ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId,
String containerIdStr, Token<ApplicationTokenIdentifier> appToken, String containerIdStr, Token<AMRMTokenIdentifier> appToken,
Token<ClientTokenIdentifier> clientToken, TestDispatcher dispatcher) Token<ClientTokenIdentifier> clientToken, TestDispatcher dispatcher)
throws Exception { throws Exception {
@ -214,7 +214,7 @@ public class TestRMStateStore {
RMAppAttempt mockAttempt = mock(RMAppAttempt.class); RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId); when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
when(mockAttempt.getMasterContainer()).thenReturn(container); when(mockAttempt.getMasterContainer()).thenReturn(container);
when(mockAttempt.getApplicationToken()).thenReturn(appToken); when(mockAttempt.getAMRMToken()).thenReturn(appToken);
when(mockAttempt.getClientToken()).thenReturn(clientToken); when(mockAttempt.getClientToken()).thenReturn(clientToken);
dispatcher.attemptId = attemptId; dispatcher.attemptId = attemptId;
dispatcher.storedException = null; dispatcher.storedException = null;
@ -231,8 +231,8 @@ public class TestRMStateStore {
TestDispatcher dispatcher = new TestDispatcher(); TestDispatcher dispatcher = new TestDispatcher();
store.setDispatcher(dispatcher); store.setDispatcher(dispatcher);
ApplicationTokenSecretManager appTokenMgr = AMRMTokenSecretManager appTokenMgr =
new ApplicationTokenSecretManager(conf); new AMRMTokenSecretManager(conf);
ClientToAMTokenSecretManagerInRM clientTokenMgr = ClientToAMTokenSecretManagerInRM clientTokenMgr =
new ClientToAMTokenSecretManagerInRM(); new ClientToAMTokenSecretManagerInRM();
@ -249,7 +249,7 @@ public class TestRMStateStore {
ContainerId containerId1 = storeAttempt(store, attemptId1, ContainerId containerId1 = storeAttempt(store, attemptId1,
"container_1352994193343_0001_01_000001", "container_1352994193343_0001_01_000001",
(Token<ApplicationTokenIdentifier>) (appAttemptToken1.get(0)), (Token<AMRMTokenIdentifier>) (appAttemptToken1.get(0)),
(Token<ClientTokenIdentifier>)(appAttemptToken1.get(1)), (Token<ClientTokenIdentifier>)(appAttemptToken1.get(1)),
dispatcher); dispatcher);
@ -265,7 +265,7 @@ public class TestRMStateStore {
ContainerId containerId2 = storeAttempt(store, attemptId2, ContainerId containerId2 = storeAttempt(store, attemptId2,
"container_1352994193343_0001_02_000001", "container_1352994193343_0001_02_000001",
(Token<ApplicationTokenIdentifier>) (appAttemptToken2.get(0)), (Token<AMRMTokenIdentifier>) (appAttemptToken2.get(0)),
(Token<ClientTokenIdentifier>)(appAttemptToken2.get(1)), (Token<ClientTokenIdentifier>)(appAttemptToken2.get(1)),
dispatcher); dispatcher);
@ -372,12 +372,12 @@ public class TestRMStateStore {
} }
private List<Token<?>> generateTokens(ApplicationAttemptId attemptId, private List<Token<?>> generateTokens(ApplicationAttemptId attemptId,
ApplicationTokenSecretManager appTokenMgr, AMRMTokenSecretManager appTokenMgr,
ClientToAMTokenSecretManagerInRM clientTokenMgr, Configuration conf) { ClientToAMTokenSecretManagerInRM clientTokenMgr, Configuration conf) {
ApplicationTokenIdentifier appTokenId = AMRMTokenIdentifier appTokenId =
new ApplicationTokenIdentifier(attemptId); new AMRMTokenIdentifier(attemptId);
Token<ApplicationTokenIdentifier> appToken = Token<AMRMTokenIdentifier> appToken =
new Token<ApplicationTokenIdentifier>(appTokenId, appTokenMgr); new Token<AMRMTokenIdentifier>(appTokenId, appTokenMgr);
appToken.setService(new Text("appToken service")); appToken.setService(new Text("appToken service"));
ClientTokenIdentifier clientTokenId = new ClientTokenIdentifier(attemptId); ClientTokenIdentifier clientTokenId = new ClientTokenIdentifier(attemptId);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
View File

@ -51,7 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
@ -145,7 +145,7 @@ public class TestRMAppTransitions {
this.rmContext = this.rmContext =
new RMContextImpl(rmDispatcher, store, new RMContextImpl(rmDispatcher, store,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
null, new ApplicationTokenSecretManager(conf), null, new AMRMTokenSecretManager(conf),
new RMContainerTokenSecretManager(conf), new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf), new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM()); new ClientToAMTokenSecretManagerInRM());

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
View File

@ -75,7 +75,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@ -167,7 +167,7 @@ public class TestRMAppAttemptTransitions {
rmContext = rmContext =
new RMContextImpl(rmDispatcher, new RMContextImpl(rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
null, new ApplicationTokenSecretManager(conf), null, new AMRMTokenSecretManager(conf),
new RMContainerTokenSecretManager(conf), new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf), new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM()); new ClientToAMTokenSecretManagerInRM());

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
View File

@ -46,7 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@ -86,7 +86,7 @@ public class TestUtils {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
RMContext rmContext = RMContext rmContext =
new RMContextImpl(nullDispatcher, cae, null, null, null, new RMContextImpl(nullDispatcher, cae, null, null, null,
new ApplicationTokenSecretManager(conf), new AMRMTokenSecretManager(conf),
new RMContainerTokenSecretManager(conf), new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf), new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM()); new ClientToAMTokenSecretManagerInRM());

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
View File

@ -47,9 +47,9 @@ import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
public class TestApplicationTokens { public class TestAMRMTokens {
private static final Log LOG = LogFactory.getLog(TestApplicationTokens.class); private static final Log LOG = LogFactory.getLog(TestAMRMTokens.class);
private static final Configuration confWithSecurityEnabled = private static final Configuration confWithSecurityEnabled =
new Configuration(); new Configuration();
@ -212,8 +212,8 @@ public class TestApplicationTokens {
rmClient.allocate(allocateRequest).getAMCommand() == null); rmClient.allocate(allocateRequest).getAMCommand() == null);
// Simulate a master-key-roll-over // Simulate a master-key-roll-over
ApplicationTokenSecretManager appTokenSecretManager = AMRMTokenSecretManager appTokenSecretManager =
rm.getRMContext().getApplicationTokenSecretManager(); rm.getRMContext().getAMRMTokenSecretManager();
SecretKey oldKey = appTokenSecretManager.getMasterKey(); SecretKey oldKey = appTokenSecretManager.getMasterKey();
appTokenSecretManager.rollMasterKey(); appTokenSecretManager.rollMasterKey();
SecretKey newKey = appTokenSecretManager.getMasterKey(); SecretKey newKey = appTokenSecretManager.getMasterKey();

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
View File

@ -67,13 +67,13 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
@ -459,14 +459,14 @@ public class TestContainerManagerSecurity {
final InetSocketAddress schedulerAddr = final InetSocketAddress schedulerAddr =
resourceManager.getApplicationMasterService().getBindAddress(); resourceManager.getApplicationMasterService().getBindAddress();
if (UserGroupInformation.isSecurityEnabled()) { if (UserGroupInformation.isSecurityEnabled()) {
ApplicationTokenIdentifier appTokenIdentifier = new ApplicationTokenIdentifier( AMRMTokenIdentifier appTokenIdentifier = new AMRMTokenIdentifier(
appAttempt.getAppAttemptId()); appAttempt.getAppAttemptId());
ApplicationTokenSecretManager appTokenSecretManager = AMRMTokenSecretManager appTokenSecretManager =
new ApplicationTokenSecretManager(conf); new AMRMTokenSecretManager(conf);
appTokenSecretManager.setMasterKey(resourceManager appTokenSecretManager.setMasterKey(resourceManager
.getApplicationTokenSecretManager().getMasterKey()); .getAMRMTokenSecretManager().getMasterKey());
Token<ApplicationTokenIdentifier> appToken = Token<AMRMTokenIdentifier> appToken =
new Token<ApplicationTokenIdentifier>(appTokenIdentifier, new Token<AMRMTokenIdentifier>(appTokenIdentifier,
appTokenSecretManager); appTokenSecretManager);
SecurityUtil.setTokenService(appToken, schedulerAddr); SecurityUtil.setTokenService(appToken, schedulerAddr);
currentUser.addToken(appToken); currentUser.addToken(appToken);