Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-1536. Cleanup: Get rid of ResourceManager#get*SecretManager() methods and use the RMContext methods instead. (Anubhav Dhoot via kasha) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1580129 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Karthik Kambatla 2014-03-22 02:38:23 +00:00
parent 751bba3d14
commit 1bb24f2c18
17 changed files with 61 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -315,6 +315,9 @@ Release 2.4.0 - UNRELEASED
YARN-1570. Fixed formatting of the lines in YarnCommands.apt.vm docs source.
(Akira Ajisaka via vinodkv)
YARN-1536. Cleanup: Get rid of ResourceManager#get*SecretManager() methods
and use the RMContext methods instead. (Anubhav Dhoot via kasha)
OPTIMIZATIONS
BUG FIXES

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
View File

@ -86,12 +86,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher;
@ -960,7 +956,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
this.applicationACLsManager, this.queueACLsManager,
getRMDTSecretManager());
this.rmContext.getRMDelegationTokenSecretManager());
}
protected ApplicationMasterService createApplicationMasterService() {
@ -1013,30 +1009,10 @@ public class ResourceManager extends CompositeService implements Recoverable {
return this.queueACLsManager;
}
@Private
public RMContainerTokenSecretManager getRMContainerTokenSecretManager() {
return this.rmContext.getContainerTokenSecretManager();
}
@Private
public NMTokenSecretManagerInRM getRMNMTokenSecretManager() {
return this.rmContext.getNMTokenSecretManager();
}
@Private
public AMRMTokenSecretManager getAMRMTokenSecretManager(){
return this.rmContext.getAMRMTokenSecretManager();
}
@Private
public RMDelegationTokenSecretManager getRMDTSecretManager(){
return this.rmContext.getRMDelegationTokenSecretManager();
}
@Override
public void recover(RMState state) throws Exception {
// recover RMdelegationTokenSecretManager
getRMDTSecretManager().recover(state);
rmContext.getRMDelegationTokenSecretManager().recover(state);
// recover applications
rmAppManager.recover(state);

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
View File

@ -386,7 +386,7 @@ public class MockRM extends ResourceManager {
protected ClientRMService createClientRMService() {
return new ClientRMService(getRMContext(), getResourceScheduler(),
rmAppManager, applicationACLsManager, queueACLsManager,
getRMDTSecretManager()) {
getRMContext().getRMDelegationTokenSecretManager()) {
@Override
protected void serviceStart() {
// override to not start rpc handler
@ -404,10 +404,10 @@ public class MockRM extends ResourceManager {
Configuration conf = new Configuration();
RMContainerTokenSecretManager containerTokenSecretManager =
getRMContainerTokenSecretManager();
getRMContext().getContainerTokenSecretManager();
containerTokenSecretManager.rollMasterKey();
NMTokenSecretManagerInRM nmTokenSecretManager =
getRMNMTokenSecretManager();
getRMContext().getNMTokenSecretManager();
nmTokenSecretManager.rollMasterKey();
return new ResourceTrackerService(getRMContext(), nodesListManager,
this.nmLivelinessMonitor, containerTokenSecretManager,

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java
View File

@ -86,7 +86,7 @@ public abstract class QueueACLsTestBase {
protected ClientRMService createClientRMService() {
return new ClientRMService(getRMContext(), this.scheduler,
this.rmAppManager, this.applicationACLsManager,
this.queueACLsManager, getRMDTSecretManager());
this.queueACLsManager, getRMContext().getRMDelegationTokenSecretManager());
};
@Override

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -159,7 +159,7 @@ public class TestClientRMService {
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler,
this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
this.getRMDTSecretManager());
this.getRMContext().getRMDelegationTokenSecretManager());
};
};
rm.start();

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestKillApplicationWithRMHA.java
View File

@ -195,7 +195,7 @@ public class TestKillApplicationWithRMHA extends RMHATestBase{
protected ClientRMService createClientRMService() {
return new MyClientRMService(this.rmContext, this.scheduler,
this.rmAppManager, this.applicationACLsManager,
this.queueACLsManager, getRMDTSecretManager());
this.queueACLsManager, getRMContext().getRMDelegationTokenSecretManager());
}
};

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java
View File

@ -53,8 +53,8 @@ public class TestMoveApplication {
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
resourceManager = new ResourceManager();
resourceManager.init(conf);
resourceManager.getRMContainerTokenSecretManager().rollMasterKey();
resourceManager.getRMNMTokenSecretManager().rollMasterKey();
resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
resourceManager.start();
failMove = false;
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
View File

@ -198,7 +198,7 @@ public class TestRM {
// am container Id not equal to 1.
Assert.assertTrue(attempt.getMasterContainer().getId().getId() != 1);
// NMSecretManager doesn't record the node on which the am is allocated.
Assert.assertFalse(rm.getRMNMTokenSecretManager()
Assert.assertFalse(rm.getRMContext().getNMTokenSecretManager()
.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(),
nm1.getNodeId()));
am.registerAppAttempt();

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
View File

@ -1101,7 +1101,7 @@ public class TestRMRestart {
userText1);
Token<RMDelegationTokenIdentifier> token1 =
new Token<RMDelegationTokenIdentifier>(dtId1,
rm1.getRMDTSecretManager());
rm1.getRMContext().getRMDelegationTokenSecretManager());
SecurityUtil.setTokenService(token1, rmAddr);
ts.addToken(userText1, token1);
tokenSet.add(token1);
@ -1112,7 +1112,7 @@ public class TestRMRestart {
userText2);
Token<RMDelegationTokenIdentifier> token2 =
new Token<RMDelegationTokenIdentifier>(dtId2,
rm1.getRMDTSecretManager());
rm1.getRMContext().getRMDelegationTokenSecretManager());
SecurityUtil.setTokenService(token2, rmAddr);
ts.addToken(userText2, token2);
tokenSet.add(token2);
@ -1255,7 +1255,7 @@ public class TestRMRestart {
// assert AMRMTokenSecretManager also knows about the AMRMToken password
Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
Assert.assertArrayEquals(amrmToken.getPassword(),
rm2.getAMRMTokenSecretManager().retrievePassword(
rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword(
amrmToken.decodeIdentifier()));
rm1.stop();
rm2.stop();
@ -1314,19 +1314,20 @@ public class TestRMRestart {
Assert.assertNotNull(appState);
// assert all master keys are saved
Set<DelegationKey> allKeysRM1 = rm1.getRMDTSecretManager().getAllMasterKeys();
Set<DelegationKey> allKeysRM1 = rm1.getRMContext()
.getRMDelegationTokenSecretManager().getAllMasterKeys();
Assert.assertEquals(allKeysRM1, rmDTMasterKeyState);
// assert all tokens are saved
Map<RMDelegationTokenIdentifier, Long> allTokensRM1 =
rm1.getRMDTSecretManager().getAllTokens();
rm1.getRMContext().getRMDelegationTokenSecretManager().getAllTokens();
Assert.assertEquals(tokenIdentSet, allTokensRM1.keySet());
Assert.assertEquals(allTokensRM1, rmDTState);
// assert sequence number is saved
Assert.assertEquals(
rm1.getRMDTSecretManager().getLatestDTSequenceNumber(),
rmState.getRMDTSecretManagerState().getDTSequenceNumber());
Assert.assertEquals(rm1.getRMContext().getRMDelegationTokenSecretManager()
.getLatestDTSequenceNumber(), rmState.getRMDTSecretManagerState()
.getDTSequenceNumber());
// request one more token
GetDelegationTokenRequest request2 =
@ -1341,16 +1342,15 @@ public class TestRMRestart {
// cancel token2
try{
rm1.getRMDTSecretManager().cancelToken(token2,
rm1.getRMContext().getRMDelegationTokenSecretManager().cancelToken(token2,
UserGroupInformation.getCurrentUser().getUserName());
} catch(Exception e) {
Assert.fail();
}
// Assert the token which has the latest delegationTokenSequenceNumber is removed
Assert.assertEquals(
rm1.getRMDTSecretManager().getLatestDTSequenceNumber(),
dtId2.getSequenceNumber());
Assert.assertEquals(rm1.getRMContext().getRMDelegationTokenSecretManager()
.getLatestDTSequenceNumber(), dtId2.getSequenceNumber());
Assert.assertFalse(rmDTState.containsKey(dtId2));
// start new RM
@ -1359,16 +1359,17 @@ public class TestRMRestart {
// assert master keys and tokens are populated back to DTSecretManager
Map<RMDelegationTokenIdentifier, Long> allTokensRM2 =
rm2.getRMDTSecretManager().getAllTokens();
rm2.getRMContext().getRMDelegationTokenSecretManager().getAllTokens();
Assert.assertEquals(allTokensRM2.keySet(), allTokensRM1.keySet());
// rm2 has its own master keys when it starts, we use containsAll here
Assert.assertTrue(rm2.getRMDTSecretManager().getAllMasterKeys()
.containsAll(allKeysRM1));
Assert.assertTrue(rm2.getRMContext().getRMDelegationTokenSecretManager()
.getAllMasterKeys().containsAll(allKeysRM1));
// assert sequenceNumber is properly recovered,
// even though the token which has max sequenceNumber is not stored
Assert.assertEquals(rm1.getRMDTSecretManager().getLatestDTSequenceNumber(),
rm2.getRMDTSecretManager().getLatestDTSequenceNumber());
Assert.assertEquals(rm1.getRMContext().getRMDelegationTokenSecretManager()
.getLatestDTSequenceNumber(), rm2.getRMContext()
.getRMDelegationTokenSecretManager().getLatestDTSequenceNumber());
// renewDate before renewing
Long renewDateBeforeRenew = allTokensRM2.get(dtId1);
@ -1376,12 +1377,14 @@ public class TestRMRestart {
// Sleep for one millisecond to make sure renewDataAfterRenew is greater
Thread.sleep(1);
// renew recovered token
rm2.getRMDTSecretManager().renewToken(token1, "renewer1");
rm2.getRMContext().getRMDelegationTokenSecretManager().renewToken(
token1, "renewer1");
} catch(Exception e) {
Assert.fail();
}
allTokensRM2 = rm2.getRMDTSecretManager().getAllTokens();
allTokensRM2 = rm2.getRMContext().getRMDelegationTokenSecretManager()
.getAllTokens();
Long renewDateAfterRenew = allTokensRM2.get(dtId1);
// assert token is renewed
Assert.assertTrue(renewDateAfterRenew > renewDateBeforeRenew);
@ -1392,14 +1395,15 @@ public class TestRMRestart {
Assert.assertFalse(rmDTState.containsValue(renewDateBeforeRenew));
try{
rm2.getRMDTSecretManager().cancelToken(token1,
rm2.getRMContext().getRMDelegationTokenSecretManager().cancelToken(token1,
UserGroupInformation.getCurrentUser().getUserName());
} catch(Exception e) {
Assert.fail();
}
// assert token is removed from state after its cancelled
allTokensRM2 = rm2.getRMDTSecretManager().getAllTokens();
allTokensRM2 = rm2.getRMContext().getRMDelegationTokenSecretManager()
.getAllTokens();
Assert.assertFalse(allTokensRM2.containsKey(dtId1));
Assert.assertFalse(rmDTState.containsKey(dtId1));
@ -1874,7 +1878,8 @@ public class TestRMRestart {
@Override
protected ClientRMService createClientRMService() {
return new ClientRMService(getRMContext(), getResourceScheduler(),
rmAppManager, applicationACLsManager, null, getRMDTSecretManager()){
rmAppManager, applicationACLsManager, null,
getRMContext().getRMDelegationTokenSecretManager()){
@Override
protected void serviceStart() throws Exception {
// do nothing

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
View File

@ -54,8 +54,8 @@ public class TestResourceManager {
Configuration conf = new YarnConfiguration();
resourceManager = new ResourceManager();
resourceManager.init(conf);
resourceManager.getRMContainerTokenSecretManager().rollMasterKey();
resourceManager.getRMNMTokenSecretManager().rollMasterKey();
resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
}
@After

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
View File

@ -109,8 +109,8 @@ public class TestCapacityScheduler {
conf.setClass(YarnConfiguration.RM_SCHEDULER,
CapacityScheduler.class, ResourceScheduler.class);
resourceManager.init(conf);
resourceManager.getRMContainerTokenSecretManager().rollMasterKey();
resourceManager.getRMNMTokenSecretManager().rollMasterKey();
resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
mockContext = mock(RMContext.class);
when(mockContext.getConfigurationProvider()).thenReturn(

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
View File

@ -149,7 +149,7 @@ public class TestFairScheduler {
resourceManager.getRMContext().getStateStore().start();
// to initialize the master key
resourceManager.getRMContainerTokenSecretManager().rollMasterKey();
resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
}
@After

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
View File

@ -168,7 +168,7 @@ public class TestClientToAMTokens {
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler,
this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
getRMDTSecretManager());
getRMContext().getRMDelegationTokenSecretManager());
};
@Override

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
View File

@ -85,7 +85,8 @@ public class TestRMDelegationTokens {
// the other is created on the first run of
// tokenRemoverThread.rollMasterKey()
RMDelegationTokenSecretManager dtSecretManager = rm1.getRMDTSecretManager();
RMDelegationTokenSecretManager dtSecretManager =
rm1.getRMContext().getRMDelegationTokenSecretManager();
// assert all master keys are saved
Assert.assertEquals(dtSecretManager.getAllMasterKeys(), rmDTMasterKeyState);
Set<DelegationKey> expiringKeys = new HashSet<DelegationKey>();
@ -140,7 +141,8 @@ public class TestRMDelegationTokens {
MockRM rm1 = new MyMockRM(conf, memStore);
rm1.start();
RMDelegationTokenSecretManager dtSecretManager = rm1.getRMDTSecretManager();
RMDelegationTokenSecretManager dtSecretManager =
rm1.getRMContext().getRMDelegationTokenSecretManager();
// assert all master keys are saved
Assert.assertEquals(dtSecretManager.getAllMasterKeys(), rmDTMasterKeyState);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
View File

@ -81,8 +81,8 @@ public class TestRMWebServicesNodes extends JerseyTest {
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
rm = new MockRM(new Configuration());
rm.getRMContainerTokenSecretManager().rollMasterKey();
rm.getRMNMTokenSecretManager().rollMasterKey();
rm.getRMContext().getContainerTokenSecretManager().rollMasterKey();
rm.getRMContext().getNMTokenSecretManager().rollMasterKey();
bind(ResourceManager.class).toInstance(rm);
bind(RMContext.class).toInstance(rm.getRMContext());
bind(ApplicationACLsManager.class).toInstance(

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
View File

@ -164,7 +164,8 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
NMTokenSecretManagerInNM nmTokenSecretManagerNM =
yarnCluster.getNodeManager(0).getNMContext().getNMTokenSecretManager();
RMContainerTokenSecretManager containerTokenSecretManager =
yarnCluster.getResourceManager().getRMContainerTokenSecretManager();
yarnCluster.getResourceManager().getRMContext().
getContainerTokenSecretManager();
NodeManager nm = yarnCluster.getNodeManager(0);
@ -573,7 +574,8 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
// Creating a tampered Container Token
RMContainerTokenSecretManager containerTokenSecretManager =
yarnCluster.getResourceManager().getRMContainerTokenSecretManager();
yarnCluster.getResourceManager().getRMContext().
getContainerTokenSecretManager();
RMContainerTokenSecretManager tamperedContainerTokenSecretManager =
new RMContainerTokenSecretManager(conf);

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java
View File

@ -109,8 +109,8 @@ public class TestRMNMSecretKeys {
dispatcher.await();
// Let's force a roll-over
rm.getRMContainerTokenSecretManager().rollMasterKey();
rm.getRMNMTokenSecretManager().rollMasterKey();
rm.getRMContext().getContainerTokenSecretManager().rollMasterKey();
rm.getRMContext().getNMTokenSecretManager().rollMasterKey();
// Heartbeats after roll-over and before activation should be fine.
response = nm.nodeHeartbeat(true);
@ -141,8 +141,8 @@ public class TestRMNMSecretKeys {
dispatcher.await();
// Let's force activation
rm.getRMContainerTokenSecretManager().activateNextMasterKey();
rm.getRMNMTokenSecretManager().activateNextMasterKey();
rm.getRMContext().getContainerTokenSecretManager().activateNextMasterKey();
rm.getRMContext().getNMTokenSecretManager().activateNextMasterKey();
response = nm.nodeHeartbeat(true);
Assert.assertNull(containerToken