YARN-1915. Fixed a race condition that client could use the ClientToAMToken to contact with AM before AM actually receives the ClientToAMTokenMasterKey. Contributed by Jason Lowe

This commit is contained in:
Jian He 2014-10-24 22:47:16 -07:00
parent 683897fd02
commit 5864dd99a4
3 changed files with 148 additions and 3 deletions

View File

@ -742,6 +742,10 @@ Release 2.6.0 - UNRELEASED
YARN-2724. Skipped uploading a local log file to HDFS if exception is raised
when opening it. (Xuan Gong via zjshen)
YARN-1915. Fixed a race condition that client could use the ClientToAMToken
to contact with AM before AM actually receives the ClientToAMTokenMasterKey.
(Jason Lowe via jianhe)
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES

View File

@ -36,9 +36,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@Evolving
public class ClientToAMTokenSecretManager extends
BaseClientToAMTokenSecretManager {
private static final int MASTER_KEY_WAIT_MSEC = 10 * 1000;
// Only one master-key for AM
private SecretKey masterKey;
private volatile SecretKey masterKey;
public ClientToAMTokenSecretManager(
ApplicationAttemptId applicationAttemptID, byte[] key) {
@ -51,6 +52,23 @@ public class ClientToAMTokenSecretManager extends
}
@Override
public byte[] retrievePassword(ClientToAMTokenIdentifier identifier)
throws InvalidToken {
if (this.masterKey == null) {
synchronized (this) {
while (masterKey == null) {
try {
wait(MASTER_KEY_WAIT_MSEC);
break;
} catch (InterruptedException e) {
}
}
}
}
return super.retrievePassword(identifier);
}
@Override
public SecretKey getMasterKey(ApplicationAttemptId applicationAttemptID) {
// Only one master-key for AM, just return that.
@ -58,6 +76,9 @@ public class ClientToAMTokenSecretManager extends
}
public void setMasterKey(byte[] key) {
this.masterKey = SecretManager.createSecretKey(key);
synchronized (this) {
this.masterKey = SecretManager.createSecretKey(key);
notifyAll();
}
}
}

View File

@ -30,9 +30,12 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Timer;
import java.util.TimerTask;
import javax.security.sasl.SaslException;
@ -169,6 +172,10 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
this.address = NetUtils.getConnectAddress(server);
super.serviceStart();
}
public void setClientSecretKey(byte[] key) {
secretMgr.setMasterKey(key);
}
}
@Test
@ -291,7 +298,7 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
// Verify for a new version token
verifyNewVersionToken(conf, am, token, rm);
am.stop();
rm.stop();
}
@ -410,4 +417,117 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
}
});
}
@Test(timeout=20000)
public void testClientTokenRace() throws Exception {
final Configuration conf = new Configuration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
UserGroupInformation.setConfiguration(conf);
ContainerManagementProtocol containerManager =
mock(ContainerManagementProtocol.class);
StartContainersResponse mockResponse = mock(StartContainersResponse.class);
when(containerManager.startContainers((StartContainersRequest) any()))
.thenReturn(mockResponse);
final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler,
this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
getRMContext().getRMDelegationTokenSecretManager());
};
@Override
protected Dispatcher createDispatcher() {
return dispatcher;
}
@Override
protected void doSecureLogin() throws IOException {
}
};
rm.start();
// Submit an app
RMApp app = rm.submitApp(1024);
// Set up a node.
MockNM nm1 = rm.registerNode("localhost:1234", 3072);
nm1.nodeHeartbeat(true);
dispatcher.await();
nm1.nodeHeartbeat(true);
dispatcher.await();
ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
final MockAM mockAM =
new MockAM(rm.getRMContext(), rm.getApplicationMasterService(),
app.getCurrentAppAttempt().getAppAttemptId());
UserGroupInformation appUgi =
UserGroupInformation.createRemoteUser(appAttempt.toString());
RegisterApplicationMasterResponse response =
appUgi.doAs(new PrivilegedAction<RegisterApplicationMasterResponse>() {
@Override
public RegisterApplicationMasterResponse run() {
RegisterApplicationMasterResponse response = null;
try {
response = mockAM.registerAppAttempt();
} catch (Exception e) {
Assert.fail("Exception was not expected");
}
return response;
}
});
// Get the app-report.
GetApplicationReportRequest request =
Records.newRecord(GetApplicationReportRequest.class);
request.setApplicationId(app.getApplicationId());
GetApplicationReportResponse reportResponse =
rm.getClientRMService().getApplicationReport(request);
ApplicationReport appReport = reportResponse.getApplicationReport();
org.apache.hadoop.yarn.api.records.Token originalClientToAMToken =
appReport.getClientToAMToken();
// ClientToAMToken master key should have been received on register
// application master response.
final ByteBuffer clientMasterKey = response.getClientToAMTokenMasterKey();
Assert.assertNotNull(clientMasterKey);
Assert.assertTrue(clientMasterKey.array().length > 0);
// Start the AM with the correct shared-secret.
ApplicationAttemptId appAttemptId =
app.getAppAttempts().keySet().iterator().next();
Assert.assertNotNull(appAttemptId);
final CustomAM am = new CustomAM(appAttemptId, null);
am.init(conf);
am.start();
// Now the real test!
// Set up clients to be able to pick up correct tokens.
SecurityUtil.setSecurityInfoProviders(new CustomSecurityInfo());
Token<ClientToAMTokenIdentifier> token =
ConverterUtils.convertFromYarn(originalClientToAMToken, am.address);
// Schedule the key to be set after a significant delay
Timer timer = new Timer();
TimerTask timerTask = new TimerTask() {
@Override
public void run() {
am.setClientSecretKey(clientMasterKey.array());
}
};
timer.schedule(timerTask, 250);
// connect should pause waiting for the master key to arrive
verifyValidToken(conf, am, token);
am.stop();
rm.stop();
}
}