YARN-3857: Memory leak in ResourceManager with SIMPLE mode. Contributed by mujunchao.
(cherry picked from commit 3a76a010b8
)
Conflicts:
hadoop-yarn-project/CHANGES.txt
This commit is contained in:
parent
843dac5353
commit
70289432f7
|
@ -14,6 +14,8 @@ Release 2.6.4 - UNRELEASED
|
||||||
|
|
||||||
YARN-2975. FSLeafQueue app lists are accessed without required locks. (kasha)
|
YARN-2975. FSLeafQueue app lists are accessed without required locks. (kasha)
|
||||||
|
|
||||||
|
YARN-3857. Memory leak in ResourceManager with SIMPLE mode. (mujunchao via zxu)
|
||||||
|
|
||||||
Release 2.6.3 - UNRELEASED
|
Release 2.6.3 - UNRELEASED
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
|
|
@ -1262,11 +1262,13 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
||||||
|
|
||||||
// register the ClientTokenMasterKey after it is saved in the store,
|
// register the ClientTokenMasterKey after it is saved in the store,
|
||||||
// otherwise client may hold an invalid ClientToken after RM restarts.
|
// otherwise client may hold an invalid ClientToken after RM restarts.
|
||||||
|
if (UserGroupInformation.isSecurityEnabled()) {
|
||||||
appAttempt.rmContext.getClientToAMTokenSecretManager()
|
appAttempt.rmContext.getClientToAMTokenSecretManager()
|
||||||
.registerApplication(appAttempt.getAppAttemptId(),
|
.registerApplication(appAttempt.getAppAttemptId(),
|
||||||
appAttempt.getClientTokenMasterKey());
|
appAttempt.getClientTokenMasterKey());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean shouldCountTowardsMaxAttemptRetry() {
|
public boolean shouldCountTowardsMaxAttemptRetry() {
|
||||||
|
|
|
@ -22,6 +22,7 @@ import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import javax.crypto.SecretKey;
|
import javax.crypto.SecretKey;
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager;
|
import org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager;
|
||||||
|
@ -61,4 +62,10 @@ public class ClientToAMTokenSecretManagerInRM extends
|
||||||
ApplicationAttemptId applicationAttemptID) {
|
ApplicationAttemptId applicationAttemptID) {
|
||||||
return this.masterKeys.get(applicationAttemptID);
|
return this.masterKeys.get(applicationAttemptID);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
public synchronized boolean hasMasterKey(
|
||||||
|
ApplicationAttemptId applicationAttemptID) {
|
||||||
|
return this.masterKeys.containsKey(applicationAttemptID);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1363,6 +1363,38 @@ public class TestRMAppAttemptTransitions {
|
||||||
Assert.assertNull(token);
|
Assert.assertNull(token);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// this is to test master key is saved in the secret manager only after
|
||||||
|
// attempt is launched and in secure-mode
|
||||||
|
@Test
|
||||||
|
public void testApplicationAttemptMasterKey() throws Exception {
|
||||||
|
Container amContainer = allocateApplicationAttempt();
|
||||||
|
ApplicationAttemptId appid = applicationAttempt.getAppAttemptId();
|
||||||
|
boolean isMasterKeyExisted = false;
|
||||||
|
|
||||||
|
// before attempt is launched, can not get MasterKey
|
||||||
|
isMasterKeyExisted = clientToAMTokenManager.hasMasterKey(appid);
|
||||||
|
Assert.assertFalse(isMasterKeyExisted);
|
||||||
|
|
||||||
|
launchApplicationAttempt(amContainer);
|
||||||
|
// after attempt is launched and in secure mode, can get MasterKey
|
||||||
|
isMasterKeyExisted = clientToAMTokenManager.hasMasterKey(appid);
|
||||||
|
if (isSecurityEnabled) {
|
||||||
|
Assert.assertTrue(isMasterKeyExisted);
|
||||||
|
Assert.assertNotNull(clientToAMTokenManager.getMasterKey(appid));
|
||||||
|
} else {
|
||||||
|
Assert.assertFalse(isMasterKeyExisted);
|
||||||
|
}
|
||||||
|
|
||||||
|
applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
|
||||||
|
.getAppAttemptId(), RMAppAttemptEventType.KILL));
|
||||||
|
assertEquals(YarnApplicationAttemptState.LAUNCHED,
|
||||||
|
applicationAttempt.createApplicationAttemptState());
|
||||||
|
sendAttemptUpdateSavedEvent(applicationAttempt);
|
||||||
|
// after attempt is killed, can not get MasterKey
|
||||||
|
isMasterKeyExisted = clientToAMTokenManager.hasMasterKey(appid);
|
||||||
|
Assert.assertFalse(isMasterKeyExisted);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testFailedToFailed() {
|
public void testFailedToFailed() {
|
||||||
// create a failed attempt.
|
// create a failed attempt.
|
||||||
|
|
Loading…
Reference in New Issue