Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-11253. Add Configuration to delegationToken RemoverScanInterval. (#4751)

This commit is contained in:
slfan1989 2022-08-28 01:02:59 +08:00 committed by GitHub
parent 5567154f71
commit 4031b0774e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 31 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -793,6 +793,10 @@ public class YarnConfiguration extends Configuration {
RM_PREFIX + "delegation.token.max-lifetime";
public static final long RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
7*24*60*60*1000; // 7 days
public static final String RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY =
RM_PREFIX + "delegation.token.remove-scan-interval";
public static final long RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT =
60*60*1000; // 1 hour
public static final String RM_DELEGATION_TOKEN_MAX_CONF_SIZE =
RM_PREFIX + "delegation-token.max-conf-size-bytes";

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1077,6 +1077,18 @@
<value>86400000</value>
</property>
<property>
<description>
This configuration is used for
how often the tokens are scanned for expired tokens in milliseconds.
the background thread(delegation token remover thread)
will delete expired tokens after the configured time.
the default value is 1h.
</description>
<name>yarn.resourcemanager.delegation.token.remove-scan-interval</name>
<value>1h</value>
</property>
<property>
<description>
RM DelegationTokenRenewer thread timeout

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
View File

@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSe
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
public class RMSecretManagerService extends AbstractService {
@ -135,9 +136,13 @@ public class RMSecretManagerService extends AbstractService {
long tokenRenewInterval =
conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
long removeScanInterval =
conf.getTimeDuration(YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY,
YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
return new RMDelegationTokenSecretManager(secretKeyInterval,
tokenMaxLifetime, tokenRenewInterval, 3600000, rmContext);
tokenMaxLifetime, tokenRenewInterval, removeScanInterval, rmContext);
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java
View File

@ -34,6 +34,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException;
@ -124,9 +125,13 @@ public class TestClientRMTokens {
long initialInterval = 10000l;
long maxLifetime= 20000l;
long renewInterval = 10000l;
long delegationTokenRemoverScanInterval =
conf.getTimeDuration(YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY,
YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
RMDelegationTokenSecretManager rmDtSecretManager = createRMDelegationTokenSecretManager(
initialInterval, maxLifetime, renewInterval);
initialInterval, maxLifetime, renewInterval, delegationTokenRemoverScanInterval);
rmDtSecretManager.startThreads();
LOG.info("Creating DelegationTokenSecretManager with initialInterval: "
+ initialInterval + ", maxLifetime: " + maxLifetime
@ -574,7 +579,8 @@ public class TestClientRMTokens {
private static RMDelegationTokenSecretManager
createRMDelegationTokenSecretManager(long secretKeyInterval,
long tokenMaxLifetime, long tokenRenewInterval) {
long tokenMaxLifetime, long tokenRenewInterval,
long delegationTokenRemoverScanInterval) {
ResourceManager rm = mock(ResourceManager.class);
RMContext rmContext = mock(RMContext.class);
when(rmContext.getStateStore()).thenReturn(new NullRMStateStore());
@ -583,7 +589,7 @@ public class TestClientRMTokens {
RMDelegationTokenSecretManager rmDtSecretManager =
new RMDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime,
tokenRenewInterval, 3600000, rmContext);
tokenRenewInterval, delegationTokenRemoverScanInterval, rmContext);
return rmDtSecretManager;
}
}