From 41bcef94869cab9c030cc685e81008a4932d1be5 Mon Sep 17 00:00:00 2001 From: Eric Badger Date: Mon, 13 Jul 2020 23:12:18 +0000 Subject: [PATCH] YARN-10348. Allow RM to always cancel tokens after app completes. Contributed by Jim Brennan (cherry picked from commit 48f90115b5ecb37f814af281f09bb404361b2bba) --- .../hadoop/yarn/conf/YarnConfiguration.java | 3 + .../src/main/resources/yarn-default.xml | 10 +++ .../security/DelegationTokenRenewer.java | 8 +- .../security/TestDelegationTokenRenewer.java | 75 ++++++++++++++++++- 4 files changed, 93 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index aa1ff29b1df..25c2dd68c79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -748,6 +748,9 @@ public class YarnConfiguration extends Configuration { RM_PREFIX + "delegation-token.max-conf-size-bytes"; public static final int DEFAULT_RM_DELEGATION_TOKEN_MAX_CONF_SIZE_BYTES = 12800; + public static final String RM_DELEGATION_TOKEN_ALWAYS_CANCEL = + RM_PREFIX + "delegation-token.always-cancel"; + public static final boolean DEFAULT_RM_DELEGATION_TOKEN_ALWAYS_CANCEL = false; public static final String RM_DT_RENEWER_THREAD_TIMEOUT = RM_PREFIX + "delegation-token-renewer.thread-timeout"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index f09186ecf4c..48164b3a85a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -804,6 +804,16 @@ 12800 + + If true, ResourceManager will always try to cancel delegation + tokens after the application completes, even if the client sets + shouldCancelAtEnd false. References to delegation tokens are tracked, + so they will not be canceled until all sub-tasks are done using them. + + yarn.resourcemanager.delegation-token.always-cancel + false + + If true, ResourceManager will have proxy-user privileges. Use case: In a secure cluster, YARN requires the user hdfs delegation-tokens to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index fd8935debbc..4c21b55e24c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -115,6 +115,7 @@ public class DelegationTokenRenewer extends AbstractService { private volatile boolean isServiceStarted; private LinkedBlockingQueue pendingEventQueue; + private boolean alwaysCancelDelegationTokens; private boolean tokenKeepAliveEnabled; private boolean hasProxyUserPrivileges; private long credentialsValidTimeRemaining; @@ -137,6 +138,9 @@ public class DelegationTokenRenewer extends AbstractService { @Override protected void serviceInit(Configuration conf) throws Exception { + this.alwaysCancelDelegationTokens = + conf.getBoolean(YarnConfiguration.RM_DELEGATION_TOKEN_ALWAYS_CANCEL, + YarnConfiguration.DEFAULT_RM_DELEGATION_TOKEN_ALWAYS_CANCEL); this.hasProxyUserPrivileges = conf.getBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, YarnConfiguration.DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED); @@ -268,7 +272,7 @@ public class DelegationTokenRenewer extends AbstractService { * */ @VisibleForTesting - protected static class DelegationTokenToRenew { + protected class DelegationTokenToRenew { public final Token token; public final Collection referringAppIds; public final Configuration conf; @@ -298,7 +302,7 @@ public class DelegationTokenRenewer extends AbstractService { this.conf = conf; this.expirationDate = expirationDate; this.timerTask = null; - this.shouldCancelAtEnd = shouldCancelAtEnd; + this.shouldCancelAtEnd = shouldCancelAtEnd | alwaysCancelDelegationTokens; } public void setTimerTask(RenewalTimerTask tTask) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 31a87cb71be..01cf3b6f5bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -217,6 +217,8 @@ public class TestDelegationTokenRenewer { conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); conf.set("override_token_expire_time", "3000"); + conf.setBoolean(YarnConfiguration.RM_DELEGATION_TOKEN_ALWAYS_CANCEL, + false); UserGroupInformation.setConfiguration(conf); eventQueue = new LinkedBlockingQueue(); dispatcher = new AsyncDispatcher(eventQueue); @@ -608,6 +610,77 @@ public class TestDelegationTokenRenewer { token1.renew(conf); } + /** + * Basic idea of the test: + * 1. Verify that YarnConfiguration.RM_DELEGATION_TOKEN_ALWAYS_CANCEL = true + * overrides shouldCancelAtEnd + * 2. register a token for 2 seconds with shouldCancelAtEnd = false + * 3. cancel it immediately + * 4. check that token was canceled + * @throws IOException + * @throws URISyntaxException + */ + @Test(timeout=60000) + public void testDTRenewalWithNoCancelAlwaysCancel() throws Exception { + Configuration lconf = new Configuration(conf); + lconf.setBoolean(YarnConfiguration.RM_DELEGATION_TOKEN_ALWAYS_CANCEL, + true); + + DelegationTokenRenewer localDtr = + createNewDelegationTokenRenewer(lconf, counter); + RMContext mockContext = mock(RMContext.class); + when(mockContext.getSystemCredentialsForApps()).thenReturn( + new ConcurrentHashMap()); + ClientRMService mockClientRMService = mock(ClientRMService.class); + when(mockContext.getClientRMService()).thenReturn(mockClientRMService); + when(mockContext.getDelegationTokenRenewer()).thenReturn( + localDtr); + when(mockContext.getDispatcher()).thenReturn(dispatcher); + InetSocketAddress sockAddr = + InetSocketAddress.createUnresolved("localhost", 1234); + when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); + localDtr.setDelegationTokenRenewerPoolTracker(false); + localDtr.setRMContext(mockContext); + localDtr.init(lconf); + localDtr.start(); + + MyFS dfs = (MyFS)FileSystem.get(lconf); + LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode()); + + Credentials ts = new Credentials(); + MyToken token1 = dfs.getDelegationToken("user1"); + + //to cause this one to be set for renew in 2 secs + Renewer.tokenToRenewIn2Sec = token1; + LOG.info("token="+token1+" should be renewed for 2 secs"); + + String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0"; + ts.addToken(new Text(nn1), token1); + + ApplicationId applicationId = BuilderUtils.newApplicationId(0, 1); + localDtr.addApplicationAsync(applicationId, ts, false, "user", + new Configuration()); + waitForEventsToGetProcessed(localDtr); + localDtr.applicationFinished(applicationId); + waitForEventsToGetProcessed(localDtr); + + int numberOfExpectedRenewals = Renewer.counter; // number of renewals so far + try { + Thread.sleep(6*1000); // sleep 6 seconds, so it has time to renew + } catch (InterruptedException e) {} + LOG.info("Counter = " + Renewer.counter + ";t="+ Renewer.lastRenewed); + + // counter and the token should still be the old ones + assertEquals("renew wasn't called as many times as expected", + numberOfExpectedRenewals, Renewer.counter); + + // The token should have been cancelled at this point. Renewal will fail. + try { + token1.renew(lconf); + fail("Renewal of cancelled token should have failed"); + } catch (InvalidToken ite) {} + } + /** * Basic idea of the test: * 0. Setup token KEEP_ALIVE @@ -1616,7 +1689,7 @@ public class TestDelegationTokenRenewer { // Ensure incrTokenSequenceNo has been called for new token request Mockito.verify(mockContext, Mockito.times(1)).incrTokenSequenceNo(); - DelegationTokenToRenew dttr = new DelegationTokenToRenew(appIds, + DelegationTokenToRenew dttr = dtr.new DelegationTokenToRenew(appIds, expectedToken, conf, 1000, false, "user1"); dtr.requestNewHdfsDelegationTokenIfNeeded(dttr);