From 9e326f4eeb4f3ca2c385fc5cde12c8e2017545f3 Mon Sep 17 00:00:00 2001 From: Varun Vasudev Date: Wed, 20 Jul 2016 12:48:32 +0530 Subject: [PATCH] YARN-5309. Fix SSLFactory truststore reloader thread leak in TimelineClientImpl. Contributed by Weiwei Yang. (cherry picked from commit 9ccf935b2cba5ac8a3e8124360d2203931397391) Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java --- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 5 ++ .../client/api/impl/TimelineClientImpl.java | 21 +++++---- .../client/api/impl/TestTimelineClient.java | 47 +++++++++++++++++++ 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index 2877bcb5c2c..f6bd613b69e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -144,6 +144,11 @@ junit test + + org.bouncycastle + bcprov-jdk16 + test + com.sun.jersey.jersey-test-framework jersey-test-framework-grizzly2 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 8c600416954..1f662dda449 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -111,6 +111,7 @@ public class TimelineClientImpl extends TimelineClient { private Configuration configuration; private float timelineServiceVersion; private TimelineWriter timelineWriter; + private SSLFactory sslFactory; @Private @VisibleForTesting @@ -269,7 +270,7 @@ public class TimelineClientImpl extends TimelineClient { } ClientConfig cc = new DefaultClientConfig(); cc.getClasses().add(YarnJacksonJaxbJsonProvider.class); - connConfigurator = newConnConfigurator(conf); + connConfigurator = initConnConfigurator(conf); if (UserGroupInformation.isSecurityEnabled()) { authenticator = new KerberosDelegationTokenAuthenticator(); } else { @@ -325,6 +326,9 @@ public class TimelineClientImpl extends TimelineClient { if (this.timelineWriter != null) { this.timelineWriter.close(); } + if (this.sslFactory != null) { + this.sslFactory.destroy(); + } super.serviceStop(); } @@ -477,9 +481,9 @@ public class TimelineClientImpl extends TimelineClient { } - private static ConnectionConfigurator newConnConfigurator(Configuration conf) { + private ConnectionConfigurator initConnConfigurator(Configuration conf) { try { - return newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf); + return initSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf); } catch (Exception e) { LOG.debug("Cannot load customized ssl related configuration. " + "Fallback to system-generic settings.", e); @@ -497,16 +501,15 @@ public class TimelineClientImpl extends TimelineClient { } }; - private static ConnectionConfigurator newSslConnConfigurator(final int timeout, + private ConnectionConfigurator initSslConnConfigurator(final int timeout, Configuration conf) throws IOException, GeneralSecurityException { - final SSLFactory factory; final SSLSocketFactory sf; final HostnameVerifier hv; - factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); - factory.init(); - sf = factory.createSSLSocketFactory(); - hv = factory.getHostnameVerifier(); + sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + sslFactory.init(); + sf = sslFactory.createSSLSocketFactory(); + hv = sslFactory.getHostnameVerifier(); return new ConnectionConfigurator() { @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index 41b788dcbac..a1d44494026 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import java.io.File; import java.io.IOException; import java.net.ConnectException; import java.net.SocketTimeoutException; @@ -35,8 +36,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.test.TestGenericTestUtils; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; @@ -434,6 +437,50 @@ public class TestTimelineClient { return client; } + @Test + public void testTimelineClientCleanup() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 0); + + File testDir = TestGenericTestUtils.getTestDir(); + String sslConfDir = + KeyStoreTestUtil.getClasspathDir(TestTimelineClient.class); + KeyStoreTestUtil.setupSSLConfig(testDir.getAbsolutePath(), + sslConfDir, conf, false); + client = createTimelineClient(conf); + + ThreadGroup threadGroup = Thread.currentThread().getThreadGroup(); + + while (threadGroup.getParent() != null) { + threadGroup = threadGroup.getParent(); + } + + Thread[] threads = new Thread[threadGroup.activeCount()]; + + threadGroup.enumerate(threads); + Thread reloaderThread = null; + for (Thread thread : threads) { + if ((thread.getName() != null) + && (thread.getName().contains("Truststore reloader thread"))) { + reloaderThread = thread; + } + } + Assert.assertTrue("Reloader is not alive", reloaderThread.isAlive()); + + client.close(); + + boolean reloaderStillAlive = true; + for (int i = 0; i < 10; i++) { + reloaderStillAlive = reloaderThread.isAlive(); + if (!reloaderStillAlive) { + break; + } + Thread.sleep(1000); + } + Assert.assertFalse("Reloader is still alive", reloaderStillAlive); + } + private static class TestTimlineDelegationTokenSecretManager extends AbstractDelegationTokenSecretManager {