From 478a2bb1854167ad7ac806cff2b99b15a3b796b8 Mon Sep 17 00:00:00 2001 From: Varun Vasudev Date: Wed, 20 Jul 2016 13:02:34 +0530 Subject: [PATCH] YARN-5309. Fix SSLFactory truststore reloader thread leak in TimelineClientImpl. Contributed by Weiwei Yang. --- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 5 ++ .../client/api/impl/TimelineClientImpl.java | 21 ++++---- .../client/api/impl/TestTimelineClient.java | 53 +++++++++++++++++++ 3 files changed, 70 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index 3d74ac0e75b..a982ab2fae3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -144,6 +144,11 @@ junit test + + org.bouncycastle + bcprov-jdk16 + test + com.sun.jersey.jersey-test-framework jersey-test-framework-grizzly2 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 8c600416954..1f662dda449 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -111,6 +111,7 @@ public class TimelineClientImpl extends TimelineClient { private Configuration configuration; private float timelineServiceVersion; private TimelineWriter timelineWriter; + private SSLFactory sslFactory; @Private @VisibleForTesting @@ -269,7 +270,7 @@ public class TimelineClientImpl extends TimelineClient { } ClientConfig cc = new DefaultClientConfig(); cc.getClasses().add(YarnJacksonJaxbJsonProvider.class); - connConfigurator = newConnConfigurator(conf); + connConfigurator = initConnConfigurator(conf); if (UserGroupInformation.isSecurityEnabled()) { authenticator = new KerberosDelegationTokenAuthenticator(); } else { @@ -325,6 +326,9 @@ public class TimelineClientImpl extends TimelineClient { if (this.timelineWriter != null) { this.timelineWriter.close(); } + if (this.sslFactory != null) { + this.sslFactory.destroy(); + } super.serviceStop(); } @@ -477,9 +481,9 @@ public class TimelineClientImpl extends TimelineClient { } - private static ConnectionConfigurator newConnConfigurator(Configuration conf) { + private ConnectionConfigurator initConnConfigurator(Configuration conf) { try { - return newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf); + return initSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf); } catch (Exception e) { LOG.debug("Cannot load customized ssl related configuration. " + "Fallback to system-generic settings.", e); @@ -497,16 +501,15 @@ public class TimelineClientImpl extends TimelineClient { } }; - private static ConnectionConfigurator newSslConnConfigurator(final int timeout, + private ConnectionConfigurator initSslConnConfigurator(final int timeout, Configuration conf) throws IOException, GeneralSecurityException { - final SSLFactory factory; final SSLSocketFactory sf; final HostnameVerifier hv; - factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); - factory.init(); - sf = factory.createSSLSocketFactory(); - hv = factory.getHostnameVerifier(); + sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + sslFactory.init(); + sf = sslFactory.createSSLSocketFactory(); + hv = sslFactory.getHostnameVerifier(); return new ConnectionConfigurator() { @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index 41b788dcbac..ae9fa46f2d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import java.io.File; import java.io.IOException; import java.net.ConnectException; import java.net.SocketTimeoutException; @@ -33,8 +34,10 @@ import java.security.PrivilegedExceptionAction; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; @@ -58,9 +61,15 @@ public class TestTimelineClient { private TimelineClientImpl client; private TimelineWriter spyTimelineWriter; + private static final File testDir = new File( + System.getProperty("test.build.data", + System.getProperty("java.io.tmpdir")), + "TestTimelineClient"); @Before public void setup() { + FileUtil.fullyDelete(testDir); + testDir.mkdirs(); YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.0f); @@ -69,6 +78,7 @@ public class TestTimelineClient { @After public void tearDown() { + FileUtil.fullyDelete(testDir); if (client != null) { client.stop(); } @@ -434,6 +444,49 @@ public class TestTimelineClient { return client; } + @Test + public void testTimelineClientCleanup() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 0); + + String sslConfDir = + KeyStoreTestUtil.getClasspathDir(TestTimelineClient.class); + KeyStoreTestUtil.setupSSLConfig(testDir.getAbsolutePath(), + sslConfDir, conf, false); + client = createTimelineClient(conf); + + ThreadGroup threadGroup = Thread.currentThread().getThreadGroup(); + + while (threadGroup.getParent() != null) { + threadGroup = threadGroup.getParent(); + } + + Thread[] threads = new Thread[threadGroup.activeCount()]; + + threadGroup.enumerate(threads); + Thread reloaderThread = null; + for (Thread thread : threads) { + if ((thread.getName() != null) + && (thread.getName().contains("Truststore reloader thread"))) { + reloaderThread = thread; + } + } + Assert.assertTrue("Reloader is not alive", reloaderThread.isAlive()); + + client.close(); + + boolean reloaderStillAlive = true; + for (int i = 0; i < 10; i++) { + reloaderStillAlive = reloaderThread.isAlive(); + if (!reloaderStillAlive) { + break; + } + Thread.sleep(1000); + } + Assert.assertFalse("Reloader is still alive", reloaderStillAlive); + } + private static class TestTimlineDelegationTokenSecretManager extends AbstractDelegationTokenSecretManager {