From 086577c67a458b466acc3595f5c74beffbcaab50 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Mon, 24 Oct 2016 08:56:14 -0500 Subject: [PATCH] HDFS-11042. Add missing cleanupSSLConfig() call for tests that use setupSSLConfig(). Contributed by Kuhu Shukla. (cherry picked from commit b18f35f097a3ee33f449e506676961f9b9456bbd) Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java --- .../TestHttpFSFWithSWebhdfsFileSystem.java | 6 ++++-- .../hdfs/TestSecureEncryptionZoneWithKMS.java | 9 ++++++--- .../sasl/SaslDataTransferTestCase.java | 9 ++++++--- .../hdfs/qjournal/TestSecureNNWithQJM.java | 11 ++++++++--- .../hdfs/server/balancer/TestBalancer.java | 16 ++++++++++++++-- .../TestNameNodeRespectsBindHostKeys.java | 10 ++++++++-- .../hadoop/hdfs/web/TestWebHdfsTokens.java | 5 +++-- 7 files changed, 49 insertions(+), 17 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFWithSWebhdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFWithSWebhdfsFileSystem.java index d8f860a94dd..d53bb50f400 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFWithSWebhdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFWithSWebhdfsFileSystem.java @@ -40,6 +40,7 @@ public class TestHttpFSFWithSWebhdfsFileSystem private static String classpathDir; private static final String BASEDIR = GenericTestUtils.getTempPath(UUID.randomUUID().toString()); + private static String keyStoreDir; private static Configuration sslConf; @@ -57,7 +58,7 @@ public class TestHttpFSFWithSWebhdfsFileSystem File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); - String keyStoreDir = new File(BASEDIR).getAbsolutePath(); + keyStoreDir = new File(BASEDIR).getAbsolutePath(); try { sslConf = new Configuration(); KeyStoreTestUtil.setupSSLConfig(keyStoreDir, classpathDir, sslConf, false); @@ -69,9 +70,10 @@ public class TestHttpFSFWithSWebhdfsFileSystem } @AfterClass - public static void cleanUp() { + public static void cleanUp() throws Exception { new File(classpathDir, "ssl-client.xml").delete(); new File(classpathDir, "ssl-server.xml").delete(); + KeyStoreTestUtil.cleanupSSLConfig(keyStoreDir, classpathDir); } public TestHttpFSFWithSWebhdfsFileSystem(Operation operation) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java index 96e39ba0bfa..7c4763c13fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java @@ -86,6 +86,8 @@ public class TestSecureEncryptionZoneWithKMS { private static final Path TEST_PATH = new Path("/test-dir"); private static HdfsConfiguration baseConf; private static File baseDir; + private static String keystoresDir; + private static String sslConfDir; private static final EnumSet< CreateEncryptionZoneFlag > NO_TRASH = EnumSet.of(CreateEncryptionZoneFlag.NO_TRASH); @@ -189,8 +191,8 @@ public static void init() throws Exception { baseConf.set(KMS_CLIENT_ENC_KEY_CACHE_SIZE, "4"); baseConf.set(KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK, "0.5"); - String keystoresDir = baseDir.getAbsolutePath(); - String sslConfDir = KeyStoreTestUtil.getClasspathDir( + keystoresDir = baseDir.getAbsolutePath(); + sslConfDir = KeyStoreTestUtil.getClasspathDir( TestSecureEncryptionZoneWithKMS.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, baseConf, false); baseConf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, @@ -225,7 +227,7 @@ public static void init() throws Exception { } @AfterClass - public static void destroy() { + public static void destroy() throws Exception { if (kdc != null) { kdc.stop(); } @@ -233,6 +235,7 @@ public static void destroy() { miniKMS.stop(); } FileUtil.fullyDelete(baseDir); + KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); } @Before diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java index 4eade6ad5e6..d03d0953999 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java @@ -50,6 +50,8 @@ public abstract class SaslDataTransferTestCase { private static File baseDir; + private static String keystoresDir; + private static String sslConfDir; private static String hdfsPrincipal; private static String userPrincipal; private static MiniKdc kdc; @@ -99,11 +101,12 @@ public static void initKdc() throws Exception { } @AfterClass - public static void shutdownKdc() { + public static void shutdownKdc() throws Exception { if (kdc != null) { kdc.stop(); } FileUtil.fullyDelete(baseDir); + KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); } /** @@ -129,8 +132,8 @@ protected HdfsConfiguration createSecureConfig( conf.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY, 10); - String keystoresDir = baseDir.getAbsolutePath(); - String sslConfDir = KeyStoreTestUtil.getClasspathDir(this.getClass()); + keystoresDir = baseDir.getAbsolutePath(); + sslConfDir = KeyStoreTestUtil.getClasspathDir(this.getClass()); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); conf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, KeyStoreTestUtil.getClientSSLConfigFileName()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java index 46b016f8fad..18adc4ecff8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.qjournal; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import static org.junit.Assert.*; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY; @@ -71,6 +73,8 @@ public class TestSecureNNWithQJM { private static HdfsConfiguration baseConf; private static File baseDir; + private static String keystoresDir; + private static String sslConfDir; private static MiniKdc kdc; private MiniDFSCluster cluster; @@ -127,8 +131,8 @@ public static void init() throws Exception { baseConf.set(DFS_JOURNALNODE_HTTPS_ADDRESS_KEY, "localhost:0"); baseConf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY, 10); - String keystoresDir = baseDir.getAbsolutePath(); - String sslConfDir = KeyStoreTestUtil.getClasspathDir( + keystoresDir = baseDir.getAbsolutePath(); + sslConfDir = KeyStoreTestUtil.getClasspathDir( TestSecureNNWithQJM.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, baseConf, false); baseConf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, @@ -138,11 +142,12 @@ public static void init() throws Exception { } @AfterClass - public static void destroy() { + public static void destroy() throws Exception { if (kdc != null) { kdc.stop(); } FileUtil.fullyDelete(baseDir); + KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); } @Before diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 7235372bfcc..d89a3ed49fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -43,6 +43,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY; +import org.junit.AfterClass; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -136,6 +137,8 @@ public class TestBalancer { final static private String username = "balancer"; private static String principal; private static File baseDir; + private static String keystoresDir; + private static String sslConfDir; private static MiniKdc kdc; private static File keytabFile; private MiniDFSCluster cluster; @@ -183,8 +186,8 @@ static void initSecureConf(Configuration conf) throws Exception { conf.set(DFS_BALANCER_KEYTAB_FILE_KEY, keytab); conf.set(DFS_BALANCER_KERBEROS_PRINCIPAL_KEY, principal); - String keystoresDir = baseDir.getAbsolutePath(); - String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestBalancer.class); + keystoresDir = baseDir.getAbsolutePath(); + sslConfDir = KeyStoreTestUtil.getClasspathDir(TestBalancer.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); conf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, @@ -245,6 +248,15 @@ static void initConfWithRamDisk(Configuration conf, conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L); } + @AfterClass + public static void destroy() throws Exception { + if (kdc != null) { + kdc.stop(); + } + FileUtil.fullyDelete(baseDir); + KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); + } + /* create a file with a length of fileLen */ static void createFile(MiniDFSCluster cluster, Path filePath, long fileLen, short replicationFactor, int nnIndex) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java index 4353e6186b3..f687691e051 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java @@ -56,6 +56,8 @@ public class TestNameNodeRespectsBindHostKeys { public static final Log LOG = LogFactory.getLog(TestNameNodeRespectsBindHostKeys.class); private static final String WILDCARD_ADDRESS = "0.0.0.0"; private static final String LOCALHOST_SERVER_ADDRESS = "127.0.0.1:0"; + private static String keystoresDir; + private static String sslConfDir; private static String getRpcServerAddress(MiniDFSCluster cluster) { NameNodeRpcServer rpcServer = (NameNodeRpcServer) cluster.getNameNodeRpc(); @@ -252,8 +254,8 @@ private static void setupSsl() throws Exception { File base = new File(BASEDIR); FileUtil.fullyDelete(base); assertTrue(base.mkdirs()); - final String keystoresDir = new File(BASEDIR).getAbsolutePath(); - final String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestNameNodeRespectsBindHostKeys.class); + keystoresDir = new File(BASEDIR).getAbsolutePath(); + sslConfDir = KeyStoreTestUtil.getClasspathDir(TestNameNodeRespectsBindHostKeys.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); } @@ -311,6 +313,10 @@ public void testHttpsBindHostKey() throws Exception { if (cluster != null) { cluster.shutdown(); } + if (keystoresDir != null && !keystoresDir.isEmpty() + && sslConfDir != null && !sslConfDir.isEmpty()) { + KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java index 16f20a87950..2802cb7ebda 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java @@ -196,6 +196,8 @@ public void testLazyTokenFetchForWebhdfs() throws Exception { public void testLazyTokenFetchForSWebhdfs() throws Exception { MiniDFSCluster cluster = null; SWebHdfsFileSystem fs = null; + String keystoresDir; + String sslConfDir; try { final Configuration clusterConf = new HdfsConfiguration(conf); SecurityUtil.setAuthenticationMethod(SIMPLE, clusterConf); @@ -203,8 +205,6 @@ public void testLazyTokenFetchForSWebhdfs() throws Exception { .DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); String baseDir = GenericTestUtils.getTempPath(TestWebHdfsTokens.class.getSimpleName()); - String keystoresDir; - String sslConfDir; clusterConf.setBoolean(HdfsClientConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); clusterConf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); @@ -242,6 +242,7 @@ public void testLazyTokenFetchForSWebhdfs() throws Exception { cluster.shutdown(); } } + KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); } @Test