From b0b97dfbef87132ce1c54455d369619d51f6fe09 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Tue, 13 Oct 2015 19:00:08 +0530 Subject: [PATCH] HDFS-9139. Enable parallel JUnit tests for HDFS Pre-commit (Contributed by Chris Nauroth and Vinayakumar B) (cherry picked from commit 39581e3be2aaeb1eeb7fb98b6bdecd8d4e3c7269) Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestSeveralNameNodes.java --- dev-support/test-patch.sh | 10 +----- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 35 +++++++++++++++---- ...TestSWebHdfsFileContextMainOperations.java | 11 +++--- .../sasl/SaslDataTransferTestCase.java | 6 ++++ .../hdfs/qjournal/TestSecureNNWithQJM.java | 6 ++++ .../server/datanode/TestBlockScanner.java | 10 ++---- .../server/datanode/TestDataNodeMetrics.java | 5 +-- .../server/namenode/TestFSNamesystem.java | 1 + .../namenode/TestNameNodeHttpServer.java | 4 +++ .../TestNameNodeRespectsBindHostKeys.java | 4 +++ .../server/namenode/ha/TestEditLogTailer.java | 21 +++++++---- .../hdfs/server/namenode/ha/TestHAAppend.java | 2 +- .../hadoop/hdfs/web/TestHttpsFileSystem.java | 4 +++ .../hadoop/hdfs/web/TestWebHdfsTokens.java | 6 +++- ...ientCredentialTimeBasedTokenRefresher.java | 3 +- 16 files changed, 91 insertions(+), 40 deletions(-) diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index 3a6001abb7c..b689889bd17 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -2255,16 +2255,8 @@ function check_unittests test_logfile=${PATCH_DIR}/testrun_${module_suffix}.txt echo " Running tests in ${module_suffix}" - # Temporary hack to run the parallel tests profile only for hadoop-common. - # This code will be removed once hadoop-hdfs is ready for parallel test - # execution. - if [[ ${module} == "hadoop-common-project/hadoop-common" ]] ; then - OPTIONAL_PARALLEL_TESTS_PROFILE=${PARALLEL_TESTS_PROFILE} - else - unset OPTIONAL_PARALLEL_TESTS_PROFILE - fi # shellcheck disable=2086 - echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} ${OPTIONAL_PARALLEL_TESTS_PROFILE} ${TESTS_THREAD_COUNT} -D${PROJECT_NAME}PatchProcess + echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} ${PARALLEL_TESTS_PROFILE} ${TESTS_THREAD_COUNT} -D${PROJECT_NAME}PatchProcess test_build_result=$? add_jira_footer "${module_suffix} test log" "@@BASE@@/testrun_${module_suffix}.txt" diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3c54d92d7d7..11c98b5960a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -714,6 +714,9 @@ Release 2.8.0 - UNRELEASED DataTransferProtocol sockets and allow configuring auto-tuning (He Tianyi via Colin P. McCabe) + HDFS-9139. Enable parallel JUnit tests for HDFS Pre-commit + (Chris Nauroth and Vinayakumar B via vinayakumarb) + BUG FIXES HDFS-8091: ACLStatus and XAttributes should be presented to diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 2da1f51180f..5ece421b4ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -424,14 +424,23 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> maven-antrun-plugin - create-parallel-tests-dirs + create-parallel-tests-dirs test-compile - - - - + @@ -445,10 +454,24 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> maven-surefire-plugin ${testsThreadCount} - -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -DminiClusterDedicatedDirs=true + false + ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true ${test.build.data}/${surefire.forkNumber} + ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} + + + + + + ${test.build.data} + + + + + + fork-${surefire.forkNumber} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java index 53513fd1504..02110ac7031 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java @@ -57,8 +57,6 @@ public class TestSWebHdfsFileContextMainOperations protected static final byte[] data = getFileData(numBlocks, getDefaultBlockSize()); - private static Configuration sslConf; - @BeforeClass public static void clusterSetupAtBeginning() throws IOException, LoginException, URISyntaxException { @@ -67,15 +65,18 @@ public static void clusterSetupAtBeginning() FileUtil.fullyDelete(base); base.mkdirs(); keystoresDir = new File(BASEDIR).getAbsolutePath(); - sslConf = new Configuration(); - try { sslConfDir = KeyStoreTestUtil .getClasspathDir(TestSWebHdfsFileContextMainOperations.class); - KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, sslConf, false); + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, CONF, false); + CONF.set(DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + CONF.set(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); } catch (Exception ex) { throw new RuntimeException(ex); } + CONF.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, "HTTPS_ONLY"); CONF.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); CONF.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java index 9e660b23d13..37b2a2c0606 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY; @@ -27,6 +28,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY; import static org.junit.Assert.*; @@ -129,6 +131,10 @@ protected HdfsConfiguration createSecureConfig( String keystoresDir = baseDir.getAbsolutePath(); String sslConfDir = KeyStoreTestUtil.getClasspathDir(this.getClass()); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); + conf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + conf.set(DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); return conf; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java index b40e52a0f30..f95594a81ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY; @@ -35,6 +36,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY; import java.io.File; @@ -128,6 +130,10 @@ public static void init() throws Exception { String sslConfDir = KeyStoreTestUtil.getClasspathDir( TestSecureNNWithQJM.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, baseConf, false); + baseConf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + baseConf.set(DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); } @AfterClass diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java index b01597a4deb..acd5a476f9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java @@ -612,14 +612,8 @@ public void testMultipleBlockPoolScanning() throws Exception { // We scan 5 bytes per file (1 byte in file, 4 bytes of checksum) final int BYTES_SCANNED_PER_FILE = 5; - final int NUM_FILES[] = new int[] { 1, 5, 10 }; - int TOTAL_FILES = 0; - for (int i = 0; i < NUM_FILES.length; i++) { - TOTAL_FILES += NUM_FILES[i]; - } - ctx.createFiles(0, NUM_FILES[0], 1); - ctx.createFiles(0, NUM_FILES[1], 1); - ctx.createFiles(0, NUM_FILES[2], 1); + int TOTAL_FILES = 16; + ctx.createFiles(0, TOTAL_FILES, 1); // start scanning final TestScanResultHandler.Info info = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java index 0b473442acc..5c0a9712b17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java @@ -277,13 +277,14 @@ public void testDataNodeTimeSpend() throws Exception { } for (int x =0; x < 50; x++) { - String s = DFSTestUtil.readFile(fs, new Path("/time.txt." + x)); + DFSTestUtil.readFile(fs, new Path("/time.txt." + x)); } MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name()); long endWriteValue = getLongCounter("TotalWriteTime", rbNew); long endReadValue = getLongCounter("TotalReadTime", rbNew); - + // Lets Metric system update latest metrics + Thread.sleep(100); assertTrue(endReadValue > startReadValue); assertTrue(endWriteValue > startWriteValue); } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java index af9a6f78120..d533cf6d5b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java @@ -263,6 +263,7 @@ public void run() { } latch.await(); + Thread.sleep(10); // Lets all threads get BLOCKED Assert.assertEquals("Expected number of blocked thread not found", threadCount, rwLock.getQueueLength()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java index 975a6649e60..7dc719e564a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java @@ -74,6 +74,10 @@ public static void setUp() throws Exception { KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); connectionFactory = URLConnectionFactory .newDefaultURLConnectionFactory(conf); + conf.set(DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + conf.set(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); } @AfterClass diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java index e2a292ea97c..ed00e37eef6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java @@ -223,6 +223,10 @@ public void testHttpsBindHostKey() throws Exception { LOG.info("Testing behavior without " + DFS_NAMENODE_HTTPS_BIND_HOST_KEY); setupSsl(); + conf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + conf.set(DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java index 1916b78bbfc..d486920f3e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; import org.junit.Test; @@ -122,9 +123,11 @@ private static void testStandbyTriggersLogRolls(int activeIndex) // Have to specify IPC ports so the NNs can talk to each other. MiniDFSNNTopology topology = new MiniDFSNNTopology() - .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10031)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10032))); + .addNameservice(new MiniDFSNNTopology.NSConf("ns1") + .addNN(new MiniDFSNNTopology.NNConf("nn1") + .setIpcPort(ServerSocketUtil.getPort(0, 100))) + .addNN(new MiniDFSNNTopology.NNConf("nn2") + .setIpcPort(ServerSocketUtil.getPort(0, 100)))); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) @@ -146,13 +149,17 @@ private static void waitForLogRollInSharedDir(MiniDFSCluster cluster, long startTxId) throws Exception { URI sharedUri = cluster.getSharedEditsDir(0, 1); File sharedDir = new File(sharedUri.getPath(), "current"); - final File expectedLog = new File(sharedDir, - NNStorage.getInProgressEditsFileName(startTxId)); - + final File expectedInProgressLog = + new File(sharedDir, NNStorage.getInProgressEditsFileName(startTxId)); + final File expectedFinalizedLog = new File(sharedDir, + NNStorage.getFinalizedEditsFileName(startTxId, startTxId + 1)); + // There is a chance that multiple rolling happens by multiple NameNodes + // And expected inprogress file would have also finalized. So look for the + // finalized edits file as well GenericTestUtils.waitFor(new Supplier() { @Override public Boolean get() { - return expectedLog.exists(); + return expectedInProgressLog.exists() || expectedFinalizedLog.exists(); } }, 100, 10000); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java index 7b7100b0a1d..1fccb3c51fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java @@ -123,7 +123,7 @@ public void testMultipleAppendsDuringCatchupTailing() throws Exception { if (!isTruncateReady) { TestFileTruncate.checkBlockRecovery(fileToTruncate, - cluster.getFileSystem(1)); + cluster.getFileSystem(1), 300, 200); } AppendTestUtil.checkFullFile(fs, fileToTruncate, truncatePos[0], data, fileToTruncate.toString()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java index f8cb6bd22b8..3afe2119881 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java @@ -65,6 +65,10 @@ public static void setUp() throws Exception { sslConfDir = KeyStoreTestUtil.getClasspathDir(TestHttpsFileSystem.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); + conf.set(DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + conf.set(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java index 4d8911e6bea..d7a87173394 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java @@ -216,7 +216,11 @@ public void testLazyTokenFetchForSWebhdfs() throws Exception { keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestWebHdfsTokens.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, clusterConf, false); - + clusterConf.set(DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + clusterConf.set(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); + // trick the NN into thinking security is enabled w/o it trying // to login from a keytab UserGroupInformation.setConfiguration(clusterConf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java index c259b30aae8..d796753999c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.web.oauth2; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.util.Timer; import org.apache.http.HttpStatus; import org.codehaus.jackson.map.ObjectMapper; @@ -74,7 +75,7 @@ public Configuration buildConf(String credential, String tokenExpires, @Test public void refreshUrlIsCorrect() throws IOException { - final int PORT = 7552; + final int PORT = ServerSocketUtil.getPort(0, 20); final String REFRESH_ADDRESS = "http://localhost:" + PORT + "/refresh"; long tokenExpires = 0;