From 3e00835dadabdf10295803c972d8755cc5afe93e Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Sat, 27 Apr 2013 16:16:19 +0000 Subject: [PATCH] HDFS-4705. Address HDFS test failures on Windows because of invalid dfs.namenode.name.dir. Contributed by Ivan Mitic. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1476610 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/namenode/TestAllowFormat.java | 2 ++ .../hdfs/server/namenode/TestCheckpoint.java | 9 ++++++--- .../hdfs/server/namenode/TestFSNamesystem.java | 13 +++++++++++++ .../server/namenode/TestNNThroughputBenchmark.java | 12 ++++++++++++ .../hdfs/server/namenode/TestNameEditsConfigs.java | 7 +++++++ .../TestValidateConfigurationSettings.java | 14 ++++++++++++++ 7 files changed, 57 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 11f28e96604..15d7b00a645 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -323,6 +323,9 @@ Trunk (Unreleased) HDFS-4722. TestGetConf#testFederation times out on Windows. (Ivan Mitic via suresh) + HDFS-4705. Address HDFS test failures on Windows because of invalid + dfs.namenode.name.dir. (Ivan Mitic via suresh) + BREAKDOWN OF HDFS-347 SUBTASKS AND RELATED JIRAS HDFS-4353. Encapsulate connections to peers in Peer and PeerServer classes. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java index d18b343c749..41ac73a63fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java @@ -169,6 +169,8 @@ public void testFormatShouldBeIgnoredForNonFileBasedDirs() throws Exception { InetSocketAddress nnAddr2 = new InetSocketAddress(localhost, 9020); HATestUtil.setFailoverConfigurations(conf, logicalName, nnAddr1, nnAddr2); + conf.set(DFS_NAMENODE_NAME_DIR_KEY, + new File(hdfsDir, "name").getAbsolutePath()); conf.setBoolean(DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY, true); conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_NAMENODE_EDITS_PLUGIN_PREFIX, "dummy"), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java index d84283fa7d3..1dfc4d5e75c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java @@ -750,9 +750,12 @@ public void testNameDirLocking() throws IOException { @Test public void testSeparateEditsDirLocking() throws IOException { Configuration conf = new HdfsConfiguration(); - File editsDir = new File(MiniDFSCluster.getBaseDirectory() + - "/testSeparateEditsDirLocking"); - + File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name"); + File editsDir = new File(MiniDFSCluster.getBaseDirectory(), + "testSeparateEditsDirLocking"); + + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + nameDir.getAbsolutePath()); conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, editsDir.getAbsolutePath()); MiniDFSCluster cluster = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java index 4a66cba7b45..4b3dd1c3ba9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java @@ -19,20 +19,30 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; import static org.junit.Assert.assertEquals; +import java.io.File; import java.io.IOException; import java.net.URI; import java.util.Collection; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; +import org.junit.After; import org.junit.Test; public class TestFSNamesystem { + @After + public void cleanUp() { + FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); + } + /** * Tests that the namenode edits dirs are gotten with duplicates removed */ @@ -54,6 +64,9 @@ public void testUniqueEditDirs() throws IOException { @Test public void testFSNamespaceClearLeases() throws Exception { Configuration conf = new HdfsConfiguration(); + File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name"); + conf.set(DFS_NAMENODE_NAME_DIR_KEY, nameDir.getAbsolutePath()); + NameNode.initMetrics(conf, NamenodeRole.NAMENODE); DFSTestUtil.formatNameNode(conf); FSNamesystem fsn = FSNamesystem.loadFromDisk(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java index 8203292155f..d964230da52 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java @@ -17,23 +17,35 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import java.io.File; import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.After; import org.junit.Test; public class TestNNThroughputBenchmark { + @After + public void cleanUp() { + FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); + } + /** * This test runs all benchmarks defined in {@link NNThroughputBenchmark}. */ @Test public void testNNThroughput() throws Exception { Configuration conf = new HdfsConfiguration(); + File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name"); + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + nameDir.getAbsolutePath()); FileSystem.setDefaultUri(conf, "hdfs://localhost:" + 0); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); DFSTestUtil.formatNameNode(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java index b2368a1458f..4e4914b3ab6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java @@ -322,12 +322,15 @@ public void testNameEditsRequiredConfigs() throws IOException { MiniDFSCluster cluster = null; File nameAndEditsDir = new File(base_dir, "name_and_edits"); File nameAndEditsDir2 = new File(base_dir, "name_and_edits2"); + File nameDir = new File(base_dir, "name"); // 1 // Bad configuration. Add a directory to dfs.namenode.edits.dir.required // without adding it to dfs.namenode.edits.dir. try { Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + nameDir.getAbsolutePath()); conf.set( DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY, nameAndEditsDir2.toURI().toString()); @@ -353,6 +356,8 @@ public void testNameEditsRequiredConfigs() throws IOException { // and dfs.namenode.edits.dir. try { Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + nameDir.getAbsolutePath()); conf.setStrings( DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameAndEditsDir.toURI().toString(), @@ -375,6 +380,8 @@ public void testNameEditsRequiredConfigs() throws IOException { // dfs.namenode.edits.dir.required. try { Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + nameDir.getAbsolutePath()); conf.setStrings( DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameAndEditsDir.toURI().toString(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java index 4e9efd5a57e..54e4f7acbc7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.test.GenericTestUtils; +import org.junit.After; import org.junit.Test; /** @@ -39,6 +40,11 @@ */ public class TestValidateConfigurationSettings { + @After + public void cleanUp() { + FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); + } + /** * Tests setting the rpc port to the same as the web port to test that * an exception @@ -49,6 +55,10 @@ public void testThatMatchingRPCandHttpPortsThrowException() throws IOException { Configuration conf = new HdfsConfiguration(); + File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name"); + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + nameDir.getAbsolutePath()); + // set both of these to port 9000, should fail FileSystem.setDefaultUri(conf, "hdfs://localhost:9000"); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "127.0.0.1:9000"); @@ -72,6 +82,10 @@ public void testThatDifferentRPCandHttpPortsAreOK() throws IOException { Configuration conf = new HdfsConfiguration(); + File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name"); + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + nameDir.getAbsolutePath()); + FileSystem.setDefaultUri(conf, "hdfs://localhost:8000"); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "127.0.0.1:9000"); DFSTestUtil.formatNameNode(conf);