From f411de6a79a0a87f03c09366cfe7a7d0726ed932 Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Mon, 23 Apr 2018 11:13:18 -0700 Subject: [PATCH] HDFS-13408. MiniDFSCluster to support being built on randomized base directory. Contributed by Xiao Liang --- .../apache/hadoop/hdfs/MiniDFSCluster.java | 22 ++++++++++++++++++- .../hadoop/hdfs/TestMiniDFSCluster.java | 16 ++++++-------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 4c3aed714e1..acb720e544d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -202,8 +202,28 @@ public class MiniDFSCluster implements AutoCloseable { this.conf = conf; this.storagesPerDatanode = FsDatasetTestUtils.Factory.getFactory(conf).getDefaultNumOfDataDirs(); + if (null == conf.get(HDFS_MINIDFS_BASEDIR)) { + conf.set(HDFS_MINIDFS_BASEDIR, + new File(getBaseDirectory()).getAbsolutePath()); + } } - + + public Builder(Configuration conf, File basedir) { + this.conf = conf; + this.storagesPerDatanode = + FsDatasetTestUtils.Factory.getFactory(conf).getDefaultNumOfDataDirs(); + if (null == basedir) { + throw new IllegalArgumentException( + "MiniDFSCluster base directory cannot be null"); + } + String cdir = conf.get(HDFS_MINIDFS_BASEDIR); + if (cdir != null) { + throw new IllegalArgumentException( + "MiniDFSCluster base directory already defined (" + cdir + ")"); + } + conf.set(HDFS_MINIDFS_BASEDIR, basedir.getAbsolutePath()); + } + /** * Default: 0 */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java index d631b68ddfc..afc977fc0e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java @@ -201,9 +201,8 @@ public class TestMiniDFSCluster { public void testIsClusterUpAfterShutdown() throws Throwable { Configuration conf = new HdfsConfiguration(); File testDataCluster4 = new File(testDataPath, CLUSTER_4); - String c4Path = testDataCluster4.getAbsolutePath(); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c4Path); - MiniDFSCluster cluster4 = new MiniDFSCluster.Builder(conf).build(); + MiniDFSCluster cluster4 = + new MiniDFSCluster.Builder(conf, testDataCluster4).build(); try { DistributedFileSystem dfs = cluster4.getFileSystem(); dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); @@ -222,12 +221,11 @@ public class TestMiniDFSCluster { Configuration conf = new HdfsConfiguration(); conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "MYHOST"); File testDataCluster5 = new File(testDataPath, CLUSTER_5); - String c5Path = testDataCluster5.getAbsolutePath(); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c5Path); - try (MiniDFSCluster cluster5 = new MiniDFSCluster.Builder(conf) - .numDataNodes(1) - .checkDataNodeHostConfig(true) - .build()) { + try (MiniDFSCluster cluster5 = + new MiniDFSCluster.Builder(conf, testDataCluster5) + .numDataNodes(1) + .checkDataNodeHostConfig(true) + .build()) { assertEquals("DataNode hostname config not respected", "MYHOST", cluster5.getDataNodes().get(0).getDatanodeId().getHostName()); }