From e82f048d7b17b871f813bf6f4aa7ecda226d1b19 Mon Sep 17 00:00:00 2001 From: stack Date: Tue, 16 Dec 2014 16:41:27 -0800 Subject: [PATCH] HBASE-12645 HBaseTestingUtility is using ${$HOME} for rootDir (Varun Saxena) --- .../hadoop/hbase/HBaseTestingUtility.java | 122 +++++++++++++++--- .../client/TestTableSnapshotScanner.java | 2 +- .../TableSnapshotInputFormatTestBase.java | 2 +- .../replication/TestReplicationSource.java | 8 +- .../hadoop/hbase/util/TestMergeTool.java | 5 +- 5 files changed, 111 insertions(+), 28 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index bf61208d77f..5a729657f20 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -444,7 +444,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } /** - * Sets up a path in test filesystem to be used by tests + * Sets up a path in test filesystem to be used by tests. + * Creates a new directory if not already setup. */ private void setupDataTestDirOnTestFS() throws IOException { if (dataTestDirOnTestFS != null) { @@ -452,22 +453,30 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { + dataTestDirOnTestFS.toString()); return; } + dataTestDirOnTestFS = getNewDataTestDirOnTestFS(); + } + /** + * Sets up a new path in test filesystem to be used by tests. + */ + private Path getNewDataTestDirOnTestFS() throws IOException { //The file system can be either local, mini dfs, or if the configuration //is supplied externally, it can be an external cluster FS. If it is a local //file system, the tests should use getBaseTestDir, otherwise, we can use //the working directory, and create a unique sub dir there FileSystem fs = getTestFileSystem(); + Path newDataTestDir = null; if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) { File dataTestDir = new File(getDataTestDir().toString()); if (deleteOnExit()) dataTestDir.deleteOnExit(); - dataTestDirOnTestFS = new Path(dataTestDir.getAbsolutePath()); + newDataTestDir = new Path(dataTestDir.getAbsolutePath()); } else { Path base = getBaseTestDirOnTestFS(); String randomStr = UUID.randomUUID().toString(); - dataTestDirOnTestFS = new Path(base, randomStr); - if (deleteOnExit()) fs.deleteOnExit(dataTestDirOnTestFS); + newDataTestDir = new Path(base, randomStr); + if (deleteOnExit()) fs.deleteOnExit(newDataTestDir); } + return newDataTestDir; } /** @@ -741,6 +750,19 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { return startMiniCluster(1, 1); } + /** + * Start up a minicluster of hbase, dfs, and zookeeper. + * Set the create flag to create root or data directory path or not + * (will overwrite if dir already exists) + * @throws Exception + * @return Mini hbase cluster instance created. + * @see {@link #shutdownMiniDFSCluster()} + */ + public MiniHBaseCluster startMiniCluster(final int numSlaves, boolean create) + throws Exception { + return startMiniCluster(1, numSlaves, create); + } + /** * Start up a minicluster of hbase, optionally dfs, and zookeeper. * Modifies Configuration. Homes the cluster data directory under a random @@ -756,9 +778,21 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { */ public MiniHBaseCluster startMiniCluster(final int numSlaves) throws Exception { - return startMiniCluster(1, numSlaves); + return startMiniCluster(1, numSlaves, false); } + /** + * Start minicluster. Whether to create a new root or data dir path even if such a path + * has been created earlier is decided based on flag create + * @throws Exception + * @see {@link #shutdownMiniCluster()} + * @return Mini hbase cluster instance created. + */ + public MiniHBaseCluster startMiniCluster(final int numMasters, + final int numSlaves, boolean create) + throws Exception { + return startMiniCluster(numMasters, numSlaves, null, create); + } /** * start minicluster @@ -769,7 +803,14 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { public MiniHBaseCluster startMiniCluster(final int numMasters, final int numSlaves) throws Exception { - return startMiniCluster(numMasters, numSlaves, null); + return startMiniCluster(numMasters, numSlaves, null, false); + } + + public MiniHBaseCluster startMiniCluster(final int numMasters, + final int numSlaves, final String[] dataNodeHosts, boolean create) + throws Exception { + return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts, + null, null, create); } /** @@ -798,7 +839,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { */ public MiniHBaseCluster startMiniCluster(final int numMasters, final int numSlaves, final String[] dataNodeHosts) throws Exception { - return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts, null, null); + return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts, + null, null); } /** @@ -845,15 +887,27 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { numMasters, numSlaves, numSlaves, dataNodeHosts, masterClass, regionserverClass); } + public MiniHBaseCluster startMiniCluster(final int numMasters, + final int numSlaves, int numDataNodes, final String[] dataNodeHosts, + Class masterClass, + Class regionserverClass) + throws Exception { + return startMiniCluster(numMasters, numSlaves, numDataNodes, dataNodeHosts, + masterClass, regionserverClass, false); + } + /** * Same as {@link #startMiniCluster(int, int, String[], Class, Class)}, but with custom * number of datanodes. * @param numDataNodes Number of data nodes. + * @param create Set this flag to create a new + * root or data directory path or not (will overwrite if exists already). */ public MiniHBaseCluster startMiniCluster(final int numMasters, final int numSlaves, int numDataNodes, final String[] dataNodeHosts, Class masterClass, - Class regionserverClass) + Class regionserverClass, + boolean create) throws Exception { if (dataNodeHosts != null && dataNodeHosts.length != 0) { numDataNodes = dataNodeHosts.length; @@ -881,12 +935,13 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } // Start the MiniHBaseCluster - return startMiniHBaseCluster(numMasters, numSlaves, masterClass, regionserverClass); + return startMiniHBaseCluster(numMasters, numSlaves, masterClass, + regionserverClass, create); } public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves) throws IOException, InterruptedException{ - return startMiniHBaseCluster(numMasters, numSlaves, null, null); + return startMiniHBaseCluster(numMasters, numSlaves, null, null, false); } /** @@ -895,6 +950,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { * Usually you won't want this. You'll usually want {@link #startMiniCluster()}. * @param numMasters * @param numSlaves + * @param create Whether to create a + * root or data directory path or not; will overwrite if exists already. * @return Reference to the hbase mini hbase cluster. * @throws IOException * @throws InterruptedException @@ -902,10 +959,11 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { */ public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves, Class masterClass, - Class regionserverClass) + Class regionserverClass, + boolean create) throws IOException, InterruptedException { // Now do the mini hbase cluster. Set the hbase.rootdir in config. - createRootDir(); + createRootDir(create); // These settings will make the server waits until this exact number of // regions servers are connected. @@ -1028,14 +1086,30 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } /** - * Returns the path to the default root dir the minicluster uses. + * Returns the path to the default root dir the minicluster uses. If getNewDirPathIfExists + * is true, a new root directory path is fetched irrespective of whether it has been fetched + * before or not. If false, previous path is used. + * Note: this does not cause the root dir to be created. + * @return Fully qualified path for the default hbase root dir + * @throws IOException + */ + public Path getDefaultRootDirPath(boolean create) throws IOException { + if (!create) { + return getDataTestDirOnTestFS(); + } else { + return getNewDataTestDirOnTestFS(); + } + } + + /** + * Same as {{@link HBaseTestingUtility#getDefaultRootDirPath(boolean getNewDirPathIfExists)} + * except that getNewDirPathIfExists flag is false. * Note: this does not cause the root dir to be created. * @return Fully qualified path for the default hbase root dir * @throws IOException */ public Path getDefaultRootDirPath() throws IOException { - FileSystem fs = FileSystem.get(this.conf); - return new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase"); + return getDefaultRootDirPath(false); } /** @@ -1043,18 +1117,32 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { * version file. Normally you won't make use of this method. Root hbasedir * is created for you as part of mini cluster startup. You'd only use this * method if you were doing manual operation. + * @param getNewDirPathIfExists This flag decides whether to get a new + * root or data directory path or not, if it has been fetched already. + * Note : Directory will be made irrespective of whether path has been fetched or not. + * If directory already exists, it will be overwritten * @return Fully qualified path to hbase root dir * @throws IOException */ - public Path createRootDir() throws IOException { + public Path createRootDir(boolean getNewDirPathIfExists) throws IOException { FileSystem fs = FileSystem.get(this.conf); - Path hbaseRootdir = getDefaultRootDirPath(); + Path hbaseRootdir = getDefaultRootDirPath(getNewDirPathIfExists); FSUtils.setRootDir(this.conf, hbaseRootdir); fs.mkdirs(hbaseRootdir); FSUtils.setVersion(fs, hbaseRootdir); return hbaseRootdir; } + /** + * Same as {@link HBaseTestingUtility#createRootDir(boolean getNewDirPathIfExists)} + * except that getNewDirPathIfExists flag is false. + * @return Fully qualified path to hbase root dir + * @throws IOException + */ + public Path createRootDir() throws IOException { + return createRootDir(false); + } + /** * Flushes all caches in the mini hbase cluster * @throws IOException diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java index 10f7b9f37a4..3e915e1b376 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java @@ -56,7 +56,7 @@ public class TestTableSnapshotScanner { public void setupCluster() throws Exception { setupConf(UTIL.getConfiguration()); - UTIL.startMiniCluster(NUM_REGION_SERVERS); + UTIL.startMiniCluster(NUM_REGION_SERVERS, true); rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); fs = rootDir.getFileSystem(UTIL.getConfiguration()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java index 7c55ad083b5..1f8ccae75ae 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java @@ -50,7 +50,7 @@ public abstract class TableSnapshotInputFormatTestBase { public void setupCluster() throws Exception { setupConf(UTIL.getConfiguration()); - UTIL.startMiniCluster(NUM_REGION_SERVERS); + UTIL.startMiniCluster(NUM_REGION_SERVERS, true); rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); fs = rootDir.getFileSystem(UTIL.getConfiguration()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java index 1fb03124df5..458819d12f4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java @@ -62,12 +62,10 @@ public class TestReplicationSource { public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniDFSCluster(1); FS = TEST_UTIL.getDFSCluster().getFileSystem(); - FSUtils.setRootDir(TEST_UTIL.getConfiguration(), FS.getHomeDirectory()); - oldLogDir = new Path(FS.getHomeDirectory(), - HConstants.HREGION_OLDLOGDIR_NAME); + Path rootDir = TEST_UTIL.createRootDir(); + oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); if (FS.exists(oldLogDir)) FS.delete(oldLogDir, true); - logDir = new Path(FS.getHomeDirectory(), - HConstants.HREGION_LOGDIR_NAME); + logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); if (FS.exists(logDir)) FS.delete(logDir, true); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java index ca09b39ddda..d4c7fa885c6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java @@ -135,10 +135,7 @@ public class TestMergeTool extends HBaseTestCase { this.fs = this.dfsCluster.getFileSystem(); System.out.println("fs=" + this.fs); FSUtils.setFsDefault(this.conf, new Path(fs.getUri())); - Path parentdir = fs.getHomeDirectory(); - FSUtils.setRootDir(conf, parentdir); - fs.mkdirs(parentdir); - FSUtils.setVersion(fs, parentdir); + TEST_UTIL.createRootDir(); // Note: we must call super.setUp after starting the mini cluster or // we will end up with a local file system