From b987a52e3652b25b7ce061e9da85fb95dd3dfa06 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Fri, 13 Apr 2012 22:45:25 +0000 Subject: [PATCH] HBASE-5747 Forward port "hbase-5708 [89-fb] Make MiniMapRedCluster directory a subdirectory of target/test" git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1326000 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/hbase/HConstants.java | 6 + .../hadoop/hbase/master/SplitLogManager.java | 8 + .../apache/hadoop/hbase/HBaseTestCase.java | 39 +-- .../hadoop/hbase/HBaseTestingUtility.java | 246 +++++++++++------- .../hbase/TestFullLogReconstruction.java | 2 +- .../hadoop/hbase/TestHBaseTestingUtility.java | 12 +- .../apache/hadoop/hbase/TestZooKeeper.java | 4 +- .../hbase/coprocessor/TestClassLoading.java | 2 + .../coprocessor/TestCoprocessorEndpoint.java | 4 - .../hbase/io/hfile/TestFixedFileTrailer.java | 3 +- .../hadoop/hbase/io/hfile/TestHFile.java | 5 +- .../io/hfile/TestHFileDataBlockEncoder.java | 8 - .../hadoop/hbase/io/hfile/TestHFileSeek.java | 2 + .../hfile/TestScannerSelectionUsingTTL.java | 5 +- .../mapreduce/TestHFileOutputFormat.java | 12 +- .../hbase/mapreduce/TestTimeRangeMapRed.java | 2 +- .../hbase/master/TestSplitLogManager.java | 17 +- .../hbase/regionserver/TestStoreFile.java | 8 +- .../hbase/regionserver/wal/TestHLog.java | 14 +- .../util/TestMiniClusterLoadSequential.java | 3 + .../hbase/zookeeper/TestZKLeaderManager.java | 8 +- src/test/resources/hbase-site.xml | 7 + 22 files changed, 219 insertions(+), 198 deletions(-) diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java index 9291a274cd9..5f8a5385726 100644 --- a/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -103,6 +103,10 @@ public final class HConstants { /** by default every master is a possible primary master unless the conf explicitly overrides it */ public static final boolean DEFAULT_MASTER_TYPE_BACKUP = false; + /** Parameter name for ZooKeeper session time out.*/ + public static final String ZOOKEEPER_SESSION_TIMEOUT = + "zookeeper.session.timeout"; + /** Name of ZooKeeper quorum configuration parameter. */ public static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; @@ -655,6 +659,8 @@ public final class HConstants { /** Region in Transition metrics threshold time */ public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD="hbase.metrics.rit.stuck.warning.threshold"; + public static final String LOAD_BALANCER_SLOP_KEY = "hbase.regions.slop"; + private HConstants() { // Can't be instantiated with this ctor. } diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java index b94fea313a6..919c65f2b05 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java +++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java @@ -1004,6 +1004,10 @@ public class SplitLogManager extends ZooKeeperListener { Stat stat) { tot_mgr_get_data_result.incrementAndGet(); if (rc != 0) { + if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) { + LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries."); + return; + } if (rc == KeeperException.Code.NONODE.intValue()) { tot_mgr_get_data_nonode.incrementAndGet(); // The task znode has been deleted. Must be some pending delete @@ -1085,6 +1089,10 @@ public class SplitLogManager extends ZooKeeperListener { @Override public void processResult(int rc, String path, Object ctx, String name) { if (rc != 0) { + if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) { + LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries."); + return; + } Long retry_count = (Long)ctx; LOG.warn("rc=" + KeeperException.Code.get(rc) + " for "+ path + " remaining retries=" + retry_count); diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java index 2897a8c542e..ab1c5f330b8 100644 --- a/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java +++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java @@ -19,7 +19,6 @@ */ package org.apache.hadoop.hbase; -import java.io.File; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.Iterator; @@ -54,16 +53,6 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; public abstract class HBaseTestCase extends TestCase { private static final Log LOG = LogFactory.getLog(HBaseTestCase.class); - /** configuration parameter name for test directory - * @deprecated see HBaseTestingUtility#TEST_DIRECTORY_KEY - **/ - private static final String TEST_DIRECTORY_KEY = "test.build.data"; - -/* - protected final static byte [] fam1 = Bytes.toBytes("colfamily1"); - protected final static byte [] fam2 = Bytes.toBytes("colfamily2"); - protected final static byte [] fam3 = Bytes.toBytes("colfamily3"); -*/ protected final static byte [] fam1 = Bytes.toBytes("colfamily11"); protected final static byte [] fam2 = Bytes.toBytes("colfamily21"); protected final static byte [] fam3 = Bytes.toBytes("colfamily31"); @@ -82,9 +71,7 @@ public abstract class HBaseTestCase extends TestCase { protected String START_KEY; protected static final int MAXVERSIONS = 3; - static { - initialize(); - } + protected final HBaseTestingUtility testUtil = new HBaseTestingUtility(); public volatile Configuration conf; @@ -161,19 +148,12 @@ public abstract class HBaseTestCase extends TestCase { * @return directory to use for this test */ protected Path getUnitTestdir(String testName) { - return new Path( - System.getProperty( - HBaseTestingUtility.BASE_TEST_DIRECTORY_KEY, - HBaseTestingUtility.DEFAULT_BASE_TEST_DIRECTORY - ), - testName - ); + return testUtil.getDataTestDir(testName); } protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey, byte [] endKey) throws IOException { - FileSystem filesystem = FileSystem.get(conf); HRegionInfo hri = new HRegionInfo(desc.getName(), startKey, endKey); return HRegion.createHRegion(hri, testDir, conf, desc); } @@ -628,21 +608,6 @@ public abstract class HBaseTestCase extends TestCase { } } - /** - * Initializes parameters used in the test environment: - * - * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set. - * Sets the boolean debugging if "DEBUGGING" is set in the environment. - * If debugging is enabled, reconfigures logging so that the root log level is - * set to WARN and the logging level for the package is set to DEBUG. - */ - public static void initialize() { - if (System.getProperty(TEST_DIRECTORY_KEY) == null) { - System.setProperty(TEST_DIRECTORY_KEY, new File( - "build/hbase/test").getAbsolutePath()); - } - } - /** * Common method to close down a MiniDFSCluster and the associated file system * diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index a2339d9a76a..9aa5c1e5329 100644 --- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.net.InetAddress; import java.net.ServerSocket; import java.net.Socket; @@ -84,8 +85,8 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; +import org.apache.hadoop.mapred.TaskLog; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.WatchedEvent; @@ -124,11 +125,15 @@ public class HBaseTestingUtility { private MiniHBaseCluster hbaseCluster = null; private MiniMRCluster mrCluster = null; - // Directory where we put the data for this instance of HBaseTestingUtility + /** If there is a mini cluster running for this testing utility instance. */ + private boolean miniClusterRunning; + + private String hadoopLogDir; + + // Directory where we put the data for this instance of HBaseTestingUtility. private File dataTestDir = null; - // Directory (usually a subdirectory of dataTestDir) used by the dfs cluster - // if any + // Directory (a subdirectory of dataTestDir) used by the dfs cluster if any private File clusterTestDir = null; /** @@ -151,6 +156,9 @@ public class HBaseTestingUtility { */ public static final String DEFAULT_BASE_TEST_DIRECTORY = "target/test-data"; + /** Filesystem URI used for map-reduce mini-cluster setup */ + private static String FS_URI; + /** Compression algorithms to use in parameterized JUnit 4 tests */ public static final List COMPRESSION_ALGORITHMS_PARAMETERIZED = Arrays.asList(new Object[][] { @@ -236,24 +244,42 @@ public class HBaseTestingUtility { * @see #getTestFileSystem() */ public Path getDataTestDir() { - if (dataTestDir == null){ + if (this.dataTestDir == null){ setupDataTestDir(); } - return new Path(dataTestDir.getAbsolutePath()); + return new Path(this.dataTestDir.getAbsolutePath()); } /** * @return Where the DFS cluster will write data on the local subsystem. - * Creates it if it does not exist already. + * Creates it if it does not exist already. A subdir of {@link #getBaseTestDir()} * @see #getTestFileSystem() */ - public Path getClusterTestDir() { + Path getClusterTestDir() { if (clusterTestDir == null){ setupClusterTestDir(); } return new Path(clusterTestDir.getAbsolutePath()); } + /** + * Creates a directory for the DFS cluster, under the test data + */ + private void setupClusterTestDir() { + if (clusterTestDir != null) { + return; + } + + // Using randomUUID ensures that multiple clusters can be launched by + // a same test, if it stops & starts them + Path testDir = getDataTestDir("dfscluster_" + UUID.randomUUID().toString()); + clusterTestDir = new File(testDir.toString()).getAbsoluteFile(); + // Have it cleaned up on exit + clusterTestDir.deleteOnExit(); + conf.set(TEST_DIRECTORY_KEY, clusterTestDir.getPath()); + LOG.info("Created new mini-cluster data directory: " + clusterTestDir); + } + /** * @param subdirName * @return Path to a subdirectory named subdirName under @@ -281,7 +307,7 @@ public class HBaseTestingUtility { * @return The calculated data test build directory. */ private void setupDataTestDir() { - if (dataTestDir != null) { + if (this.dataTestDir != null) { LOG.warn("Data test dir already setup in " + dataTestDir.getAbsolutePath()); return; @@ -290,8 +316,8 @@ public class HBaseTestingUtility { String randomStr = UUID.randomUUID().toString(); Path testPath= new Path(getBaseTestDir(), randomStr); - dataTestDir = new File(testPath.toString()).getAbsoluteFile(); - dataTestDir.deleteOnExit(); + this.dataTestDir = new File(testPath.toString()).getAbsoluteFile(); + this.dataTestDir.deleteOnExit(); createSubDirAndSystemProperty( "hadoop.log.dir", @@ -329,7 +355,7 @@ public class HBaseTestingUtility { // There is already a value set. So we do nothing but hope // that there will be no conflicts LOG.info("System.getProperty(\""+propertyName+"\") already set to: "+ - sysValue + " so I do NOT create it in "+dataTestDir.getAbsolutePath()); + sysValue + " so I do NOT create it in " + this.dataTestDir.getAbsolutePath()); String confValue = conf.get(propertyName); if (confValue != null && !confValue.endsWith(sysValue)){ LOG.warn( @@ -346,33 +372,6 @@ public class HBaseTestingUtility { } } - /** - * Creates a directory for the DFS cluster, under the test data - */ - private void setupClusterTestDir() { - if (clusterTestDir != null) { - LOG.warn("Cluster test dir already setup in " + - clusterTestDir.getAbsolutePath()); - return; - } - - // Using randomUUID ensures that multiple clusters can be launched by - // a same test, if it stops & starts them - Path testDir = getDataTestDir("dfscluster_" + UUID.randomUUID().toString()); - clusterTestDir = new File(testDir.toString()).getAbsoluteFile(); - // Have it cleaned up on exit - clusterTestDir.deleteOnExit(); - } - - /** - * @throws IOException If a cluster -- zk, dfs, or hbase -- already running. - */ - public void isRunningCluster() throws IOException { - if (dfsCluster == null) return; - throw new IOException("Cluster already running at " + - this.clusterTestDir); - } - /** * Start a minidfscluster. * @param servers How many DNs to start. @@ -396,7 +395,7 @@ public class HBaseTestingUtility { * @return The mini dfs cluster created. */ public MiniDFSCluster startMiniDFSCluster(final String hosts[]) - throws Exception { + throws Exception { if ( hosts != null && hosts.length != 0) { return startMiniDFSCluster(hosts.length, hosts); } else { @@ -415,24 +414,7 @@ public class HBaseTestingUtility { */ public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[]) throws Exception { - - // Check that there is not already a cluster running - isRunningCluster(); - - // Initialize the local directory used by the MiniDFS - if (clusterTestDir == null) { - setupClusterTestDir(); - } - - // We have to set this property as it is used by MiniCluster - System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.toString()); - - // Some tests also do this: - // System.getProperty("test.cache.data", "build/test/cache"); - // It's also deprecated - System.setProperty("test.cache.data", this.clusterTestDir.toString()); - - // Ok, now we can start + createDirsAndSetProperties(); this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true, true, null, null, hosts, null); @@ -448,18 +430,46 @@ public class HBaseTestingUtility { return this.dfsCluster; } + public MiniDFSCluster startMiniDFSClusterForTestHLog(int namenodePort) throws IOException { + createDirsAndSetProperties(); + dfsCluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null, + null, null, null); + return dfsCluster; + } + + /** This is used before starting HDFS and map-reduce mini-clusters */ + private void createDirsAndSetProperties() { + setupClusterTestDir(); + System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath()); + createDirAndSetProperty("cache_data", "test.cache.data"); + createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir"); + hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir"); + createDirAndSetProperty("mapred_output", "mapred.output.dir"); + createDirAndSetProperty("mapred_local", "mapred.local.dir"); + createDirAndSetProperty("mapred_system", "mapred.system.dir"); + createDirAndSetProperty("mapred_temp", "mapred.temp.dir"); + } + + private String createDirAndSetProperty(final String relPath, String property) { + String path = clusterTestDir.getPath() + "/" + relPath; + System.setProperty(property, path); + conf.set(property, path); + new File(path).mkdirs(); + LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf"); + return path; + } + /** * Shuts down instance created by call to {@link #startMiniDFSCluster(int)} * or does nothing. - * @throws Exception + * @throws IOException */ - public void shutdownMiniDFSCluster() throws Exception { + public void shutdownMiniDFSCluster() throws IOException { if (this.dfsCluster != null) { // The below throws an exception per dn, AsynchronousCloseException. this.dfsCluster.shutdown(); dfsCluster = null; } - } /** @@ -483,8 +493,8 @@ public class HBaseTestingUtility { */ public MiniZooKeeperCluster startMiniZKCluster(int zooKeeperServerNum) throws Exception { - File zkClusterFile = new File(getClusterTestDir().toString()); - return startMiniZKCluster(zkClusterFile, zooKeeperServerNum); + setupClusterTestDir(); + return startMiniZKCluster(clusterTestDir, zooKeeperServerNum); } private MiniZooKeeperCluster startMiniZKCluster(final File dir) @@ -597,7 +607,13 @@ public class HBaseTestingUtility { numSlaves + " regionserver(s) and " + numDataNodes + " datanode(s)"); // If we already put up a cluster, fail. - isRunningCluster(); + if (miniClusterRunning) { + throw new IllegalStateException("A mini-cluster is already running"); + } + miniClusterRunning = true; + + setupClusterTestDir(); + System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.getPath()); // Bring up mini dfs cluster. This spews a bunch of warnings about missing // scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'. @@ -691,15 +707,8 @@ public class HBaseTestingUtility { } shutdownMiniDFSCluster(); - // Clean up our directory. - if (this.clusterTestDir != null && this.clusterTestDir.exists()) { - // Need to use deleteDirectory because File.delete required dir is empty. - if (!FSUtils.deleteDirectory(FileSystem.getLocal(this.conf), - new Path(this.clusterTestDir.toString()))) { - LOG.warn("Failed delete of " + this.clusterTestDir.toString()); - } - this.clusterTestDir = null; - } + cleanupTestDir(); + miniClusterRunning = false; LOG.info("Minicluster is down"); } @@ -1265,28 +1274,59 @@ public class HBaseTestingUtility { * * @throws IOException When starting the cluster fails. */ - public void startMiniMapReduceCluster() throws IOException { + public MiniMRCluster startMiniMapReduceCluster() throws IOException { startMiniMapReduceCluster(2); + return mrCluster; } /** - * Starts a MiniMRCluster. - * + * Tasktracker has a bug where changing the hadoop.log.dir system property + * will not change its internal static LOG_DIR variable. + */ + private void forceChangeTaskLogDir() { + Field logDirField; + try { + logDirField = TaskLog.class.getDeclaredField("LOG_DIR"); + logDirField.setAccessible(true); + + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(logDirField, logDirField.getModifiers() & ~Modifier.FINAL); + + logDirField.set(null, new File(hadoopLogDir, "userlogs")); + } catch (SecurityException e) { + throw new RuntimeException(e); + } catch (NoSuchFieldException e) { + // TODO Auto-generated catch block + throw new RuntimeException(e); + } catch (IllegalArgumentException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + /** + * Starts a MiniMRCluster. Call {@link #setFileSystemURI(String)} to use a different + * filesystem. * @param servers The number of TaskTracker's to start. * @throws IOException When starting the cluster fails. */ - public void startMiniMapReduceCluster(final int servers) throws IOException { - LOG.info("Starting mini mapreduce cluster..."); - if (dataTestDir == null) { - setupDataTestDir(); + private void startMiniMapReduceCluster(final int servers) throws IOException { + if (mrCluster != null) { + throw new IllegalStateException("MiniMRCluster is already running"); } + LOG.info("Starting mini mapreduce cluster..."); + setupClusterTestDir(); + createDirsAndSetProperties(); - // These are needed for the new and improved Map/Reduce framework - conf.set("mapred.output.dir", conf.get("hadoop.tmp.dir")); - mrCluster = new MiniMRCluster(0, 0, servers, - FileSystem.get(conf).getUri().toString(), 1, null, null, null, new JobConf(conf)); + forceChangeTaskLogDir(); + + // Allow the user to override FS URI for this map-reduce cluster to use. + mrCluster = new MiniMRCluster(servers, + FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1); mrCluster.getJobTrackerRunner().getJobTracker().getConf().set("mapred.local.dir", - conf.get("mapred.local.dir")); //Hadoop MiniMR overwrites this while it should not + conf.get("mapred.local.dir")); //Hadoop MiniMR overwrites this while it should not LOG.info("Mini mapreduce cluster started"); conf.set("mapred.job.tracker", mrCluster.createJobConf().get("mapred.job.tracker")); @@ -1524,17 +1564,15 @@ public class HBaseTestingUtility { } /** - * @return True if we removed the test dir + * @return True if we removed the test dirs * @throws IOException */ - public boolean cleanupTestDir() throws IOException { - if (dataTestDir == null ){ - return false; - } else { - boolean ret = deleteDir(getDataTestDir()); - dataTestDir = null; - return ret; + boolean cleanupTestDir() throws IOException { + if (deleteDir(this.dataTestDir)) { + this.dataTestDir = null; + return true; } + return false; } /** @@ -1542,11 +1580,11 @@ public class HBaseTestingUtility { * @return True if we removed the test dir * @throws IOException */ - public boolean cleanupTestDir(final String subdir) throws IOException { - if (dataTestDir == null){ + boolean cleanupTestDir(final String subdir) throws IOException { + if (this.dataTestDir == null){ return false; } - return deleteDir(getDataTestDir(subdir)); + return deleteDir(new File(this.dataTestDir, subdir)); } /** @@ -1554,10 +1592,15 @@ public class HBaseTestingUtility { * @return True if we deleted it. * @throws IOException */ - public boolean deleteDir(final Path dir) throws IOException { - FileSystem fs = getTestFileSystem(); - if (fs.exists(dir)) { - return fs.delete(getDataTestDir(), true); + boolean deleteDir(final File dir) throws IOException { + if (dir != null && dir.exists()) { + // Need to use deleteDirectory because File.delete required dir is empty. + if (!FSUtils.deleteDirectory(FileSystem.getLocal(this.conf), + new Path(dir.getAbsolutePath()))) { + LOG.warn("Failed delete of " + dir.toString()); + } else { + return true; + } } return false; } @@ -2037,4 +2080,7 @@ public class HBaseTestingUtility { return region; } + public void setFileSystemURI(String fsURI) { + FS_URI = fsURI; + } } diff --git a/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java b/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java index e725d434577..7210f297b97 100644 --- a/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java +++ b/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java @@ -58,6 +58,7 @@ public class TestFullLogReconstruction { // faster failover with cluster.shutdown();fs.close() idiom c.setInt("ipc.client.connect.max.retries", 1); c.setInt("dfs.client.block.recovery.retries", 1); + c.setInt(HConstants.ZOOKEEPER_SESSION_TIMEOUT, 1000); TEST_UTIL.startMiniCluster(2); } @@ -129,4 +130,3 @@ public class TestFullLogReconstruction { public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); } - diff --git a/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java index 3ed5d52e008..c398aed144d 100644 --- a/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java +++ b/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java @@ -24,9 +24,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.File; -import java.io.IOException; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; @@ -38,10 +35,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -194,7 +187,7 @@ public class TestHBaseTestingUtility { @Test public void testMiniDFSCluster() throws Exception { HBaseTestingUtility hbt = new HBaseTestingUtility(); - MiniDFSCluster cluster = hbt.startMiniDFSCluster(1); + MiniDFSCluster cluster = hbt.startMiniDFSCluster(null); FileSystem dfs = cluster.getFileSystem(); Path dir = new Path("dir"); Path qualifiedDir = dfs.makeQualified(dir); @@ -213,12 +206,11 @@ public class TestHBaseTestingUtility { assertFalse(fs.exists(testdir)); - hbt.startMiniDFSCluster(1); + hbt.startMiniDFSCluster(null); assertTrue(fs.exists(testdir)); hbt.shutdownMiniCluster(); assertFalse(fs.exists(testdir)); - } @Test public void testTestDir() throws Exception { diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java index 6e655c5a2c5..d496d48f4e3 100644 --- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java +++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java @@ -68,8 +68,10 @@ public class TestZooKeeper { @BeforeClass public static void setUpBeforeClass() throws Exception { // Test we can first start the ZK cluster by itself + Configuration conf = TEST_UTIL.getConfiguration(); TEST_UTIL.startMiniZKCluster(); - TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + conf.setBoolean("dfs.support.append", true); + conf.setInt(HConstants.ZOOKEEPER_SESSION_TIMEOUT, 1000); TEST_UTIL.startMiniCluster(2); } diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java index 1918b1e7e4a..d039be359c0 100644 --- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java @@ -176,6 +176,8 @@ public class TestClassLoading { String classpath = currentDir + Path.SEPARATOR + "target"+ Path.SEPARATOR + "classes" + System.getProperty("path.separator") + + // Note that the below trick only works if mvn is running the test; + // doesn't work in eclipse for example. System.getProperty("surefire.test.class.path"); options.add(classpath); LOG.debug("Setting classpath to: "+classpath); diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java index e38cdcd4d7c..5a5d00a2138 100644 --- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java @@ -67,7 +67,6 @@ public class TestCoprocessorEndpoint { private static byte[][] ROWS = makeN(ROW, ROWSIZE); private static HBaseTestingUtility util = new HBaseTestingUtility(); - private static MiniHBaseCluster cluster = null; @BeforeClass public static void setupBeforeClass() throws Exception { @@ -76,10 +75,7 @@ public class TestCoprocessorEndpoint { conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint", "org.apache.hadoop.hbase.coprocessor.GenericEndpoint"); - util.startMiniCluster(2); - cluster = util.getMiniHBaseCluster(); - HTable table = util.createTable(TEST_TABLE, TEST_FAMILY); util.createMultiRegions(util.getConfiguration(), table, TEST_FAMILY, new byte[][] { HConstants.EMPTY_BYTE_ARRAY, diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java index a9288ce979e..de401470e9a 100644 --- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java @@ -130,8 +130,7 @@ public class TestFixedFileTrailer { } // Now check what happens if the trailer is corrupted. - Path trailerPath = new Path(util.getDataTestDir(), "trailer_" - + version); + Path trailerPath = new Path(util.getDataTestDir(), "trailer_" + version); { for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1, diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index bb992b8a66d..6614e671faa 100644 --- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -52,7 +52,9 @@ import org.junit.experimental.categories.Category; public class TestHFile extends HBaseTestCase { static final Log LOG = LogFactory.getLog(TestHFile.class); - private String ROOT_DIR; + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static String ROOT_DIR = + TEST_UTIL.getDataTestDir("TestHFile").toString(); private final int minBlockSize = 512; private static String localFormatter = "%010d"; private static CacheConfig cacheConf = null; @@ -61,7 +63,6 @@ public class TestHFile extends HBaseTestCase { @Override public void setUp() throws Exception { startingMetrics = SchemaMetrics.getMetricsSnapshot(); - ROOT_DIR = this.getUnitTestdir("TestHFile").toString(); super.setUp(); } diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java index 24974e57eb9..ff97fe61d06 100644 --- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java @@ -78,14 +78,6 @@ public class TestHFileDataBlockEncoder { SchemaMetrics.configureGlobally(conf); } - /** - * Cleanup after JUnit test. - */ - @After - public void tearDown() throws IOException { - TEST_UTIL.cleanupTestDir(); - } - /** * Test putting and taking out blocks into cache with different * encoding options. diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java index 8798999b221..c4d259a40a8 100644 --- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java @@ -66,6 +66,8 @@ public class TestHFileSeek extends TestCase { private RandomDistribution.DiscreteRNG keyLenGen; private KVGenerator kvGen; + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + @Override public void setUp() throws IOException { if (options == null) { diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java index 17284a3880a..bc7071b8314 100644 --- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; -import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType; import org.apache.hadoop.hbase.util.Bytes; @@ -66,7 +65,7 @@ public class TestScannerSelectionUsingTTL { private static String FAMILY = "myCF"; private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY); - private static final int TTL_SECONDS = 2; + private static final int TTL_SECONDS = 10; private static final int TTL_MS = TTL_SECONDS * 1000; private static final int NUM_EXPIRED_FILES = 2; @@ -106,7 +105,7 @@ public class TestScannerSelectionUsingTTL { htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(Bytes.toBytes(TABLE)); HRegion region = - HRegion.createHRegion(info, TEST_UTIL.getClusterTestDir(), + HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(info.getEncodedName()), TEST_UTIL.getConfiguration(), htd); for (int iFile = 0; iFile < totalNumFiles; ++iFile) { diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java index e43451c2b80..0e265124648 100644 --- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java @@ -77,7 +77,6 @@ import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; @@ -159,12 +158,6 @@ public class TestHFileOutputFormat { } } - @Before - public void cleanupDir() throws IOException { - util.cleanupTestDir(); - } - - private void setupRandomGeneratorMapper(Job job) { job.setInputFormatClass(NMapInputFormat.class); job.setMapperClass(RandomKVGeneratingMapper.class); @@ -370,16 +363,19 @@ public class TestHFileOutputFormat { @Test public void testMRIncrementalLoad() throws Exception { + LOG.info("\nStarting test testMRIncrementalLoad\n"); doIncrementalLoadTest(false); } @Test public void testMRIncrementalLoadWithSplit() throws Exception { + LOG.info("\nStarting test testMRIncrementalLoadWithSplit\n"); doIncrementalLoadTest(true); } private void doIncrementalLoadTest( boolean shouldChangeRegions) throws Exception { + util = new HBaseTestingUtility(); Configuration conf = util.getConfiguration(); Path testDir = util.getDataTestDir("testLocalMRIncrementalLoad"); byte[][] startKeys = generateRandomStartKeys(5); @@ -442,9 +438,7 @@ public class TestHFileOutputFormat { expectedRows, util.countRows(table)); Scan scan = new Scan(); ResultScanner results = table.getScanner(scan); - int count = 0; for (Result res : results) { - count++; assertEquals(FAMILIES.length, res.raw().length); KeyValue first = res.raw()[0]; for (KeyValue kv : res.raw()) { diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java index 74ea4ec35fb..e051f6eaa6f 100644 --- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java @@ -165,7 +165,7 @@ public class TestTimeRangeMapRed { private void runTestOnTable() throws IOException, InterruptedException, ClassNotFoundException { - UTIL.startMiniMapReduceCluster(1); + UTIL.startMiniMapReduceCluster(); Job job = null; try { job = new Job(UTIL.getConfiguration(), "test123"); diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java index 6602134aace..0f7d54e3146 100644 --- a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java +++ b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java @@ -69,6 +69,12 @@ public class TestSplitLogManager { private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + /** + * Additional amount of time we wait for events to happen. Added where unit + * test failures have been observed. + */ + private static final int EXTRA_TOLERANCE_MS = 200; + static Stoppable stopper = new Stoppable() { @Override public void stop(String why) { @@ -94,7 +100,8 @@ public class TestSplitLogManager { public void setup() throws Exception { TEST_UTIL.startMiniZKCluster(); conf = TEST_UTIL.getConfiguration(); - zkw = new ZooKeeperWatcher(conf, "split-log-manager-tests", null); + // Use a different ZK wrapper instance for each tests. + zkw = new ZooKeeperWatcher(conf, "split-log-manager-tests" + UUID.randomUUID().toString(), null); ZKUtil.deleteChildrenRecursively(zkw, zkw.baseZNode); ZKUtil.createAndFailSilent(zkw, zkw.baseZNode); assertTrue(ZKUtil.checkExists(zkw, zkw.baseZNode) != -1); @@ -211,7 +218,7 @@ public class TestSplitLogManager { assertTrue((task.last_update <= curt) && (task.last_update > (curt - 1000))); LOG.info("waiting for manager to resubmit the orphan task"); - waitForCounter(tot_mgr_resubmit, 0, 1, to + 100); + waitForCounter(tot_mgr_resubmit, 0, 1, to + 300); assertTrue(task.isUnassigned()); waitForCounter(tot_mgr_rescan, 0, 1, to + 100); } @@ -265,7 +272,7 @@ public class TestSplitLogManager { ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker1")); waitForCounter(tot_mgr_heartbeat, 0, 1, 1000); - waitForCounter(tot_mgr_resubmit, 0, 1, to + 100); + waitForCounter(tot_mgr_resubmit, 0, 1, to + EXTRA_TOLERANCE_MS); int version1 = ZKUtil.checkExists(zkw, tasknode); assertTrue(version1 > version); ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker2")); @@ -275,8 +282,8 @@ public class TestSplitLogManager { assertTrue(version2 > version1); ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker3")); waitForCounter(tot_mgr_heartbeat, 1, 2, 1000); - waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + 100); - Thread.sleep(to + 100); + waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + EXTRA_TOLERANCE_MS); + Thread.sleep(to + EXTRA_TOLERANCE_MS); assertEquals(2L, tot_mgr_resubmit.get()); } diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java index 76c329fa4bf..988d0bf4427 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.client.Scan; @@ -66,8 +67,10 @@ import com.google.common.collect.Lists; @Category(SmallTests.class) public class TestStoreFile extends HBaseTestCase { static final Log LOG = LogFactory.getLog(TestStoreFile.class); - private CacheConfig cacheConf = new CacheConfig(conf); - private String ROOT_DIR; + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration()); + private static String ROOT_DIR = + TEST_UTIL.getDataTestDir("TestStoreFile").toString(); private Map startingMetrics; private static final ChecksumType CKTYPE = ChecksumType.CRC32; @@ -77,7 +80,6 @@ public class TestStoreFile extends HBaseTestCase { public void setUp() throws Exception { super.setUp(); startingMetrics = SchemaMetrics.getMetricsSnapshot(); - ROOT_DIR = new Path(this.testDir, "TestStoreFile").toString(); } @Override diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java index 81592f85fe5..29377652fe6 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java @@ -19,10 +19,7 @@ */ package org.apache.hadoop.hbase.regionserver.wal; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import java.io.IOException; import java.lang.reflect.Method; @@ -51,7 +48,6 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.LeaseManager; import org.apache.hadoop.io.SequenceFile; import org.apache.log4j.Level; @@ -405,6 +401,7 @@ public class TestHLog { LOG.error("Waiting for cluster to go down"); Thread.sleep(1000); } + assertFalse(cluster.isClusterUp()); // Workaround a strange issue with Hadoop's RPC system - if we don't // sleep here, the new datanodes will pick up a cached IPC connection to @@ -412,11 +409,12 @@ public class TestHLog { // the idle time threshold configured in the conf above Thread.sleep(2000); - cluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null, null, null, null); - TEST_UTIL.setDFSCluster(cluster); + LOG.info("Waiting a few seconds before re-starting HDFS"); + Thread.sleep(5000); + cluster = TEST_UTIL.startMiniDFSClusterForTestHLog(namenodePort); cluster.waitActive(); fs = cluster.getFileSystem(); - LOG.info("START second instance."); + LOG.info("STARTED second instance."); } // set the lease period to be 1 second so that the diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java index d24648c014d..53a58be91fa 100644 --- a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java +++ b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java @@ -76,6 +76,9 @@ public class TestMiniClusterLoadSequential { this.isMultiPut = isMultiPut; this.dataBlockEncoding = dataBlockEncoding; conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024); + + // We don't want any region reassignments by the load balancer during the test. + conf.setFloat(HConstants.LOAD_BALANCER_SLOP_KEY, 10.0f); } @Parameters diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java index 43046809e25..6551a705335 100644 --- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java +++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java @@ -93,7 +93,7 @@ public class TestZKLeaderManager { while (master.get() && !stopped) { try { - Thread.sleep(200); + Thread.sleep(10); } catch (InterruptedException ignored) {} } } @@ -204,8 +204,8 @@ public class TestZKLeaderManager { private MockLeader getCurrentLeader() throws Exception { MockLeader currentLeader = null; outer: - // wait up to 2 secs for initial leader - for (int i = 0; i < 20; i++) { + // Wait up to 10 secs for initial leader + for (int i = 0; i < 1000; i++) { for (int j = 0; j < CANDIDATES.length; j++) { if (CANDIDATES[j].isMaster()) { // should only be one leader @@ -218,7 +218,7 @@ public class TestZKLeaderManager { if (currentLeader != null) { break outer; } - Thread.sleep(100); + Thread.sleep(10); } return currentLeader; } diff --git a/src/test/resources/hbase-site.xml b/src/test/resources/hbase-site.xml index 84b56126592..e7b711237d6 100644 --- a/src/test/resources/hbase-site.xml +++ b/src/test/resources/hbase-site.xml @@ -142,4 +142,11 @@ version is X.X.X-SNAPSHOT" + + hbase.client.retries.number + 100 + + Use a lot of retries in unit tests. + +