HBASE-5747 Forward port "hbase-5708 [89-fb] Make MiniMapRedCluster directory a subdirectory of target/test"

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1326000 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2012-04-13 22:45:25 +00:00
parent 4d04f85308
commit b987a52e36
22 changed files with 219 additions and 198 deletions

View File

@ -103,6 +103,10 @@ public final class HConstants {
/** by default every master is a possible primary master unless the conf explicitly overrides it */
public static final boolean DEFAULT_MASTER_TYPE_BACKUP = false;
/** Parameter name for ZooKeeper session time out.*/
public static final String ZOOKEEPER_SESSION_TIMEOUT =
"zookeeper.session.timeout";
/** Name of ZooKeeper quorum configuration parameter. */
public static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
@ -655,6 +659,8 @@ public final class HConstants {
/** Region in Transition metrics threshold time */
public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD="hbase.metrics.rit.stuck.warning.threshold";
public static final String LOAD_BALANCER_SLOP_KEY = "hbase.regions.slop";
private HConstants() {
// Can't be instantiated with this ctor.
}

View File

@ -1004,6 +1004,10 @@ public class SplitLogManager extends ZooKeeperListener {
Stat stat) {
tot_mgr_get_data_result.incrementAndGet();
if (rc != 0) {
if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) {
LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries.");
return;
}
if (rc == KeeperException.Code.NONODE.intValue()) {
tot_mgr_get_data_nonode.incrementAndGet();
// The task znode has been deleted. Must be some pending delete
@ -1085,6 +1089,10 @@ public class SplitLogManager extends ZooKeeperListener {
@Override
public void processResult(int rc, String path, Object ctx, String name) {
if (rc != 0) {
if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) {
LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries.");
return;
}
Long retry_count = (Long)ctx;
LOG.warn("rc=" + KeeperException.Code.get(rc) + " for "+ path +
" remaining retries=" + retry_count);

View File

@ -19,7 +19,6 @@
*/
package org.apache.hadoop.hbase;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Iterator;
@ -54,16 +53,6 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
public abstract class HBaseTestCase extends TestCase {
private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
/** configuration parameter name for test directory
* @deprecated see HBaseTestingUtility#TEST_DIRECTORY_KEY
**/
private static final String TEST_DIRECTORY_KEY = "test.build.data";
/*
protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
*/
protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
@ -82,9 +71,7 @@ public abstract class HBaseTestCase extends TestCase {
protected String START_KEY;
protected static final int MAXVERSIONS = 3;
static {
initialize();
}
protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
public volatile Configuration conf;
@ -161,19 +148,12 @@ public abstract class HBaseTestCase extends TestCase {
* @return directory to use for this test
*/
protected Path getUnitTestdir(String testName) {
return new Path(
System.getProperty(
HBaseTestingUtility.BASE_TEST_DIRECTORY_KEY,
HBaseTestingUtility.DEFAULT_BASE_TEST_DIRECTORY
),
testName
);
return testUtil.getDataTestDir(testName);
}
protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
byte [] endKey)
throws IOException {
FileSystem filesystem = FileSystem.get(conf);
HRegionInfo hri = new HRegionInfo(desc.getName(), startKey, endKey);
return HRegion.createHRegion(hri, testDir, conf, desc);
}
@ -628,21 +608,6 @@ public abstract class HBaseTestCase extends TestCase {
}
}
/**
* Initializes parameters used in the test environment:
*
* Sets the configuration parameter TEST_DIRECTORY_KEY if not already set.
* Sets the boolean debugging if "DEBUGGING" is set in the environment.
* If debugging is enabled, reconfigures logging so that the root log level is
* set to WARN and the logging level for the package is set to DEBUG.
*/
public static void initialize() {
if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
System.setProperty(TEST_DIRECTORY_KEY, new File(
"build/hbase/test").getAbsolutePath());
}
}
/**
* Common method to close down a MiniDFSCluster and the associated file system
*

View File

@ -25,6 +25,7 @@ import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
@ -84,8 +85,8 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.TaskLog;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.apache.zookeeper.WatchedEvent;
@ -124,11 +125,15 @@ public class HBaseTestingUtility {
private MiniHBaseCluster hbaseCluster = null;
private MiniMRCluster mrCluster = null;
// Directory where we put the data for this instance of HBaseTestingUtility
/** If there is a mini cluster running for this testing utility instance. */
private boolean miniClusterRunning;
private String hadoopLogDir;
// Directory where we put the data for this instance of HBaseTestingUtility.
private File dataTestDir = null;
// Directory (usually a subdirectory of dataTestDir) used by the dfs cluster
// if any
// Directory (a subdirectory of dataTestDir) used by the dfs cluster if any
private File clusterTestDir = null;
/**
@ -151,6 +156,9 @@ public class HBaseTestingUtility {
*/
public static final String DEFAULT_BASE_TEST_DIRECTORY = "target/test-data";
/** Filesystem URI used for map-reduce mini-cluster setup */
private static String FS_URI;
/** Compression algorithms to use in parameterized JUnit 4 tests */
public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
Arrays.asList(new Object[][] {
@ -236,24 +244,42 @@ public class HBaseTestingUtility {
* @see #getTestFileSystem()
*/
public Path getDataTestDir() {
if (dataTestDir == null){
if (this.dataTestDir == null){
setupDataTestDir();
}
return new Path(dataTestDir.getAbsolutePath());
return new Path(this.dataTestDir.getAbsolutePath());
}
/**
* @return Where the DFS cluster will write data on the local subsystem.
* Creates it if it does not exist already.
* Creates it if it does not exist already. A subdir of {@link #getBaseTestDir()}
* @see #getTestFileSystem()
*/
public Path getClusterTestDir() {
Path getClusterTestDir() {
if (clusterTestDir == null){
setupClusterTestDir();
}
return new Path(clusterTestDir.getAbsolutePath());
}
/**
* Creates a directory for the DFS cluster, under the test data
*/
private void setupClusterTestDir() {
if (clusterTestDir != null) {
return;
}
// Using randomUUID ensures that multiple clusters can be launched by
// a same test, if it stops & starts them
Path testDir = getDataTestDir("dfscluster_" + UUID.randomUUID().toString());
clusterTestDir = new File(testDir.toString()).getAbsoluteFile();
// Have it cleaned up on exit
clusterTestDir.deleteOnExit();
conf.set(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
LOG.info("Created new mini-cluster data directory: " + clusterTestDir);
}
/**
* @param subdirName
* @return Path to a subdirectory named <code>subdirName</code> under
@ -281,7 +307,7 @@ public class HBaseTestingUtility {
* @return The calculated data test build directory.
*/
private void setupDataTestDir() {
if (dataTestDir != null) {
if (this.dataTestDir != null) {
LOG.warn("Data test dir already setup in " +
dataTestDir.getAbsolutePath());
return;
@ -290,8 +316,8 @@ public class HBaseTestingUtility {
String randomStr = UUID.randomUUID().toString();
Path testPath= new Path(getBaseTestDir(), randomStr);
dataTestDir = new File(testPath.toString()).getAbsoluteFile();
dataTestDir.deleteOnExit();
this.dataTestDir = new File(testPath.toString()).getAbsoluteFile();
this.dataTestDir.deleteOnExit();
createSubDirAndSystemProperty(
"hadoop.log.dir",
@ -329,7 +355,7 @@ public class HBaseTestingUtility {
// There is already a value set. So we do nothing but hope
// that there will be no conflicts
LOG.info("System.getProperty(\""+propertyName+"\") already set to: "+
sysValue + " so I do NOT create it in "+dataTestDir.getAbsolutePath());
sysValue + " so I do NOT create it in " + this.dataTestDir.getAbsolutePath());
String confValue = conf.get(propertyName);
if (confValue != null && !confValue.endsWith(sysValue)){
LOG.warn(
@ -346,33 +372,6 @@ public class HBaseTestingUtility {
}
}
/**
* Creates a directory for the DFS cluster, under the test data
*/
private void setupClusterTestDir() {
if (clusterTestDir != null) {
LOG.warn("Cluster test dir already setup in " +
clusterTestDir.getAbsolutePath());
return;
}
// Using randomUUID ensures that multiple clusters can be launched by
// a same test, if it stops & starts them
Path testDir = getDataTestDir("dfscluster_" + UUID.randomUUID().toString());
clusterTestDir = new File(testDir.toString()).getAbsoluteFile();
// Have it cleaned up on exit
clusterTestDir.deleteOnExit();
}
/**
* @throws IOException If a cluster -- zk, dfs, or hbase -- already running.
*/
public void isRunningCluster() throws IOException {
if (dfsCluster == null) return;
throw new IOException("Cluster already running at " +
this.clusterTestDir);
}
/**
* Start a minidfscluster.
* @param servers How many DNs to start.
@ -396,7 +395,7 @@ public class HBaseTestingUtility {
* @return The mini dfs cluster created.
*/
public MiniDFSCluster startMiniDFSCluster(final String hosts[])
throws Exception {
throws Exception {
if ( hosts != null && hosts.length != 0) {
return startMiniDFSCluster(hosts.length, hosts);
} else {
@ -415,24 +414,7 @@ public class HBaseTestingUtility {
*/
public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[])
throws Exception {
// Check that there is not already a cluster running
isRunningCluster();
// Initialize the local directory used by the MiniDFS
if (clusterTestDir == null) {
setupClusterTestDir();
}
// We have to set this property as it is used by MiniCluster
System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.toString());
// Some tests also do this:
// System.getProperty("test.cache.data", "build/test/cache");
// It's also deprecated
System.setProperty("test.cache.data", this.clusterTestDir.toString());
// Ok, now we can start
createDirsAndSetProperties();
this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
true, null, null, hosts, null);
@ -448,18 +430,46 @@ public class HBaseTestingUtility {
return this.dfsCluster;
}
public MiniDFSCluster startMiniDFSClusterForTestHLog(int namenodePort) throws IOException {
createDirsAndSetProperties();
dfsCluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null,
null, null, null);
return dfsCluster;
}
/** This is used before starting HDFS and map-reduce mini-clusters */
private void createDirsAndSetProperties() {
setupClusterTestDir();
System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
createDirAndSetProperty("cache_data", "test.cache.data");
createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir");
createDirAndSetProperty("mapred_output", "mapred.output.dir");
createDirAndSetProperty("mapred_local", "mapred.local.dir");
createDirAndSetProperty("mapred_system", "mapred.system.dir");
createDirAndSetProperty("mapred_temp", "mapred.temp.dir");
}
private String createDirAndSetProperty(final String relPath, String property) {
String path = clusterTestDir.getPath() + "/" + relPath;
System.setProperty(property, path);
conf.set(property, path);
new File(path).mkdirs();
LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf");
return path;
}
/**
* Shuts down instance created by call to {@link #startMiniDFSCluster(int)}
* or does nothing.
* @throws Exception
* @throws IOException
*/
public void shutdownMiniDFSCluster() throws Exception {
public void shutdownMiniDFSCluster() throws IOException {
if (this.dfsCluster != null) {
// The below throws an exception per dn, AsynchronousCloseException.
this.dfsCluster.shutdown();
dfsCluster = null;
}
}
/**
@ -483,8 +493,8 @@ public class HBaseTestingUtility {
*/
public MiniZooKeeperCluster startMiniZKCluster(int zooKeeperServerNum)
throws Exception {
File zkClusterFile = new File(getClusterTestDir().toString());
return startMiniZKCluster(zkClusterFile, zooKeeperServerNum);
setupClusterTestDir();
return startMiniZKCluster(clusterTestDir, zooKeeperServerNum);
}
private MiniZooKeeperCluster startMiniZKCluster(final File dir)
@ -597,7 +607,13 @@ public class HBaseTestingUtility {
numSlaves + " regionserver(s) and " + numDataNodes + " datanode(s)");
// If we already put up a cluster, fail.
isRunningCluster();
if (miniClusterRunning) {
throw new IllegalStateException("A mini-cluster is already running");
}
miniClusterRunning = true;
setupClusterTestDir();
System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.getPath());
// Bring up mini dfs cluster. This spews a bunch of warnings about missing
// scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'.
@ -691,15 +707,8 @@ public class HBaseTestingUtility {
}
shutdownMiniDFSCluster();
// Clean up our directory.
if (this.clusterTestDir != null && this.clusterTestDir.exists()) {
// Need to use deleteDirectory because File.delete required dir is empty.
if (!FSUtils.deleteDirectory(FileSystem.getLocal(this.conf),
new Path(this.clusterTestDir.toString()))) {
LOG.warn("Failed delete of " + this.clusterTestDir.toString());
}
this.clusterTestDir = null;
}
cleanupTestDir();
miniClusterRunning = false;
LOG.info("Minicluster is down");
}
@ -1265,28 +1274,59 @@ public class HBaseTestingUtility {
*
* @throws IOException When starting the cluster fails.
*/
public void startMiniMapReduceCluster() throws IOException {
public MiniMRCluster startMiniMapReduceCluster() throws IOException {
startMiniMapReduceCluster(2);
return mrCluster;
}
/**
* Starts a <code>MiniMRCluster</code>.
*
* Tasktracker has a bug where changing the hadoop.log.dir system property
* will not change its internal static LOG_DIR variable.
*/
private void forceChangeTaskLogDir() {
Field logDirField;
try {
logDirField = TaskLog.class.getDeclaredField("LOG_DIR");
logDirField.setAccessible(true);
Field modifiersField = Field.class.getDeclaredField("modifiers");
modifiersField.setAccessible(true);
modifiersField.setInt(logDirField, logDirField.getModifiers() & ~Modifier.FINAL);
logDirField.set(null, new File(hadoopLogDir, "userlogs"));
} catch (SecurityException e) {
throw new RuntimeException(e);
} catch (NoSuchFieldException e) {
// TODO Auto-generated catch block
throw new RuntimeException(e);
} catch (IllegalArgumentException e) {
throw new RuntimeException(e);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
/**
* Starts a <code>MiniMRCluster</code>. Call {@link #setFileSystemURI(String)} to use a different
* filesystem.
* @param servers The number of <code>TaskTracker</code>'s to start.
* @throws IOException When starting the cluster fails.
*/
public void startMiniMapReduceCluster(final int servers) throws IOException {
LOG.info("Starting mini mapreduce cluster...");
if (dataTestDir == null) {
setupDataTestDir();
private void startMiniMapReduceCluster(final int servers) throws IOException {
if (mrCluster != null) {
throw new IllegalStateException("MiniMRCluster is already running");
}
LOG.info("Starting mini mapreduce cluster...");
setupClusterTestDir();
createDirsAndSetProperties();
// These are needed for the new and improved Map/Reduce framework
conf.set("mapred.output.dir", conf.get("hadoop.tmp.dir"));
mrCluster = new MiniMRCluster(0, 0, servers,
FileSystem.get(conf).getUri().toString(), 1, null, null, null, new JobConf(conf));
forceChangeTaskLogDir();
// Allow the user to override FS URI for this map-reduce cluster to use.
mrCluster = new MiniMRCluster(servers,
FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1);
mrCluster.getJobTrackerRunner().getJobTracker().getConf().set("mapred.local.dir",
conf.get("mapred.local.dir")); //Hadoop MiniMR overwrites this while it should not
conf.get("mapred.local.dir")); //Hadoop MiniMR overwrites this while it should not
LOG.info("Mini mapreduce cluster started");
conf.set("mapred.job.tracker",
mrCluster.createJobConf().get("mapred.job.tracker"));
@ -1524,17 +1564,15 @@ public class HBaseTestingUtility {
}
/**
* @return True if we removed the test dir
* @return True if we removed the test dirs
* @throws IOException
*/
public boolean cleanupTestDir() throws IOException {
if (dataTestDir == null ){
return false;
} else {
boolean ret = deleteDir(getDataTestDir());
dataTestDir = null;
return ret;
boolean cleanupTestDir() throws IOException {
if (deleteDir(this.dataTestDir)) {
this.dataTestDir = null;
return true;
}
return false;
}
/**
@ -1542,11 +1580,11 @@ public class HBaseTestingUtility {
* @return True if we removed the test dir
* @throws IOException
*/
public boolean cleanupTestDir(final String subdir) throws IOException {
if (dataTestDir == null){
boolean cleanupTestDir(final String subdir) throws IOException {
if (this.dataTestDir == null){
return false;
}
return deleteDir(getDataTestDir(subdir));
return deleteDir(new File(this.dataTestDir, subdir));
}
/**
@ -1554,10 +1592,15 @@ public class HBaseTestingUtility {
* @return True if we deleted it.
* @throws IOException
*/
public boolean deleteDir(final Path dir) throws IOException {
FileSystem fs = getTestFileSystem();
if (fs.exists(dir)) {
return fs.delete(getDataTestDir(), true);
boolean deleteDir(final File dir) throws IOException {
if (dir != null && dir.exists()) {
// Need to use deleteDirectory because File.delete required dir is empty.
if (!FSUtils.deleteDirectory(FileSystem.getLocal(this.conf),
new Path(dir.getAbsolutePath()))) {
LOG.warn("Failed delete of " + dir.toString());
} else {
return true;
}
}
return false;
}
@ -2037,4 +2080,7 @@ public class HBaseTestingUtility {
return region;
}
public void setFileSystemURI(String fsURI) {
FS_URI = fsURI;
}
}

View File

@ -58,6 +58,7 @@ public class TestFullLogReconstruction {
// faster failover with cluster.shutdown();fs.close() idiom
c.setInt("ipc.client.connect.max.retries", 1);
c.setInt("dfs.client.block.recovery.retries", 1);
c.setInt(HConstants.ZOOKEEPER_SESSION_TIMEOUT, 1000);
TEST_UTIL.startMiniCluster(2);
}
@ -129,4 +130,3 @@ public class TestFullLogReconstruction {
public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
}

View File

@ -24,9 +24,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
@ -38,10 +35,6 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@ -194,7 +187,7 @@ public class TestHBaseTestingUtility {
@Test public void testMiniDFSCluster() throws Exception {
HBaseTestingUtility hbt = new HBaseTestingUtility();
MiniDFSCluster cluster = hbt.startMiniDFSCluster(1);
MiniDFSCluster cluster = hbt.startMiniDFSCluster(null);
FileSystem dfs = cluster.getFileSystem();
Path dir = new Path("dir");
Path qualifiedDir = dfs.makeQualified(dir);
@ -213,12 +206,11 @@ public class TestHBaseTestingUtility {
assertFalse(fs.exists(testdir));
hbt.startMiniDFSCluster(1);
hbt.startMiniDFSCluster(null);
assertTrue(fs.exists(testdir));
hbt.shutdownMiniCluster();
assertFalse(fs.exists(testdir));
}
@Test public void testTestDir() throws Exception {

View File

@ -68,8 +68,10 @@ public class TestZooKeeper {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
// Test we can first start the ZK cluster by itself
Configuration conf = TEST_UTIL.getConfiguration();
TEST_UTIL.startMiniZKCluster();
TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
conf.setBoolean("dfs.support.append", true);
conf.setInt(HConstants.ZOOKEEPER_SESSION_TIMEOUT, 1000);
TEST_UTIL.startMiniCluster(2);
}

View File

@ -176,6 +176,8 @@ public class TestClassLoading {
String classpath =
currentDir + Path.SEPARATOR + "target"+ Path.SEPARATOR + "classes" +
System.getProperty("path.separator") +
// Note that the below trick only works if mvn is running the test;
// doesn't work in eclipse for example.
System.getProperty("surefire.test.class.path");
options.add(classpath);
LOG.debug("Setting classpath to: "+classpath);

View File

@ -67,7 +67,6 @@ public class TestCoprocessorEndpoint {
private static byte[][] ROWS = makeN(ROW, ROWSIZE);
private static HBaseTestingUtility util = new HBaseTestingUtility();
private static MiniHBaseCluster cluster = null;
@BeforeClass
public static void setupBeforeClass() throws Exception {
@ -76,10 +75,7 @@ public class TestCoprocessorEndpoint {
conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
"org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint",
"org.apache.hadoop.hbase.coprocessor.GenericEndpoint");
util.startMiniCluster(2);
cluster = util.getMiniHBaseCluster();
HTable table = util.createTable(TEST_TABLE, TEST_FAMILY);
util.createMultiRegions(util.getConfiguration(), table, TEST_FAMILY,
new byte[][] { HConstants.EMPTY_BYTE_ARRAY,

View File

@ -130,8 +130,7 @@ public class TestFixedFileTrailer {
}
// Now check what happens if the trailer is corrupted.
Path trailerPath = new Path(util.getDataTestDir(), "trailer_"
+ version);
Path trailerPath = new Path(util.getDataTestDir(), "trailer_" + version);
{
for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1,

View File

@ -52,7 +52,9 @@ import org.junit.experimental.categories.Category;
public class TestHFile extends HBaseTestCase {
static final Log LOG = LogFactory.getLog(TestHFile.class);
private String ROOT_DIR;
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static String ROOT_DIR =
TEST_UTIL.getDataTestDir("TestHFile").toString();
private final int minBlockSize = 512;
private static String localFormatter = "%010d";
private static CacheConfig cacheConf = null;
@ -61,7 +63,6 @@ public class TestHFile extends HBaseTestCase {
@Override
public void setUp() throws Exception {
startingMetrics = SchemaMetrics.getMetricsSnapshot();
ROOT_DIR = this.getUnitTestdir("TestHFile").toString();
super.setUp();
}

View File

@ -78,14 +78,6 @@ public class TestHFileDataBlockEncoder {
SchemaMetrics.configureGlobally(conf);
}
/**
* Cleanup after JUnit test.
*/
@After
public void tearDown() throws IOException {
TEST_UTIL.cleanupTestDir();
}
/**
* Test putting and taking out blocks into cache with different
* encoding options.

View File

@ -66,6 +66,8 @@ public class TestHFileSeek extends TestCase {
private RandomDistribution.DiscreteRNG keyLenGen;
private KVGenerator kvGen;
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@Override
public void setUp() throws IOException {
if (options == null) {

View File

@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType;
import org.apache.hadoop.hbase.util.Bytes;
@ -66,7 +65,7 @@ public class TestScannerSelectionUsingTTL {
private static String FAMILY = "myCF";
private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
private static final int TTL_SECONDS = 2;
private static final int TTL_SECONDS = 10;
private static final int TTL_MS = TTL_SECONDS * 1000;
private static final int NUM_EXPIRED_FILES = 2;
@ -106,7 +105,7 @@ public class TestScannerSelectionUsingTTL {
htd.addFamily(hcd);
HRegionInfo info = new HRegionInfo(Bytes.toBytes(TABLE));
HRegion region =
HRegion.createHRegion(info, TEST_UTIL.getClusterTestDir(),
HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(info.getEncodedName()),
TEST_UTIL.getConfiguration(), htd);
for (int iFile = 0; iFile < totalNumFiles; ++iFile) {

View File

@ -77,7 +77,6 @@ import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.mockito.Mockito;
@ -159,12 +158,6 @@ public class TestHFileOutputFormat {
}
}
@Before
public void cleanupDir() throws IOException {
util.cleanupTestDir();
}
private void setupRandomGeneratorMapper(Job job) {
job.setInputFormatClass(NMapInputFormat.class);
job.setMapperClass(RandomKVGeneratingMapper.class);
@ -370,16 +363,19 @@ public class TestHFileOutputFormat {
@Test
public void testMRIncrementalLoad() throws Exception {
LOG.info("\nStarting test testMRIncrementalLoad\n");
doIncrementalLoadTest(false);
}
@Test
public void testMRIncrementalLoadWithSplit() throws Exception {
LOG.info("\nStarting test testMRIncrementalLoadWithSplit\n");
doIncrementalLoadTest(true);
}
private void doIncrementalLoadTest(
boolean shouldChangeRegions) throws Exception {
util = new HBaseTestingUtility();
Configuration conf = util.getConfiguration();
Path testDir = util.getDataTestDir("testLocalMRIncrementalLoad");
byte[][] startKeys = generateRandomStartKeys(5);
@ -442,9 +438,7 @@ public class TestHFileOutputFormat {
expectedRows, util.countRows(table));
Scan scan = new Scan();
ResultScanner results = table.getScanner(scan);
int count = 0;
for (Result res : results) {
count++;
assertEquals(FAMILIES.length, res.raw().length);
KeyValue first = res.raw()[0];
for (KeyValue kv : res.raw()) {

View File

@ -165,7 +165,7 @@ public class TestTimeRangeMapRed {
private void runTestOnTable()
throws IOException, InterruptedException, ClassNotFoundException {
UTIL.startMiniMapReduceCluster(1);
UTIL.startMiniMapReduceCluster();
Job job = null;
try {
job = new Job(UTIL.getConfiguration(), "test123");

View File

@ -69,6 +69,12 @@ public class TestSplitLogManager {
private final static HBaseTestingUtility TEST_UTIL =
new HBaseTestingUtility();
/**
* Additional amount of time we wait for events to happen. Added where unit
* test failures have been observed.
*/
private static final int EXTRA_TOLERANCE_MS = 200;
static Stoppable stopper = new Stoppable() {
@Override
public void stop(String why) {
@ -94,7 +100,8 @@ public class TestSplitLogManager {
public void setup() throws Exception {
TEST_UTIL.startMiniZKCluster();
conf = TEST_UTIL.getConfiguration();
zkw = new ZooKeeperWatcher(conf, "split-log-manager-tests", null);
// Use a different ZK wrapper instance for each tests.
zkw = new ZooKeeperWatcher(conf, "split-log-manager-tests" + UUID.randomUUID().toString(), null);
ZKUtil.deleteChildrenRecursively(zkw, zkw.baseZNode);
ZKUtil.createAndFailSilent(zkw, zkw.baseZNode);
assertTrue(ZKUtil.checkExists(zkw, zkw.baseZNode) != -1);
@ -211,7 +218,7 @@ public class TestSplitLogManager {
assertTrue((task.last_update <= curt) &&
(task.last_update > (curt - 1000)));
LOG.info("waiting for manager to resubmit the orphan task");
waitForCounter(tot_mgr_resubmit, 0, 1, to + 100);
waitForCounter(tot_mgr_resubmit, 0, 1, to + 300);
assertTrue(task.isUnassigned());
waitForCounter(tot_mgr_rescan, 0, 1, to + 100);
}
@ -265,7 +272,7 @@ public class TestSplitLogManager {
ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker1"));
waitForCounter(tot_mgr_heartbeat, 0, 1, 1000);
waitForCounter(tot_mgr_resubmit, 0, 1, to + 100);
waitForCounter(tot_mgr_resubmit, 0, 1, to + EXTRA_TOLERANCE_MS);
int version1 = ZKUtil.checkExists(zkw, tasknode);
assertTrue(version1 > version);
ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker2"));
@ -275,8 +282,8 @@ public class TestSplitLogManager {
assertTrue(version2 > version1);
ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker3"));
waitForCounter(tot_mgr_heartbeat, 1, 2, 1000);
waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + 100);
Thread.sleep(to + 100);
waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + EXTRA_TOLERANCE_MS);
Thread.sleep(to + EXTRA_TOLERANCE_MS);
assertEquals(2L, tot_mgr_resubmit.get());
}

View File

@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.client.Scan;
@ -66,8 +67,10 @@ import com.google.common.collect.Lists;
@Category(SmallTests.class)
public class TestStoreFile extends HBaseTestCase {
static final Log LOG = LogFactory.getLog(TestStoreFile.class);
private CacheConfig cacheConf = new CacheConfig(conf);
private String ROOT_DIR;
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
private static String ROOT_DIR =
TEST_UTIL.getDataTestDir("TestStoreFile").toString();
private Map<String, Long> startingMetrics;
private static final ChecksumType CKTYPE = ChecksumType.CRC32;
@ -77,7 +80,6 @@ public class TestStoreFile extends HBaseTestCase {
public void setUp() throws Exception {
super.setUp();
startingMetrics = SchemaMetrics.getMetricsSnapshot();
ROOT_DIR = new Path(this.testDir, "TestStoreFile").toString();
}
@Override

View File

@ -19,10 +19,7 @@
*/
package org.apache.hadoop.hbase.regionserver.wal;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
import static org.junit.Assert.*;
import java.io.IOException;
import java.lang.reflect.Method;
@ -51,7 +48,6 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
import org.apache.hadoop.io.SequenceFile;
import org.apache.log4j.Level;
@ -405,6 +401,7 @@ public class TestHLog {
LOG.error("Waiting for cluster to go down");
Thread.sleep(1000);
}
assertFalse(cluster.isClusterUp());
// Workaround a strange issue with Hadoop's RPC system - if we don't
// sleep here, the new datanodes will pick up a cached IPC connection to
@ -412,11 +409,12 @@ public class TestHLog {
// the idle time threshold configured in the conf above
Thread.sleep(2000);
cluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null, null, null, null);
TEST_UTIL.setDFSCluster(cluster);
LOG.info("Waiting a few seconds before re-starting HDFS");
Thread.sleep(5000);
cluster = TEST_UTIL.startMiniDFSClusterForTestHLog(namenodePort);
cluster.waitActive();
fs = cluster.getFileSystem();
LOG.info("START second instance.");
LOG.info("STARTED second instance.");
}
// set the lease period to be 1 second so that the

View File

@ -76,6 +76,9 @@ public class TestMiniClusterLoadSequential {
this.isMultiPut = isMultiPut;
this.dataBlockEncoding = dataBlockEncoding;
conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
// We don't want any region reassignments by the load balancer during the test.
conf.setFloat(HConstants.LOAD_BALANCER_SLOP_KEY, 10.0f);
}
@Parameters

View File

@ -93,7 +93,7 @@ public class TestZKLeaderManager {
while (master.get() && !stopped) {
try {
Thread.sleep(200);
Thread.sleep(10);
} catch (InterruptedException ignored) {}
}
}
@ -204,8 +204,8 @@ public class TestZKLeaderManager {
private MockLeader getCurrentLeader() throws Exception {
MockLeader currentLeader = null;
outer:
// wait up to 2 secs for initial leader
for (int i = 0; i < 20; i++) {
// Wait up to 10 secs for initial leader
for (int i = 0; i < 1000; i++) {
for (int j = 0; j < CANDIDATES.length; j++) {
if (CANDIDATES[j].isMaster()) {
// should only be one leader
@ -218,7 +218,7 @@ public class TestZKLeaderManager {
if (currentLeader != null) {
break outer;
}
Thread.sleep(100);
Thread.sleep(10);
}
return currentLeader;
}

View File

@ -142,4 +142,11 @@
version is X.X.X-SNAPSHOT"
</description>
</property>
<property>
<name>hbase.client.retries.number</name>
<value>100</value>
<description>
Use a lot of retries in unit tests.
</description>
</property>
</configuration>