HBASE-8900 TestRSKilledWhenMasterInitializing.testCorrectnessWhenMasterFailOver is flakey; PUTTING BACK THIS PATCH ON TRUNK
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1501542 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack
parent
ed997ccdeb
commit
b7b840bf59
|
|
@ -0,0 +1,221 @@
|
||||||
|
/*
|
||||||
|
* Copyright The Apache Software Foundation
|
||||||
|
*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.regionserver;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||||
|
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.MiniHBaseCluster;
|
||||||
|
import org.apache.hadoop.hbase.ServerName;
|
||||||
|
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
||||||
|
import org.apache.hadoop.hbase.client.HTable;
|
||||||
|
import org.apache.hadoop.hbase.client.Put;
|
||||||
|
import org.apache.hadoop.hbase.client.ResultScanner;
|
||||||
|
import org.apache.hadoop.hbase.client.Scan;
|
||||||
|
import org.apache.hadoop.hbase.master.HMaster;
|
||||||
|
import org.apache.hadoop.hbase.master.MasterFileSystem;
|
||||||
|
import org.apache.hadoop.hbase.master.ServerManager;
|
||||||
|
import org.apache.hadoop.hbase.master.TestMasterFailover;
|
||||||
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
|
||||||
|
import org.apache.hadoop.hbase.util.Threads;
|
||||||
|
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
|
||||||
|
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
|
||||||
|
import org.apache.hadoop.hbase.LargeTests;
|
||||||
|
import org.apache.zookeeper.KeeperException;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.experimental.categories.Category;
|
||||||
|
|
||||||
|
@Category(LargeTests.class)
|
||||||
|
public class TestRSKilledWhenMasterInitializing {
|
||||||
|
private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
|
||||||
|
|
||||||
|
private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
|
||||||
|
private static final int NUM_MASTERS = 1;
|
||||||
|
private static final int NUM_RS = 4;
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void setUpBeforeClass() throws Exception {
|
||||||
|
// Set it so that this test runs with my custom master
|
||||||
|
Configuration conf = TESTUTIL.getConfiguration();
|
||||||
|
conf.setClass(HConstants.MASTER_IMPL, TestingMaster.class, HMaster.class);
|
||||||
|
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3);
|
||||||
|
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 4);
|
||||||
|
|
||||||
|
// Start up the cluster.
|
||||||
|
TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void tearDownAfterClass() throws Exception {
|
||||||
|
if (!TESTUTIL.getHBaseCluster().getMaster().isInitialized()) {
|
||||||
|
// master is not initialized and is waiting something forever.
|
||||||
|
for (MasterThread mt : TESTUTIL.getHBaseCluster().getLiveMasterThreads()) {
|
||||||
|
mt.interrupt();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TESTUTIL.shutdownMiniCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An HMaster instance used in this test. If 'TestingMaster.sleep' is set in
|
||||||
|
* the Configuration, then we'll sleep after log is split and we'll also
|
||||||
|
* return a custom RegionServerTracker.
|
||||||
|
*/
|
||||||
|
public static class TestingMaster extends HMaster {
|
||||||
|
private boolean logSplit = false;
|
||||||
|
|
||||||
|
public TestingMaster(Configuration conf) throws IOException,
|
||||||
|
KeeperException, InterruptedException {
|
||||||
|
super(conf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 120000)
|
||||||
|
public void testCorrectnessWhenMasterFailOver() throws Exception {
|
||||||
|
final byte[] TABLENAME = Bytes.toBytes("testCorrectnessWhenMasterFailOver");
|
||||||
|
final byte[] FAMILY = Bytes.toBytes("family");
|
||||||
|
final byte[][] SPLITKEYS = { Bytes.toBytes("b"), Bytes.toBytes("i") };
|
||||||
|
|
||||||
|
MiniHBaseCluster cluster = TESTUTIL.getHBaseCluster();
|
||||||
|
|
||||||
|
HTableDescriptor desc = new HTableDescriptor(TABLENAME);
|
||||||
|
desc.addFamily(new HColumnDescriptor(FAMILY));
|
||||||
|
HBaseAdmin hbaseAdmin = TESTUTIL.getHBaseAdmin();
|
||||||
|
hbaseAdmin.createTable(desc, SPLITKEYS);
|
||||||
|
|
||||||
|
assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
|
||||||
|
|
||||||
|
HTable table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
|
||||||
|
List<Put> puts = new ArrayList<Put>();
|
||||||
|
Put put1 = new Put(Bytes.toBytes("a"));
|
||||||
|
put1.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
|
||||||
|
Put put2 = new Put(Bytes.toBytes("h"));
|
||||||
|
put2.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
|
||||||
|
Put put3 = new Put(Bytes.toBytes("o"));
|
||||||
|
put3.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
|
||||||
|
puts.add(put1);
|
||||||
|
puts.add(put2);
|
||||||
|
puts.add(put3);
|
||||||
|
table.put(puts);
|
||||||
|
ResultScanner resultScanner = table.getScanner(new Scan());
|
||||||
|
int count = 0;
|
||||||
|
while (resultScanner.next() != null) {
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
resultScanner.close();
|
||||||
|
table.close();
|
||||||
|
assertEquals(3, count);
|
||||||
|
|
||||||
|
/* Starting test */
|
||||||
|
cluster.getConfiguration().setBoolean("TestingMaster.sleep", true);
|
||||||
|
cluster.getConfiguration().setInt("TestingMaster.sleep.duration", 10000);
|
||||||
|
|
||||||
|
/* NO.1 .META. region correctness */
|
||||||
|
// First abort master
|
||||||
|
abortMaster(cluster);
|
||||||
|
TestingMaster master = startMasterAndWaitTillMetaRegionAssignment(cluster);
|
||||||
|
|
||||||
|
// Second kill meta server
|
||||||
|
int metaServerNum = cluster.getServerWithMeta();
|
||||||
|
HRegionServer metaRS = cluster.getRegionServer(metaServerNum);
|
||||||
|
LOG.debug("Killing metaRS");
|
||||||
|
metaRS.kill();
|
||||||
|
metaRS.join();
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Sleep double time of TestingMaster.sleep.duration, so we can ensure that
|
||||||
|
* master has already assigned META or is blocking on assigning
|
||||||
|
* META
|
||||||
|
*/
|
||||||
|
Thread.sleep(10000 * 2);
|
||||||
|
|
||||||
|
waitUntilMasterIsInitialized(master);
|
||||||
|
|
||||||
|
// Third check whether data is correct in meta region
|
||||||
|
assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
|
||||||
|
|
||||||
|
/* NO.2 data region correctness */
|
||||||
|
ServerManager serverManager = cluster.getMaster().getServerManager();
|
||||||
|
while (serverManager.areDeadServersInProgress()) {
|
||||||
|
Thread.sleep(100);
|
||||||
|
}
|
||||||
|
// Create a ZKW to use in the test
|
||||||
|
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTUTIL);
|
||||||
|
ZKAssign.blockUntilNoRIT(zkw);
|
||||||
|
|
||||||
|
table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
|
||||||
|
resultScanner = table.getScanner(new Scan());
|
||||||
|
count = 0;
|
||||||
|
while (resultScanner.next() != null) {
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
resultScanner.close();
|
||||||
|
table.close();
|
||||||
|
assertEquals(3, count);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void abortMaster(MiniHBaseCluster cluster)
|
||||||
|
throws InterruptedException {
|
||||||
|
for (MasterThread mt : cluster.getLiveMasterThreads()) {
|
||||||
|
if (mt.getMaster().isActiveMaster()) {
|
||||||
|
mt.getMaster().abort("Aborting for tests", new Exception("Trace info"));
|
||||||
|
mt.join();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LOG.debug("Master is aborted");
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestingMaster startMasterAndWaitTillMetaRegionAssignment(MiniHBaseCluster cluster)
|
||||||
|
throws IOException, InterruptedException {
|
||||||
|
TestingMaster master = (TestingMaster) cluster.startMaster().getMaster();
|
||||||
|
while (!master.isInitializationStartsMetaRegionAssignment()) {
|
||||||
|
Thread.sleep(100);
|
||||||
|
}
|
||||||
|
return master;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void waitUntilMasterIsInitialized(HMaster master)
|
||||||
|
throws InterruptedException {
|
||||||
|
while (!master.isInitialized()) {
|
||||||
|
Thread.sleep(100);
|
||||||
|
}
|
||||||
|
while (master.getServerManager().areDeadServersInProgress()) {
|
||||||
|
Thread.sleep(100);
|
||||||
|
}
|
||||||
|
LOG.debug("master isInitialized");
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue