HBASE-8900 TestRSKilledWhenMasterInitializing.testCorrectnessWhenMasterFailOver is flakey
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1501022 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack
parent
d291f13132
commit
784e45548f
|
|
@ -1,221 +0,0 @@
|
|||
/*
|
||||
* Copyright The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.regionserver;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.HRegionInfo;
|
||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||
import org.apache.hadoop.hbase.MiniHBaseCluster;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
||||
import org.apache.hadoop.hbase.client.HTable;
|
||||
import org.apache.hadoop.hbase.client.Put;
|
||||
import org.apache.hadoop.hbase.client.ResultScanner;
|
||||
import org.apache.hadoop.hbase.client.Scan;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.MasterFileSystem;
|
||||
import org.apache.hadoop.hbase.master.ServerManager;
|
||||
import org.apache.hadoop.hbase.master.TestMasterFailover;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
|
||||
import org.apache.hadoop.hbase.util.Threads;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
|
||||
import org.apache.hadoop.hbase.LargeTests;
|
||||
import org.apache.zookeeper.KeeperException;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
|
||||
@Category(LargeTests.class)
|
||||
public class TestRSKilledWhenMasterInitializing {
|
||||
private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
|
||||
|
||||
private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
|
||||
private static final int NUM_MASTERS = 1;
|
||||
private static final int NUM_RS = 4;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUpBeforeClass() throws Exception {
|
||||
// Set it so that this test runs with my custom master
|
||||
Configuration conf = TESTUTIL.getConfiguration();
|
||||
conf.setClass(HConstants.MASTER_IMPL, TestingMaster.class, HMaster.class);
|
||||
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3);
|
||||
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 4);
|
||||
|
||||
// Start up the cluster.
|
||||
TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDownAfterClass() throws Exception {
|
||||
if (!TESTUTIL.getHBaseCluster().getMaster().isInitialized()) {
|
||||
// master is not initialized and is waiting something forever.
|
||||
for (MasterThread mt : TESTUTIL.getHBaseCluster().getLiveMasterThreads()) {
|
||||
mt.interrupt();
|
||||
}
|
||||
}
|
||||
TESTUTIL.shutdownMiniCluster();
|
||||
}
|
||||
|
||||
/**
|
||||
* An HMaster instance used in this test. If 'TestingMaster.sleep' is set in
|
||||
* the Configuration, then we'll sleep after log is split and we'll also
|
||||
* return a custom RegionServerTracker.
|
||||
*/
|
||||
public static class TestingMaster extends HMaster {
|
||||
private boolean logSplit = false;
|
||||
|
||||
public TestingMaster(Configuration conf) throws IOException,
|
||||
KeeperException, InterruptedException {
|
||||
super(conf);
|
||||
}
|
||||
}
|
||||
|
||||
@Test(timeout = 120000)
|
||||
public void testCorrectnessWhenMasterFailOver() throws Exception {
|
||||
final byte[] TABLENAME = Bytes.toBytes("testCorrectnessWhenMasterFailOver");
|
||||
final byte[] FAMILY = Bytes.toBytes("family");
|
||||
final byte[][] SPLITKEYS = { Bytes.toBytes("b"), Bytes.toBytes("i") };
|
||||
|
||||
MiniHBaseCluster cluster = TESTUTIL.getHBaseCluster();
|
||||
|
||||
HTableDescriptor desc = new HTableDescriptor(TABLENAME);
|
||||
desc.addFamily(new HColumnDescriptor(FAMILY));
|
||||
HBaseAdmin hbaseAdmin = TESTUTIL.getHBaseAdmin();
|
||||
hbaseAdmin.createTable(desc, SPLITKEYS);
|
||||
|
||||
assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
|
||||
|
||||
HTable table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
|
||||
List<Put> puts = new ArrayList<Put>();
|
||||
Put put1 = new Put(Bytes.toBytes("a"));
|
||||
put1.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
|
||||
Put put2 = new Put(Bytes.toBytes("h"));
|
||||
put2.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
|
||||
Put put3 = new Put(Bytes.toBytes("o"));
|
||||
put3.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
|
||||
puts.add(put1);
|
||||
puts.add(put2);
|
||||
puts.add(put3);
|
||||
table.put(puts);
|
||||
ResultScanner resultScanner = table.getScanner(new Scan());
|
||||
int count = 0;
|
||||
while (resultScanner.next() != null) {
|
||||
count++;
|
||||
}
|
||||
resultScanner.close();
|
||||
table.close();
|
||||
assertEquals(3, count);
|
||||
|
||||
/* Starting test */
|
||||
cluster.getConfiguration().setBoolean("TestingMaster.sleep", true);
|
||||
cluster.getConfiguration().setInt("TestingMaster.sleep.duration", 10000);
|
||||
|
||||
/* NO.1 .META. region correctness */
|
||||
// First abort master
|
||||
abortMaster(cluster);
|
||||
TestingMaster master = startMasterAndWaitTillMetaRegionAssignment(cluster);
|
||||
|
||||
// Second kill meta server
|
||||
int metaServerNum = cluster.getServerWithMeta();
|
||||
HRegionServer metaRS = cluster.getRegionServer(metaServerNum);
|
||||
LOG.debug("Killing metaRS");
|
||||
metaRS.kill();
|
||||
metaRS.join();
|
||||
|
||||
/*
|
||||
* Sleep double time of TestingMaster.sleep.duration, so we can ensure that
|
||||
* master has already assigned META or is blocking on assigning
|
||||
* META
|
||||
*/
|
||||
Thread.sleep(10000 * 2);
|
||||
|
||||
waitUntilMasterIsInitialized(master);
|
||||
|
||||
// Third check whether data is correct in meta region
|
||||
assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
|
||||
|
||||
/* NO.2 data region correctness */
|
||||
ServerManager serverManager = cluster.getMaster().getServerManager();
|
||||
while (serverManager.areDeadServersInProgress()) {
|
||||
Thread.sleep(100);
|
||||
}
|
||||
// Create a ZKW to use in the test
|
||||
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTUTIL);
|
||||
ZKAssign.blockUntilNoRIT(zkw);
|
||||
|
||||
table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
|
||||
resultScanner = table.getScanner(new Scan());
|
||||
count = 0;
|
||||
while (resultScanner.next() != null) {
|
||||
count++;
|
||||
}
|
||||
resultScanner.close();
|
||||
table.close();
|
||||
assertEquals(3, count);
|
||||
}
|
||||
|
||||
private void abortMaster(MiniHBaseCluster cluster)
|
||||
throws InterruptedException {
|
||||
for (MasterThread mt : cluster.getLiveMasterThreads()) {
|
||||
if (mt.getMaster().isActiveMaster()) {
|
||||
mt.getMaster().abort("Aborting for tests", new Exception("Trace info"));
|
||||
mt.join();
|
||||
break;
|
||||
}
|
||||
}
|
||||
LOG.debug("Master is aborted");
|
||||
}
|
||||
|
||||
private TestingMaster startMasterAndWaitTillMetaRegionAssignment(MiniHBaseCluster cluster)
|
||||
throws IOException, InterruptedException {
|
||||
TestingMaster master = (TestingMaster) cluster.startMaster().getMaster();
|
||||
while (!master.isInitializationStartsMetaRegionAssignment()) {
|
||||
Thread.sleep(100);
|
||||
}
|
||||
return master;
|
||||
}
|
||||
|
||||
private void waitUntilMasterIsInitialized(HMaster master)
|
||||
throws InterruptedException {
|
||||
while (!master.isInitialized()) {
|
||||
Thread.sleep(100);
|
||||
}
|
||||
while (master.getServerManager().areDeadServersInProgress()) {
|
||||
Thread.sleep(100);
|
||||
}
|
||||
LOG.debug("master isInitialized");
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue