HBASE-8900 TestRSKilledWhenMasterInitializing.testCorrectnessWhenMasterFailOver is flakey
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1501022 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack
parent
d291f13132
commit
784e45548f
|
|
@ -1,221 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright The Apache Software Foundation
|
|
||||||
*
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.hadoop.hbase.regionserver;
|
|
||||||
|
|
||||||
import static org.junit.Assert.assertEquals;
|
|
||||||
import static org.junit.Assert.assertTrue;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
|
||||||
import org.apache.commons.logging.LogFactory;
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
|
||||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
|
||||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
|
||||||
import org.apache.hadoop.hbase.MiniHBaseCluster;
|
|
||||||
import org.apache.hadoop.hbase.ServerName;
|
|
||||||
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
|
||||||
import org.apache.hadoop.hbase.client.HTable;
|
|
||||||
import org.apache.hadoop.hbase.client.Put;
|
|
||||||
import org.apache.hadoop.hbase.client.ResultScanner;
|
|
||||||
import org.apache.hadoop.hbase.client.Scan;
|
|
||||||
import org.apache.hadoop.hbase.master.HMaster;
|
|
||||||
import org.apache.hadoop.hbase.master.MasterFileSystem;
|
|
||||||
import org.apache.hadoop.hbase.master.ServerManager;
|
|
||||||
import org.apache.hadoop.hbase.master.TestMasterFailover;
|
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
|
||||||
import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
|
|
||||||
import org.apache.hadoop.hbase.util.Threads;
|
|
||||||
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
|
|
||||||
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
|
|
||||||
import org.apache.hadoop.hbase.LargeTests;
|
|
||||||
import org.apache.zookeeper.KeeperException;
|
|
||||||
import org.junit.AfterClass;
|
|
||||||
import org.junit.BeforeClass;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.experimental.categories.Category;
|
|
||||||
|
|
||||||
@Category(LargeTests.class)
|
|
||||||
public class TestRSKilledWhenMasterInitializing {
|
|
||||||
private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
|
|
||||||
|
|
||||||
private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
|
|
||||||
private static final int NUM_MASTERS = 1;
|
|
||||||
private static final int NUM_RS = 4;
|
|
||||||
|
|
||||||
@BeforeClass
|
|
||||||
public static void setUpBeforeClass() throws Exception {
|
|
||||||
// Set it so that this test runs with my custom master
|
|
||||||
Configuration conf = TESTUTIL.getConfiguration();
|
|
||||||
conf.setClass(HConstants.MASTER_IMPL, TestingMaster.class, HMaster.class);
|
|
||||||
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3);
|
|
||||||
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 4);
|
|
||||||
|
|
||||||
// Start up the cluster.
|
|
||||||
TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
|
|
||||||
}
|
|
||||||
|
|
||||||
@AfterClass
|
|
||||||
public static void tearDownAfterClass() throws Exception {
|
|
||||||
if (!TESTUTIL.getHBaseCluster().getMaster().isInitialized()) {
|
|
||||||
// master is not initialized and is waiting something forever.
|
|
||||||
for (MasterThread mt : TESTUTIL.getHBaseCluster().getLiveMasterThreads()) {
|
|
||||||
mt.interrupt();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TESTUTIL.shutdownMiniCluster();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An HMaster instance used in this test. If 'TestingMaster.sleep' is set in
|
|
||||||
* the Configuration, then we'll sleep after log is split and we'll also
|
|
||||||
* return a custom RegionServerTracker.
|
|
||||||
*/
|
|
||||||
public static class TestingMaster extends HMaster {
|
|
||||||
private boolean logSplit = false;
|
|
||||||
|
|
||||||
public TestingMaster(Configuration conf) throws IOException,
|
|
||||||
KeeperException, InterruptedException {
|
|
||||||
super(conf);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test(timeout = 120000)
|
|
||||||
public void testCorrectnessWhenMasterFailOver() throws Exception {
|
|
||||||
final byte[] TABLENAME = Bytes.toBytes("testCorrectnessWhenMasterFailOver");
|
|
||||||
final byte[] FAMILY = Bytes.toBytes("family");
|
|
||||||
final byte[][] SPLITKEYS = { Bytes.toBytes("b"), Bytes.toBytes("i") };
|
|
||||||
|
|
||||||
MiniHBaseCluster cluster = TESTUTIL.getHBaseCluster();
|
|
||||||
|
|
||||||
HTableDescriptor desc = new HTableDescriptor(TABLENAME);
|
|
||||||
desc.addFamily(new HColumnDescriptor(FAMILY));
|
|
||||||
HBaseAdmin hbaseAdmin = TESTUTIL.getHBaseAdmin();
|
|
||||||
hbaseAdmin.createTable(desc, SPLITKEYS);
|
|
||||||
|
|
||||||
assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
|
|
||||||
|
|
||||||
HTable table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
|
|
||||||
List<Put> puts = new ArrayList<Put>();
|
|
||||||
Put put1 = new Put(Bytes.toBytes("a"));
|
|
||||||
put1.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
|
|
||||||
Put put2 = new Put(Bytes.toBytes("h"));
|
|
||||||
put2.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
|
|
||||||
Put put3 = new Put(Bytes.toBytes("o"));
|
|
||||||
put3.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
|
|
||||||
puts.add(put1);
|
|
||||||
puts.add(put2);
|
|
||||||
puts.add(put3);
|
|
||||||
table.put(puts);
|
|
||||||
ResultScanner resultScanner = table.getScanner(new Scan());
|
|
||||||
int count = 0;
|
|
||||||
while (resultScanner.next() != null) {
|
|
||||||
count++;
|
|
||||||
}
|
|
||||||
resultScanner.close();
|
|
||||||
table.close();
|
|
||||||
assertEquals(3, count);
|
|
||||||
|
|
||||||
/* Starting test */
|
|
||||||
cluster.getConfiguration().setBoolean("TestingMaster.sleep", true);
|
|
||||||
cluster.getConfiguration().setInt("TestingMaster.sleep.duration", 10000);
|
|
||||||
|
|
||||||
/* NO.1 .META. region correctness */
|
|
||||||
// First abort master
|
|
||||||
abortMaster(cluster);
|
|
||||||
TestingMaster master = startMasterAndWaitTillMetaRegionAssignment(cluster);
|
|
||||||
|
|
||||||
// Second kill meta server
|
|
||||||
int metaServerNum = cluster.getServerWithMeta();
|
|
||||||
HRegionServer metaRS = cluster.getRegionServer(metaServerNum);
|
|
||||||
LOG.debug("Killing metaRS");
|
|
||||||
metaRS.kill();
|
|
||||||
metaRS.join();
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Sleep double time of TestingMaster.sleep.duration, so we can ensure that
|
|
||||||
* master has already assigned META or is blocking on assigning
|
|
||||||
* META
|
|
||||||
*/
|
|
||||||
Thread.sleep(10000 * 2);
|
|
||||||
|
|
||||||
waitUntilMasterIsInitialized(master);
|
|
||||||
|
|
||||||
// Third check whether data is correct in meta region
|
|
||||||
assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
|
|
||||||
|
|
||||||
/* NO.2 data region correctness */
|
|
||||||
ServerManager serverManager = cluster.getMaster().getServerManager();
|
|
||||||
while (serverManager.areDeadServersInProgress()) {
|
|
||||||
Thread.sleep(100);
|
|
||||||
}
|
|
||||||
// Create a ZKW to use in the test
|
|
||||||
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTUTIL);
|
|
||||||
ZKAssign.blockUntilNoRIT(zkw);
|
|
||||||
|
|
||||||
table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
|
|
||||||
resultScanner = table.getScanner(new Scan());
|
|
||||||
count = 0;
|
|
||||||
while (resultScanner.next() != null) {
|
|
||||||
count++;
|
|
||||||
}
|
|
||||||
resultScanner.close();
|
|
||||||
table.close();
|
|
||||||
assertEquals(3, count);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void abortMaster(MiniHBaseCluster cluster)
|
|
||||||
throws InterruptedException {
|
|
||||||
for (MasterThread mt : cluster.getLiveMasterThreads()) {
|
|
||||||
if (mt.getMaster().isActiveMaster()) {
|
|
||||||
mt.getMaster().abort("Aborting for tests", new Exception("Trace info"));
|
|
||||||
mt.join();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
LOG.debug("Master is aborted");
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestingMaster startMasterAndWaitTillMetaRegionAssignment(MiniHBaseCluster cluster)
|
|
||||||
throws IOException, InterruptedException {
|
|
||||||
TestingMaster master = (TestingMaster) cluster.startMaster().getMaster();
|
|
||||||
while (!master.isInitializationStartsMetaRegionAssignment()) {
|
|
||||||
Thread.sleep(100);
|
|
||||||
}
|
|
||||||
return master;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void waitUntilMasterIsInitialized(HMaster master)
|
|
||||||
throws InterruptedException {
|
|
||||||
while (!master.isInitialized()) {
|
|
||||||
Thread.sleep(100);
|
|
||||||
}
|
|
||||||
while (master.getServerManager().areDeadServersInProgress()) {
|
|
||||||
Thread.sleep(100);
|
|
||||||
}
|
|
||||||
LOG.debug("master isInitialized");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue