HBASE-8266-Master cannot start if TableNotFoundException is thrown while partial table recovery(Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1466565 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
ramkrishna 2013-04-10 16:59:39 +00:00
parent 1dda0cbc91
commit cc8e0d3341
4 changed files with 66 additions and 11 deletions

View File

@ -346,16 +346,22 @@ public class ZKTable {
/**
* If the table is found in ENABLING state the inmemory state is removed. This
* helps in cases where CreateTable is to be retried by the client incase of
* failures
* failures. If deleteZNode is true - the znode is also deleted
*
* @param tableName
* @param deleteZNode
* @throws KeeperException
*/
public void removeEnablingTable(final String tableName) {
public void removeEnablingTable(final String tableName, boolean deleteZNode)
throws KeeperException {
synchronized (this.cache) {
if (isEnablingTable(tableName)) {
this.cache.remove(tableName);
if (deleteZNode) {
ZKUtil.deleteNodeFailSilent(this.watcher,
ZKUtil.joinZNode(this.watcher.tableZNode, tableName));
}
}
}
}

View File

@ -172,8 +172,14 @@ public class CreateTableHandler extends EventHandler {
// If this does not happen then if the client tries to create the table
// again with the same Active master
// It will block the creation saying TableAlreadyExists.
this.assignmentManager.getZKTable().removeEnablingTable(
this.hTableDescriptor.getNameAsString());
try {
this.assignmentManager.getZKTable().removeEnablingTable(
this.hTableDescriptor.getNameAsString(), false);
} catch (KeeperException e) {
// Keeper exception should not happen here
LOG.error("Got a keeper exception while removing the ENABLING table znode "
+ this.hTableDescriptor.getNameAsString(), e);
}
}
}

View File

@ -86,7 +86,17 @@ public class EnableTableHandler extends EventHandler {
try {
// Check if table exists
if (!MetaReader.tableExists(catalogTracker, this.tableNameStr)) {
throw new TableNotFoundException(Bytes.toString(tableName));
// retainAssignment is true only during recovery. In normal case it is false
if (!this.retainAssignment) {
throw new TableNotFoundException(tableNameStr);
}
try {
this.assignmentManager.getZKTable().removeEnablingTable(tableNameStr, true);
} catch (KeeperException e) {
// TODO : Use HBCK to clear such nodes
LOG.warn("Failed to delete the ENABLING node for the table " + tableNameStr
+ ". The table will remain unusable. Run HBCK to manually fix the problem.");
}
}
// There could be multiple client requests trying to disable or enable

View File

@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -41,7 +42,9 @@ import org.apache.hadoop.hbase.master.TestMaster;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.zookeeper.ZKTable;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@ -52,16 +55,18 @@ public class TestCreateTableHandler {
private static final Log LOG = LogFactory.getLog(TestCreateTableHandler.class);
private static final byte[] TABLENAME = Bytes.toBytes("TestCreateTableHandler");
private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
public static boolean throwException = false;
private static boolean throwException = false;
@BeforeClass
public static void setUp() throws Exception {
@Before
public void setUp() throws Exception {
TEST_UTIL.startMiniCluster(1);
}
@AfterClass
public static void tearDown() throws Exception {
@After
public void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
throwException = false;
}
@Test
@ -90,6 +95,34 @@ public class TestCreateTableHandler {
assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(TABLENAME));
}
@Test (timeout=5000)
public void testMasterRestartAfterEnablingNodeIsCreated() throws Exception {
byte[] tableName = Bytes.toBytes("testMasterRestartAfterEnablingNodeIsCreated");
final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
final HMaster m = cluster.getMaster();
final HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(FAMILYNAME));
final HRegionInfo[] hRegionInfos = new HRegionInfo[] { new HRegionInfo(desc.getName(), null,
null) };
CustomCreateTableHandler handler = new CustomCreateTableHandler(m, m.getMasterFileSystem(),
desc, cluster.getConfiguration(), hRegionInfos, m);
handler.prepare();
throwException = true;
handler.process();
abortAndStartNewMaster(cluster);
assertTrue(cluster.getLiveMasterThreads().size() == 1);
}
private void abortAndStartNewMaster(final MiniHBaseCluster cluster) throws IOException {
cluster.abortMaster(0);
cluster.waitOnMaster(0);
LOG.info("Starting new master");
cluster.startMaster();
LOG.info("Waiting for master to become active.");
cluster.waitForActiveAndReadyMaster();
}
private static class CustomCreateTableHandler extends CreateTableHandler {
public CustomCreateTableHandler(Server server, MasterFileSystem fileSystemManager,