HBASE-8266-Master cannot start if TableNotFoundException is thrown while partial table recovery(Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1466565 13f79535-47bb-0310-9956-ffa450edef68

hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java

@@ -346,16 +346,22 @@ public class ZKTable {
   /**
    * If the table is found in ENABLING state the inmemory state is removed. This
    * helps in cases where CreateTable is to be retried by the client incase of
-   * failures
+   * failures.  If deleteZNode is true - the znode is also deleted
    * 
    * @param tableName
+   * @param deleteZNode
+   * @throws KeeperException
    */
-  public void removeEnablingTable(final String tableName) {
+  public void removeEnablingTable(final String tableName, boolean deleteZNode)
+      throws KeeperException {
     synchronized (this.cache) {
       if (isEnablingTable(tableName)) {
         this.cache.remove(tableName);
+        if (deleteZNode) {
+          ZKUtil.deleteNodeFailSilent(this.watcher,
+              ZKUtil.joinZNode(this.watcher.tableZNode, tableName));
+        }
       }
-
     }
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java

@@ -172,8 +172,14 @@ public class CreateTableHandler extends EventHandler {
       // If this does not happen then if the client tries to create the table
       // again with the same Active master
       // It will block the creation saying TableAlreadyExists.
+      try {
         this.assignmentManager.getZKTable().removeEnablingTable(
-          this.hTableDescriptor.getNameAsString());
+            this.hTableDescriptor.getNameAsString(), false);
+      } catch (KeeperException e) {
+        // Keeper exception should not happen here
+        LOG.error("Got a keeper exception while removing the ENABLING table znode "
+            + this.hTableDescriptor.getNameAsString(), e);
+      }
     }
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java

@@ -86,7 +86,17 @@ public class EnableTableHandler extends EventHandler {
     try {
       // Check if table exists
       if (!MetaReader.tableExists(catalogTracker, this.tableNameStr)) {
-        throw new TableNotFoundException(Bytes.toString(tableName));
+        // retainAssignment is true only during recovery.  In normal case it is false
+        if (!this.retainAssignment) {
+          throw new TableNotFoundException(tableNameStr);
+        } 
+        try {
+          this.assignmentManager.getZKTable().removeEnablingTable(tableNameStr, true);
+        } catch (KeeperException e) {
+          // TODO : Use HBCK to clear such nodes
+          LOG.warn("Failed to delete the ENABLING node for the table " + tableNameStr
+              + ".  The table will remain unusable. Run HBCK to manually fix the problem.");
+        }
       }
 
       // There could be multiple client requests trying to disable or enable

hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java

@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -41,7 +42,9 @@ import org.apache.hadoop.hbase.master.TestMaster;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.zookeeper.ZKTable;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -52,16 +55,18 @@ public class TestCreateTableHandler {
   private static final Log LOG = LogFactory.getLog(TestCreateTableHandler.class);
   private static final byte[] TABLENAME = Bytes.toBytes("TestCreateTableHandler");
   private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
-  public static boolean throwException = false;
+  private static boolean throwException = false;
   
-  @BeforeClass
-  public static void setUp() throws Exception {
+
+  @Before
+  public void setUp() throws Exception {
     TEST_UTIL.startMiniCluster(1);
   }
 
-  @AfterClass
-  public static void tearDown() throws Exception {
+  @After
+  public void tearDown() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
+    throwException = false;
   }
 
   @Test
@@ -91,6 +96,34 @@ public class TestCreateTableHandler {
 
   }
   
+  @Test (timeout=5000)
+  public void testMasterRestartAfterEnablingNodeIsCreated() throws Exception {
+    byte[] tableName = Bytes.toBytes("testMasterRestartAfterEnablingNodeIsCreated");
+    final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    final HMaster m = cluster.getMaster();
+    final HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.addFamily(new HColumnDescriptor(FAMILYNAME));
+    final HRegionInfo[] hRegionInfos = new HRegionInfo[] { new HRegionInfo(desc.getName(), null,
+        null) };
+    CustomCreateTableHandler handler = new CustomCreateTableHandler(m, m.getMasterFileSystem(),
+        desc, cluster.getConfiguration(), hRegionInfos, m);
+    handler.prepare();
+    throwException = true;
+    handler.process();
+    abortAndStartNewMaster(cluster);
+    assertTrue(cluster.getLiveMasterThreads().size() == 1);
+    
+  }
+  
+  private void abortAndStartNewMaster(final MiniHBaseCluster cluster) throws IOException {
+    cluster.abortMaster(0);
+    cluster.waitOnMaster(0);
+    LOG.info("Starting new master");
+    cluster.startMaster();
+    LOG.info("Waiting for master to become active.");
+    cluster.waitForActiveAndReadyMaster();
+  }
+
   private static class CustomCreateTableHandler extends CreateTableHandler {
     public CustomCreateTableHandler(Server server, MasterFileSystem fileSystemManager,
         HTableDescriptor hTableDescriptor, Configuration conf, HRegionInfo[] newRegions,