HBASE-13194 TableNamespaceManager not ready cause MasterQuotaManager initialization fail

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -288,7 +288,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
   // monitor for distributed procedures
   MasterProcedureManagerHost mpmHost;
 
-  private MasterQuotaManager quotaManager;
+  // it is assigned after 'initialized' guard set to true, so should be volatile
+  private volatile MasterQuotaManager quotaManager;
 
   // handle table states
   private TableStateManager tableStateManager;
@@ -897,9 +898,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
   }
 
   void initQuotaManager() throws IOException {
-    quotaManager = new MasterQuotaManager(this);
+    MasterQuotaManager quotaManager = new MasterQuotaManager(this);
     this.assignmentManager.setRegionStateListener((RegionStateListener)quotaManager);
     quotaManager.start();
+    this.quotaManager = quotaManager;
   }
 
   boolean isCatalogJanitorEnabled() {

hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java

@@ -86,31 +86,26 @@ public class TableNamespaceManager {
 
   public void start() throws IOException {
     if (!MetaTableAccessor.tableExists(masterServices.getConnection(),
-        TableName.NAMESPACE_TABLE_NAME)) {
+      TableName.NAMESPACE_TABLE_NAME)) {
       LOG.info("Namespace table not found. Creating...");
       createNamespaceTable(masterServices);
     }
 
     try {
-      // Wait for the namespace table to be assigned.
+      // Wait for the namespace table to be initialized.
-      // If timed out, we will move ahead without initializing it.
-      // So that it should be initialized later on lazily.
       long startTime = EnvironmentEdgeManager.currentTime();
       int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT);
-      while (!(isTableAssigned() && isTableEnabled())) {
+      while (!isTableAvailableAndInitialized()) {
         if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) {
           // We can't do anything if ns is not online.
-          throw new IOException("Timedout " + timeout + "ms waiting for namespace table to " +
+          throw new IOException("Timedout " + timeout + "ms waiting for namespace table to "
-            "be assigned and enabled: " + getTableState());
+              + "be assigned and enabled: " + getTableState());
         }
         Thread.sleep(100);
       }
     } catch (InterruptedException e) {
-      throw (InterruptedIOException)new InterruptedIOException().initCause(e);
+      throw (InterruptedIOException) new InterruptedIOException().initCause(e);
     }
-
-    // initialize namespace table
-    isTableAvailableAndInitialized();
   }
 
   private synchronized Table getNamespaceTable() throws IOException {

hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java

@@ -18,7 +18,6 @@
 package org.apache.hadoop.hbase.namespace;
 
 import java.io.IOException;
-import java.io.InterruptedIOException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -31,7 +30,6 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.quotas.QuotaExceededException;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -55,19 +53,6 @@ public class NamespaceAuditor {
 
   public void start() throws IOException {
     stateManager.start();
-    long startTime = EnvironmentEdgeManager.currentTime();
-    int timeout = masterServices.getConfiguration().getInt(NS_AUDITOR_INIT_TIMEOUT,
-      DEFAULT_NS_AUDITOR_INIT_TIMEOUT);
-    try {
-      while (!stateManager.isInitialized()) {
-        if (EnvironmentEdgeManager.currentTime() - startTime + 1000 > timeout) {
-          throw new HBaseIOException("Timed out waiting for namespace auditor to be initialized.");
-        }
-        Thread.sleep(1000);
-      }
-    } catch (InterruptedException e) {
-      throw (InterruptedIOException) new InterruptedIOException().initCause(e);
-    }
     LOG.info("NamespaceAuditor started.");
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java

@@ -183,27 +183,22 @@ class NamespaceStateManager {
   /**
    * Initialize namespace state cache by scanning meta table.
    */
-  void initialize() {
+  private void initialize() throws IOException {
-    try {
+    List<NamespaceDescriptor> namespaces = this.master.listNamespaceDescriptors();
-      List<NamespaceDescriptor> namespaces = this.master.listNamespaceDescriptors();
+    for (NamespaceDescriptor namespace : namespaces) {
-      for (NamespaceDescriptor namespace : namespaces) {
+      addNamespace(namespace.getName());
-        addNamespace(namespace.getName());
+      List<TableName> tables = this.master.listTableNamesByNamespace(namespace.getName());
-        List<TableName> tables = this.master.listTableNamesByNamespace(namespace.getName());
+      for (TableName table : tables) {
-        for (TableName table : tables) {
+        if (table.isSystemTable()) {
-          if (table.isSystemTable()) {
+          continue;
-            continue;
-          }
-          List<HRegionInfo> regions = MetaTableAccessor.getTableRegions(
-              this.master.getConnection(), table, true);
-          addTable(table, regions.size());
         }
+        List<HRegionInfo> regions =
+            MetaTableAccessor.getTableRegions(this.master.getConnection(), table, true);
+        addTable(table, regions.size());
       }
-      LOG.info("Finished updating state of " + nsStateCache.size() + " namespaces. ");
-      initialized = true;
-    } catch (IOException e) {
-      LOG.error("Error while update namespace state.", e);
-      initialized = false;
     }
+    LOG.info("Finished updating state of " + nsStateCache.size() + " namespaces. ");
+    initialized = true;
   }
 
   boolean isInitialized() {

hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java

@@ -102,12 +102,7 @@ public class TestNamespaceAuditor {
     conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
       RegionServerObserver.class);
     UTIL.startMiniCluster(1, 1);
-    UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
+    waitForQuotaEnabled();
-      @Override
-      public boolean evaluate() throws Exception {
-        return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().isQuotaEnabled();
-      }
-    });
     ADMIN = UTIL.getHBaseAdmin();
   }
 
@@ -543,10 +538,7 @@ public class TestNamespaceAuditor {
         .getTables().size(), after.getTables().size());
   }
 
-  private void restartMaster() throws Exception {
+  private static void waitForQuotaEnabled() throws Exception {
-    UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
-    UTIL.getHBaseCluster().waitOnMaster(0);
-    UTIL.getHBaseCluster().startMaster();
     UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
       @Override
       public boolean evaluate() throws Exception {
@@ -560,6 +552,13 @@ public class TestNamespaceAuditor {
     });
   }
 
+  private void restartMaster() throws Exception {
+    UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
+    UTIL.getHBaseCluster().waitOnMaster(0);
+    UTIL.getHBaseCluster().startMaster();
+    waitForQuotaEnabled();
+  }
+
   private NamespaceAuditor getQuotaManager() {
     return UTIL.getHBaseCluster().getMaster()
         .getMasterQuotaManager().getNamespaceQuotaManager();