HBASE-4093 When verifyAndAssignRoot throws exception, the deadServers state

cannot be changed (fulin wang via Ted Yu)


git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1148174 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -166,6 +166,8 @@ Release 0.91.0 - Unreleased
    HBASE-4112  Creating table may throw NullPointerException (Jinchao via Ted Yu)
    HBASE-4095  Hlog may not be rolled in a long time if checkLowReplication's
                request of LogRoll is blocked (Jieshan via Ted Yu)
+   HBASE-4093  When verifyAndAssignRoot throws exception, the deadServers state
+               cannot be changed (fulin wang via Ted Yu)
 
   IMPROVEMENTS
    HBASE-3290  Max Compaction Size (Nicolas Spiegelberg via Stack)  

src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java

@@ -92,6 +92,43 @@ public class ServerShutdownHandler extends EventHandler {
     }
   }
   
+  /**
+   * Failed many times, shutdown processing
+   * @throws IOException
+   */
+  private void verifyAndAssignRootWithRetries() throws IOException {
+    int iTimes = this.server.getConfiguration().getInt(
+        "hbase.catalog.verification.retries", 10);
+
+    long waitTime = this.server.getConfiguration().getLong(
+        "hbase.catalog.verification.timeout", 1000);
+
+    int iFlag = 0;
+    while (true) {
+      try {
+        verifyAndAssignRoot();
+        break;
+      } catch (KeeperException e) {
+        this.server.abort("In server shutdown processing, assigning root", e);
+        throw new IOException("Aborting", e);
+      } catch (Exception e) {
+        if (iFlag >= iTimes) {
+          this.server.abort("verifyAndAssignRoot failed after" + iTimes
+              + " times retries, aborting", e);
+          throw new IOException("Aborting", e);
+        }
+        try {
+          Thread.sleep(waitTime);
+        } catch (InterruptedException e1) {
+          LOG.warn("Interrupted when is the thread sleep", e1);
+          Thread.currentThread().interrupt();
+          throw new IOException("Interrupted", e1);
+        }
+        iFlag++;
+      }
+    }
+  }
+  
   /**
    * @return True if the server we are processing was carrying <code>-ROOT-</code>
    */
@@ -131,16 +168,7 @@ public class ServerShutdownHandler extends EventHandler {
 
     // Assign root and meta if we were carrying them.
     if (isCarryingRoot()) { // -ROOT-
-      try {
-        verifyAndAssignRoot();
-      } catch (KeeperException e) {
-        this.server.abort("In server shutdown processing, assigning root", e);
-        throw new IOException("Aborting", e);
-      } catch (InterruptedException e1) {
-        LOG.warn("Interrupted while verifying root region's location", e1);
-        Thread.currentThread().interrupt();
-        throw new IOException(e1);  
-      }
+      verifyAndAssignRootWithRetries();
     }
 
     // Carrying meta?