From f2499505de1473d4bb9e448408cf7ccbf6dccee1 Mon Sep 17 00:00:00 2001
From: Zhihong Yu <tedyu@apache.org>
Date: Tue, 19 Jul 2011 05:20:02 +0000
Subject: [PATCH] HBASE-4093  When verifyAndAssignRoot throws exception, the
 deadServers state                cannot be changed (fulin wang via Ted Yu)

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1148174 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt                                   |  2 +
 .../master/handler/ServerShutdownHandler.java | 50 +++++++++++++++----
 2 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 4c4c4a1c7f3..15d838d7732 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -166,6 +166,8 @@ Release 0.91.0 - Unreleased
    HBASE-4112  Creating table may throw NullPointerException (Jinchao via Ted Yu)
    HBASE-4095  Hlog may not be rolled in a long time if checkLowReplication's
                request of LogRoll is blocked (Jieshan via Ted Yu)
+   HBASE-4093  When verifyAndAssignRoot throws exception, the deadServers state
+               cannot be changed (fulin wang via Ted Yu)
 
   IMPROVEMENTS
    HBASE-3290  Max Compaction Size (Nicolas Spiegelberg via Stack)  
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 6da68c1fc67..c478e9f26b9 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -92,6 +92,43 @@ public class ServerShutdownHandler extends EventHandler {
     }
   }
   
+  /**
+   * Failed many times, shutdown processing
+   * @throws IOException
+   */
+  private void verifyAndAssignRootWithRetries() throws IOException {
+    int iTimes = this.server.getConfiguration().getInt(
+        "hbase.catalog.verification.retries", 10);
+
+    long waitTime = this.server.getConfiguration().getLong(
+        "hbase.catalog.verification.timeout", 1000);
+
+    int iFlag = 0;
+    while (true) {
+      try {
+        verifyAndAssignRoot();
+        break;
+      } catch (KeeperException e) {
+        this.server.abort("In server shutdown processing, assigning root", e);
+        throw new IOException("Aborting", e);
+      } catch (Exception e) {
+        if (iFlag >= iTimes) {
+          this.server.abort("verifyAndAssignRoot failed after" + iTimes
+              + " times retries, aborting", e);
+          throw new IOException("Aborting", e);
+        }
+        try {
+          Thread.sleep(waitTime);
+        } catch (InterruptedException e1) {
+          LOG.warn("Interrupted when is the thread sleep", e1);
+          Thread.currentThread().interrupt();
+          throw new IOException("Interrupted", e1);
+        }
+        iFlag++;
+      }
+    }
+  }
+  
   /**
    * @return True if the server we are processing was carrying <code>-ROOT-</code>
    */
@@ -131,16 +168,7 @@ public class ServerShutdownHandler extends EventHandler {
 
     // Assign root and meta if we were carrying them.
     if (isCarryingRoot()) { // -ROOT-
-      try {
-        verifyAndAssignRoot();
-      } catch (KeeperException e) {
-        this.server.abort("In server shutdown processing, assigning root", e);
-        throw new IOException("Aborting", e);
-      } catch (InterruptedException e1) {
-        LOG.warn("Interrupted while verifying root region's location", e1);
-        Thread.currentThread().interrupt();
-        throw new IOException(e1);  
-      }
+      verifyAndAssignRootWithRetries();
     }
 
     // Carrying meta?
@@ -351,4 +379,4 @@ public class ServerShutdownHandler extends EventHandler {
       return false;
     }
   }
-}
\ No newline at end of file
+}