diff --git a/CHANGES.txt b/CHANGES.txt index 4c4c4a1c7f3..15d838d7732 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -166,6 +166,8 @@ Release 0.91.0 - Unreleased HBASE-4112 Creating table may throw NullPointerException (Jinchao via Ted Yu) HBASE-4095 Hlog may not be rolled in a long time if checkLowReplication's request of LogRoll is blocked (Jieshan via Ted Yu) + HBASE-4093 When verifyAndAssignRoot throws exception, the deadServers state + cannot be changed (fulin wang via Ted Yu) IMPROVEMENTS HBASE-3290 Max Compaction Size (Nicolas Spiegelberg via Stack) diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java index 6da68c1fc67..c478e9f26b9 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java +++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java @@ -92,6 +92,43 @@ public class ServerShutdownHandler extends EventHandler { } } + /** + * Failed many times, shutdown processing + * @throws IOException + */ + private void verifyAndAssignRootWithRetries() throws IOException { + int iTimes = this.server.getConfiguration().getInt( + "hbase.catalog.verification.retries", 10); + + long waitTime = this.server.getConfiguration().getLong( + "hbase.catalog.verification.timeout", 1000); + + int iFlag = 0; + while (true) { + try { + verifyAndAssignRoot(); + break; + } catch (KeeperException e) { + this.server.abort("In server shutdown processing, assigning root", e); + throw new IOException("Aborting", e); + } catch (Exception e) { + if (iFlag >= iTimes) { + this.server.abort("verifyAndAssignRoot failed after" + iTimes + + " times retries, aborting", e); + throw new IOException("Aborting", e); + } + try { + Thread.sleep(waitTime); + } catch (InterruptedException e1) { + LOG.warn("Interrupted when is the thread sleep", e1); + Thread.currentThread().interrupt(); + throw new IOException("Interrupted", e1); + } + iFlag++; + } + } + } + /** * @return True if the server we are processing was carrying -ROOT- */ @@ -131,16 +168,7 @@ public class ServerShutdownHandler extends EventHandler { // Assign root and meta if we were carrying them. if (isCarryingRoot()) { // -ROOT- - try { - verifyAndAssignRoot(); - } catch (KeeperException e) { - this.server.abort("In server shutdown processing, assigning root", e); - throw new IOException("Aborting", e); - } catch (InterruptedException e1) { - LOG.warn("Interrupted while verifying root region's location", e1); - Thread.currentThread().interrupt(); - throw new IOException(e1); - } + verifyAndAssignRootWithRetries(); } // Carrying meta? @@ -351,4 +379,4 @@ public class ServerShutdownHandler extends EventHandler { return false; } } -} \ No newline at end of file +}