From 96e750fdac1e81803efe3342b6b1f0b89e479f65 Mon Sep 17 00:00:00 2001
From: jeffreyz <jeffreyz@unknown>
Date: Mon, 5 Aug 2013 17:28:13 +0000
Subject: [PATCH] HBASE-9099: logReplay could trigger double region assignment

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1510615 13f79535-47bb-0310-9956-ffa450edef68
---
 .../hbase/master/AssignmentManager.java       |  3 +-
 .../handler/MetaServerShutdownHandler.java    |  4 ++-
 .../master/handler/ServerShutdownHandler.java | 30 +++++++++----------
 3 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 671fd7ccd99..df8c6090f30 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2776,11 +2776,10 @@ public class AssignmentManager extends ZooKeeperListener {
    * @param hri
    * @param timeOut Milliseconds to wait for current region to be out of transition state.
    * @return True when a region clears regions-in-transition before timeout otherwise false
-   * @throws IOException
    * @throws InterruptedException
    */
   public boolean waitOnRegionToClearRegionsInTransition(final HRegionInfo hri, long timeOut)
-      throws IOException, InterruptedException {
+      throws InterruptedException {
     if (!regionStates.isRegionInTransition(hri)) return true;
     RegionState rs = null;
     long end = (timeOut <= 0) ? Long.MAX_VALUE : EnvironmentEdgeManager.currentTimeMillis()
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
index 693b831f113..87620921ae4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
@@ -85,7 +85,9 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler {
         if (this.shouldSplitHlog && this.distributedLogReplay) {
           if (!am.waitOnRegionToClearRegionsInTransition(HRegionInfo.FIRST_META_REGIONINFO,
             regionAssignmentWaitTimeout)) {
-            throw new IOException("Region " + HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()
+            // Wait here is to avoid log replay hits current dead server and incur a RPC timeout
+            // when replay happens before region assignment completes.
+            LOG.warn("Region " + HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()
                 + " didn't complete assignment in time");
           }
           this.services.getMasterFileSystem().splitMetaLog(serverName);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
index 75f77369adf..8695f9f7acd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.master.handler;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -278,26 +279,25 @@ public class ServerShutdownHandler extends EventHandler {
         throw new IOException(ie);
       }
 
-      try {
-        if (this.shouldSplitHlog && this.distributedLogReplay) {
-          // wait for region assignment completes
-          for (HRegionInfo hri : toAssignRegions) {
+      if (this.shouldSplitHlog && this.distributedLogReplay) {
+        // wait for region assignment completes
+        for (HRegionInfo hri : toAssignRegions) {
+          try {
             if (!am.waitOnRegionToClearRegionsInTransition(hri, regionAssignmentWaitTimeout)) {
-              throw new IOException("Region " + hri.getEncodedName()
+              // Wait here is to avoid log replay hits current dead server and incur a RPC timeout
+              // when replay happens before region assignment completes.
+              LOG.warn("Region " + hri.getEncodedName()
                   + " didn't complete assignment in time");
             }
+          } catch (InterruptedException ie) {
+            throw new InterruptedIOException("Caught " + ie
+                + " during waitOnRegionToClearRegionsInTransition");
           }
-          // submit logReplay work
-          this.services.getExecutorService().submit(
-            new LogReplayHandler(this.server, this.services, this.deadServers, this.serverName));
-          hasLogReplayWork = true;
-        }
-      } catch (Exception ex) {
-        if (ex instanceof IOException) {
-          resubmit(serverName, (IOException)ex);
-        } else {
-          throw new IOException(ex);
         }
+        // submit logReplay work
+        this.services.getExecutorService().submit(
+          new LogReplayHandler(this.server, this.services, this.deadServers, this.serverName));
+        hasLogReplayWork = true;
       }
     } finally {
       this.deadServers.finish(serverName);