HBASE-7698 race between RS shutdown thread and openregionhandler causes region to get stuck (Ram)

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1443637 13f79535-47bb-0310-9956-ffa450edef68

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java

@@ -87,6 +87,7 @@ public class OpenRegionHandler extends EventHandler {
   @Override
   public void process() throws IOException {
     boolean openSuccessful = false;
+    boolean transitionToFailedOpen = false;
     final String regionName = regionInfo.getRegionNameAsString();
 
     try {
@@ -130,6 +131,7 @@ public class OpenRegionHandler extends EventHandler {
       HRegion region = openRegion();
       if (region == null) {
         tryTransitionFromOpeningToFailedOpen(regionInfo);
+        transitionToFailedOpen = true;
         return;
       }
       boolean failed = true;
@@ -142,6 +144,7 @@ public class OpenRegionHandler extends EventHandler {
           this.rsServices.isStopping()) {
         cleanupFailedOpen(region);
         tryTransitionFromOpeningToFailedOpen(regionInfo);
+        transitionToFailedOpen = true;
         return;
       }
 
@@ -154,6 +157,7 @@ public class OpenRegionHandler extends EventHandler {
         // In all cases, we try to transition to failed_open to be safe.
         cleanupFailedOpen(region);
         tryTransitionFromOpeningToFailedOpen(regionInfo);
+        transitionToFailedOpen = true;
         return;
       }
 
@@ -197,6 +201,8 @@ public class OpenRegionHandler extends EventHandler {
               " should be closed is now opened."
           );
         }
+      } else if (transitionToFailedOpen == false) {
+        tryTransitionFromOpeningToFailedOpen(regionInfo);
       }
     }
   }
@@ -455,7 +461,7 @@ public class OpenRegionHandler extends EventHandler {
     return region;
   }
 
-  private void cleanupFailedOpen(final HRegion region) throws IOException {
+  void cleanupFailedOpen(final HRegion region) throws IOException {
     if (region != null) region.close();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java

@@ -184,6 +184,33 @@ public class TestOpenRegionHandler {
     assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType());
   }
   
+  @Test
+  public void testTransitionToFailedOpenEvenIfCleanupFails() throws Exception {
+    Server server = new MockServer(HTU);
+    RegionServerServices rsServices = new MockRegionServerServices();
+    // Create it OFFLINE, which is what it expects
+    ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName());
+    // Create the handler
+    OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD) {
+      @Override
+      boolean updateMeta(HRegion r) {
+        return false;
+      };
 
+      @Override
+      void cleanupFailedOpen(HRegion region) throws IOException {
+        throw new IOException("FileSystem got closed.");
+      }
+    };
+    rsServices.getRegionsInTransitionInRS().put(TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE);
+    try {
+      handler.process();
+    } catch (Exception e) {
+      // Ignore the IOException that we have thrown from cleanupFailedOpen
+    }
+    RegionTransition rt = RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(),
+        TEST_HRI.getEncodedName()));
+    assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType());
+  }
 }