HBASE-1573 Holes in master state change; updated startcode and server go into .META. but catalog scanner just got old values

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@799014 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -302,6 +302,8 @@ Release 0.20.0 - Unreleased
                have (Doğacan Güney via Ryan Rawson and Stack)
    HBASE-1718  Reuse of KeyValue during log replay could cause the wrong
                data to be used
+   HBASE-1573  Holes in master state change; updated startcode and server
+               go into .META. but catalog scanner just got old values (redux)
 
   IMPROVEMENTS
    HBASE-1089  Add count of regions on filesystem to master UI; add percentage

src/java/org/apache/hadoop/hbase/master/BaseScanner.java

@@ -347,14 +347,11 @@ abstract class BaseScanner extends Chore implements HConstants {
     HServerInfo storedInfo = null;
     synchronized (this.master.regionManager) {
       /* We don't assign regions that are offline, in transition or were on
-       * a dead server (unless they have an empty serverName which would imply
+       * a dead server. Regions that were on a dead server will get reassigned
-       * they haven't been assigned in the first place OR it was closed from
+       * by ProcessServerShutdown
-       * the shell with 'close_region' which deletes server and startcode
-       * from .META. so region gets reassigned). Regions that were on a dead
-       * server will get reassigned by ProcessServerShutdown
        */
       if (info.isOffline() ||
-        (serverName != null && this.master.regionManager.regionIsInTransition(info.getRegionNameAsString())) ||
+        this.master.regionManager.regionIsInTransition(info.getRegionNameAsString()) ||
           (serverName != null && this.master.serverManager.isDead(serverName))) {
         return;
       }
@@ -368,8 +365,8 @@ abstract class BaseScanner extends Chore implements HConstants {
         // The current assignment is invalid
         if (LOG.isDebugEnabled()) {
           LOG.debug("Current assignment of " + info.getRegionNameAsString() +
-            " is not valid; " + " Server '" + serverAddress + "' startCode: " +
+            " is not valid; " + " serverAddress=" + serverAddress +
-            startCode + " unknown.");
+            ", startCode=" + startCode + " unknown.");
         }
         // Now get the region assigned
         this.master.regionManager.setUnassigned(info, true);

src/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -688,7 +688,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
         DEFAULT_HOST)) {
       String rsAddress = HBaseServer.getRemoteAddress();
       serverInfo.setServerAddress(new HServerAddress(rsAddress,
-      serverInfo.getServerAddress().getPort()));
+        serverInfo.getServerAddress().getPort()));
     }
     // Register with server manager
     this.serverManager.regionServerStartup(serverInfo);
@@ -1025,10 +1025,16 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
         servername = 
           Bytes.toString(rr.getValue(CATALOG_FAMILY, SERVER_QUALIFIER));
       }
+      // Need to make up a HServerInfo 'servername' for that is how
+      // items are keyed in regionmanager Maps.
+      HServerAddress addr = new HServerAddress(servername);
+      long startCode =
+        Bytes.toLong(rr.getValue(CATALOG_FAMILY, STARTCODE_QUALIFIER));
+      String name = HServerInfo.getServerName(addr, startCode);
       LOG.info("Marking " + hri.getRegionNameAsString() +
-        " as closed on " + servername + "; cleaning SERVER + STARTCODE; " +
+        " as closing on " + name + "; cleaning SERVER + STARTCODE; " +
           "master will tell regionserver to close region on next heartbeat");
-      this.regionManager.setClosing(servername, hri, hri.isOffline());
+      this.regionManager.setClosing(name, hri, hri.isOffline());
       MetaRegion meta = this.regionManager.getMetaRegionForRow(regionname);
       HRegionInterface srvr = getMETAServer(meta);
       HRegion.cleanRegionInMETA(srvr, meta.getRegionName(), hri);

src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java

@@ -25,11 +25,8 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.RegionHistorian;
-import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.util.Bytes;
 
 /** 
@@ -77,16 +74,15 @@ class ProcessRegionOpen extends ProcessRegionStatusChange {
         serverInfo.getServerAddress().toString());
 
     // Register the newly-available Region's location.
-    LOG.info("updating row " + regionInfo.getRegionNameAsString() +
-        " in region " + Bytes.toString(metaRegionName) +
-        " with startcode " + serverInfo.getStartCode() + " and server " +
-        serverInfo.getServerAddress());
     Put p = new Put(regionInfo.getRegionName());
     p.add(CATALOG_FAMILY, SERVER_QUALIFIER,
-        Bytes.toBytes(serverInfo.getServerAddress().toString()));
+      Bytes.toBytes(serverInfo.getServerAddress().toString()));
     p.add(CATALOG_FAMILY, STARTCODE_QUALIFIER,
-        Bytes.toBytes(serverInfo.getStartCode()));
+      Bytes.toBytes(serverInfo.getStartCode()));
     server.put(metaRegionName, p);
+    LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
+      " in region " + Bytes.toString(metaRegionName) + " with startcode=" +
+      serverInfo.getStartCode() + ", server=" + serverInfo.getServerAddress());
     if (!historian.isOnline()) {
       // This is safest place to do the onlining of the historian in
       // the master.  When we get to here, we know there is a .META.