HADOOP-2276 Address regression caused by HADOOP-2274, fix HADOOP-2173 (When the master times out a region servers lease, the region server may not restart)

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@598535 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -34,6 +34,9 @@ Trunk (unreleased changes)
    HADOOP-2274 Excess synchronization introduced by HADOOP-2139 negatively
                impacts performance
    HADOOP-2196 Fix how hbase sits in hadoop 'package' product
+   HADOOP-2276 Address regression caused by HADOOP-2274, fix HADOOP-2173 (When
+               the master times out a region servers lease, the region server
+               may not restart)
 
   IMPROVEMENTS
     HADOOP-2401 Add convenience put method that takes writable

src/java/org/apache/hadoop/hbase/HMaster.java

@@ -1606,16 +1606,20 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
   private void assignRegions(HServerInfo info, String serverName,
       ArrayList<HMsg> returnMsgs) {
     
+    synchronized (this.assignAttempts) {
+      
+      // We need to hold a lock on assign attempts while we figure out what to
+      // do so that multiple threads do not execute this method in parallel
+      // resulting in assigning the same region to multiple servers.
+      
       long now = System.currentTimeMillis();
       Set<Text> regionsToAssign = new HashSet<Text>();
-    synchronized (this.assignAttempts) {
       for (Map.Entry<Text, Long> e: this.assignAttempts.entrySet()) {
         long diff = now - e.getValue().longValue();
         if (diff > this.maxRegionOpenTime) {
           regionsToAssign.add(e.getKey());
         }
       }
-    }
       int nRegionsToAssign = regionsToAssign.size();
       if (nRegionsToAssign <= 0) {
         // No regions to assign.  Return.
@@ -1703,6 +1707,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
         }
       }
     }
+  }
   
   /*
    * @param nRegionsToAssign
@@ -2092,14 +2097,18 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
       if (numberOfMetaRegions.get() != onlineMetaRegions.size()) {
         // We can't proceed because not all of the meta regions are online.
         // We can't block either because that would prevent the meta region
-        // online message from being processed. So return false to have this
+        // online message from being processed. In order to prevent spinning
-        // operation requeued.
+        // in the run queue, put this request on the delay queue to give
-        
+        // other threads the opportunity to get the meta regions on-line.
         if (LOG.isDebugEnabled()) {
           LOG.debug(
               "Requeuing shutdown because not all meta regions are online");
         }
-        return false;
+        this.expire = System.currentTimeMillis() + leaseTimeout / 2;
+        shutdownQueue.put(this);
+        
+        // Return true so run() does not put us back on the msgQueue
+        return true;
       }
       for (int tries = 0; tries < numRetries; tries++) {
         try {

src/java/org/apache/hadoop/hbase/HRegionServer.java

@@ -642,7 +642,7 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
       false, conf);
     this.serverInfo = new HServerInfo(new HServerAddress(
       new InetSocketAddress(getThisIP(),
-      this.server.getListenerAddress().getPort())), this.rand.nextLong(),
+      this.server.getListenerAddress().getPort())), System.currentTimeMillis(),
       this.conf.getInt("hbase.regionserver.info.port", 60030));
      this.leases = new Leases(
        conf.getInt("hbase.regionserver.lease.period", 3 * 60 * 1000),
@@ -704,7 +704,12 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
                     synchronized (logRollerLock) {
                       try {
                         log.closeAndDelete();
-                        serverInfo.setStartCode(rand.nextLong());
+
+                      } catch (Exception e) {
+                        LOG.error("error closing and deleting HLog", e);
+                      }
+                      try {
+                        serverInfo.setStartCode(System.currentTimeMillis());
                         log = setupHLog();
                       } catch (IOException e) {
                         this.abortRequested = true;