HBASE-12 when hbase regionserver restarts, it says "impossible state for createLease()"

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@648110 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -4,6 +4,8 @@ Hbase Change Log
    HBASE-573   HBase does not read hadoop-*.xml for dfs configuration after 
                moving out hadoop/contrib
    HBASE-11    Unexpected exits corrupt DFS
+   HBASE-12    When hbase regionserver restarts, it says "impossible state for
+               createLease()"
 
 Release 0.1.1 - 04/11/2008
 

src/java/org/apache/hadoop/hbase/Leases.java

@@ -29,6 +29,8 @@ import java.util.concurrent.Delayed;
 import java.util.concurrent.DelayQueue;
 import java.util.concurrent.TimeUnit;
 
+import java.io.IOException;
+
 /**
  * Leases
  *
@@ -125,8 +127,10 @@ public class Leases extends Thread {
    * 
    * @param leaseName name of the lease
    * @param listener listener that will process lease expirations
+   * @throws LeaseStillHeldException 
    */
-  public void createLease(String leaseName, final LeaseListener listener) {
+  public void createLease(String leaseName, final LeaseListener listener)
+  throws LeaseStillHeldException {
     if (stopRequested) {
       return;
     }
@@ -134,13 +138,28 @@ public class Leases extends Thread {
         System.currentTimeMillis() + leasePeriod);
     synchronized (leaseQueue) {
       if (leases.containsKey(leaseName)) {
-        throw new IllegalStateException("lease '" + leaseName +
+        throw new LeaseStillHeldException(leaseName);
-            "' already exists");
       }
       leases.put(leaseName, lease);
       leaseQueue.add(lease);
     }
   }
+
+  /**
+   * Thrown if we are asked create a lease but lease on passed name already
+   * exists.
+   */
+  public static class LeaseStillHeldException extends IOException {
+    private final String leaseName;
+    
+    public LeaseStillHeldException(final String name) {
+      this.leaseName = name;
+    }
+    
+    public String getName() {
+      return this.leaseName;
+    }
+  }
   
   /**
    * Renew a lease
@@ -229,4 +248,4 @@ public class Leases extends Thread {
       this.expirationTime = expirationTime;
     }
   }
-}
+}

src/java/org/apache/hadoop/hbase/master/ServerManager.java

@@ -90,7 +90,12 @@ class ServerManager implements HConstants {
     // server expecially if it just shutdown and came back up near-immediately
     // after.
     if (!master.closed.get()) {
-      serverLeases.createLease(s, new ServerExpirer(s));
+      try {
+        serverLeases.createLease(s, new ServerExpirer(s));
+      } catch (Leases.LeaseStillHeldException e) {
+        LOG.debug("Lease still held on " + e.getName());
+        return;
+      }
     }
     HServerLoad load = serversToLoad.remove(s);
     if (load != null) {

src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

@@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.regionserver;
 import java.io.IOException;
 import java.lang.Thread.UncaughtExceptionHandler;
 import java.lang.reflect.Constructor;
+import java.lang.reflect.Member;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
 import java.util.Arrays;
@@ -46,6 +47,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.AlreadyBeingCreatedException;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -685,11 +687,17 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
           LOG.debug("Done telling master we are up");
         }
         break;
-      } catch(IOException e) {
+      } catch (Leases.LeaseStillHeldException e) {
+        LOG.info("Lease " + e.getName() + " already held on master. Check " +
+          "DNS configuration so that all region servers are" +
+          "reporting their true IPs and not 127.0.0.1. Otherwise, this" +
+          "problem should resolve itself after the lease period of " +
+          this.conf.get("hbase.master.lease.period")
+          + " seconds expires over on the master");
+      } catch (IOException e) {
         LOG.warn("error telling master we are up", e);
-        sleeper.sleep(lastMsg);
-        continue;
       }
+      sleeper.sleep(lastMsg);
     }
     return result;
   }