HBASE=3755 Catch zk's ConnectionLossException and augment error

message with more help
Fixup of CHANGES


git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1091607 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -150,12 +150,11 @@ Release 0.91.0 - Unreleased
                (Ted Yu via Stack)
    HBASE-3764  Book.xml - adding 2 FAQs (SQL and arch question)
 
-  TASK
+  TASKS
    HBASE-3559  Move report of split to master OFF the heartbeat channel
    HBASE-3573  Move shutdown messaging OFF hearbeat; prereq for fix of
                hbase-1502
    HBASE-3071  Graceful decommissioning of a regionserver
-   HBASE-3748  Add rolling of thrift/rest daemons to graceful_stop.sh script
 
 
   NEW FEATURES
@@ -191,11 +190,15 @@ Release 0.90.3 - Unreleased
                (Ted Yu via Stack)
    HBASE-3750  HTablePool.putTable() should call releaseHTableInterface()
                for discarded tables (Ted Yu via garyh)
+   HBASE=3755  Catch zk's ConnectionLossException and augment error
+               message with more help
 
   IMPROVEMENTS
    HBASE-3747  ReplicationSource should differanciate remote and local exceptions
    HBASE-3652  Speed up tests by lowering some sleeps
 
+  TASKS
+   HBASE-3748  Add rolling of thrift/rest daemons to graceful_stop.sh script
 
 Release 0.90.2 - 20110408
   

src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java

@@ -43,7 +43,7 @@ public class ZooKeeperConnectionException extends IOException {
    * Constructor taking another exception.
    * @param e Exception to grab data from.
    */
-  public ZooKeeperConnectionException(Exception e) {
+  public ZooKeeperConnectionException(String message, Exception e) {
-    super(e);
+    super(message, e);
   }
 }

src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java

@@ -1000,8 +1000,11 @@ public class HConnectionManager {
       if(zooKeeper == null) {
         try {
           this.zooKeeper = new ZooKeeperWatcher(conf, "hconnection", this);
+        } catch(ZooKeeperConnectionException zce) {
+          throw zce;
         } catch (IOException e) {
-          throw new ZooKeeperConnectionException(e);
+          throw new ZooKeeperConnectionException("An error is preventing" +
+              " HBase from connecting to ZooKeeper", e);
         }
       }
       return zooKeeper;

src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java

@@ -146,13 +146,28 @@ public class ZooKeeperWatcher implements Watcher, Abortable {
         }
       } while (isFinishedRetryingRecoverable(finished));
       // Convert connectionloss exception to ZKCE.
-      if (ke != null) throw new ZooKeeperConnectionException(ke);
+      if (ke != null) {
+        try {
+          // If we don't close it, the zk connection managers won't be killed
+          this.zooKeeper.close();
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+          LOG.warn("Interrupted while closing", e);
+        }
+        throw new ZooKeeperConnectionException("HBase is able to connect to" +
+            " ZooKeeper but the connection closes immediately. This could be" +
+            " a sign that the server has too many connections (30 is the" +
+            " default). Consider inspecting your ZK server logs for that" +
+            " error and then make sure you are reusing HBaseConfiguration" +
+            " as often as you can. See HTable's javadoc for more information.",
+            ke);
+      }
       ZKUtil.createAndFailSilent(this, assignmentZNode);
       ZKUtil.createAndFailSilent(this, rsZNode);
       ZKUtil.createAndFailSilent(this, tableZNode);
     } catch (KeeperException e) {
-      LOG.error(prefix("Unexpected KeeperException creating base node"), e);
+      throw new ZooKeeperConnectionException(
-      throw new IOException(e);
+          prefix("Unexpected KeeperException creating base node"), e);
     }
   }