HBASE-12238 A few ugly exceptions on startup

Conflicts:
	hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java

hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java

@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.zookeeper.KeeperException;
 
@@ -48,6 +49,9 @@ import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.rmi.UnknownHostException;
 
+import java.util.List;
+import java.util.ArrayList;
+
 /**
  * Utility class to perform operation (get/wait for/verify/set/delete) on znode in ZooKeeper
  * which keeps hbase:meta region server location.
@@ -78,15 +82,38 @@ public class MetaTableLocator {
    * @return true if meta region location is available, false if not
    */
   public boolean isLocationAvailable(ZooKeeperWatcher zkw) {
-    try {
+    return getMetaRegionLocation(zkw) != null;
-      return ZKUtil.getData(zkw, zkw.metaServerZNode) != null;
+  }
-    } catch(KeeperException e) {
+
-      LOG.error("ZK error trying to get hbase:meta from ZooKeeper");
+  /**
-      return false;
+   * @param zkw ZooKeeper watcher to be used
-    } catch (InterruptedException e) {
+   * @return meta table regions and their locations.
-      LOG.error("ZK error trying to get hbase:meta from ZooKeeper");
+   */
-      return false;
+  public List<Pair<HRegionInfo, ServerName>> getMetaRegionsAndLocations(ZooKeeperWatcher zkw) {
+    ServerName serverName = new MetaTableLocator().getMetaRegionLocation(zkw);
+    List<Pair<HRegionInfo, ServerName>> list = new ArrayList<Pair<HRegionInfo, ServerName>>();
+    list.add(new Pair<HRegionInfo, ServerName>(HRegionInfo.FIRST_META_REGIONINFO, serverName));
+    return list;
+  }
+
+  /**
+   * @param zkw ZooKeeper watcher to be used
+   * @return List of meta regions
+   */
+  public List<HRegionInfo> getMetaRegions(ZooKeeperWatcher zkw) {
+    List<Pair<HRegionInfo, ServerName>> result;
+    result = getMetaRegionsAndLocations(zkw);
+    return getListOfHRegionInfos(result);
+  }
+
+  private List<HRegionInfo> getListOfHRegionInfos(
+      final List<Pair<HRegionInfo, ServerName>> pairs) {
+    if (pairs == null || pairs.isEmpty()) return null;
+    List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
+    for (Pair<HRegionInfo, ServerName> pair: pairs) {
+      result.add(pair.getFirst());
     }
+    return result;
   }
 
   /**
@@ -239,7 +266,7 @@ public class MetaTableLocator {
       }
     }
     LOG.info("Failed verification of " + Bytes.toStringBinary(regionName) +
-      " at address=" + address + ", exception=" + t);
+      " at address=" + address + ", exception=" + t.getMessage());
     return false;
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -256,7 +256,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
    * </ol>
    * <p>
    * Remaining steps of initialization occur in
-   * {@link #finishActiveMasterInitialization(MonitoredTask)} after
+   * #finishActiveMasterInitialization(MonitoredTask) after
    * the master becomes the active one.
    *
    * @throws InterruptedException
@@ -286,6 +286,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
     this.metricsMaster = new MetricsMaster( new MetricsMasterWrapperImpl(this));
 
     // Do we publish the status?
+    
     boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
         HConstants.STATUS_PUBLISHED_DEFAULT);
     Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
@@ -1261,8 +1262,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
     * this node explicitly.  If we crash before then, ZooKeeper will delete
     * this node for us since it is ephemeral.
     */
-    LOG.info("Adding ZNode for " + backupZNode + " in backup master directory");
+    LOG.info("Adding backup master ZNode " + backupZNode);
-    MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName);
+    if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName)) {
+      LOG.warn("Failed create of " + backupZNode + " by " + serverName);
+    }
 
     activeMasterManager = new ActiveMasterManager(zooKeeper, serverName, this);
     // Start a thread to try to become the active master, so we won't block here

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java

@@ -166,7 +166,10 @@ public class HMasterCommandLine extends ServerCommandLine {
         // login the zookeeper server principal (if using security)
         ZKUtil.loginServer(conf, "hbase.zookeeper.server.keytab.file",
           "hbase.zookeeper.server.kerberos.principal", null);
-
+        int localZKClusterSessionTimeout =
+          conf.getInt(HConstants.ZK_SESSION_TIMEOUT + ".localHBaseCluster", 10*1000);
+        conf.setInt(HConstants.ZK_SESSION_TIMEOUT, localZKClusterSessionTimeout);
+        LOG.info("Starting a zookeeper cluster");
         int clientPort = zooKeeperCluster.startup(zkDataPath);
         if (clientPort != zkClientPort) {
           String errorMsg = "Could not start ZK at requested port of " +
@@ -176,13 +179,15 @@ public class HMasterCommandLine extends ServerCommandLine {
           System.err.println(errorMsg);
           throw new IOException(errorMsg);
         }
-        conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
+        conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort));
-                 Integer.toString(clientPort));
-        conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 10 *1000);
         // Need to have the zk cluster shutdown when master is shutdown.
         // Run a subclass that does the zk cluster shutdown on its way out.
-        LocalHBaseCluster cluster = new LocalHBaseCluster(conf, conf.getInt("hbase.masters", 1),
+        int mastersCount = conf.getInt("hbase.masters", 1);
-          conf.getInt("hbase.regionservers", 1), LocalHMaster.class, HRegionServer.class);
+        int regionServersCount = conf.getInt("hbase.regionservers", 1);
+        LOG.info("Starting up instance of localHBaseCluster; master=" + mastersCount +
+          ", regionserversCount=" + regionServersCount);
+        LocalHBaseCluster cluster = new LocalHBaseCluster(conf, mastersCount, regionServersCount,
+          LocalHMaster.class, HRegionServer.class);
         ((LocalHMaster)cluster.getMaster(0)).setZKCluster(zooKeeperCluster);
         cluster.startup();
         waitOnMasterThreads(cluster);

hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java

@@ -165,8 +165,7 @@ public class MiniZooKeeperCluster {
           standaloneServerFactory = new NIOServerCnxnFactory();
           standaloneServerFactory.configure(
             new InetSocketAddress(tentativePort),
-            configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
+            configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, 1000));
-              1000));
         } catch (BindException e) {
           LOG.debug("Failed binding ZK Server to client port: " +
               tentativePort, e);
@@ -181,6 +180,7 @@ public class MiniZooKeeperCluster {
 
       // Start up this ZK server
       standaloneServerFactory.startup(server);
+      // Runs a 'stat' against the servers.
       if (!waitForServerUp(tentativePort, CONNECTION_TIMEOUT)) {
         throw new IOException("Waiting for startup of standalone server");
       }
@@ -196,8 +196,8 @@ public class MiniZooKeeperCluster {
     activeZKServerIndex = 0;
     started = true;
     int clientPort = clientPortList.get(activeZKServerIndex);
-    LOG.info("Started MiniZK Cluster and connect 1 ZK server " +
+    LOG.info("Started MiniZooKeeperCluster and ran successful 'stat' " +
-        "on client port: " + clientPort);
+        "on client port=" + clientPort);
     return clientPort;
   }