HBASE-1357 If one sets the hbase.master to 0.0.0.0 non local regionservers

can't find the master
HBASE-1445  Add the ability to start a master from any machine



git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@781099 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -19,6 +19,8 @@ Release 0.20.0 - Unreleased
    HBASE-1367  Get rid of Thrift exception 'NotFound'
    HBASE-1381  Remove onelab and bloom filters files from hbase
    HBASE-1411  Remove HLogEdit.
+   HBASE-1357  If one sets the hbase.master to 0.0.0.0 non local regionservers
+               can't find the master
 
   BUG FIXES
    HBASE-1140  "ant clean test" fails (Nitay Joffe via Stack)
@@ -297,6 +299,7 @@ Release 0.20.0 - Unreleased
    HBASE-1464  Add hbase.regionserver.logroll.period to hbase-default
    HBASE-1192  LRU-style map for the block cache (Jon Gray and Ryan Rawson
                via Stack)
+   HBASE-1445  Add the ability to start a master from any machine
 
   OPTIMIZATIONS
    HBASE-1412  Change values for delete column and column family in KeyValue

conf/hbase-default.xml

@@ -30,18 +30,19 @@
     E.g: hdfs://NAMENODE_SERVER:PORT/HBASE_ROOTDIR
     </description>
   </property>
-  <property>
-    <name>hbase.master.hostname</name>
-    <value>local</value>
-    <description>The host that the HBase master runs at.
-      A value of 'local' runs the master and regionserver in a single process.
-    </description>
-  </property>
   <property>
     <name>hbase.master.port</name>
     <value>60000</value>
     <description>The port master should bind to.</description>
   </property>
+  <property>
+    <name>hbase.cluster.distributed</name>
+    <value>false</value>
+    <description>The mode the cluster will be in. Possible values are
+      false: standalone and pseudo-distributed setups with managed Zookeeper
+      true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh)
+    </description>
+  </property>
   <property>
     <name>hbase.tmp.dir</name>
     <value>/tmp/hbase-${user.name}</value>
@@ -213,6 +214,21 @@
       master for communication and display purposes.
     </description>
   </property>
+  <property>
+    <name>hbase.master.dns.interface</name>
+    <value>default</value>
+    <description>The name of the Network Interface from which a master 
+      should report its IP address.
+    </description>
+  </property>
+  <property>
+    <name>hbase.master.dns.nameserver</name>
+    <value>default</value>
+    <description>The host name or IP address of the name server (DNS)
+      which a master should use to determine the host name used 
+      for communication and display purposes.
+    </description>
+  </property>
   <property>
     <name>hbase.regionserver.globalMemcache.upperLimit</name>
     <value>0.4</value>

conf/zoo.cfg

@@ -11,4 +11,4 @@ dataDir=${hbase.tmp.dir}/zookeeper
 # the port at which the clients will connect
 clientPort=2181
 
-server.0=${hbase.master.hostname}:2888:3888
+server.0=localhost:2888:3888

src/java/org/apache/hadoop/hbase/HConstants.java

@@ -52,21 +52,16 @@ public interface HConstants {
   
   // Configuration parameters
   
-  // TODO: URL for hbase master like hdfs URLs with host and port.
+  //TODO: Is having HBase homed on port 60k OK?
-  // Like jdbc URLs?  URLs could be used to refer to table cells?
-  // jdbc:mysql://[host][,failoverhost...][:port]/[database]
-  // jdbc:mysql://[host][,failoverhost...][:port]/[database][?propertyName1][=propertyValue1][&propertyName2][=propertyValue2]...
   
-  // Key into HBaseConfiguration for the hbase.master address.
+  /** Cluster is in distributed mode or not */
-  // TODO: Support 'local': i.e. default of all running in single
+  static final String CLUSTER_DISTRIBUTED = "hbase.cluster.distributed";
-  // process.  Same for regionserver. TODO: Is having HBase homed
-  // on port 60k OK?
   
-  /** Parameter name for master address */
+  /** Cluster is standalone or pseudo-distributed */
-  static final String MASTER_ADDRESS = "hbase.master";
+  static final String CLUSTER_IS_LOCAL = "false";
   
-  /** Parameter name for master host name. */
+  /** Cluster is fully-distributed */
-  static final String MASTER_HOST_NAME = "hbase.master.hostname";
+  static final String CLUSTER_IS_DISTRIBUTED = "true";
 
   /** default host address */
   static final String DEFAULT_HOST = "0.0.0.0";

src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java

@@ -90,7 +90,6 @@ public class LocalHBaseCluster implements HConstants {
     final int noRegionServers)
   throws IOException {
     this.conf = conf;
-    doLocal(conf);
     // Create the master
     this.master = new HMaster(conf);
     // Start the HRegionServers.  Always have region servers come up on
@@ -316,47 +315,13 @@ public class LocalHBaseCluster implements HConstants {
     }
   }
 
-  /**
-   * Changes <code>hbase.master</code> from 'local' to 'localhost:PORT' in
-   * passed Configuration instance.
-   * @param c
-   * @return The passed <code>c</code> configuration modified if hbase.master
-   * value was 'local' otherwise, unaltered.
-   */
-  private static HBaseConfiguration doLocal(final HBaseConfiguration c) {
-    if (!isLocal(c)) {
-      return c;
-    }
-
-    // Need to rewrite address in Configuration if not done already.
-    String address = c.get(MASTER_ADDRESS);
-    if (address != null) {
-      String port = address.startsWith(LOCAL_COLON)?
-        address.substring(LOCAL_COLON.length()):
-        Integer.toString(DEFAULT_MASTER_PORT);
-      c.set(MASTER_ADDRESS, "localhost:" + port);
-    }
-
-    // Need to rewrite host in Configuration if not done already.
-    String host = c.get(MASTER_HOST_NAME);
-    if (host != null && host.equals(LOCAL)) {
-      c.set(MASTER_HOST_NAME, "localhost");
-    }
-
-    return c;
-  }
-
   /**
    * @param c Configuration to check.
    * @return True if a 'local' address in hbase.master value.
    */
   public static boolean isLocal(final Configuration c) {
-    String address = c.get(MASTER_ADDRESS);
+    String mode = c.get(CLUSTER_DISTRIBUTED);
-    boolean addressIsLocal = address == null || address.equals(LOCAL) ||
+    return mode == null || mode.equals(CLUSTER_IS_LOCAL);
-                             address.startsWith(LOCAL_COLON);
-    String host = c.get(MASTER_HOST_NAME);
-    boolean hostIsLocal = host == null || host.equals(LOCAL);
-    return addressIsLocal && hostIsLocal;
   }
 
   /**

src/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -85,6 +85,7 @@ import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.net.DNS;
 import org.apache.hadoop.util.StringUtils;
 
 /**
@@ -161,17 +162,12 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
    * @throws IOException
    */
   public HMaster(HBaseConfiguration conf) throws IOException {
-    // find out our address. If it's set in config, use that, otherwise look it
+    // find out our address up in DNS.
-    // up in DNS.
+    String addressStr = DNS.getDefaultHost(
-    String addressStr = conf.get(MASTER_ADDRESS);
+        conf.get("hbase.master.dns.interface","default"),
-    if (addressStr == null) {
+        conf.get("hbase.master.dns.nameserver","default"));
-      addressStr = conf.get(MASTER_HOST_NAME);
+    addressStr += ":" + 
-      if (addressStr == null) {
+      conf.get("hbase.master.port", Integer.toString(DEFAULT_MASTER_PORT));
-        addressStr = InetAddress.getLocalHost().getCanonicalHostName();
-      }
-      addressStr += ":";
-      addressStr += conf.get("hbase.master.port", Integer.toString(DEFAULT_MASTER_PORT));
-    }
     HServerAddress address = new HServerAddress(addressStr);
     LOG.info("My address is " + address);
 
@@ -531,7 +527,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
           this.connection.getHRegionConnection(address, false);
         HServerInfo info = hri.getHServerInfo();
         LOG.debug("Inspection found server " + info.getName());
-        serverManager.recordNewServer(info);
+        serverManager.recordNewServer(info, true);
         HRegionInfo[] regions = hri.getRegionsAssignment();
         for (HRegionInfo region : regions) {
           if(region.isRootRegion()) {
@@ -1075,13 +1071,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
 
     // Process command-line args. TODO: Better cmd-line processing
     // (but hopefully something not as painful as cli options).
-
-    final String addressArgKey = "--bind=";
     for (String cmd: args) {
-      if (cmd.startsWith(addressArgKey)) {
-        conf.set(MASTER_ADDRESS, cmd.substring(addressArgKey.length()));
-        continue;
-      }
 
       if (cmd.equals("start")) {
         try {

src/java/org/apache/hadoop/hbase/master/ServerManager.java

@@ -194,12 +194,23 @@ class ServerManager implements HConstants {
     recordNewServer(info);
   }
   
+  
   /**
-   * Adds the HSI to the RS list
+   * Adds the HSI to the RS list and creates an empty load
    * @param info The region server informations
    */
   public void recordNewServer(HServerInfo info) {
-    HServerLoad load = new HServerLoad();
+    recordNewServer(info, false);
+  }
+  
+  /**
+   * Adds the HSI to the RS list
+   * @param info The region server informations
+   * @param useInfoLoad True if the load from the info should be used
+   *                    like under a master failover
+   */
+  public void recordNewServer(HServerInfo info, boolean useInfoLoad) {
+    HServerLoad load = useInfoLoad ? info.getLoad() : new HServerLoad();
     String serverName = HServerInfo.getServerName(info);
     info.setLoad(load);
     // We must set this watcher here because it can be set on a fresh start

src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java

@@ -101,6 +101,7 @@ public class HQuorumPeer implements HConstants {
     }
     for (Entry<Object, Object> entry : properties.entrySet()) {
       String value = entry.getValue().toString().trim();
+      String key = entry.getKey().toString().trim();
       StringBuilder newValue = new StringBuilder();
       int varStart = value.indexOf(VARIABLE_START);
       int varEnd = 0;
@@ -123,19 +124,24 @@ public class HQuorumPeer implements HConstants {
           LOG.fatal(msg);
           throw new IOException(msg);
         }
-        // Special case for 'hbase.master.hostname' property being 'local'
+        
-        if (variable.equals(HConstants.MASTER_HOST_NAME) && substituteValue.equals("local")) {
-          substituteValue = "localhost";
-        }
         newValue.append(substituteValue);
 
         varEnd += VARIABLE_END_LENGTH;
         varStart = value.indexOf(VARIABLE_START, varEnd);
       }
-
+      // Special case for 'hbase.cluster.distributed' property being 'true'
+      if (key.startsWith("server.")) {
+        if(conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) && 
+            value.startsWith("localhost")) {
+           String msg = "The server in zoo.cfg cannot be set to localhost " +
+              "in a fully-distributed setup because it won't be reachable. " +
+              "See \"Getting Started\" for more information.";
+           LOG.fatal(msg);
+           throw new IOException(msg);
+        }
+      }
       newValue.append(value.substring(varEnd));
-
-      String key = entry.getKey().toString().trim();
       properties.setProperty(key, newValue.toString());
     }
     return properties;

src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java

@@ -152,8 +152,8 @@ public class ZooKeeperWrapper implements HConstants {
     try {
       properties = HQuorumPeer.parseZooKeeperConfig();
     } catch (IOException e) {
-      LOG.error("fail to read properties from " + ZOOKEEPER_CONFIG_NAME);
+      LOG.fatal("Fail to read properties from " + ZOOKEEPER_CONFIG_NAME, e);
-      return;
+      System.exit(-1);
     }
 
     String clientPort = null;
@@ -532,9 +532,9 @@ public class ZooKeeperWrapper implements HConstants {
         zooKeeper.delete(joinPath(this.rsZNode, node), -1);
       }
     } catch (KeeperException e) {
-      LOG.warn("Failed to delete " + rsZNode + " znode in ZooKeeper: " + e);
+      LOG.warn("Failed to delete " + rsZNode + " znodes in ZooKeeper: " + e);
     } catch (InterruptedException e) {
-      LOG.warn("Failed to delete " + rsZNode + " znode in ZooKeeper: " + e);
+      LOG.warn("Failed to delete " + rsZNode + " znodes in ZooKeeper: " + e);
     }
   }
   

src/java/overview.html

@@ -79,15 +79,9 @@ operation, continue reading.
 </p>
 
 <h2><a name="distributed">Distributed Operation</a></h2>
-<p>Distributed mode requires an instance of the Hadoop Distributed File System (DFS) and a ZooKeeper cluster.
+<p>Distributed mode requires an instance of the Hadoop Distributed File System (DFS).
 See the Hadoop <a href="http://lucene.apache.org/hadoop/api/overview-summary.html#overview_description">
 requirements and instructions</a> for how to set up a DFS.
-See the ZooKeeeper <a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html">Getting Started Guide</a>
-for information about the ZooKeeper distributed coordination service.
-If you do not configure a ZooKeeper cluster, HBase will manage a single instance
-ZooKeeper service for you running on the master node.
-This is intended for development and local testing only.
-It SHOULD NOT be used in a fully-distributed production operation.
 </p>
 
 <h3><a name="pseudo-distrib">Pseudo-Distributed Operation</a></h3>
@@ -122,21 +116,19 @@ create them if you let it).
 For running a fully-distributed operation on more than one host, the following
 configurations must be made <i>in addition</i> to those described in the
 <a href="#pseudo-distrib">pseudo-distributed operation</a> section above.
+A Zookeeper cluster is also required to ensure higher availability.
 In <code>hbase-site.xml</code>, you must also configure
-<code>hbase.master.hostname</code> to the host on which the HBase master runs
+<code>hbase.cluster.distributed</code> to 'true'. 
-(<a href="http://wiki.apache.org/lucene-hadoop/Hbase/HbaseArchitecture">read 
-about the HBase master, regionservers, etc</a>).
-For example, adding the below to your <code>hbase-site.xml</code> says the
-master is up on the host example.org:
 </p>
 <pre>
 &lt;configuration&gt;
   ...
   &lt;property&gt;
-    &lt;name&gt;hbase.master.hostname&lt;/name&gt;
+    &lt;name&gt;hbase.cluster.distributed&lt;/name&gt;
-    &lt;value&gt;example.org&lt;/value&gt;
+    &lt;value&gt;true&lt;/value&gt;
-    &lt;description&gt;The host that the HBase master runs at.
+    &lt;description&gt;The mode the cluster will be in. Possible values are
-      A value of 'local' runs the master and regionserver in a single process.
+      false: standalone and pseudo-distributed setups with managed Zookeeper
+      true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh)
     &lt;/description&gt;
   &lt;/property&gt;
   ...
@@ -151,14 +143,35 @@ operation requires that you also modify <code>${HBASE_HOME}/conf/regionservers</
 in HBase is like the hadoop slaves file at <code>${HADOOP_HOME}/conf/slaves</code>).
 </p>
 <p>
-Furthermore, you should configure a distributed ZooKeeper cluster.
+Furthermore, you have to configure a distributed ZooKeeper cluster.
 The ZooKeeper configuration file is stored at <code>${HBASE_HOME}/conf/zoo.cfg</code>.
 See the ZooKeeper <a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html"> Getting Started Guide</a> for information about the format and options of that file.
 Specifically, look at the <a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html#sc_RunningReplicatedZooKeeper">Running Replicated ZooKeeper</a> section.
-In <code>${HBASE_HOME}/conf/hbase-env.sh</code>, set <code>HBASE_MANAGES_ZK=false</code> to tell HBase not to manage its own single instance ZooKeeper service.
+In <code>${HBASE_HOME}/conf/hbase-env.sh</code>, set the following to tell HBase not to manage its own single instance of ZooKeeper.
+<pre>
+  ...
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+export HBASE_MANAGES_ZK=false
+</pre>
+</p>
+<p>
+It's still possible to use HBase in order to start a single Zookeeper instance in fully-distributed operation.
+The first thing to do is still to change <code>${HBASE_HOME}/conf/zoo.cfg</code> and set a single node.
+Note that leaving the value "localhost" will make it impossible to start HBase.
+<pre>
+  ...
+server.0=example.org:2888:3888
+</pre>
+Then on the example.org server do the following <i>before</i> running HBase. 
+<pre>
+${HBASE_HOME}/bin/hbase-daemon.sh start zookeeper
+</pre>
+Be aware that this option is only recommanded for testing purposes as a failure
+on that node would render HBase <b>unusable</b>.
 </p>
 
-<p>Of note, if you have made <i>HDFS client configuration</i> on your hadoop cluster, hbase will not
+
+<p>Of note, if you have made <i>HDFS client configuration</i> on your hadoop cluster, HBase will not
 see this configuration unless you do one of the following:
 <ul>
     <li>Add a pointer to your <code>HADOOP_CONF_DIR</code> to <code>CLASSPATH</code> in <code>hbase-env.sh</code></li>
@@ -167,7 +180,7 @@ see this configuration unless you do one of the following:
 </ul>
 An example of such an HDFS client configuration is <code>dfs.replication</code>.  If for example,
 you want to run with a replication factor of 5, hbase will create files with the default of 3 unless 
-you do the above to make the configuration available to hbase.
+you do the above to make the configuration available to HBase.
 </p>
 
 <h2><a name="runandconfirm">Running and Confirming Your Installation</a></h2>

src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java

@@ -768,12 +768,6 @@ public class PerformanceEvaluation implements HConstants {
           break;
         }
        
-        final String masterArgKey = "--master=";
-        if (cmd.startsWith(masterArgKey)) {
-          this.conf.set(MASTER_ADDRESS, cmd.substring(masterArgKey.length()));
-          continue;
-        }
-       
         final String miniClusterArgKey = "--miniCluster";
         if (cmd.startsWith(miniClusterArgKey)) {
           this.miniCluster = true;