diff --git a/bin/rolling-restart.sh b/bin/rolling-restart.sh
index 8934e55c277..81a837d7480 100644
--- a/bin/rolling-restart.sh
+++ b/bin/rolling-restart.sh
@@ -49,7 +49,7 @@ then
 fi
 
 # quick function to get a value from the HBase config file
-distMode=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool hbase.cluster.distributed`
+distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed`
 if [ "$distMode" == 'false' ]; then
   "$bin"/hbase-daemon.sh restart master
 else 
@@ -59,9 +59,9 @@ else
     --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup
 
   # make sure the master znode has been deleted before continuing
-  zparent=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool zookeeper.znode.parent`
+  zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent`
   if [ "$zparent" == "null" ]; then zparent="/hbase"; fi
-  zmaster=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool zookeeper.znode.master`
+  zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master`
   if [ "$zmaster" == "null" ]; then zmaster="master"; fi
   zmaster=$zparent/$zmaster
   echo -n "Waiting for Master ZNode to expire"
@@ -134,7 +134,7 @@ then
 fi
 
 # quick function to get a value from the HBase config file
-distMode=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool hbase.cluster.distributed`
+distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed`
 if [ "$distMode" == 'false' ]; then
   "$bin"/hbase-daemon.sh restart master
 else 
@@ -144,9 +144,9 @@ else
     --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup
 
   # make sure the master znode has been deleted before continuing
-  zparent=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool zookeeper.znode.parent`
+  zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent`
   if [ "$zparent" == "null" ]; then zparent="/hbase"; fi
-  zmaster=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool zookeeper.znode.master`
+  zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master`
   if [ "$zmaster" == "null" ]; then zmaster="master"; fi
   zmaster=$zparent/$zmaster
   echo -n "Waiting for Master ZNode to expire"
diff --git a/bin/start-hbase.sh b/bin/start-hbase.sh
index f413cb07dcc..15e952e91e7 100755
--- a/bin/start-hbase.sh
+++ b/bin/start-hbase.sh
@@ -38,7 +38,7 @@ then
   exit $errCode
 fi
 
-distMode=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool hbase.cluster.distributed`
+distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed`
 
 
 if [ "$distMode" == 'false' ] 
diff --git a/bin/stop-hbase.sh b/bin/stop-hbase.sh
index b6d7c885179..4d9220b3a20 100755
--- a/bin/stop-hbase.sh
+++ b/bin/stop-hbase.sh
@@ -57,7 +57,7 @@ while kill -0 `cat $pid` > /dev/null 2>&1; do
 done
 
 # distributed == false means that the HMaster will kill ZK when it exits
-distMode=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool hbase.cluster.distributed`
+distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed`
 if [ "$distMode" == 'true' ] 
 then
   # TODO: store backup masters in ZooKeeper and have the primary send them a shutdown message
diff --git a/conf/hbase-site.xml b/conf/hbase-site.xml
index c39db3e5619..af4c3009521 100644
--- a/conf/hbase-site.xml
+++ b/conf/hbase-site.xml
@@ -2,7 +2,7 @@
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <!--
 /**
- * Copyright 2009 The Apache Software Foundation
+ * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
diff --git a/conf/hbase-site.xml.psuedo-distributed.template b/conf/hbase-site.xml.psuedo-distributed.template
deleted file mode 100644
index a960eb7b7da..00000000000
--- a/conf/hbase-site.xml.psuedo-distributed.template
+++ /dev/null
@@ -1,77 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Copyright 2009 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration>
-
-<!-- NEEDED WHETHER OR NOT YOU ARE RUNNING OVER HDFS -->
-<property>
-  <name>hbase.cluster.distributed</name>
-  <value>true</value>
-  <description>For psuedo-distributed, you want to set this to true.
-  false means that HBase tries to put Master + RegionServers in one process.
-  Pseudo-distributed = seperate processes/pids</description>
-</property> <property>
-  <name>hbase.regionserver.hlog.replication</name>
-  <value>1</value>
-  <description>For HBase to offer good data durability, we roll logs if
-  filesystem replication falls below a certain amount.  In psuedo-distributed
-  mode, you normally only have the local filesystem or 1 HDFS DataNode, so you
-  don't want to roll logs constantly.</description>
-</property>
-<property>
-  <name>hbase.tmp.dir</name>
-  <value>/tmp/hbase-testing</value>
-  <description>Temporary directory on the local filesystem.</description>
-</property>
-
-<!-- DEFAULT = use local filesystem, not HDFS
-     ADD THESE LINES if you have a copy of HDFS source and want to run HBase
-     psuedo-distributed over a psuedo-distributed HDFS cluster.
-     For HDFS psuedo-distributed setup, see their documentation:
-
-     http://hadoop.apache.org/common/docs/r0.20.2/quickstart.html#PseudoDistributed
-
-
-<property>
-  <name>hbase.rootdir</name>
-  <value>hdfs://localhost:9000/hbase-testing</value>
-  <description>The directory shared by region servers.
-  Should be fully-qualified to include the filesystem to use.
-  E.g: hdfs://NAMENODE_SERVER:PORT/HBASE_ROOTDIR
-  </description>
-</property>
--->
-
-<!-- OPTIONAL: You might want to add these options depending upon your use case
-
-
-<property>
-  <name>dfs.support.append</name>
-  <value>true</value>
-  <description>Allow append support (if you want to test data durability with HDFS)
-  </description>
-</property>
--->
-
-
-</configuration>
diff --git a/pom.xml b/pom.xml
index 2d3d75a51c2..7686d7cdb08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -782,11 +782,11 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-test</artifactId>
     </dependency>
-     <dependency>
-       <groupId>com.google.guava</groupId>
-       <artifactId>guava</artifactId>
-       <version>${guava.version}</version>
-    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>${guava.version}</version>
+   </dependency>
   </dependencies>
 
   <!--
@@ -961,6 +961,7 @@
       <plugin>
         <groupId>org.apache.rat</groupId>
         <artifactId>apache-rat-plugin</artifactId>
+        <version>0.6</version>
       </plugin>
     </plugins>
   </reporting>
diff --git a/src/main/java/org/apache/hadoop/hbase/ValueOverMaxLengthException.java b/src/main/java/org/apache/hadoop/hbase/Abortable.java
similarity index 62%
rename from src/main/java/org/apache/hadoop/hbase/ValueOverMaxLengthException.java
rename to src/main/java/org/apache/hadoop/hbase/Abortable.java
index 383c9dbf610..b4fba8816b8 100644
--- a/src/main/java/org/apache/hadoop/hbase/ValueOverMaxLengthException.java
+++ b/src/main/java/org/apache/hadoop/hbase/Abortable.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2008 The Apache Software Foundation
+ * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -20,24 +20,18 @@
 package org.apache.hadoop.hbase;
 
 /**
- * Thrown when a value is longer than the specified LENGTH
+ * Interface to support the aborting of a given server or client.
+ * <p>
+ * This is used primarily for ZooKeeper usage when we could get an unexpected
+ * and fatal exception, requiring an abort.
+ * <p>
+ * Implemented by the Master, RegionServer, and TableServers (client).
  */
-public class ValueOverMaxLengthException extends DoNotRetryIOException {
-
-  private static final long serialVersionUID = -5525656352372008316L;
-
+public interface Abortable {
   /**
-   * default constructor
+   * Abort the server or client.
+   * @param why Why we're aborting.
+   * @param e Throwable that caused abort. Can be null.
    */
-  public ValueOverMaxLengthException() {
-    super();
-  }
-
-  /**
-   * @param message
-   */
-  public ValueOverMaxLengthException(String message) {
-    super(message);
-  }
-
-}
+  public void abort(String why, Throwable e);
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/Chore.java b/src/main/java/org/apache/hadoop/hbase/Chore.java
index 25bf1a0e398..df1514a6a84 100644
--- a/src/main/java/org/apache/hadoop/hbase/Chore.java
+++ b/src/main/java/org/apache/hadoop/hbase/Chore.java
@@ -19,8 +19,6 @@
  */
 package org.apache.hadoop.hbase;
 
-import java.util.concurrent.atomic.AtomicBoolean;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.util.Sleeper;
@@ -32,24 +30,24 @@ import org.apache.hadoop.hbase.util.Sleeper;
  * Implementers just need to add checking if there is work to be done and if
  * so, do it.  Its the base of most of the chore threads in hbase.
  *
- * Don't subclass Chore if the task relies on being woken up for something to
+ * <p>Don't subclass Chore if the task relies on being woken up for something to
  * do, such as an entry being added to a queue, etc.
  */
 public abstract class Chore extends Thread {
   private final Log LOG = LogFactory.getLog(this.getClass());
   private final Sleeper sleeper;
-  protected volatile AtomicBoolean stop;
+  protected final Stoppable stopper;
 
   /**
    * @param p Period at which we should run.  Will be adjusted appropriately
    * should we find work and it takes time to complete.
-   * @param s When this flag is set to true, this thread will cleanup and exit
-   * cleanly.
+   * @param stopper When {@link Stoppable#isStopped()} is true, this thread will
+   * cleanup and exit cleanly.
    */
-  public Chore(String name, final int p, final AtomicBoolean s) {
+  public Chore(String name, final int p, final Stoppable stopper) {
     super(name);
-    this.sleeper = new Sleeper(p, s);
-    this.stop = s;
+    this.sleeper = new Sleeper(p, stopper);
+    this.stopper = stopper;
   }
 
   /**
@@ -59,7 +57,7 @@ public abstract class Chore extends Thread {
   public void run() {
     try {
       boolean initialChoreComplete = false;
-      while (!this.stop.get()) {
+      while (!this.stopper.isStopped()) {
         long startTime = System.currentTimeMillis();
         try {
           if (!initialChoreComplete) {
@@ -69,15 +67,14 @@ public abstract class Chore extends Thread {
           }
         } catch (Exception e) {
           LOG.error("Caught exception", e);
-          if (this.stop.get()) {
+          if (this.stopper.isStopped()) {
             continue;
           }
         }
         this.sleeper.sleep(startTime);
       }
     } catch (Throwable t) {
-      LOG.fatal("Caught error. Starting shutdown.", t);
-      this.stop.set(true);
+      LOG.fatal(getName() + "error", t);
     } finally {
       LOG.info(getName() + " exiting");
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java b/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
index d1026433a18..789cad4b77f 100644
--- a/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
+++ b/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
@@ -27,9 +27,9 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
-import java.util.NavigableMap;
 import java.util.TreeMap;
 
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
 import org.apache.hadoop.io.VersionedWritable;
 
 /**
@@ -53,7 +53,7 @@ public class ClusterStatus extends VersionedWritable {
   private String hbaseVersion;
   private Collection<HServerInfo> liveServerInfo;
   private Collection<String> deadServers;
-  private NavigableMap<String, String> intransition;
+  private Map<String, RegionState> intransition;
 
   /**
    * Constructor, for Writable
@@ -186,11 +186,11 @@ public class ClusterStatus extends VersionedWritable {
     this.deadServers = deadServers;
   }
 
-  public Map<String, String> getRegionsInTransition() {
+  public Map<String, RegionState> getRegionsInTransition() {
     return this.intransition;
   }
 
-  public void setRegionsInTransition(final NavigableMap<String, String> m) {
+  public void setRegionsInTransition(final Map<String, RegionState> m) {
     this.intransition = m;
   }
 
@@ -210,9 +210,9 @@ public class ClusterStatus extends VersionedWritable {
       out.writeUTF(server);
     }
     out.writeInt(this.intransition.size());
-    for (Map.Entry<String, String> e: this.intransition.entrySet()) {
+    for (Map.Entry<String, RegionState> e: this.intransition.entrySet()) {
       out.writeUTF(e.getKey());
-      out.writeUTF(e.getValue());
+      e.getValue().write(out);
     }
   }
 
@@ -232,11 +232,12 @@ public class ClusterStatus extends VersionedWritable {
       deadServers.add(in.readUTF());
     }
     count = in.readInt();
-    this.intransition = new TreeMap<String, String>();
+    this.intransition = new TreeMap<String, RegionState>();
     for (int i = 0; i < count; i++) {
       String key = in.readUTF();
-      String value = in.readUTF();
-      this.intransition.put(key, value);
+      RegionState regionState = new RegionState();
+      regionState.readFields(in);
+      this.intransition.put(key, regionState);
     }
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java
index 5ddfd0b3876..9b1d021c853 100644
--- a/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java
+++ b/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java
@@ -38,4 +38,4 @@ public class DroppedSnapshotException extends IOException {
   public DroppedSnapshotException() {
     super();
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index bfaa4a1d179..a6692903fe1 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -132,6 +132,15 @@ public final class HConstants {
   /** Parameter name for how often threads should wake up */
   public static final String THREAD_WAKE_FREQUENCY = "hbase.server.thread.wakefrequency";
 
+  /** Default value for thread wake frequency */
+  public static final int DEFAULT_THREAD_WAKE_FREQUENCY = 10 * 1000;
+  
+  /** Number of retries for the client */
+  public static final String NUM_CLIENT_RETRIES = "hbase.client.retries.number";
+
+  /** Default number of retries for the client */
+  public static final int DEFAULT_NUM_CLIENT_RETRIES = 2;
+
   /** Parameter name for how often a region should should perform a major compaction */
   public static final String MAJOR_COMPACTION_PERIOD = "hbase.hregion.majorcompaction";
 
@@ -197,9 +206,6 @@ public final class HConstants {
   /** The catalog family */
   public static final byte [] CATALOG_FAMILY = Bytes.toBytes(CATALOG_FAMILY_STR);
 
-  /** The catalog historian family */
-  public static final byte [] CATALOG_HISTORIAN_FAMILY = Bytes.toBytes("historian");
-
   /** The regioninfo column qualifier */
   public static final byte [] REGIONINFO_QUALIFIER = Bytes.toBytes("regioninfo");
 
@@ -343,7 +349,8 @@ public final class HConstants {
    * HRegion server lease period in milliseconds. Clients must report in within this period
    * else they are considered dead. Unit measured in ms (milliseconds).
    */
-  public static String HBASE_REGIONSERVER_LEASE_PERIOD_KEY   = "hbase.regionserver.lease.period";
+  public static String HBASE_REGIONSERVER_LEASE_PERIOD_KEY =
+    "hbase.regionserver.lease.period";
 
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/HMsg.java b/src/main/java/org/apache/hadoop/hbase/HMsg.java
index 9c311bb3da0..c53460f3766 100644
--- a/src/main/java/org/apache/hadoop/hbase/HMsg.java
+++ b/src/main/java/org/apache/hadoop/hbase/HMsg.java
@@ -27,106 +27,37 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.Writable;
 
 /**
- * HMsg is for communicating instructions between the HMaster and the
- * HRegionServers.
+ * HMsg is used to send messages between master and regionservers.  Messages are
+ * sent as payload on the regionserver-to-master heartbeats.  Region assignment
+ * does not use this mechanism.  It goes via zookeeper.
  *
- * Most of the time the messages are simple but some messages are accompanied
- * by the region affected.  HMsg may also carry optional message.
+ * <p>Most of the time the messages are simple but some messages are accompanied
+ * by the region affected.  HMsg may also carry an optional message.
+ * 
+ * <p>TODO: Clean out all messages that go from master to regionserver; by
+ * design, these are to go via zk from here on out.
  */
 public class HMsg implements Writable {
-  public static final HMsg REGIONSERVER_QUIESCE =
-    new HMsg(Type.MSG_REGIONSERVER_QUIESCE);
-  public static final HMsg REGIONSERVER_STOP =
-    new HMsg(Type.MSG_REGIONSERVER_STOP);
+  public static final HMsg [] STOP_REGIONSERVER_ARRAY =
+    new HMsg [] {new HMsg(Type.STOP_REGIONSERVER)};
   public static final HMsg [] EMPTY_HMSG_ARRAY = new HMsg[0];
 
-  /**
-   * Message types sent between master and regionservers
-   */
   public static enum Type {
-    /** null message */
-    MSG_NONE,
-
-    // Message types sent from master to region server
-    /** Start serving the specified region */
-    MSG_REGION_OPEN,
-
-    /** Stop serving the specified region */
-    MSG_REGION_CLOSE,
-
-    /** Split the specified region */
-    MSG_REGION_SPLIT,
-
-    /** Compact the specified region */
-    MSG_REGION_COMPACT,
-
-    /** Master tells region server to stop */
-    MSG_REGIONSERVER_STOP,
-
-    /** Stop serving the specified region and don't report back that it's
-     * closed
+    /** Master tells region server to stop.
      */
-    MSG_REGION_CLOSE_WITHOUT_REPORT,
-
-    /** Stop serving user regions */
-    MSG_REGIONSERVER_QUIESCE,
-
-    // Message types sent from the region server to the master
-    /** region server is now serving the specified region */
-    MSG_REPORT_OPEN,
-
-    /** region server is no longer serving the specified region */
-    MSG_REPORT_CLOSE,
-
-    /** region server is processing open request */
-    MSG_REPORT_PROCESS_OPEN,
+    STOP_REGIONSERVER,
 
     /**
      * Region server split the region associated with this message.
-     *
-     * Note that this message is immediately followed by two MSG_REPORT_OPEN
-     * messages, one for each of the new regions resulting from the split
-     * @deprecated See MSG_REPORT_SPLIT_INCLUDES_DAUGHTERS
      */
-    MSG_REPORT_SPLIT,
-
-    /**
-     * Region server is shutting down
-     *
-     * Note that this message is followed by MSG_REPORT_CLOSE messages for each
-     * region the region server was serving, unless it was told to quiesce.
-     */
-    MSG_REPORT_EXITING,
-
-    /** Region server has closed all user regions but is still serving meta
-     * regions
-     */
-    MSG_REPORT_QUIESCED,
-
-    /**
-     * Flush
-     */
-    MSG_REGION_FLUSH,
-
-    /**
-     * Run Major Compaction
-     */
-    MSG_REGION_MAJOR_COMPACT,
-
-    /**
-     * Region server split the region associated with this message.
-     *
-     * Its like MSG_REPORT_SPLIT only it carries the daughters in the message
-     * rather than send them individually in MSG_REPORT_OPEN messages.
-     */
-    MSG_REPORT_SPLIT_INCLUDES_DAUGHTERS,
+    REGION_SPLIT,
 
     /**
      * When RegionServer receives this message, it goes into a sleep that only
      * an exit will cure.  This message is sent by unit tests simulating
      * pathological states.
      */
-    TESTING_MSG_BLOCK_RS,
+    TESTING_BLOCK_REGIONSERVER,
   }
 
   private Type type = null;
@@ -137,7 +68,7 @@ public class HMsg implements Writable {
 
   /** Default constructor. Used during deserialization */
   public HMsg() {
-    this(Type.MSG_NONE);
+    this(null);
   }
 
   /**
@@ -181,9 +112,6 @@ public class HMsg implements Writable {
    */
   public HMsg(final HMsg.Type type, final HRegionInfo hri,
       final HRegionInfo daughterA, final HRegionInfo daughterB, final byte[] msg) {
-    if (type == null) {
-      throw new NullPointerException("Message type cannot be null");
-    }
     this.type = type;
     if (hri == null) {
       throw new NullPointerException("Region cannot be null");
@@ -301,7 +229,7 @@ public class HMsg implements Writable {
        out.writeBoolean(true);
        Bytes.writeByteArray(out, this.message);
      }
-     if (this.type.equals(Type.MSG_REPORT_SPLIT_INCLUDES_DAUGHTERS)) {
+     if (this.type.equals(Type.REGION_SPLIT)) {
        this.daughterA.write(out);
        this.daughterB.write(out);
      }
@@ -318,7 +246,7 @@ public class HMsg implements Writable {
      if (hasMessage) {
        this.message = Bytes.readByteArray(in);
      }
-     if (this.type.equals(Type.MSG_REPORT_SPLIT_INCLUDES_DAUGHTERS)) {
+     if (this.type.equals(Type.REGION_SPLIT)) {
        this.daughterA = new HRegionInfo();
        this.daughterB = new HRegionInfo();
        this.daughterA.readFields(in);
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index ee946900a51..d95c03386ac 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -103,13 +103,28 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
       // old format region name. ROOT and first META region also 
       // use this format.EncodedName is the JenkinsHash value.
       int hashVal = Math.abs(JenkinsHash.getInstance().hash(regionName,
-                                                            regionName.length,
-                                                            0));
+        regionName.length, 0));
       encodedName = String.valueOf(hashVal);
     }
     return encodedName;
   }
 
+  /**
+   * Use logging.
+   * @param encodedRegionName The encoded regionname.
+   * @return <code>-ROOT-</code> if passed <code>70236052</code> or
+   * <code>.META.</code> if passed </code>1028785192</code> else returns
+   * <code>encodedRegionName</code>
+   */
+  public static String prettyPrint(final String encodedRegionName) {
+    if (encodedRegionName.equals("70236052")) {
+      return encodedRegionName + "/-ROOT-";
+    } else if (encodedRegionName.equals("1028785192")) {
+      return encodedRegionName + "/.META.";
+    }
+    return encodedRegionName;
+  }
+
   /** delimiter used between portions of a region name */
   public static final int DELIMITER = ',';
 
@@ -133,6 +148,7 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
   //TODO: Move NO_HASH to HStoreFile which is really the only place it is used.
   public static final String NO_HASH = null;
   private volatile String encodedName = NO_HASH;
+  private byte [] encodedNameAsBytes = null;
 
   private void setHashCode() {
     int result = Arrays.hashCode(this.regionName);
@@ -315,6 +331,24 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
     return b;
   }
 
+  /**
+   * Gets the table name from the specified region name.
+   * @param regionName
+   * @return
+   */
+  public static byte [] getTableName(byte [] regionName) {
+    int offset = -1;
+    for (int i = 0; i < regionName.length; i++) {
+      if (regionName[i] == DELIMITER) {
+        offset = i;
+        break;
+      }
+    }
+    byte [] tableName = new byte[offset];
+    System.arraycopy(regionName, 0, tableName, 0, offset);
+    return tableName;
+  }
+
   /**
    * Separate elements of a regionName.
    * @param regionName
@@ -393,6 +427,13 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
     return this.encodedName;
   }
 
+  public synchronized byte [] getEncodedNameAsBytes() {
+    if (this.encodedNameAsBytes == null) {
+      this.encodedNameAsBytes = Bytes.toBytes(getEncodedName());
+    }
+    return this.encodedNameAsBytes;
+  }
+
   /** @return the startKey */
   public byte [] getStartKey(){
     return startKey;
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java b/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
index 722991b2c4c..06a4e430d32 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
@@ -24,6 +24,7 @@ package org.apache.hadoop.hbase;
  * HRegionServer serving the region
  */
 public class HRegionLocation implements Comparable<HRegionLocation> {
+  // TODO: Is this class necessary?  Why not just have a Pair?
   private HRegionInfo regionInfo;
   private HServerAddress serverAddress;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/HServerAddress.java b/src/main/java/org/apache/hadoop/hbase/HServerAddress.java
index 763eca29d29..3859968717f 100644
--- a/src/main/java/org/apache/hadoop/hbase/HServerAddress.java
+++ b/src/main/java/org/apache/hadoop/hbase/HServerAddress.java
@@ -46,7 +46,7 @@ public class HServerAddress implements WritableComparable<HServerAddress> {
    */
   public HServerAddress(InetSocketAddress address) {
     this.address = address;
-    this.stringValue = address.getAddress().getHostAddress() + ":" +
+    this.stringValue = address.getAddress().getHostName() + ":" +
       address.getPort();
     checkBindAddressCanBeResolved();
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/HServerInfo.java b/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
index 3457f9e6de7..aaf4835158e 100644
--- a/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
@@ -23,6 +23,7 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.util.Comparator;
 import java.util.Set;
 
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -147,6 +148,8 @@ public class HServerInfo implements WritableComparable<HServerInfo> {
   }
 
   /**
+   * Gets the unique server instance name.  Includes the hostname, port, and
+   * start code.
    * @return Server name made of the concatenation of hostname, port and
    * startcode formatted as <code>&lt;hostname> ',' &lt;port> ',' &lt;startcode></code>
    */
@@ -242,6 +245,17 @@ public class HServerInfo implements WritableComparable<HServerInfo> {
     return this.getServerName().compareTo(o.getServerName());
   }
 
+  /**
+   * Orders HServerInfos by load then name.  Natural/ascending order.
+   */
+  public static class LoadComparator implements Comparator<HServerInfo> {
+    @Override
+    public int compare(HServerInfo left, HServerInfo right) {
+      int loadCompare = left.getLoad().compareTo(right.getLoad());
+      return loadCompare != 0 ? loadCompare : left.compareTo(right);
+    }
+  }
+
   /**
    * Utility method that does a find of a servername or a hostandport combination
    * in the passed Set.
diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 0d57270ba04..63107883d08 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -678,10 +678,5 @@ public class HTableDescriptor implements WritableComparable<HTableDescriptor> {
             10, // Ten is arbitrary number.  Keep versions to help debuggging.
             Compression.Algorithm.NONE.getName(), true, true, 8 * 1024,
             HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
-            HConstants.REPLICATION_SCOPE_LOCAL),
-          new HColumnDescriptor(HConstants.CATALOG_HISTORIAN_FAMILY,
-            HConstants.ALL_VERSIONS, Compression.Algorithm.NONE.getName(),
-            false, false,  8 * 1024,
-            HConstants.WEEK_IN_SECONDS,StoreFile.BloomType.NONE.toString(),
             HConstants.REPLICATION_SCOPE_LOCAL)});
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java b/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java
new file mode 100644
index 00000000000..bb2b6669d87
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java
@@ -0,0 +1,50 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+/**
+ * Thrown if a request is table schema modification is requested but
+ * made for an invalid family name.
+ */
+public class InvalidFamilyOperationException extends IOException {
+  private static final long serialVersionUID = 1L << 22 - 1L;
+  /** default constructor */
+  public InvalidFamilyOperationException() {
+    super();
+  }
+
+  /**
+   * Constructor
+   * @param s message
+   */
+  public InvalidFamilyOperationException(String s) {
+    super(s);
+  }
+
+  /**
+   * Constructor taking another exception.
+   * @param e Exception to grab data from.
+   */
+  public InvalidFamilyOperationException(Exception e) {
+    super(e);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/MasterAddressTracker.java b/src/main/java/org/apache/hadoop/hbase/MasterAddressTracker.java
new file mode 100644
index 00000000000..345cc1998cc
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/MasterAddressTracker.java
@@ -0,0 +1,91 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+
+/**
+ * Manages the location of the current active Master for this RegionServer.
+ * <p>
+ * Listens for ZooKeeper events related to the master address. The node
+ * <code>/master</code> will contain the address of the current master.
+ * This listener is interested in
+ * <code>NodeDeleted</code> and <code>NodeCreated</code> events on
+ * <code>/master</code>.
+ * <p>
+ * Utilizes {@link ZooKeeperNodeTracker} for zk interactions.
+ * <p>
+ * You can get the current master via {@link #getMasterAddress()} or the
+ * blocking method {@link #waitMasterAddress()}.
+ */
+public class MasterAddressTracker extends ZooKeeperNodeTracker {
+  /**
+   * Construct a master address listener with the specified
+   * <code>zookeeper</code> reference.
+   * <p>
+   * This constructor does not trigger any actions, you must call methods
+   * explicitly.  Normally you will just want to execute {@link #start()} to
+   * begin tracking of the master address.
+   *
+   * @param watcher zk reference and watcher
+   * @param abortable abortable in case of fatal error
+   */
+  public MasterAddressTracker(ZooKeeperWatcher watcher, Abortable abortable) {
+    super(watcher, watcher.masterAddressZNode, abortable);
+  }
+
+  /**
+   * Get the address of the current master if one is available.  Returns null
+   * if no current master.
+   *
+   * Use {@link #waitMasterAddress} if you want to block until the master is
+   * available.
+   * @return server address of current active master, or null if none available
+   */
+  public HServerAddress getMasterAddress() {
+    byte [] data = super.getData();
+    return data == null ? null : new HServerAddress(Bytes.toString(data));
+  }
+
+  /**
+   * Check if there is a master available.
+   * @return true if there is a master set, false if not.
+   */
+  public boolean hasMaster() {
+    return super.getData() != null;
+  }
+
+  /**
+   * Get the address of the current master.  If no master is available, method
+   * will block until one is available, the thread is interrupted, or timeout
+   * has passed.
+   *
+   * @param timeout maximum time to wait for master in millis, 0 for forever
+   * @return server address of current active master, null if timed out
+   * @throws InterruptedException if the thread is interrupted while waiting
+   */
+  public synchronized HServerAddress waitForMaster(long timeout)
+  throws InterruptedException {
+    byte [] data = super.blockUntilAvailable();
+    return data == null ? null : new HServerAddress(Bytes.toString(data));
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/NotAllMetaRegionsOnlineException.java b/src/main/java/org/apache/hadoop/hbase/NotAllMetaRegionsOnlineException.java
similarity index 96%
rename from src/main/java/org/apache/hadoop/hbase/master/NotAllMetaRegionsOnlineException.java
rename to src/main/java/org/apache/hadoop/hbase/NotAllMetaRegionsOnlineException.java
index 0e0ae274c8b..2c275e3586a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/NotAllMetaRegionsOnlineException.java
+++ b/src/main/java/org/apache/hadoop/hbase/NotAllMetaRegionsOnlineException.java
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hbase.master;
+package org.apache.hadoop.hbase;
 
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 
@@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.DoNotRetryIOException;
  */
 public class NotAllMetaRegionsOnlineException extends DoNotRetryIOException {
   private static final long serialVersionUID = 6439786157874827523L;
-
   /**
    * default constructor
    */
@@ -41,5 +40,4 @@ public class NotAllMetaRegionsOnlineException extends DoNotRetryIOException {
   public NotAllMetaRegionsOnlineException(String message) {
     super(message);
   }
-
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java b/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java
index 5f63de4e829..e3a9315fd49 100644
--- a/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java
+++ b/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java
@@ -26,9 +26,9 @@ import java.io.IOException;
  * and restarted so fast that the master still hasn't processed the server
  * shutdown of the first instance.
  */
+@SuppressWarnings("serial")
 public class PleaseHoldException extends IOException {
-
   public PleaseHoldException(String message) {
     super(message);
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/Server.java b/src/main/java/org/apache/hadoop/hbase/Server.java
new file mode 100644
index 00000000000..df396fad3f4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/Server.java
@@ -0,0 +1,54 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+
+/**
+ * Defines the set of shared functions implemented by HBase servers (Masters
+ * and RegionServers).
+ */
+public interface Server extends Abortable, Stoppable {
+  /**
+   * Gets the configuration object for this server.
+   */
+  public Configuration getConfiguration();
+
+  /**
+   * Gets the ZooKeeper instance for this server.
+   */
+  public ZooKeeperWatcher getZooKeeper();
+
+  /**
+   * @return Master's instance of {@link CatalogTracker}
+   */
+  public CatalogTracker getCatalogTracker();
+
+  /**
+   * Gets the unique server name for this server.
+   * If a RegionServer, it returns a concatenation of hostname, port and
+   * startcode formatted as <code>&lt;hostname> ',' &lt;port> ',' &lt;startcode></code>.
+   * If the master, it returns <code>&lt;hostname> ':' &lt;port>'.
+   * @return unique server name
+   */
+  public String getServerName();
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/WritableComparator.java b/src/main/java/org/apache/hadoop/hbase/Stoppable.java
similarity index 72%
rename from src/main/java/org/apache/hadoop/hbase/WritableComparator.java
rename to src/main/java/org/apache/hadoop/hbase/Stoppable.java
index 8faa1894e4c..74d4f4a689f 100644
--- a/src/main/java/org/apache/hadoop/hbase/WritableComparator.java
+++ b/src/main/java/org/apache/hadoop/hbase/Stoppable.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2008 The Apache Software Foundation
+ * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -19,11 +19,18 @@
  */
 package org.apache.hadoop.hbase;
 
-import java.util.Comparator;
-
-import org.apache.hadoop.io.Writable;
-
 /**
- * Interface that brings writable and comparable together
+ * Implementers are Stoppable.
  */
-public interface WritableComparator<T> extends Writable, Comparator<T> {}
+public interface Stoppable {
+  /**
+   * Stop this service.
+   * @param why Why we're stopping.
+   */
+  public void stop(String why);
+
+  /**
+   * @return True if {@link #stop(String)} has been closed.
+   */
+  public boolean isStopped();
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/ColumnNameParseException.java b/src/main/java/org/apache/hadoop/hbase/UnknownRegionException.java
similarity index 67%
rename from src/main/java/org/apache/hadoop/hbase/ColumnNameParseException.java
rename to src/main/java/org/apache/hadoop/hbase/UnknownRegionException.java
index 943c1e373c6..e87f42a6013 100644
--- a/src/main/java/org/apache/hadoop/hbase/ColumnNameParseException.java
+++ b/src/main/java/org/apache/hadoop/hbase/UnknownRegionException.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2008 The Apache Software Foundation
+ * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -19,22 +19,15 @@
  */
 package org.apache.hadoop.hbase;
 
+import java.io.IOException;
+
 /**
- * Thrown if issue with passed column name.
+ * Thrown when we are asked to operate on a region we know nothing about.
  */
-public class ColumnNameParseException extends DoNotRetryIOException {
+public class UnknownRegionException extends IOException {
+  private static final long serialVersionUID = 1968858760475205392L;
 
-  private static final long serialVersionUID = -2897373353949942302L;
-
-  /** default constructor */
-  public ColumnNameParseException() {
-    super();
-  }
-
-  /**
-   * @param message
-   */
-  public ColumnNameParseException(String message) {
-    super(message);
+  public UnknownRegionException(String regionName) {
+    super(regionName);
   }
 }
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java b/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java
index 8cb39858f44..8ca50a9d807 100644
--- a/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java
+++ b/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java
@@ -38,4 +38,4 @@ public class UnknownRowLockException extends DoNotRetryIOException {
   public UnknownRowLockException(String s) {
     super(s);
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java b/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java
index 4089a084526..fcd2ccdd2ed 100644
--- a/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java
+++ b/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java
@@ -26,9 +26,9 @@ import java.io.IOException;
  * already being processed as dead. This can happen when a region server loses
  * its session but didn't figure it yet.
  */
+@SuppressWarnings("serial")
 public class YouAreDeadException extends IOException {
-
   public YouAreDeadException(String message) {
     super(message);
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/InvalidColumnNameException.java b/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java
similarity index 66%
rename from src/main/java/org/apache/hadoop/hbase/master/InvalidColumnNameException.java
rename to src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java
index 797fa3a87e5..b4ce03c7b26 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/InvalidColumnNameException.java
+++ b/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java
@@ -17,18 +17,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.DoNotRetryIOException;
+package org.apache.hadoop.hbase;
 
+import java.io.IOException;
 
 /**
- * Thrown when an invalid column name is encountered
+ * Thrown if the client can't connect to zookeeper
  */
-public class InvalidColumnNameException extends DoNotRetryIOException {
-  private static final long serialVersionUID = 1L << 29 - 1L;
+public class ZooKeeperConnectionException extends IOException {
+  private static final long serialVersionUID = 1L << 23 - 1L;
   /** default constructor */
-  public InvalidColumnNameException() {
+  public ZooKeeperConnectionException() {
     super();
   }
 
@@ -36,7 +35,15 @@ public class InvalidColumnNameException extends DoNotRetryIOException {
    * Constructor
    * @param s message
    */
-  public InvalidColumnNameException(String s) {
+  public ZooKeeperConnectionException(String s) {
     super(s);
   }
-}
\ No newline at end of file
+
+  /**
+   * Constructor taking another exception.
+   * @param e Exception to grab data from.
+   */
+  public ZooKeeperConnectionException(Exception e) {
+    super(e);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java b/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java
index d8cd00af464..197009adb05 100644
--- a/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java
@@ -20,10 +20,7 @@ package org.apache.hadoop.hbase.avro;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.Collection;
 import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
 
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericArray;
@@ -33,33 +30,17 @@ import org.apache.avro.specific.SpecificResponder;
 import org.apache.avro.util.Utf8;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.HTablePool;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-
 import org.apache.hadoop.hbase.avro.generated.AClusterStatus;
-import org.apache.hadoop.hbase.avro.generated.AColumnValue;
-import org.apache.hadoop.hbase.avro.generated.ACompressionAlgorithm;
 import org.apache.hadoop.hbase.avro.generated.ADelete;
 import org.apache.hadoop.hbase.avro.generated.AFamilyDescriptor;
 import org.apache.hadoop.hbase.avro.generated.AGet;
-import org.apache.hadoop.hbase.avro.generated.AIllegalArgument;
 import org.apache.hadoop.hbase.avro.generated.AIOError;
+import org.apache.hadoop.hbase.avro.generated.AIllegalArgument;
 import org.apache.hadoop.hbase.avro.generated.AMasterNotRunning;
 import org.apache.hadoop.hbase.avro.generated.APut;
 import org.apache.hadoop.hbase.avro.generated.AResult;
@@ -67,6 +48,12 @@ import org.apache.hadoop.hbase.avro.generated.AScan;
 import org.apache.hadoop.hbase.avro.generated.ATableDescriptor;
 import org.apache.hadoop.hbase.avro.generated.ATableExists;
 import org.apache.hadoop.hbase.avro.generated.HBase;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.HTablePool;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
 
 /**
  * Start an Avro server
@@ -135,10 +122,9 @@ public class AvroServer {
     // TODO(hammer): figure out appropriate setting of maxSize for htablePool
     /**
      * Constructs an HBaseImpl object.
-     * 
-     * @throws MasterNotRunningException
+     * @throws IOException 
      */
-    HBaseImpl() throws MasterNotRunningException {
+    HBaseImpl() throws IOException {
       conf = HBaseConfiguration.create();
       admin = new HBaseAdmin(conf);
       htablePool = new HTablePool(conf, 10);
@@ -376,8 +362,7 @@ public class AvroServer {
     // NB: Asynchronous operation
     public Void modifyFamily(ByteBuffer table, ByteBuffer familyName, AFamilyDescriptor familyDescriptor) throws AIOError {
       try {
-	admin.modifyColumn(Bytes.toBytes(table), Bytes.toBytes(familyName),
-                           AvroUtil.afdToHCD(familyDescriptor));
+	admin.modifyColumn(Bytes.toBytes(table), AvroUtil.afdToHCD(familyDescriptor));
 	return null;
       } catch (IOException e) {
 	AIOError ioe = new AIOError();
@@ -506,7 +491,6 @@ public class AvroServer {
 	  aie.message = new Utf8("scanner ID is invalid: " + scannerId);
           throw aie;
         }
-        Result[] results = null;
         return AvroUtil.resultsToAResults(scanner.next(numberOfRows));
       } catch (IOException e) {
     	AIOError ioe = new AIOError();
@@ -566,7 +550,7 @@ public class AvroServer {
     Log LOG = LogFactory.getLog("AvroServer");
     LOG.info("starting HBase Avro server on port " + Integer.toString(port));
     SpecificResponder r = new SpecificResponder(HBase.class, new HBaseImpl());
-    HttpServer server = new HttpServer(r, 9090);
+    new HttpServer(r, 9090);
     Thread.sleep(1000000);
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
new file mode 100644
index 00000000000..81a4c9b43a7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
@@ -0,0 +1,395 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.catalog;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.client.RetriesExhaustedException;
+import org.apache.hadoop.hbase.client.ServerConnection;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.MetaNodeTracker;
+import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Tracks the availability of the catalog tables <code>-ROOT-</code> and
+ * <code>.META.</code>.
+ * <p>
+ * This class is "read-only" in that the locations of the catalog tables cannot
+ * be explicitly set.  Instead, ZooKeeper is used to learn of the availability
+ * and location of ROOT.  ROOT is used to learn of the location of META.  If not
+ * available in ROOT, ZooKeeper is used to monitor for a new location of META.
+ */
+public class CatalogTracker {
+  private static final Log LOG = LogFactory.getLog(CatalogTracker.class);
+
+  private final ServerConnection connection;
+
+  private final ZooKeeperWatcher zookeeper;
+
+  private final RootRegionTracker rootRegionTracker;
+
+  private final MetaNodeTracker metaNodeTracker;
+
+  private final AtomicBoolean metaAvailable = new AtomicBoolean(false);
+  private HServerAddress metaLocation;
+
+  private final int defaultTimeout;
+
+  public static final byte [] ROOT_REGION =
+    HRegionInfo.ROOT_REGIONINFO.getRegionName();
+
+  public static final byte [] META_REGION =
+    HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
+
+  /**
+   * Constructs the catalog tracker.  Find current state of catalog tables and
+   * begin active tracking by executing {@link #start()}.
+   * @param zk
+   * @param connection server connection
+   * @param abortable if fatal exception
+   * @throws IOException 
+   */
+  public CatalogTracker(final ZooKeeperWatcher zk,
+      final ServerConnection connection, final Abortable abortable,
+      final int defaultTimeout)
+  throws IOException {
+    this.zookeeper = zk;
+    this.connection = connection;
+    this.rootRegionTracker = new RootRegionTracker(zookeeper, abortable);
+    this.metaNodeTracker = new MetaNodeTracker(zookeeper, this);
+    this.defaultTimeout = defaultTimeout;
+  }
+
+  /**
+   * Starts the catalog tracker.
+   * <p>
+   * Determines current availability of catalog tables and ensures all further
+   * transitions of either region is tracked.
+   * @throws IOException
+   * @throws InterruptedException 
+   */
+  public void start() throws IOException, InterruptedException {
+    // Register listeners with zk
+    zookeeper.registerListener(rootRegionTracker);
+    zookeeper.registerListener(metaNodeTracker);
+    // Start root tracking
+    rootRegionTracker.start();
+    // Determine meta assignment; may not work because root and meta not yet
+    // deployed.
+    getMetaServerConnection(true);
+  }
+
+  /**
+   * Gets the current location for <code>-ROOT-</code> or null if location is
+   * not currently available.
+   * @return location of root, null if not available
+   * @throws InterruptedException 
+   */
+  public HServerAddress getRootLocation() throws InterruptedException {
+    return this.rootRegionTracker.getRootRegionLocation();
+  }
+
+  /**
+   * @return Location of meta or null if not yet available.
+   */
+  public HServerAddress getMetaLocation() {
+    return this.metaLocation;
+  }
+
+  /**
+   * Waits indefinitely for availability of <code>-ROOT-</code>.  Used during
+   * cluster startup.
+   * @throws InterruptedException if interrupted while waiting
+   */
+  public void waitForRoot()
+  throws InterruptedException {
+    rootRegionTracker.getRootRegionLocation();
+  }
+
+  /**
+   * Gets the current location for <code>-ROOT-</code> if available and waits
+   * for up to the specified timeout if not immediately available.  Returns null
+   * if the timeout elapses before root is available.
+   * @param timeout maximum time to wait for root availability, in milliseconds
+   * @return location of root
+   * @throws InterruptedException if interrupted while waiting
+   * @throws NotAllMetaRegionsOnlineException if root not available before
+   *                                          timeout
+   */
+  public HServerAddress waitForRoot(final long timeout)
+  throws InterruptedException, NotAllMetaRegionsOnlineException {
+    HServerAddress address = rootRegionTracker.waitRootRegionLocation(timeout);
+    if (address == null) {
+      throw new NotAllMetaRegionsOnlineException("Timed out; " + timeout + "ms");
+    }
+    return address;
+  }
+
+  /**
+   * Gets a connection to the server hosting root, as reported by ZooKeeper,
+   * waiting up to the specified timeout for availability.
+   * @see #waitForRoot(long) for additional information
+   * @return connection to server hosting root
+   * @throws InterruptedException
+   * @throws NotAllMetaRegionsOnlineException if timed out waiting
+   * @throws IOException
+   */
+  public HRegionInterface waitForRootServerConnection(long timeout)
+  throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
+    return getCachedConnection(waitForRoot(timeout));
+  }
+
+  /**
+   * Gets a connection to the server hosting root, as reported by ZooKeeper,
+   * waiting for the default timeout specified on instantiation.
+   * @see #waitForRoot(long) for additional information
+   * @return connection to server hosting root
+   * @throws NotAllMetaRegionsOnlineException if timed out waiting
+   * @throws IOException
+   */
+  public HRegionInterface waitForRootServerConnectionDefault()
+  throws NotAllMetaRegionsOnlineException, IOException {
+    try {
+      return getCachedConnection(waitForRoot(defaultTimeout));
+    } catch (InterruptedException e) {
+      throw new NotAllMetaRegionsOnlineException("Interrupted");
+    }
+  }
+
+  /**
+   * Gets a connection to the server hosting root, as reported by ZooKeeper,
+   * if available.  Returns null if no location is immediately available.
+   * @return connection to server hosting root, null if not available
+   * @throws IOException
+   * @throws InterruptedException 
+   */
+  private HRegionInterface getRootServerConnection()
+  throws IOException, InterruptedException {
+    HServerAddress address = rootRegionTracker.getRootRegionLocation();
+    if (address == null) {
+      return null;
+    }
+    return getCachedConnection(address);
+  }
+
+  /**
+   * Gets a connection to the server currently hosting <code>.META.</code> or
+   * null if location is not currently available.
+   * <p>
+   * If a location is known, a connection to the cached location is returned.
+   * If refresh is true, the cached connection is verified first before
+   * returning.  If the connection is not valid, it is reset and rechecked.
+   * <p>
+   * If no location for meta is currently known, method checks ROOT for a new
+   * location, verifies META is currently there, and returns a cached connection
+   * to the server hosting META.
+   *
+   * @return connection to server hosting meta, null if location not available
+   * @throws IOException
+   * @throws InterruptedException 
+   */
+  private HRegionInterface getMetaServerConnection(boolean refresh)
+  throws IOException, InterruptedException {
+    synchronized(metaAvailable) {
+      if(metaAvailable.get()) {
+        HRegionInterface current = getCachedConnection(metaLocation);
+        if(!refresh) {
+          return current;
+        }
+        if(verifyRegionLocation(current, META_REGION)) {
+          return current;
+        }
+        resetMetaLocation();
+      }
+      HRegionInterface rootConnection = getRootServerConnection();
+      if(rootConnection == null) {
+        return null;
+      }
+      HServerAddress newLocation = MetaReader.readMetaLocation(rootConnection);
+      if(newLocation == null) {
+        return null;
+      }
+      HRegionInterface newConnection = getCachedConnection(newLocation);
+      if(verifyRegionLocation(newConnection, META_REGION)) {
+        setMetaLocation(newLocation);
+        return newConnection;
+      }
+      return null;
+    }
+  }
+
+  /**
+   * Waits indefinitely for availability of <code>.META.</code>.  Used during
+   * cluster startup.
+   * @throws InterruptedException if interrupted while waiting
+   */
+  public void waitForMeta() throws InterruptedException {
+    synchronized(metaAvailable) {
+      while(!metaAvailable.get()) {
+        metaAvailable.wait();
+      }
+    }
+  }
+
+  /**
+   * Gets the current location for <code>.META.</code> if available and waits
+   * for up to the specified timeout if not immediately available.  Throws an
+   * exception if timed out waiting.
+   * @param timeout maximum time to wait for meta availability, in milliseconds
+   * @return location of meta
+   * @throws InterruptedException if interrupted while waiting
+   * @throws IOException unexpected exception connecting to meta server
+   * @throws NotAllMetaRegionsOnlineException if meta not available before
+   *                                          timeout
+   */
+  public HServerAddress waitForMeta(long timeout)
+  throws InterruptedException, IOException, NotAllMetaRegionsOnlineException {
+    long stop = System.currentTimeMillis() + timeout;
+    synchronized(metaAvailable) {
+      if(getMetaServerConnection(true) != null) {
+        return metaLocation;
+      }
+      while(!metaAvailable.get() &&
+          (timeout == 0 || System.currentTimeMillis() < stop)) {
+        metaAvailable.wait(timeout);
+      }
+      if(getMetaServerConnection(true) == null) {
+        throw new NotAllMetaRegionsOnlineException(
+            "Timed out (" + timeout + "ms");
+      }
+      return metaLocation;
+    }
+  }
+
+  /**
+   * Gets a connection to the server hosting meta, as reported by ZooKeeper,
+   * waiting up to the specified timeout for availability.
+   * @see #waitForMeta(long) for additional information
+   * @return connection to server hosting meta
+   * @throws InterruptedException
+   * @throws NotAllMetaRegionsOnlineException if timed out waiting
+   * @throws IOException
+   */
+  public HRegionInterface waitForMetaServerConnection(long timeout)
+  throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
+    return getCachedConnection(waitForMeta(timeout));
+  }
+
+  /**
+   * Gets a connection to the server hosting meta, as reported by ZooKeeper,
+   * waiting up to the specified timeout for availability.
+   * @see #waitForMeta(long) for additional information
+   * @return connection to server hosting meta
+   * @throws NotAllMetaRegionsOnlineException if timed out or interrupted
+   * @throws IOException
+   */
+  public HRegionInterface waitForMetaServerConnectionDefault()
+  throws NotAllMetaRegionsOnlineException, IOException {
+    try {
+      return getCachedConnection(waitForMeta(defaultTimeout));
+    } catch (InterruptedException e) {
+      throw new NotAllMetaRegionsOnlineException("Interrupted");
+    }
+  }
+
+  private void resetMetaLocation() {
+    LOG.info("Current cached META location is not valid, resetting");
+    metaAvailable.set(false);
+    metaLocation = null;
+  }
+
+  private void setMetaLocation(HServerAddress metaLocation) {
+    LOG.info("Found new META location, " + metaLocation);
+    metaAvailable.set(true);
+    this.metaLocation = metaLocation;
+    // no synchronization because these are private and already under lock
+    metaAvailable.notifyAll();
+  }
+
+  private HRegionInterface getCachedConnection(HServerAddress address)
+  throws IOException {
+    HRegionInterface protocol = null;
+    try {
+      protocol = connection.getHRegionConnection(address, false);
+    } catch (RetriesExhaustedException e) {
+      if (e.getCause() != null && e.getCause() instanceof ConnectException) {
+        // Catch this; presume it means the cached connection has gone bad.
+      } else {
+        throw e;
+      }
+    }
+    return protocol;
+  }
+
+  private boolean verifyRegionLocation(HRegionInterface metaServer,
+      byte [] regionName) {
+    try {
+      return metaServer.getRegionInfo(regionName) != null;
+    } catch (NotServingRegionException e) {
+      return false;
+    }
+  }
+
+  /**
+   * Check if <code>hsi</code> was carrying <code>-ROOT-</code> or
+   * <code>.META.</code> and if so, clear out old locations.
+   * @param hsi Server that has crashed/shutdown.
+   * @throws InterruptedException
+   * @throws KeeperException
+   * @return Pair of booleans; if this server was carrying root, then first
+   * boolean is set, if server was carrying meta, then second boolean set.
+   */
+  public Pair<Boolean, Boolean> processServerShutdown(final HServerInfo hsi)
+  throws InterruptedException, KeeperException {
+    Pair<Boolean, Boolean> result = new Pair<Boolean, Boolean>(false, false);
+    HServerAddress rootHsa = getRootLocation();
+    if (rootHsa == null) {
+      LOG.info("-ROOT- is not assigned; continuing");
+    } else if (hsi.getServerAddress().equals(rootHsa)) {
+      result.setFirst(true);
+      LOG.info(hsi.getServerName() + " carrying -ROOT-; deleting " +
+        "-ROOT- location from meta");
+      RootLocationEditor.deleteRootLocation(this.zookeeper);
+    }
+    HServerAddress metaHsa = getMetaLocation();
+    if (metaHsa == null) {
+      LOG.info(".META. is not assigned; continuing");
+    } else if (hsi.getServerAddress().equals(metaHsa)) {
+      LOG.info(hsi.getServerName() + " carrying .META.; unsetting " +
+        ".META. location");
+      result.setSecond(true);
+      resetMetaLocation();
+    }
+    return result;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
new file mode 100644
index 00000000000..263db2d5cde
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
@@ -0,0 +1,215 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.catalog;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+
+/**
+ * Writes region and assignment information to <code>.META.</code>.
+ * <p>
+ * Uses the {@link CatalogTracker} to obtain locations and connections to
+ * catalogs.
+ */
+public class MetaEditor {
+  private static final Log LOG = LogFactory.getLog(MetaEditor.class);
+
+  /**
+   * Adds a META row for the specified new region.
+   * @param info region information
+   * @throws IOException if problem connecting or updating meta
+   */
+  public static void addRegionToMeta(CatalogTracker catalogTracker,
+      HRegionInfo regionInfo)
+  throws IOException {
+    Put put = new Put(regionInfo.getRegionName());
+    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+        Writables.getBytes(regionInfo));
+    catalogTracker.waitForMetaServerConnectionDefault().put(
+        CatalogTracker.META_REGION, put);
+    LOG.info("Added region " + regionInfo.getRegionNameAsString() + " to META");
+  }
+
+  /**
+   * Offline parent in meta.
+   * Used when splitting.
+   * @param catalogTracker
+   * @param parent
+   * @param a Split daughter region A
+   * @param b Split daughter region B
+   * @throws NotAllMetaRegionsOnlineException
+   * @throws IOException
+   */
+  public static void offlineParentInMeta(CatalogTracker catalogTracker,
+      HRegionInfo parent, final HRegionInfo a, final HRegionInfo b)
+  throws NotAllMetaRegionsOnlineException, IOException {
+    HRegionInfo copyOfParent = new HRegionInfo(parent);
+    copyOfParent.setOffline(true);
+    copyOfParent.setSplit(true);
+    Put put = new Put(copyOfParent.getRegionName());
+    addRegionInfo(put, copyOfParent);
+    put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
+        HConstants.EMPTY_BYTE_ARRAY);
+    put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
+        HConstants.EMPTY_BYTE_ARRAY);
+    put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
+      Writables.getBytes(a));
+    put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,
+      Writables.getBytes(b));
+    catalogTracker.waitForMetaServerConnectionDefault().put(CatalogTracker.META_REGION, put);
+    LOG.info("Offlined parent region " + parent.getRegionNameAsString() +
+      " in META");
+  }
+
+  public static void addDaughter(final CatalogTracker catalogTracker,
+      final HRegionInfo regionInfo, final HServerInfo serverInfo)
+  throws NotAllMetaRegionsOnlineException, IOException {
+    HRegionInterface server = catalogTracker.waitForMetaServerConnectionDefault();
+    byte [] catalogRegionName = CatalogTracker.META_REGION;
+    Put put = new Put(regionInfo.getRegionName());
+    addRegionInfo(put, regionInfo);
+    if (serverInfo != null) addLocation(put, serverInfo);
+    server.put(catalogRegionName, put);
+    LOG.info("Added daughter " + regionInfo.getRegionNameAsString() +
+      " in region " + Bytes.toString(catalogRegionName) + " with " +
+      "server=" + serverInfo.getHostnamePort() + ", " +
+      "startcode=" + serverInfo.getStartCode());
+  }
+
+  /**
+   * Updates the location of the specified META region in ROOT to be the
+   * specified server hostname and startcode.
+   * <p>
+   * Uses passed catalog tracker to get a connection to the server hosting
+   * ROOT and makes edits to that region.
+   *
+   * @param catalogTracker catalog tracker
+   * @param regionInfo region to update location of
+   * @param serverInfo server the region is located on
+   * @throws IOException
+   */
+  public static void updateMetaLocation(CatalogTracker catalogTracker,
+      HRegionInfo regionInfo, HServerInfo serverInfo)
+  throws IOException {
+    HRegionInterface server = catalogTracker.waitForRootServerConnectionDefault();
+    updateLocation(server, CatalogTracker.ROOT_REGION, regionInfo, serverInfo);
+  }
+
+  /**
+   * Updates the location of the specified region in META to be the specified
+   * server hostname and startcode.
+   * <p>
+   * Uses passed catalog tracker to get a connection to the server hosting
+   * META and makes edits to that region.
+   *
+   * @param catalogTracker catalog tracker
+   * @param regionInfo region to update location of
+   * @param serverInfo server the region is located on
+   * @throws IOException
+   */
+  public static void updateRegionLocation(CatalogTracker catalogTracker,
+      HRegionInfo regionInfo, HServerInfo serverInfo)
+  throws IOException {
+    updateLocation(catalogTracker.waitForMetaServerConnectionDefault(),
+        CatalogTracker.META_REGION, regionInfo, serverInfo);
+  }
+
+  /**
+   * Updates the location of the specified region to be the specified server.
+   * <p>
+   * Connects to the specified server which should be hosting the specified
+   * catalog region name to perform the edit.
+   *
+   * @param server connection to server hosting catalog region
+   * @param catalogRegionName name of catalog region being updated
+   * @param regionInfo region to update location of
+   * @param serverInfo server the region is located on
+   * @throws IOException
+   */
+  public static void updateLocation(HRegionInterface server,
+      byte [] catalogRegionName, HRegionInfo regionInfo, HServerInfo serverInfo)
+  throws IOException {
+    Put put = new Put(regionInfo.getRegionName());
+    addLocation(put, serverInfo);
+    server.put(catalogRegionName, put);
+    LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
+      " in region " + Bytes.toString(catalogRegionName) + " with " +
+      "server=" + serverInfo.getHostnamePort() + ", " +
+      "startcode=" + serverInfo.getStartCode());
+  }
+
+  /**
+   * Deletes the specified region from META.
+   * @param catalogTracker
+   * @param regionInfo region to be deleted from META
+   * @throws IOException
+   */
+  public static void deleteRegion(CatalogTracker catalogTracker,
+      HRegionInfo regionInfo)
+  throws IOException {
+    Delete delete = new Delete(regionInfo.getRegionName());
+    catalogTracker.waitForMetaServerConnectionDefault().delete(
+        CatalogTracker.META_REGION, delete);
+
+    LOG.info("Deleted region " + regionInfo.getRegionNameAsString() + " from META");
+  }
+
+  /**
+   * Updates the region information for the specified region in META.
+   * @param catalogTracker
+   * @param regionInfo region to be updated in META
+   * @throws IOException
+   */
+  public static void updateRegionInfo(CatalogTracker catalogTracker,
+      HRegionInfo regionInfo)
+  throws IOException {
+    Put put = new Put(regionInfo.getRegionName());
+    addRegionInfo(put, regionInfo);
+    catalogTracker.waitForMetaServerConnectionDefault().put(
+        CatalogTracker.META_REGION, put);
+    LOG.info("Updated region " + regionInfo.getRegionNameAsString() + " in META");
+  }
+
+  private static Put addRegionInfo(final Put p, final HRegionInfo hri)
+  throws IOException {
+    p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+        Writables.getBytes(hri));
+    return p;
+  }
+
+  private static Put addLocation(final Put p, final HServerInfo hsi) {
+    p.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
+      Bytes.toBytes(hsi.getHostnamePort()));
+    p.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
+      Bytes.toBytes(hsi.getStartCode()));
+    return p;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
new file mode 100644
index 00000000000..9501f51bbc4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
@@ -0,0 +1,309 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.catalog;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Writables;
+
+/**
+ * Reads region and assignment information from <code>.META.</code>.
+ * <p>
+ * Uses the {@link CatalogTracker} to obtain locations and connections to
+ * catalogs.
+ */
+public class MetaReader {
+  /**
+   * Performs a full scan of <code>.META.</code>.
+   * <p>
+   * Returns a map of every region to it's currently assigned server, according
+   * to META.  If the region does not have an assignment it will have a null
+   * value in the map.
+   *
+   * @return map of regions to their currently assigned server
+   * @throws IOException
+   */
+  public static Map<HRegionInfo,HServerAddress> fullScan(
+      CatalogTracker catalogTracker)
+  throws IOException {
+    HRegionInterface metaServer =
+      catalogTracker.waitForMetaServerConnectionDefault();
+    Map<HRegionInfo,HServerAddress> allRegions =
+      new TreeMap<HRegionInfo,HServerAddress>();
+    Scan scan = new Scan();
+    scan.addFamily(HConstants.CATALOG_FAMILY);
+    long scannerid = metaServer.openScanner(
+        HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
+    try {
+      Result data;
+      while((data = metaServer.next(scannerid)) != null) {
+        if (!data.isEmpty()) {
+          Pair<HRegionInfo,HServerAddress> region =
+            metaRowToRegionPair(data);
+          allRegions.put(region.getFirst(), region.getSecond());
+        }
+      }
+    } finally {
+      metaServer.close(scannerid);
+    }
+    return allRegions;
+  }
+
+  /**
+   * Reads the location of META from ROOT.
+   * @param metaServer connection to server hosting ROOT
+   * @return location of META in ROOT, null if not available
+   * @throws IOException
+   */
+  public static HServerAddress readMetaLocation(HRegionInterface metaServer)
+  throws IOException {
+    return readLocation(metaServer, CatalogTracker.ROOT_REGION,
+        CatalogTracker.META_REGION);
+  }
+
+  /**
+   * Reads the location of the specified region from META.
+   * @param catalogTracker
+   * @param regionName region to read location of
+   * @return location of region in META, null if not available
+   * @throws IOException
+   */
+  public static HServerAddress readRegionLocation(CatalogTracker catalogTracker,
+      byte [] regionName)
+  throws IOException {
+    return readLocation(catalogTracker.waitForMetaServerConnectionDefault(),
+        CatalogTracker.META_REGION, regionName);
+  }
+
+  private static HServerAddress readLocation(HRegionInterface metaServer,
+      byte [] catalogRegionName, byte [] regionName)
+  throws IOException {
+    Result r = null;
+    try {
+      r = metaServer.get(catalogRegionName,
+        new Get(regionName).addColumn(HConstants.CATALOG_FAMILY,
+        HConstants.SERVER_QUALIFIER));
+    } catch (java.net.ConnectException e) {
+      if (e.getMessage() != null &&
+          e.getMessage().contains("Connection refused")) {
+        // Treat this exception + message as unavailable catalog table. Catch it
+        // and fall through to return a null
+      } else {
+        throw e;
+      }
+    } catch (IOException e) {
+      if (e.getCause() != null && e.getCause() instanceof IOException &&
+          e.getCause().getMessage() != null &&
+          e.getCause().getMessage().contains("Connection reset by peer")) {
+        // Treat this exception + message as unavailable catalog table. Catch it
+        // and fall through to return a null
+      } else {
+        throw e;
+      }
+    }
+    if (r == null || r.isEmpty()) {
+      return null;
+    }
+    byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
+      HConstants.SERVER_QUALIFIER);
+    return new HServerAddress(Bytes.toString(value));
+  }
+
+  /**
+   * Gets the region info and assignment for the specified region from META.
+   * @param catalogTracker
+   * @param regionName
+   * @return region info and assignment from META, null if not available
+   * @throws IOException
+   */
+  public static Pair<HRegionInfo, HServerAddress> getRegion(
+      CatalogTracker catalogTracker, byte [] regionName)
+  throws IOException {
+    Get get = new Get(regionName);
+    get.addFamily(HConstants.CATALOG_FAMILY);
+    Result r = catalogTracker.waitForMetaServerConnectionDefault().get(
+        CatalogTracker.META_REGION, get);
+    if(r == null || r.isEmpty()) {
+      return null;
+    }
+    return metaRowToRegionPair(r);
+  }
+
+  public static Pair<HRegionInfo, HServerAddress> metaRowToRegionPair(
+      Result data) throws IOException {
+    HRegionInfo info = Writables.getHRegionInfo(
+      data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
+    final byte[] value = data.getValue(HConstants.CATALOG_FAMILY,
+      HConstants.SERVER_QUALIFIER);
+    if (value != null && value.length > 0) {
+      HServerAddress server = new HServerAddress(Bytes.toString(value));
+      return new Pair<HRegionInfo,HServerAddress>(info, server);
+    } else {
+      return new Pair<HRegionInfo, HServerAddress>(info, null);
+    }
+  }
+
+  /**
+   * Checks if the specified table exists.  Looks at the META table hosted on
+   * the specified server.
+   * @param metaServer server hosting meta
+   * @param tableName table to check
+   * @return true if the table exists in meta, false if not
+   * @throws IOException
+   */
+  public static boolean tableExists(CatalogTracker catalogTracker,
+      String tableName)
+  throws IOException {
+    HRegionInterface metaServer =
+      catalogTracker.waitForMetaServerConnectionDefault();
+    byte[] firstRowInTable = Bytes.toBytes(tableName + ",,");
+    Scan scan = new Scan(firstRowInTable);
+    scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+    long scannerid = metaServer.openScanner(
+        HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
+    try {
+      Result data = metaServer.next(scannerid);
+      if (data != null && data.size() > 0) {
+        HRegionInfo info = Writables.getHRegionInfo(
+          data.getValue(HConstants.CATALOG_FAMILY,
+              HConstants.REGIONINFO_QUALIFIER));
+        if (info.getTableDesc().getNameAsString().equals(tableName)) {
+          // A region for this table already exists. Ergo table exists.
+          return true;
+        }
+      }
+      return false;
+    } finally {
+      metaServer.close(scannerid);
+    }
+  }
+
+  /**
+   * Gets all of the regions of the specified table from META.
+   * @param catalogTracker
+   * @param tableName
+   * @return
+   * @throws IOException
+   */
+  public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
+      byte [] tableName)
+  throws IOException {
+    HRegionInterface metaServer =
+      catalogTracker.waitForMetaServerConnectionDefault();
+    List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
+    String tableString = Bytes.toString(tableName);
+    byte[] firstRowInTable = Bytes.toBytes(tableString + ",,");
+    Scan scan = new Scan(firstRowInTable);
+    scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+    long scannerid = metaServer.openScanner(
+        HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
+    try {
+      Result data;
+      while((data = metaServer.next(scannerid)) != null) {
+        if (data != null && data.size() > 0) {
+          HRegionInfo info = Writables.getHRegionInfo(
+              data.getValue(HConstants.CATALOG_FAMILY,
+                  HConstants.REGIONINFO_QUALIFIER));
+          if (info.getTableDesc().getNameAsString().equals(tableString)) {
+            regions.add(info);
+          } else {
+            break;
+          }
+        }
+      }
+      return regions;
+    } finally {
+      metaServer.close(scannerid);
+    }
+  }
+
+  public static List<Pair<HRegionInfo, HServerAddress>>
+  getTableRegionsAndLocations(CatalogTracker catalogTracker, String tableName)
+  throws IOException {
+    HRegionInterface metaServer =
+      catalogTracker.waitForMetaServerConnectionDefault();
+    List<Pair<HRegionInfo, HServerAddress>> regions =
+      new ArrayList<Pair<HRegionInfo, HServerAddress>>();
+    byte[] firstRowInTable = Bytes.toBytes(tableName + ",,");
+    Scan scan = new Scan(firstRowInTable);
+    scan.addFamily(HConstants.CATALOG_FAMILY);
+    long scannerid = metaServer.openScanner(
+        HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
+    try {
+      Result data;
+      while((data = metaServer.next(scannerid)) != null) {
+        if (data != null && data.size() > 0) {
+          Pair<HRegionInfo, HServerAddress> region = metaRowToRegionPair(data);
+          if (region.getFirst().getTableDesc().getNameAsString().equals(
+              tableName)) {
+            regions.add(region);
+          } else {
+            break;
+          }
+        }
+      }
+      return regions;
+    } finally {
+      metaServer.close(scannerid);
+    }
+  }
+
+  public static NavigableMap<HRegionInfo, Result>
+  getServerRegions(CatalogTracker catalogTracker, final HServerInfo hsi)
+  throws IOException {
+    HRegionInterface metaServer =
+      catalogTracker.waitForMetaServerConnectionDefault();
+    NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
+    Scan scan = new Scan();
+    scan.addFamily(HConstants.CATALOG_FAMILY);
+    long scannerid = metaServer.openScanner(
+        HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
+    try {
+      Result result;
+      while((result = metaServer.next(scannerid)) != null) {
+        if (result != null && result.size() > 0) {
+          Pair<HRegionInfo, HServerAddress> pair = metaRowToRegionPair(result);
+          if (!pair.getSecond().equals(hsi.getServerAddress())) continue;
+          hris.put(pair.getFirst(), result);
+        }
+      }
+      return hris;
+    } finally {
+      metaServer.close(scannerid);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java
new file mode 100644
index 00000000000..aee64c5bf1e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java
@@ -0,0 +1,72 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.catalog;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Makes changes to the location of <code>-ROOT-</code> in ZooKeeper.
+ */
+public class RootLocationEditor {
+  private static final Log LOG = LogFactory.getLog(RootLocationEditor.class);
+
+  /**
+   * Deletes the location of <code>-ROOT-</code> in ZooKeeper.
+   * @param zookeeper zookeeper reference
+   * @throws KeeperException unexpected zookeeper exception
+   */
+  public static void deleteRootLocation(ZooKeeperWatcher zookeeper)
+  throws KeeperException {
+    LOG.info("Unsetting ROOT region location in ZooKeeper");
+    try {
+      // Just delete the node.  Don't need any watches, only we will create it.
+      ZKUtil.deleteNode(zookeeper, zookeeper.rootServerZNode);
+    } catch(KeeperException.NoNodeException nne) {
+      // Has already been deleted
+    }
+  }
+
+  /**
+   * Sets the location of <code>-ROOT-</code> in ZooKeeper to the
+   * specified server address.
+   * @param zookeeper zookeeper reference
+   * @param location server address hosting root
+   * @throws KeeperException unexpected zookeeper exception
+   */
+  public static void setRootLocation(ZooKeeperWatcher zookeeper,
+      HServerAddress location)
+  throws KeeperException {
+    LOG.info("Setting ROOT region location in ZooKeeper as " + location);
+    try {
+      ZKUtil.createAndWatch(zookeeper, zookeeper.rootServerZNode,
+        Bytes.toBytes(location.toString()));
+    } catch(KeeperException.NodeExistsException nee) {
+      LOG.debug("ROOT region location already existed, updated location");
+      ZKUtil.setData(zookeeper, zookeeper.rootServerZNode,
+          Bytes.toBytes(location.toString()));
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/client/Get.java b/src/main/java/org/apache/hadoop/hbase/client/Get.java
index 50647a1be79..2246632dd21 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/Get.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/Get.java
@@ -60,7 +60,7 @@ import java.util.TreeSet;
  * <p>
  * To add a filter, execute {@link #setFilter(Filter) setFilter}.
  */
-public class Get implements Writable, Row, Comparable<Row> {
+public class Get implements Writable {
   private static final byte GET_VERSION = (byte)1;
 
   private byte [] row = null;
@@ -325,11 +325,6 @@ public class Get implements Writable, Row, Comparable<Row> {
     return sb.toString();
   }
 
-  //Row
-  public int compareTo(Row other) {
-    return Bytes.compareTo(this.getRow(), other.getRow());
-  }
-  
   //Writable
   public void readFields(final DataInput in)
   throws IOException {
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 8b01aa0c18e..b52a7e193f4 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -19,35 +19,39 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.RegionException;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.UnknownRegionException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.MetaUtils;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.ipc.RemoteException;
 
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.NavigableMap;
-
 /**
  * Provides an interface to manage HBase database table metadata + general 
  * administrative functions.  Use HBaseAdmin to create, drop, list, enable and 
@@ -55,27 +59,56 @@ import java.util.NavigableMap;
  * 
  * See {@link HTable} to add, update, and delete data from an individual table.
  */
-public class HBaseAdmin {
+public class HBaseAdmin implements Abortable {
   private final Log LOG = LogFactory.getLog(this.getClass().getName());
 //  private final HConnection connection;
   final HConnection connection;
   private volatile Configuration conf;
   private final long pause;
   private final int numRetries;
-  private volatile HMasterInterface master;
+  /**
+   * Lazily instantiated.  Use {@link #getCatalogTracker()} to ensure you get
+   * an instance rather than a null.
+   */
+  private CatalogTracker catalogTracker = null;
 
   /**
    * Constructor
    *
    * @param conf Configuration object
    * @throws MasterNotRunningException if the master is not running
+   * @throws ZooKeeperConnectionException if unable to connect to zookeeper
    */
-  public HBaseAdmin(Configuration conf) throws MasterNotRunningException {
+  public HBaseAdmin(Configuration conf)
+  throws MasterNotRunningException, ZooKeeperConnectionException {
     this.connection = HConnectionManager.getConnection(conf);
     this.conf = conf;
     this.pause = conf.getLong("hbase.client.pause", 30 * 1000);
     this.numRetries = conf.getInt("hbase.client.retries.number", 5);
-    this.master = connection.getMaster();
+    this.connection.getMaster();
+  }
+
+  private synchronized CatalogTracker getCatalogTracker()
+  throws ZooKeeperConnectionException, IOException {
+    if (this.catalogTracker == null) {
+      this.catalogTracker = new CatalogTracker(this.connection.getZooKeeperWatcher(),
+        ServerConnectionManager.getConnection(conf), this,
+        this.conf.getInt("hbase.admin.catalog.timeout", 10 * 1000));
+      try {
+        this.catalogTracker.start();
+      } catch (InterruptedException e) {
+        // Let it out as an IOE for now until we redo all so tolerate IEs
+        Thread.currentThread().interrupt();
+        throw new IOException("Interrupted", e);
+      }
+    }
+    return this.catalogTracker;
+  }
+
+  @Override
+  public void abort(String why, Throwable e) {
+    // Currently does nothing but throw the passed message and exception
+    throw new RuntimeException(why, e);
   }
 
   /** @return HConnection used by this object. */
@@ -84,15 +117,21 @@ public class HBaseAdmin {
   }
 
   /**
+   * Get a connection to the currently set master.
    * @return proxy connection to master server for this instance
    * @throws MasterNotRunningException if the master is not running
+   * @throws ZooKeeperConnectionException if unable to connect to zookeeper
    */
-  public HMasterInterface getMaster() throws MasterNotRunningException{
+  public HMasterInterface getMaster()
+  throws MasterNotRunningException, ZooKeeperConnectionException {
     return this.connection.getMaster();
   }
 
-  /** @return - true if the master server is running */
-  public boolean isMasterRunning() {
+  /** @return - true if the master server is running
+   * @throws ZooKeeperConnectionException
+   * @throws MasterNotRunningException */
+  public boolean isMasterRunning()
+  throws MasterNotRunningException, ZooKeeperConnectionException {
     return this.connection.isMasterRunning();
   }
 
@@ -100,9 +139,10 @@ public class HBaseAdmin {
    * @param tableName Table to check.
    * @return True if table exists already.
    * @throws MasterNotRunningException if the master is not running
+   * @throws ZooKeeperConnectionException if unable to connect to zookeeper
    */
   public boolean tableExists(final String tableName)
-  throws MasterNotRunningException {
+  throws MasterNotRunningException, ZooKeeperConnectionException {
     return tableExists(Bytes.toBytes(tableName));
   }
 
@@ -110,12 +150,11 @@ public class HBaseAdmin {
    * @param tableName Table to check.
    * @return True if table exists already.
    * @throws MasterNotRunningException if the master is not running
+   * @throws ZooKeeperConnectionException if unable to connect to zookeeper
    */
   public boolean tableExists(final byte [] tableName)
-  throws MasterNotRunningException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
+  throws MasterNotRunningException, ZooKeeperConnectionException {
+    connection.isMasterRunning();
     return connection.tableExists(tableName);
   }
 
@@ -147,8 +186,9 @@ public class HBaseAdmin {
 
   private long getPauseTime(int tries) {
     int triesCount = tries;
-    if (triesCount >= HConstants.RETRY_BACKOFF.length)
+    if (triesCount >= HConstants.RETRY_BACKOFF.length) {
       triesCount = HConstants.RETRY_BACKOFF.length - 1;
+    }
     return this.pause * HConstants.RETRY_BACKOFF[triesCount];
   }
 
@@ -257,7 +297,7 @@ public class HBaseAdmin {
       try {
         Thread.sleep(getPauseTime(tries));
       } catch (InterruptedException e) {
-        // continue
+        // Just continue; ignore the interruption.
       }
     }
   }
@@ -277,12 +317,9 @@ public class HBaseAdmin {
    */
   public void createTableAsync(HTableDescriptor desc, byte [][] splitKeys)
   throws IOException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
     HTableDescriptor.isLegalTableName(desc.getName());
     try {
-      this.master.createTable(desc, splitKeys);
+      getMaster().createTable(desc, splitKeys);
     } catch (RemoteException e) {
       throw RemoteExceptionHandler.decodeRemoteException(e);
     }
@@ -307,13 +344,11 @@ public class HBaseAdmin {
    * @throws IOException if a remote or network exception occurs
    */
   public void deleteTable(final byte [] tableName) throws IOException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
+    isMasterRunning();
     HTableDescriptor.isLegalTableName(tableName);
     HRegionLocation firstMetaServer = getFirstMetaServerForTable(tableName);
     try {
-      this.master.deleteTable(tableName);
+      getMaster().deleteTable(tableName);
     } catch (RemoteException e) {
       throw RemoteExceptionHandler.decodeRemoteException(e);
     }
@@ -401,21 +436,20 @@ public class HBaseAdmin {
    * @throws IOException if a remote or network exception occurs
    */
   public void enableTable(final byte [] tableName) throws IOException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
+    isMasterRunning();
 
     // Wait until all regions are enabled
     boolean enabled = false;
     for (int tries = 0; tries < this.numRetries; tries++) {
-
       try {
-        this.master.enableTable(tableName);
+        getMaster().enableTable(tableName);
       } catch (RemoteException e) {
         throw RemoteExceptionHandler.decodeRemoteException(e);
       }
       enabled = isTableEnabled(tableName);
-      if (enabled) break;
+      if (enabled) {
+        break;
+      }
       long sleep = getPauseTime(tries);
       if (LOG.isDebugEnabled()) {
         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
@@ -431,9 +465,10 @@ public class HBaseAdmin {
           Bytes.toString(tableName));
       }
     }
-    if (!enabled)
+    if (!enabled) {
       throw new IOException("Unable to enable table " +
         Bytes.toString(tableName));
+    }
     LOG.info("Enabled table " + Bytes.toString(tableName));
   }
 
@@ -458,20 +493,20 @@ public class HBaseAdmin {
    * @throws IOException if a remote or network exception occurs
    */
   public void disableTable(final byte [] tableName) throws IOException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
+    isMasterRunning();
 
     // Wait until all regions are disabled
     boolean disabled = false;
     for (int tries = 0; tries < this.numRetries; tries++) {
       try {
-        this.master.disableTable(tableName);
+        getMaster().disableTable(tableName);
       } catch (RemoteException e) {
         throw RemoteExceptionHandler.decodeRemoteException(e);
       }
       disabled = isTableDisabled(tableName);
-      if (disabled) break;
+      if (disabled) {
+        break;
+      }
       if (LOG.isDebugEnabled()) {
         LOG.debug("Sleep. Waiting for all regions to be disabled from " +
           Bytes.toString(tableName));
@@ -560,12 +595,9 @@ public class HBaseAdmin {
    */
   public void addColumn(final byte [] tableName, HColumnDescriptor column)
   throws IOException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
     HTableDescriptor.isLegalTableName(tableName);
     try {
-      this.master.addColumn(tableName, column);
+      getMaster().addColumn(tableName, column);
     } catch (RemoteException e) {
       throw RemoteExceptionHandler.decodeRemoteException(e);
     }
@@ -594,12 +626,8 @@ public class HBaseAdmin {
    */
   public void deleteColumn(final byte [] tableName, final byte [] columnName)
   throws IOException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
-    HTableDescriptor.isLegalTableName(tableName);
     try {
-      this.master.deleteColumn(tableName, columnName);
+      getMaster().deleteColumn(tableName, columnName);
     } catch (RemoteException e) {
       throw RemoteExceptionHandler.decodeRemoteException(e);
     }
@@ -613,12 +641,25 @@ public class HBaseAdmin {
    * @param columnName name of column to be modified
    * @param descriptor new column descriptor to use
    * @throws IOException if a remote or network exception occurs
+   * @deprecated The <code>columnName</code> is redundant. Use {@link #addColumn(String, HColumnDescriptor)}
    */
   public void modifyColumn(final String tableName, final String columnName,
       HColumnDescriptor descriptor)
   throws IOException {
-    modifyColumn(Bytes.toBytes(tableName), Bytes.toBytes(columnName),
-      descriptor);
+    modifyColumn(tableName,  descriptor);
+  }
+
+  /**
+   * Modify an existing column family on a table.
+   * Asynchronous operation.
+   *
+   * @param tableName name of table
+   * @param descriptor new column descriptor to use
+   * @throws IOException if a remote or network exception occurs
+   */
+  public void modifyColumn(final String tableName, HColumnDescriptor descriptor)
+  throws IOException {
+    modifyColumn(Bytes.toBytes(tableName), descriptor);
   }
 
   /**
@@ -629,56 +670,71 @@ public class HBaseAdmin {
    * @param columnName name of column to be modified
    * @param descriptor new column descriptor to use
    * @throws IOException if a remote or network exception occurs
+   * @deprecated The <code>columnName</code> is redundant. Use {@link #modifyColumn(byte[], HColumnDescriptor)}
    */
   public void modifyColumn(final byte [] tableName, final byte [] columnName,
     HColumnDescriptor descriptor)
   throws IOException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
-    HTableDescriptor.isLegalTableName(tableName);
+    modifyColumn(tableName, descriptor);
+  }
+
+  /**
+   * Modify an existing column family on a table.
+   * Asynchronous operation.
+   *
+   * @param tableName name of table
+   * @param descriptor new column descriptor to use
+   * @throws IOException if a remote or network exception occurs
+   */
+  public void modifyColumn(final byte [] tableName, HColumnDescriptor descriptor)
+  throws IOException {
     try {
-      this.master.modifyColumn(tableName, columnName, descriptor);
-    } catch (RemoteException e) {
-      throw RemoteExceptionHandler.decodeRemoteException(e);
+      getMaster().modifyColumn(tableName, descriptor);
+    } catch (RemoteException re) {
+      // Convert RE exceptions in here; client shouldn't have to deal with them,
+      // at least w/ the type of exceptions that come out of this method:
+      // TableNotFoundException, etc.
+      throw RemoteExceptionHandler.decodeRemoteException(re);
     }
   }
 
   /**
    * Close a region. For expert-admins.
-   * Asynchronous operation.
-   *
    * @param regionname region name to close
-   * @param args Optional server name.  Otherwise, we'll send close to the
-   * server registered in .META.
+   * @param hostAndPort If supplied, we'll use this location rather than
+   * the one currently in <code>.META.</code>
    * @throws IOException if a remote or network exception occurs
    */
-  public void closeRegion(final String regionname, final Object... args)
+  public void closeRegion(final String regionname, final String hostAndPort)
   throws IOException {
-    closeRegion(Bytes.toBytes(regionname), args);
+    closeRegion(Bytes.toBytes(regionname), hostAndPort);
   }
 
   /**
    * Close a region.  For expert-admins.
-   * Asynchronous operation.
-   *
    * @param regionname region name to close
-   * @param args Optional server name.  Otherwise, we'll send close to the
-   * server registered in .META.
+   * @param hostAndPort If supplied, we'll use this location rather than
+   * the one currently in <code>.META.</code>
    * @throws IOException if a remote or network exception occurs
    */
-  public void closeRegion(final byte [] regionname, final Object... args)
+  public void closeRegion(final byte [] regionname, final String hostAndPort)
   throws IOException {
-    // Be careful. Must match the handler over in HMaster at MODIFY_CLOSE_REGION
-    int len = (args == null)? 0: args.length;
-    int xtraArgsCount = 1;
-    Object [] newargs = new Object[len + xtraArgsCount];
-    newargs[0] = regionname;
-    if(args != null) {
-      System.arraycopy(args, 0, newargs, xtraArgsCount, len);
+    if (hostAndPort != null) {
+      HServerAddress hsa = new HServerAddress(hostAndPort);
+      Pair<HRegionInfo, HServerAddress> pair =
+        MetaReader.getRegion(getCatalogTracker(), regionname);
+      closeRegion(hsa, pair.getFirst());
+    } else {
+      Pair<HRegionInfo, HServerAddress> pair =
+        MetaReader.getRegion(getCatalogTracker(), regionname);
+      closeRegion(pair.getSecond(), pair.getFirst());
     }
-    modifyTable(HConstants.META_TABLE_NAME, HConstants.Modify.CLOSE_REGION,
-      newargs);
+  }
+
+  private void closeRegion(final HServerAddress hsa, final HRegionInfo hri)
+  throws IOException {
+    HRegionInterface rs = this.connection.getHRegionConnection(hsa);
+    rs.closeRegion(hri);
   }
 
   /**
@@ -700,7 +756,25 @@ public class HBaseAdmin {
    * @throws IOException if a remote or network exception occurs
    */
   public void flush(final byte [] tableNameOrRegionName) throws IOException {
-    modifyTable(tableNameOrRegionName, HConstants.Modify.TABLE_FLUSH);
+    boolean isRegionName = isRegionName(tableNameOrRegionName);
+    if (isRegionName) {
+      Pair<HRegionInfo, HServerAddress> pair =
+        MetaReader.getRegion(getCatalogTracker(), tableNameOrRegionName);
+      flush(pair.getSecond(), pair.getFirst());
+    } else {
+      List<Pair<HRegionInfo, HServerAddress>> pairs =
+        MetaReader.getTableRegionsAndLocations(getCatalogTracker(),
+          Bytes.toString(tableNameOrRegionName));
+      for (Pair<HRegionInfo, HServerAddress> pair: pairs) {
+        flush(pair.getSecond(), pair.getFirst());
+      }
+    }
+  }
+
+  private void flush(final HServerAddress hsa, final HRegionInfo hri)
+  throws IOException {
+    HRegionInterface rs = this.connection.getHRegionConnection(hsa);
+    rs.flushRegion(hri);
   }
 
   /**
@@ -722,7 +796,7 @@ public class HBaseAdmin {
    * @throws IOException if a remote or network exception occurs
    */
   public void compact(final byte [] tableNameOrRegionName) throws IOException {
-    modifyTable(tableNameOrRegionName, HConstants.Modify.TABLE_COMPACT);
+    compact(tableNameOrRegionName, false);
   }
 
   /**
@@ -746,7 +820,63 @@ public class HBaseAdmin {
    */
   public void majorCompact(final byte [] tableNameOrRegionName)
   throws IOException {
-    modifyTable(tableNameOrRegionName, HConstants.Modify.TABLE_MAJOR_COMPACT);
+    compact(tableNameOrRegionName, true);
+  }
+
+  /**
+   * Compact a table or an individual region.
+   * Asynchronous operation.
+   *
+   * @param tableNameOrRegionName table or region to compact
+   * @param major True if we are to do a major compaction.
+   * @throws IOException if a remote or network exception occurs
+   */
+  private void compact(final byte [] tableNameOrRegionName, final boolean major)
+  throws IOException {
+    if (isRegionName(tableNameOrRegionName)) {
+      Pair<HRegionInfo, HServerAddress> pair =
+        MetaReader.getRegion(getCatalogTracker(), tableNameOrRegionName);
+      compact(pair.getSecond(), pair.getFirst(), major);
+    } else {
+      List<Pair<HRegionInfo, HServerAddress>> pairs =
+        MetaReader.getTableRegionsAndLocations(getCatalogTracker(),
+          Bytes.toString(tableNameOrRegionName));
+      for (Pair<HRegionInfo, HServerAddress> pair: pairs) {
+        compact(pair.getSecond(), pair.getFirst(), major);
+      }
+    }
+  }
+
+  private void compact(final HServerAddress hsa, final HRegionInfo hri,
+      final boolean major)
+  throws IOException {
+    HRegionInterface rs = this.connection.getHRegionConnection(hsa);
+    rs.compactRegion(hri, major);
+  }
+
+  /**
+   * Move the region <code>r</code> to <code>dest</code>.
+   * @param encodedRegionName The encoded region name.
+   * @param destServerName The servername of the destination regionserver
+   * @throws UnknownRegionException Thrown if we can't find a region named
+   * <code>encodedRegionName</code>
+   * @throws ZooKeeperConnectionException 
+   * @throws MasterNotRunningException 
+   */
+  public void move(final byte [] encodedRegionName, final byte [] destServerName)
+  throws UnknownRegionException, MasterNotRunningException, ZooKeeperConnectionException {
+    getMaster().move(encodedRegionName, destServerName);
+  }
+
+  /**
+   * @param b If true, enable balancer. If false, disable balancer.
+   * @return Previous balancer value
+   * @throws ZooKeeperConnectionException 
+   * @throws MasterNotRunningException 
+   */
+  public boolean balance(final boolean b)
+  throws MasterNotRunningException, ZooKeeperConnectionException {
+    return getMaster().balance(b);
   }
 
   /**
@@ -768,32 +898,31 @@ public class HBaseAdmin {
    * @throws IOException if a remote or network exception occurs
    */
   public void split(final byte [] tableNameOrRegionName) throws IOException {
-    modifyTable(tableNameOrRegionName, HConstants.Modify.TABLE_SPLIT);
+    if (isRegionName(tableNameOrRegionName)) {
+      // Its a possible region name.
+      Pair<HRegionInfo, HServerAddress> pair =
+        MetaReader.getRegion(getCatalogTracker(), tableNameOrRegionName);
+      split(pair.getSecond(), pair.getFirst());
+    } else {
+      List<Pair<HRegionInfo, HServerAddress>> pairs =
+        MetaReader.getTableRegionsAndLocations(getCatalogTracker(),
+          Bytes.toString(tableNameOrRegionName));
+      for (Pair<HRegionInfo, HServerAddress> pair: pairs) {
+        split(pair.getSecond(), pair.getFirst());
+      }
+    }
   }
 
-  /*
-   * Call modifyTable using passed tableName or region name String.  If no
-   * such table, presume we have been passed a region name.
-   * @param tableNameOrRegionName
-   * @param op
-   * @throws IOException
-   */
-  private void modifyTable(final byte [] tableNameOrRegionName,
-      final HConstants.Modify op)
+  private void split(final HServerAddress hsa, final HRegionInfo hri)
   throws IOException {
-    if (tableNameOrRegionName == null) {
-      throw new IllegalArgumentException("Pass a table name or region name");
-    }
-    byte [] tableName = tableExists(tableNameOrRegionName)?
-      tableNameOrRegionName: null;
-    byte [] regionName = tableName == null? tableNameOrRegionName: null;
-    Object [] args = regionName == null? null: new byte [][] {regionName};
-    modifyTable(tableName == null? null: tableName, op, args);
+    HRegionInterface rs = this.connection.getHRegionConnection(hsa);
+    rs.splitRegion(hri);
   }
 
   /**
    * Modify an existing table, more IRB friendly version.
-   * Asynchronous operation.
+   * Asynchronous operation.  This means that it may be a while before your
+   * schema change is updated across all of the table.
    *
    * @param tableName name of table.
    * @param htd modified description of the table
@@ -801,107 +930,57 @@ public class HBaseAdmin {
    */
   public void modifyTable(final byte [] tableName, HTableDescriptor htd)
   throws IOException {
-    modifyTable(tableName, HConstants.Modify.TABLE_SET_HTD, htd);
-  }
-
-  /**
-   * Modify an existing table.
-   * Asynchronous operation.
-   *
-   * @param tableName name of table.  May be null if we are operating on a
-   * region.
-   * @param op table modification operation
-   * @param args operation specific arguments
-   * @throws IOException if a remote or network exception occurs
-   */
-  public void modifyTable(final byte [] tableName, HConstants.Modify op,
-      Object... args)
-      throws IOException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
-    // Let pass if its a catalog table.  Used by admins.
-    if (tableName != null && !MetaUtils.isMetaTableName(tableName)) {
-      // This will throw exception
-      HTableDescriptor.isLegalTableName(tableName);
-    }
-    Writable[] arr = null;
     try {
-      switch (op) {
-      case TABLE_SET_HTD:
-        if (args == null || args.length < 1 ||
-            !(args[0] instanceof HTableDescriptor)) {
-          throw new IllegalArgumentException("SET_HTD requires a HTableDescriptor");
-        }
-        arr = new Writable[1];
-        arr[0] = (HTableDescriptor)args[0];
-        this.master.modifyTable(tableName, op, arr);
-        break;
-
-      case TABLE_COMPACT:
-      case TABLE_SPLIT:
-      case TABLE_MAJOR_COMPACT:
-      case TABLE_FLUSH:
-        if (args != null && args.length > 0) {
-          arr = new Writable[1];
-          if (args[0] instanceof byte[]) {
-            arr[0] = new ImmutableBytesWritable((byte[])args[0]);
-          } else if (args[0] instanceof ImmutableBytesWritable) {
-            arr[0] = (ImmutableBytesWritable)args[0];
-          } else if (args[0] instanceof String) {
-            arr[0] = new ImmutableBytesWritable(Bytes.toBytes((String)args[0]));
-          } else {
-            throw new IllegalArgumentException("Requires byte[], String, or" +
-              "ImmutableBytesWritable");
-          }
-        }
-        this.master.modifyTable(tableName, op, arr);
-        break;
-
-      case CLOSE_REGION:
-        if (args == null || args.length < 1) {
-          throw new IllegalArgumentException("Requires at least a region name");
-        }
-        arr = new Writable[args.length];
-        for (int i = 0; i < args.length; i++) {
-          if (args[i] instanceof byte[]) {
-            arr[i] = new ImmutableBytesWritable((byte[])args[i]);
-          } else if (args[i] instanceof ImmutableBytesWritable) {
-            arr[i] = (ImmutableBytesWritable)args[i];
-          } else if (args[i] instanceof String) {
-            arr[i] = new ImmutableBytesWritable(Bytes.toBytes((String)args[i]));
-          } else if (args[i] instanceof Boolean) {
-            arr[i] = new BooleanWritable((Boolean) args[i]);
-          } else {
-            throw new IllegalArgumentException("Requires byte [] or " +
-              "ImmutableBytesWritable, not " + args[i]);
-          }
-        }
-        this.master.modifyTable(tableName, op, arr);
-        break;
-
-      default:
-        throw new IOException("unknown modifyTable op " + op);
-      }
-    } catch (RemoteException e) {
-      throw RemoteExceptionHandler.decodeRemoteException(e);
+      getMaster().modifyTable(tableName, htd);
+    } catch (RemoteException re) {
+      // Convert RE exceptions in here; client shouldn't have to deal with them,
+      // at least w/ the type of exceptions that come out of this method:
+      // TableNotFoundException, etc.
+      throw RemoteExceptionHandler.decodeRemoteException(re);
     }
   }
 
   /**
-   * Shuts down the HBase instance
+   * @param tableNameOrRegionName Name of a table or name of a region.
+   * @return True if <code>tableNameOrRegionName</code> is *possibly* a region
+   * name else false if a verified tablename (we call {@link #tableExists(byte[])};
+   * else we throw an exception.
+   * @throws ZooKeeperConnectionException 
+   * @throws MasterNotRunningException 
+   */
+  private boolean isRegionName(final byte [] tableNameOrRegionName)
+  throws MasterNotRunningException, ZooKeeperConnectionException {
+    if (tableNameOrRegionName == null) {
+      throw new IllegalArgumentException("Pass a table name or region name");
+    }
+    return !tableExists(tableNameOrRegionName);
+  }
+
+  /**
+   * Shuts down the HBase cluster
    * @throws IOException if a remote or network exception occurs
    */
   public synchronized void shutdown() throws IOException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
+    isMasterRunning();
     try {
-      this.master.shutdown();
+      getMaster().shutdown();
+    } catch (RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
+    }
+  }
+
+  /**
+   * Shuts down the current HBase master only.
+   * Does not shutdown the cluster.
+   * @see #shutdown()
+   * @throws IOException if a remote or network exception occurs
+   */
+  public synchronized void stopMaster() throws IOException {
+    isMasterRunning();
+    try {
+      getMaster().stopMaster();
     } catch (RemoteException e) {
       throw RemoteExceptionHandler.decodeRemoteException(e);
-    } finally {
-      this.master = null;
     }
   }
 
@@ -910,10 +989,7 @@ public class HBaseAdmin {
    * @throws IOException if a remote or network exception occurs
    */
   public ClusterStatus getClusterStatus() throws IOException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
-    return this.master.getClusterStatus();
+    return getMaster().getClusterStatus();
   }
 
   private HRegionLocation getFirstMetaServerForTable(final byte [] tableName)
@@ -926,12 +1002,13 @@ public class HBaseAdmin {
    * Check to see if HBase is running. Throw an exception if not.
    *
    * @param conf system configuration
-   * @throws MasterNotRunningException if a remote or network exception occurs
+   * @throws MasterNotRunningException if the master is not running
+   * @throws ZooKeeperConnectionException if unable to connect to zookeeper
    */
   public static void checkHBaseAvailable(Configuration conf)
-  throws MasterNotRunningException {
+  throws MasterNotRunningException, ZooKeeperConnectionException {
     Configuration copyOfConf = HBaseConfiguration.create(conf);
     copyOfConf.setInt("hbase.client.retries.number", 1);
     new HBaseAdmin(copyOfConf);
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
index 19cdec13b13..62c780fce27 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.NavigableSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -42,6 +43,7 @@ import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
@@ -73,9 +75,10 @@ public class HBaseFsck extends HBaseAdmin {
    *
    * @param conf Configuration object
    * @throws MasterNotRunningException if the master is not running
+   * @throws ZooKeeperConnectionException if unable to connect to zookeeper
    */
   public HBaseFsck(Configuration conf) 
-    throws MasterNotRunningException, IOException {
+    throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
     super(conf);
     this.conf = conf;
 
@@ -261,10 +264,10 @@ public class HBaseFsck extends HBaseAdmin {
                                     rsinfo.getServerAddress());
 
         // list all online regions from this region server
-        HRegionInfo[] regions = server.getRegionsAssignment();
+        NavigableSet<HRegionInfo> regions = server.getOnlineRegions();
         if (details) {
           System.out.print("\nRegionServer:" + rsinfo.getServerName() +
-                           " number of regions:" + regions.length);
+                           " number of regions:" + regions.size());
           for (HRegionInfo rinfo: regions) {
             System.out.print("\n\t name:" + rinfo.getRegionNameAsString() +
                              " id:" + rinfo.getRegionId() +
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
index e447b19855d..e1233fb0255 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
@@ -19,50 +19,55 @@
  */
 package org.apache.hadoop.hbase.client;
 
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+
 /**
  * Cluster connection.
  * {@link HConnectionManager} manages instances of this class.
  */
 public interface HConnection {
   /**
-   * Retrieve ZooKeeperWrapper used by the connection.
-   * @return ZooKeeperWrapper handle being used by the connection.
+   * Retrieve ZooKeeperWatcher used by the connection.
+   * @return ZooKeeperWatcher handle being used by the connection.
    * @throws IOException if a remote or network exception occurs
    */
-  public ZooKeeperWrapper getZooKeeperWrapper() throws IOException;
+  public ZooKeeperWatcher getZooKeeperWatcher() throws IOException;
 
   /**
    * @return proxy connection to master server for this instance
    * @throws MasterNotRunningException if the master is not running
+   * @throws ZooKeeperConnectionException if unable to connect to zookeeper
    */
-  public HMasterInterface getMaster() throws MasterNotRunningException;
+  public HMasterInterface getMaster()
+  throws MasterNotRunningException, ZooKeeperConnectionException;
 
   /** @return - true if the master server is running */
-  public boolean isMasterRunning();
+  public boolean isMasterRunning()
+  throws MasterNotRunningException, ZooKeeperConnectionException;
 
   /**
    * Checks if <code>tableName</code> exists.
    * @param tableName Table to check.
    * @return True if table exists already.
    * @throws MasterNotRunningException if the master is not running
+   * @throws ZooKeeperConnectionException if unable to connect to zookeeper
    */
   public boolean tableExists(final byte [] tableName)
-  throws MasterNotRunningException;
+  throws MasterNotRunningException, ZooKeeperConnectionException;
 
   /**
    * A table that isTableEnabled == false and isTableDisabled == false
@@ -131,7 +136,7 @@ public interface HConnection {
    * lives in, ignoring any value that might be in the cache.
    * @param tableName name of the table <i>row</i> is in
    * @param row row key you're trying to find the region of
-   * @return HRegionLocation that describes where to find the reigon in
+   * @return HRegionLocation that describes where to find the region in
    * question
    * @throws IOException if a remote or network exception occurs
    */
@@ -139,6 +144,25 @@ public interface HConnection {
       final byte [] row)
   throws IOException;
 
+  /**
+   * Gets the location of the region of <i>regionName</i>.
+   * @param regionName name of the region to locate
+   * @return HRegionLocation that describes where to find the region in
+   * question
+   * @throws IOException if a remote or network exception occurs
+   */
+  public HRegionLocation locateRegion(final byte [] regionName)
+  throws IOException;
+
+  /**
+   * Gets the locations of all regions in the specified table, <i>tableName</i>.
+   * @param tableName table to get regions of
+   * @return list of region locations for all regions of table
+   * @throws IOException
+   */
+  public List<HRegionLocation> locateRegions(byte[] tableName)
+  throws IOException;
+
   /**
    * Establishes a connection to the region server at the specified address.
    * @param regionServer - the server to connect to
@@ -197,21 +221,6 @@ public interface HConnection {
   public <T> T getRegionServerWithoutRetries(ServerCallable<T> callable)
   throws IOException, RuntimeException;
 
-  /**
-   * Process a mixed batch of Get, Put and Delete actions. All actions for a
-   * RegionServer are forwarded in one RPC call.
-   * 
-   * @param actions The collection of actions.
-   * @param tableName Name of the hbase table
-   * @param pool thread pool for parallel execution
-   * @param results An empty array, same size as list. If an exception is thrown,
-   * you can test here for partial results, and to determine which actions
-   * processed successfully.
-   * @throws IOException
-   */
-  public void processBatch(List<Row> actions, final byte[] tableName,
-      ExecutorService pool, Result[] results)
-  throws IOException;
 
   /**
    * Process a batch of Puts. Does the retries.
@@ -219,32 +228,20 @@ public interface HConnection {
    * @param tableName The name of the table
    * @return Count of committed Puts.  On fault, < list.size().
    * @throws IOException if a remote or network exception occurs
-   * @deprecated Use HConnectionManager::processBatch instead.
    */
-  public int processBatchOfRows(ArrayList<Put> list, byte[] tableName, ExecutorService pool)
+  public int processBatchOfRows(ArrayList<Put> list, byte[] tableName)
   throws IOException;
 
   /**
    * Process a batch of Deletes. Does the retries.
    * @param list A batch of Deletes to process.
-   * @param tableName The name of the table
    * @return Count of committed Deletes. On fault, < list.size().
+   * @param tableName The name of the table
    * @throws IOException if a remote or network exception occurs
-   * @deprecated Use HConnectionManager::processBatch instead.
    */
-  public int processBatchOfDeletes(List<Delete> list, byte[] tableName, ExecutorService pool)
+  public int processBatchOfDeletes(List<Delete> list, byte[] tableName)
   throws IOException;
 
-  /**
-   * Process a batch of Puts.
-   *
-   * @param list The collection of actions. The list is mutated: all successful Puts 
-   * are removed from the list.
-   * @param tableName Name of the hbase table
-   * @param pool Thread pool for parallel execution
-   * @throws IOException
-   * @deprecated Use HConnectionManager::processBatch instead.
-   */
   public void processBatchOfPuts(List<Put> list,
                                  final byte[] tableName, ExecutorService pool) throws IOException;
 
@@ -261,7 +258,7 @@ public interface HConnection {
   /**
    * Check whether region cache prefetch is enabled or not.
    * @param tableName name of table to check
-   * @return true if table's region cache prefetch is enabled. Otherwise
+   * @return true if table's region cache prefecth is enabled. Otherwise
    * it is disabled.
    */
   public boolean getRegionCachePrefetch(final byte[] tableName);
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 9307b03e43b..91747c5826a 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -19,37 +19,6 @@
  */
 package org.apache.hadoop.hbase.client;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.NotServingRegionException;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
-import org.apache.hadoop.hbase.ipc.HBaseRPC;
-import org.apache.hadoop.hbase.ipc.HBaseRPCProtocolVersion;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.MetaUtils;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.util.SoftValueSortedMap;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.KeeperState;
-
 import java.io.IOException;
 import java.lang.reflect.UndeclaredThrowableException;
 import java.util.ArrayList;
@@ -60,16 +29,46 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
-import java.util.Map.Entry;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.CopyOnWriteArraySet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MasterAddressTracker;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
+import org.apache.hadoop.hbase.ipc.HBaseRPC;
+import org.apache.hadoop.hbase.ipc.HBaseRPCProtocolVersion;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.MetaUtils;
+import org.apache.hadoop.hbase.util.SoftValueSortedMap;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.zookeeper.ZKTableDisable;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.zookeeper.KeeperException;
 
 /**
  * A non-instantiable class that manages connections to multiple tables in
@@ -111,16 +110,15 @@ public class HConnectionManager {
       }
   };
 
-  private static final Map<String, ClientZKWatcher> ZK_WRAPPERS =
-    new HashMap<String, ClientZKWatcher>();
-
   /**
    * Get the connection object for the instance specified by the configuration
    * If no current connection exists, create a new connection for that instance
    * @param conf configuration
    * @return HConnection object for the instance specified by the configuration
+   * @throws ZooKeeperConnectionException
    */
-  public static HConnection getConnection(Configuration conf) {
+  public static HConnection getConnection(Configuration conf)
+  throws ZooKeeperConnectionException {
     TableServers connection;
     Integer key = HBaseConfiguration.hashCode(conf);
     synchronized (HBASE_INSTANCES) {
@@ -152,6 +150,7 @@ public class HConnectionManager {
   /**
    * Delete information for all connections.
    * @param stopProxy stop the proxy as well
+   * @throws IOException
    */
   public static void deleteAllConnections(boolean stopProxy) {
     synchronized (HBASE_INSTANCES) {
@@ -161,100 +160,16 @@ public class HConnectionManager {
         }
       }
     }
-    synchronized (ZK_WRAPPERS) {
-      for (ClientZKWatcher watch : ZK_WRAPPERS.values()) {
-        watch.resetZooKeeper();
-      }
-    }
-  }
-
-  /**
-   * Get a watcher of a zookeeper connection for a given quorum address.
-   * If the connection isn't established, a new one is created.
-   * This acts like a multiton.
-   * @param conf configuration
-   * @return ZKW watcher
-   * @throws IOException if a remote or network exception occurs
-   */
-  public static synchronized ClientZKWatcher getClientZooKeeperWatcher(
-      Configuration conf) throws IOException {
-    if (!ZK_WRAPPERS.containsKey(
-        ZooKeeperWrapper.getZookeeperClusterKey(conf))) {
-      ZK_WRAPPERS.put(ZooKeeperWrapper.getZookeeperClusterKey(conf),
-          new ClientZKWatcher(conf));
-    }
-    return ZK_WRAPPERS.get(ZooKeeperWrapper.getZookeeperClusterKey(conf));
-  }
-
-  /**
-   * This class is responsible to handle connection and reconnection
-   * to a zookeeper quorum.
-   *
-   */
-  public static class ClientZKWatcher implements Watcher {
-
-    static final Log LOG = LogFactory.getLog(ClientZKWatcher.class);
-    private ZooKeeperWrapper zooKeeperWrapper;
-    private Configuration conf;
-
-    /**
-     * Takes a configuration to pass it to ZKW but won't instanciate it
-     * @param conf configuration
-     */
-    public ClientZKWatcher(Configuration conf) {
-      this.conf = conf;
-    }
-
-    /**
-     * Called by ZooKeeper when an event occurs on our connection. We use this to
-     * detect our session expiring. When our session expires, we have lost our
-     * connection to ZooKeeper. Our handle is dead, and we need to recreate it.
-     *
-     * See http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
-     * for more information.
-     *
-     * @param event WatchedEvent witnessed by ZooKeeper.
-     */
-    public void process(final WatchedEvent event) {
-      final KeeperState state = event.getState();
-      if (!state.equals(KeeperState.SyncConnected)) {
-        LOG.warn("No longer connected to ZooKeeper, current state: " + state);
-        resetZooKeeper();
-      }
-    }
-
-    /**
-     * Get this watcher's ZKW, instantiate it if necessary.
-     * @return ZKW
-     * @throws java.io.IOException if a remote or network exception occurs
-     */
-    public synchronized ZooKeeperWrapper getZooKeeperWrapper() throws IOException {
-      if (zooKeeperWrapper == null) {
-        zooKeeperWrapper =
-            ZooKeeperWrapper.createInstance(conf, HConnectionManager.class.getName());
-        zooKeeperWrapper.registerListener(this);
-      }
-      return zooKeeperWrapper;
-    }
-
-    /**
-     * Clear this connection to zookeeper.
-     */
-    private synchronized void resetZooKeeper() {
-      if (zooKeeperWrapper != null) {
-        zooKeeperWrapper.close();
-        zooKeeperWrapper = null;
-      }
-    }
   }
 
   /**
    * It is provided for unit test cases which verify the behavior of region
    * location cache prefetch.
    * @return Number of cached regions for the table.
+   * @throws ZooKeeperConnectionException
    */
   static int getCachedRegionCount(Configuration conf,
-      byte[] tableName) {
+      byte[] tableName) throws ZooKeeperConnectionException {
     TableServers connection = (TableServers)getConnection(conf);
     return connection.getNumberOfCachedRegionLocations(tableName);
   }
@@ -263,15 +178,16 @@ public class HConnectionManager {
    * It's provided for unit test cases which verify the behavior of region
    * location cache prefetch.
    * @return true if the region where the table and row reside is cached.
+   * @throws ZooKeeperConnectionException
    */
   static boolean isRegionCached(Configuration conf,
-      byte[] tableName, byte[] row) {
+      byte[] tableName, byte[] row) throws ZooKeeperConnectionException {
     TableServers connection = (TableServers)getConnection(conf);
     return connection.isRegionCached(tableName, row);
   }
 
   /* Encapsulates finding the servers for an HBase instance */
-  static class TableServers implements ServerConnection {
+  static class TableServers implements ServerConnection, Abortable {
     static final Log LOG = LogFactory.getLog(TableServers.class);
     private final Class<? extends HRegionInterface> serverInterfaceClass;
     private final long pause;
@@ -284,6 +200,10 @@ public class HConnectionManager {
     private volatile boolean closed;
     private volatile HMasterInterface master;
     private volatile boolean masterChecked;
+    // ZooKeeper reference
+    private ZooKeeperWatcher zooKeeper;
+    // ZooKeeper-based master address tracker
+    private MasterAddressTracker masterAddressTracker;
 
     private final Object rootRegionLock = new Object();
     private final Object metaRegionLock = new Object();
@@ -312,7 +232,8 @@ public class HConnectionManager {
      * @param conf Configuration object
      */
     @SuppressWarnings("unchecked")
-    public TableServers(Configuration conf) {
+    public TableServers(Configuration conf)
+    throws ZooKeeperConnectionException {
       this.conf = conf;
 
       String serverClassName =
@@ -340,14 +261,21 @@ public class HConnectionManager {
       this.prefetchRegionLimit = conf.getInt("hbase.client.prefetch.limit",
           10);
 
+      // initialize zookeeper and master address manager
+      getZooKeeperWatcher();
+      masterAddressTracker = new MasterAddressTracker(zooKeeper, this);
+      zooKeeper.registerListener(masterAddressTracker);
+      masterAddressTracker.start();
+
       this.master = null;
       this.masterChecked = false;
     }
 
     private long getPauseTime(int tries) {
       int ntries = tries;
-      if (ntries >= HConstants.RETRY_BACKOFF.length)
+      if (ntries >= HConstants.RETRY_BACKOFF.length) {
         ntries = HConstants.RETRY_BACKOFF.length - 1;
+      }
       return this.pause * HConstants.RETRY_BACKOFF[ntries];
     }
 
@@ -365,12 +293,14 @@ public class HConnectionManager {
       this.rootRegionLocation = rootRegion;
     }
 
-    public HMasterInterface getMaster() throws MasterNotRunningException {
-      ZooKeeperWrapper zk;
-      try {
-        zk = getZooKeeperWrapper();
-      } catch (IOException e) {
-        throw new MasterNotRunningException(e);
+    public HMasterInterface getMaster()
+    throws MasterNotRunningException, ZooKeeperConnectionException {
+
+      // Check if we already have a good master connection
+      if (master != null) {
+        if(master.isMasterRunning()) {
+          return master;
+        }
       }
 
       HServerAddress masterLocation = null;
@@ -382,7 +312,11 @@ public class HConnectionManager {
         tries++) {
 
           try {
-            masterLocation = zk.readMasterAddressOrThrow();
+            masterLocation = masterAddressTracker.getMasterAddress();
+            if(masterLocation == null) {
+              LOG.info("ZooKeeper available but no active master location found");
+              throw new MasterNotRunningException();
+            }
 
             HMasterInterface tryMaster = (HMasterInterface)HBaseRPC.getProxy(
                 HMasterInterface.class, HBaseRPCProtocolVersion.versionID,
@@ -424,20 +358,20 @@ public class HConnectionManager {
       return this.master;
     }
 
-    public boolean isMasterRunning() {
+    public boolean isMasterRunning()
+    throws MasterNotRunningException, ZooKeeperConnectionException {
       if (this.master == null) {
-        try {
-          getMaster();
-
-        } catch (MasterNotRunningException e) {
-          return false;
-        }
+        getMaster();
       }
-      return true;
+      boolean isRunning = master.isMasterRunning();
+      if(isRunning) {
+        return true;
+      }
+      throw new MasterNotRunningException();
     }
 
     public boolean tableExists(final byte [] tableName)
-    throws MasterNotRunningException {
+    throws MasterNotRunningException, ZooKeeperConnectionException {
       getMaster();
       if (tableName == null) {
         throw new IllegalArgumentException("Table name cannot be null");
@@ -537,15 +471,11 @@ public class HConnectionManager {
     }
 
     /*
-     * If online == true
-     *   Returns true if all regions are online
-     *   Returns false in any other case
-     * If online == false
-     *   Returns true if all regions are offline
-     *   Returns false in any other case
+     * @param True if table is online
      */
     private boolean testTableOnlineState(byte[] tableName, boolean online)
     throws IOException {
+      // TODO: Replace w/ CatalogTracker-based tableExists test.
       if (!tableExists(tableName)) {
         throw new TableNotFoundException(Bytes.toString(tableName));
       }
@@ -553,53 +483,14 @@ public class HConnectionManager {
         // The root region is always enabled
         return true;
       }
-      int rowsScanned = 0;
-      int rowsOffline = 0;
-      byte[] startKey =
-        HRegionInfo.createRegionName(tableName, null, HConstants.ZEROES, false);
-      byte[] endKey;
-      HRegionInfo currentRegion;
-      Scan scan = new Scan(startKey);
-      scan.addColumn(HConstants.CATALOG_FAMILY,
-          HConstants.REGIONINFO_QUALIFIER);
-      int rows = this.conf.getInt("hbase.meta.scanner.caching", 100);
-      scan.setCaching(rows);
-      ScannerCallable s = new ScannerCallable(this,
-          (Bytes.equals(tableName, HConstants.META_TABLE_NAME) ?
-              HConstants.ROOT_TABLE_NAME : HConstants.META_TABLE_NAME), scan);
       try {
-        // Open scanner
-        getRegionServerWithRetries(s);
-        do {
-          currentRegion = s.getHRegionInfo();
-          Result r;
-          Result [] rrs;
-          while ((rrs = getRegionServerWithRetries(s)) != null && rrs.length > 0) {
-            r = rrs[0];
-            byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
-              HConstants.REGIONINFO_QUALIFIER);
-            if (value != null) {
-              HRegionInfo info = Writables.getHRegionInfoOrNull(value);
-              if (info != null) {
-                if (Bytes.equals(info.getTableDesc().getName(), tableName)) {
-                  rowsScanned += 1;
-                  rowsOffline += info.isOffline() ? 1 : 0;
-                }
-              }
-            }
-          }
-          endKey = currentRegion.getEndKey();
-        } while (!(endKey == null ||
-            Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)));
-      } finally {
-        s.setClose();
-        // Doing below will call 'next' again and this will close the scanner
-        // Without it we leave scanners open.
-        getRegionServerWithRetries(s);
+        List<String> tables = ZKTableDisable.getDisabledTables(this.zooKeeper);
+        String searchStr = Bytes.toString(tableName);
+        boolean disabled = tables.contains(searchStr);
+        return online? !disabled: disabled;
+      } catch (KeeperException e) {
+        throw new IOException("Failed listing disabled tables", e);
       }
-      LOG.debug("Rowscanned=" + rowsScanned + ", rowsOffline=" + rowsOffline);
-      boolean onOffLine = online? rowsOffline == 0: rowsOffline == rowsScanned;
-      return rowsScanned > 0 && onOffLine;
     }
 
     private static class HTableDescriptorFinder
@@ -642,6 +533,20 @@ public class HConnectionManager {
       return result;
     }
 
+    @Override
+    public HRegionLocation locateRegion(final byte [] regionName)
+    throws IOException {
+      // TODO implement.  use old stuff or new stuff?
+      return null;
+    }
+
+    @Override
+    public List<HRegionLocation> locateRegions(final byte [] tableName)
+    throws IOException {
+      // TODO implement.  use old stuff or new stuff?
+      return null;
+    }
+
     public HRegionLocation locateRegion(final byte [] tableName,
         final byte [] row)
     throws IOException{
@@ -950,8 +855,7 @@ public class HConnectionManager {
      * Delete a cached location, if it satisfies the table name and row
      * requirements.
      */
-    void deleteCachedLocation(final byte [] tableName,
-                                      final byte [] row) {
+    void deleteCachedLocation(final byte [] tableName, final byte [] row) {
       synchronized (this.cachedRegionLocations) {
         SoftValueSortedMap<byte [], HRegionLocation> tableLocations =
             getTableLocations(tableName);
@@ -998,7 +902,7 @@ public class HConnectionManager {
      * Allows flushing the region cache.
      */
     public void clearRegionCache() {
-      cachedRegionLocations.clear();
+     cachedRegionLocations.clear();
     }
 
     /*
@@ -1033,6 +937,7 @@ public class HConnectionManager {
                 regionServer.getInetSocketAddress(), this.conf,
                 this.maxRPCAttempts, this.rpcTimeout);
           } catch (RemoteException e) {
+            LOG.warn("Remove exception connecting to RS", e);
             throw RemoteExceptionHandler.decodeRemoteException(e);
           }
           this.servers.put(regionServer.toString(), server);
@@ -1047,13 +952,27 @@ public class HConnectionManager {
       return getHRegionConnection(regionServer, false);
     }
 
-    public synchronized ZooKeeperWrapper getZooKeeperWrapper()
-        throws IOException {
-      return HConnectionManager.getClientZooKeeperWatcher(conf)
-          .getZooKeeperWrapper();
+    /**
+     * Get the ZooKeeper instance for this TableServers instance.
+     *
+     * If ZK has not been initialized yet, this will connect to ZK.
+     * @returns zookeeper reference
+     * @throws ZooKeeperConncetionException if there's a problem connecting to zk
+     */
+    public synchronized ZooKeeperWatcher getZooKeeperWatcher()
+        throws ZooKeeperConnectionException {
+      if(zooKeeper == null) {
+        try {
+          zooKeeper = new ZooKeeperWatcher(conf,
+              ZKUtil.getZooKeeperClusterKey(conf), this);
+        } catch (IOException e) {
+          throw new ZooKeeperConnectionException(e);
+        }
+      }
+      return zooKeeper;
     }
 
-    /*
+    /**
      * Repeatedly try to find the root region in ZK
      * @return HRegionLocation for root region if found
      * @throws NoServerForRegionException - if the root region can not be
@@ -1065,7 +984,12 @@ public class HConnectionManager {
 
       // We lazily instantiate the ZooKeeper object because we don't want to
       // make the constructor have to throw IOException or handle it itself.
-      ZooKeeperWrapper zk = getZooKeeperWrapper();
+      ZooKeeperWatcher zk;
+      try {
+        zk = getZooKeeperWatcher();
+      } catch (IOException e) {
+        throw new ZooKeeperConnectionException(e);
+      }
 
       HServerAddress rootRegionAddress = null;
       for (int tries = 0; tries < numRetries; tries++) {
@@ -1074,7 +998,13 @@ public class HConnectionManager {
         while (rootRegionAddress == null && localTimeouts < numRetries) {
           // Don't read root region until we're out of safe mode so we know
           // that the meta regions have been assigned.
-          rootRegionAddress = zk.readRootRegionLocation();
+          try {
+            rootRegionAddress = ZKUtil.getDataAsAddress(zk, zk.rootServerZNode);
+          } catch (KeeperException e) {
+            LOG.error("Unexpected ZooKeeper error attempting to read the root " +
+                "region server address");
+            throw new IOException(e);
+          }
           if (rootRegionAddress == null) {
             try {
               if (LOG.isDebugEnabled()) {
@@ -1214,39 +1144,177 @@ public class HConnectionManager {
       return location;
     }
 
-    /**
-     * @deprecated Use HConnectionManager::processBatch instead.
+    /*
+     * Helper class for batch updates.
+     * Holds code shared doing batch puts and batch deletes.
      */
-    public int processBatchOfRows(final ArrayList<Put> list, final byte[] tableName, ExecutorService pool)
-    throws IOException {
-      Result[] results = new Result[list.size()];
-      processBatch((List) list, tableName, pool, results);
-      int count = 0;
-      for (Result r : results) {
-        if (r != null) {
-          count++;
-        }
+    private abstract class Batch {
+      final HConnection c;
+
+      private Batch(final HConnection c) {
+        this.c = c;
+      }
+
+      /**
+       * This is the method subclasses must implement.
+       * @param currentList current list of rows
+       * @param tableName table we are processing
+       * @param row row
+       * @return Count of items processed or -1 if all.
+       * @throws IOException if a remote or network exception occurs
+       * @throws RuntimeException other undefined exception
+       */
+      abstract int doCall(final List<? extends Row> currentList,
+        final byte [] row, final byte [] tableName)
+      throws IOException, RuntimeException;
+
+      /**
+       * Process the passed <code>list</code>.
+       * @param list list of rows to process
+       * @param tableName table we are processing
+       * @return Count of how many added or -1 if all added.
+       * @throws IOException if a remote or network exception occurs
+       */
+      int process(final List<? extends Row> list, final byte[] tableName)
+      throws IOException {
+        byte [] region = getRegionName(tableName, list.get(0).getRow(), false);
+        byte [] currentRegion = region;
+        boolean isLastRow;
+        boolean retryOnlyOne = false;
+        List<Row> currentList = new ArrayList<Row>();
+        int i, tries;
+        for (i = 0, tries = 0; i < list.size() && tries < numRetries; i++) {
+          Row row = list.get(i);
+          currentList.add(row);
+          // If the next record goes to a new region, then we are to clear
+          // currentList now during this cycle.
+          isLastRow = (i + 1) == list.size();
+          if (!isLastRow) {
+            region = getRegionName(tableName, list.get(i + 1).getRow(), false);
+          }
+          if (!Bytes.equals(currentRegion, region) || isLastRow || retryOnlyOne) {
+            int index = doCall(currentList, row.getRow(), tableName);
+            // index is == -1 if all processed successfully, else its index
+            // of last record successfully processed.
+            if (index != -1) {
+              if (tries == numRetries - 1) {
+                throw new RetriesExhaustedException("Some server, retryOnlyOne=" +
+                  retryOnlyOne + ", index=" + index + ", islastrow=" + isLastRow +
+                  ", tries=" + tries + ", numtries=" + numRetries + ", i=" + i +
+                  ", listsize=" + list.size() + ", region=" +
+                  Bytes.toStringBinary(region), currentRegion, row.getRow(),
+                  tries, new ArrayList<Throwable>());
+              }
+              tries = doBatchPause(currentRegion, tries);
+              i = i - currentList.size() + index;
+              retryOnlyOne = true;
+              // Reload location.
+              region = getRegionName(tableName, list.get(i + 1).getRow(), true);
+            } else {
+              // Reset these flags/counters on successful batch Put
+              retryOnlyOne = false;
+              tries = 0;
+            }
+            currentRegion = region;
+            currentList.clear();
+          }
+        }
+        return i;
+      }
+
+      /*
+       * @param t
+       * @param r
+       * @param re
+       * @return Region name that holds passed row <code>r</code>
+       * @throws IOException
+       */
+      private byte [] getRegionName(final byte [] t, final byte [] r,
+        final boolean re)
+      throws IOException {
+        HRegionLocation location = getRegionLocationForRowWithRetries(t, r, re);
+        return location.getRegionInfo().getRegionName();
+      }
+
+      /*
+       * Do pause processing before retrying...
+       * @param currentRegion
+       * @param tries
+       * @return New value for tries.
+       */
+      private int doBatchPause(final byte [] currentRegion, final int tries) {
+        int localTries = tries;
+        long sleepTime = getPauseTime(tries);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Reloading region " + Bytes.toStringBinary(currentRegion) +
+            " location because regionserver didn't accept updates; tries=" +
+            tries + " of max=" + numRetries + ", waiting=" + sleepTime + "ms");
+        }
+        try {
+          Thread.sleep(sleepTime);
+          localTries++;
+        } catch (InterruptedException e) {
+          // continue
+        }
+        return localTries;
       }
-      return (count == list.size() ? -1 : count);
     }
 
-    /**
-     * @deprecated Use HConnectionManager::processBatch instead.
-     */
-    public int processBatchOfDeletes(final List<Delete> list,
-      final byte[] tableName, ExecutorService pool)
+    public int processBatchOfRows(final ArrayList<Put> list,
+      final byte[] tableName)
     throws IOException {
-      Result[] results = new Result[list.size()];
-      processBatch((List) list, tableName, pool, results);
-      int count = 0;
-      for (Result r : results) {
-        if (r != null) {
-          count++;
-        }
+      if (list.isEmpty()) {
+        return 0;
       }
-      return (count == list.size() ? -1 : count);
+      if (list.size() > 1) {
+        Collections.sort(list);
+      }
+      Batch b = new Batch(this) {
+        @SuppressWarnings("unchecked")
+        @Override
+        int doCall(final List<? extends Row> currentList, final byte [] row,
+          final byte [] tableName)
+        throws IOException, RuntimeException {
+          final List<Put> puts = (List<Put>)currentList;
+          return getRegionServerWithRetries(new ServerCallable<Integer>(this.c,
+              tableName, row) {
+            public Integer call() throws IOException {
+              return server.put(location.getRegionInfo().getRegionName(), puts);
+            }
+          });
+        }
+      };
+      return b.process(list, tableName);
     }
 
+    public int processBatchOfDeletes(final List<Delete> list,
+      final byte[] tableName)
+    throws IOException {
+      if (list.isEmpty()) {
+        return 0;
+      }
+      if (list.size() > 1) {
+        Collections.sort(list);
+      }
+      Batch b = new Batch(this) {
+        @SuppressWarnings("unchecked")
+        @Override
+        int doCall(final List<? extends Row> currentList, final byte [] row,
+          final byte [] tableName)
+        throws IOException, RuntimeException {
+          final List<Delete> deletes = (List<Delete>)currentList;
+          return getRegionServerWithRetries(new ServerCallable<Integer>(this.c,
+                tableName, row) {
+              public Integer call() throws IOException {
+                return server.delete(location.getRegionInfo().getRegionName(),
+                  deletes);
+              }
+            });
+          }
+        };
+        return b.process(list, tableName);
+      }
+
     void close(boolean stopProxy) {
       if (master != null) {
         if (stopProxy) {
@@ -1262,133 +1330,108 @@ public class HConnectionManager {
       }
     }
 
-    private Callable<MultiResponse> createCallable(
-        final HServerAddress address,
-        final MultiAction multi,
-        final byte [] tableName) {
-  	  final HConnection connection = this;
-  	  return new Callable<MultiResponse>() {
-  	    public MultiResponse call() throws IOException {
-  	      return getRegionServerWithoutRetries(
-  	          new ServerCallable<MultiResponse>(connection, tableName, null) {
-  	            public MultiResponse call() throws IOException {
-  	              return server.multi(multi);
-  	            }
-  	            @Override
-  	            public void instantiateServer(boolean reload) throws IOException {
-  	              server = connection.getHRegionConnection(address);
-  	            }
-  	          }
-  	      );
-  	    }
-  	  };
-  	}
-
-    public void processBatch(List<Row> list,
-        final byte[] tableName,
-        ExecutorService pool,
-        Result[] results) throws IOException {
-
-      // results must be the same size as list
-      if (results.length != list.size()) {
-        throw new IllegalArgumentException("argument results must be the same size as argument list");
-      }
-
-      if (list.size() == 0) {
-        return;
-      }
-
-      List<Row> workingList = new ArrayList<Row>(list);
-      final boolean singletonList = (list.size() == 1);
-      boolean retry = true;
+    /**
+     * Process a batch of Puts on the given executor service.
+     *
+     * @param list the puts to make - successful puts will be removed.
+     * @param pool thread pool to execute requests on
+     *
+     * In the case of an exception, we take different actions depending on the
+     * situation:
+     *  - If the exception is a DoNotRetryException, we rethrow it and leave the
+     *    'list' parameter in an indeterminate state.
+     *  - If the 'list' parameter is a singleton, we directly throw the specific
+     *    exception for that put.
+     *  - Otherwise, we throw a generic exception indicating that an error occurred.
+     *    The 'list' parameter is mutated to contain those puts that did not succeed.
+     */
+    public void processBatchOfPuts(List<Put> list,
+                                   final byte[] tableName, ExecutorService pool) throws IOException {
+      boolean singletonList = list.size() == 1;
       Throwable singleRowCause = null;
+      for ( int tries = 0 ; tries < numRetries && !list.isEmpty(); ++tries) {
+        Collections.sort(list);
+        Map<HServerAddress, MultiPut> regionPuts =
+            new HashMap<HServerAddress, MultiPut>();
+        // step 1:
+        //  break up into regionserver-sized chunks and build the data structs
+        for ( Put put : list ) {
+          byte [] row = put.getRow();
 
-      for (int tries = 0; tries < numRetries && retry; ++tries) {
+          HRegionLocation loc = locateRegion(tableName, row, true);
+          HServerAddress address = loc.getServerAddress();
+          byte [] regionName = loc.getRegionInfo().getRegionName();
 
-        // sleep first, if this is a retry
-        if (tries >= 1) {
-          long sleepTime = getPauseTime(tries);
-          LOG.debug("Retry " +tries+ ", sleep for " +sleepTime+ "ms!");
-          try {
-            Thread.sleep(sleepTime);
-          } catch (InterruptedException ignore) {
-            LOG.debug("Interupted");
-            Thread.currentThread().interrupt();
-            break;
+          MultiPut mput = regionPuts.get(address);
+          if (mput == null) {
+            mput = new MultiPut(address);
+            regionPuts.put(address, mput);
           }
+          mput.add(regionName, put);
         }
 
-        // step 1: break up into regionserver-sized chunks and build the data structs
+        // step 2:
+        //  make the requests
+        // Discard the map, just use a list now, makes error recovery easier.
+        List<MultiPut> multiPuts = new ArrayList<MultiPut>(regionPuts.values());
 
-        Map<HServerAddress, MultiAction> actionsByServer = new HashMap<HServerAddress, MultiAction>();
-        for (int i=0; i<workingList.size(); i++) {
-          Row row = workingList.get(i);
-          if (row != null) {
-            HRegionLocation loc = locateRegion(tableName, row.getRow(), true);
-            HServerAddress address = loc.getServerAddress();
-            byte[] regionName = loc.getRegionInfo().getRegionName();
-
-            MultiAction actions = actionsByServer.get(address);
-            if (actions == null) {
-              actions = new MultiAction();
-              actionsByServer.put(address, actions);
-            }
-
-            Action action = new Action(regionName, row, i);
-            actions.add(regionName, action);
-          }
+        List<Future<MultiPutResponse>> futures =
+            new ArrayList<Future<MultiPutResponse>>(regionPuts.size());
+        for ( MultiPut put : multiPuts ) {
+          futures.add(pool.submit(createPutCallable(put.address,
+              put,
+              tableName)));
         }
+        // RUN!
+        List<Put> failed = new ArrayList<Put>();
 
-        // step 2: make the requests
-
-        Map<HServerAddress,Future<MultiResponse>> futures =
-            new HashMap<HServerAddress, Future<MultiResponse>>(actionsByServer.size());
-
-        for (Entry<HServerAddress, MultiAction> e : actionsByServer.entrySet()) {
-          futures.put(e.getKey(), pool.submit(createCallable(e.getKey(), e.getValue(), tableName)));
-        }
-
-        // step 3: collect the failures and successes and prepare for retry
-
-        for (Entry<HServerAddress, Future<MultiResponse>> responsePerServer : futures.entrySet()) {
-          HServerAddress address = responsePerServer.getKey();
-
+        // step 3:
+        //  collect the failures and tries from step 1.
+        for (int i = 0; i < futures.size(); i++ ) {
+          Future<MultiPutResponse> future = futures.get(i);
+          MultiPut request = multiPuts.get(i);
           try {
-            // Gather the results for one server
-            Future<MultiResponse> future = responsePerServer.getValue();
+            MultiPutResponse resp = future.get();
 
-            // Not really sure what a reasonable timeout value is. Here's a first try.
-
-            MultiResponse resp = future.get();
-
-            if (resp == null) {
-              // Entire server failed
-              LOG.debug("Failed all for server: " + address + ", removing from cache");
-            } else {
-              // For each region
-              for (Entry<byte[], List<Pair<Integer,Result>>> e : resp.getResults().entrySet()) {
-                byte[] regionName = e.getKey();
-                List<Pair<Integer, Result>> regionResults = e.getValue();
-                for (int i = 0; i < regionResults.size(); i++) {
-                  Pair<Integer, Result> regionResult = regionResults.get(i);
-                  if (regionResult.getSecond() == null) {
-                    // failed
-                    LOG.debug("Failures for region: " + Bytes.toStringBinary(regionName) + ", removing from cache");
-                  } else {
-                    // success
-                    results[regionResult.getFirst()] = regionResult.getSecond();
-                  }
-                }
+            // For each region
+            for (Map.Entry<byte[], List<Put>> e : request.puts.entrySet()) {
+              Integer result = resp.getAnswer(e.getKey());
+              if (result == null) {
+                // failed
+                LOG.debug("Failed all for region: " +
+                    Bytes.toStringBinary(e.getKey()) + ", removing from cache");
+                failed.addAll(e.getValue());
+              } else if (result >= 0) {
+                // some failures
+                List<Put> lst = e.getValue();
+                failed.addAll(lst.subList(result, lst.size()));
+                LOG.debug("Failed past " + result + " for region: " +
+                    Bytes.toStringBinary(e.getKey()) + ", removing from cache");
               }
             }
           } catch (InterruptedException e) {
-            LOG.debug("Failed all from " + address, e);
-            Thread.currentThread().interrupt();
-            break;
+            // go into the failed list.
+            LOG.debug("Failed all from " + request.address, e);
+            failed.addAll(request.allPuts());
           } catch (ExecutionException e) {
-            LOG.debug("Failed all from " + address, e);
+            Throwable cause = e.getCause();
+            // Don't print stack trace if NSRE; NSRE is 'normal' operation.
+            if (cause instanceof NotServingRegionException) {
+              String msg = cause.getMessage();
+              if (msg != null && msg.length() > 0) {
+                // msg is the exception as a String... we just want first line.
+                msg = msg.split("[\\n\\r]+\\s*at")[0];
+              }
+              LOG.debug("Failed execution of all on " + request.address +
+                " because: " + msg);
+            } else {
+              // all go into the failed list.
+              LOG.debug("Failed execution of all on " + request.address,
+                e.getCause());
+            }
+            failed.addAll(request.allPuts());
 
-            // Just give up, leaving the batch incomplete
+            // Just give up, leaving the batch put list in an untouched/semi-committed state
             if (e.getCause() instanceof DoNotRetryIOException) {
               throw (DoNotRetryIOException) e.getCause();
             }
@@ -1399,57 +1442,56 @@ public class HConnectionManager {
             }
           }
         }
+        list.clear();
+        if (!failed.isEmpty()) {
+          for (Put failedPut: failed) {
+            deleteCachedLocation(tableName, failedPut.getRow());
+          }
 
-        // Find failures (i.e. null Result), and add them to the workingList (in
-        // order), so they can be retried.
-        retry = false;
-        workingList.clear();
-        for (int i = 0; i < results.length; i++) {
-          if (results[i] == null) {
-            retry = true;
-            Row row = list.get(i);
-            workingList.add(row);
-            deleteCachedLocation(tableName, row.getRow());
-          } else {
-            // add null to workingList, so the order remains consistent with the original list argument.
-            workingList.add(null);
+          list.addAll(failed);
+
+          long sleepTime = getPauseTime(tries);
+          LOG.debug("processBatchOfPuts had some failures, sleeping for " + sleepTime +
+              " ms!");
+          try {
+            Thread.sleep(sleepTime);
+          } catch (InterruptedException ignored) {
           }
         }
       }
-
-      if (Thread.currentThread().isInterrupted()) {
-        throw new IOException("Aborting attempt because of a thread interruption");
-      }
-
-      if (retry) {
-        // ran out of retries and didn't successfully finish everything!
-        if (singleRowCause != null) {
+      if (!list.isEmpty()) {
+        if (singletonList && singleRowCause != null) {
           throw new IOException(singleRowCause);
-        } else {
-          throw new RetriesExhaustedException("Still had " + workingList.size()
-              + " actions left after retrying " + numRetries + " times.");
         }
+
+        // ran out of retries and didnt succeed everything!
+        throw new RetriesExhaustedException("Still had " + list.size() + " puts left after retrying " +
+            numRetries + " times.");
       }
     }
 
-    /**
-     * @deprecated Use HConnectionManager::processBatch instead.
-     */
-    public void processBatchOfPuts(List<Put> list,
-        final byte[] tableName,
-        ExecutorService pool) throws IOException {
-      Result[] results = new Result[list.size()];
-      processBatch((List) list, tableName, pool, results);
 
-      // mutate list so that it is empty for complete success, or contains only failed records
-      // results are returned in the same order as the requests in list
-      // walk the list backwards, so we can remove from list without impacting the indexes of earlier members
-      for (int i = results.length - 1; i>=0; i--) {
-        // if result is not null, it succeeded
-        if (results[i] != null) {
-          list.remove(i);
+    private Callable<MultiPutResponse> createPutCallable(
+        final HServerAddress address, final MultiPut puts,
+        final byte [] tableName) {
+      final HConnection connection = this;
+      return new Callable<MultiPutResponse>() {
+        public MultiPutResponse call() throws IOException {
+          return getRegionServerWithoutRetries(
+              new ServerCallable<MultiPutResponse>(connection, tableName, null) {
+                public MultiPutResponse call() throws IOException {
+                  MultiPutResponse resp = server.multiPut(puts);
+                  resp.request = puts;
+                  return resp;
+                }
+                @Override
+                public void instantiateServer(boolean reload) throws IOException {
+                  server = connection.getHRegionConnection(address);
+                }
+              }
+          );
         }
-      }
+      };
     }
 
     private Throwable translateException(Throwable t) throws IOException {
@@ -1515,5 +1557,15 @@ public class HConnectionManager {
             new HRegionLocation(e.getKey(), e.getValue()));
       }
     }
+
+    @Override
+    public void abort(final String msg, Throwable t) {
+      if (t != null) LOG.fatal(msg, t);
+      else LOG.fatal(msg);
+      if(zooKeeper != null) {
+        zooKeeper.close();
+        zooKeeper = null;
+      }
+    }
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 5b1bbd978de..faf3286c859 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.UnknownScannerException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
@@ -45,7 +46,6 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadFactory;
@@ -77,7 +77,7 @@ public class HTable implements HTableInterface {
   private long currentWriteBufferSize;
   protected int scannerCaching;
   private int maxKeyValueSize;
-  private ExecutorService pool;  // For Multi
+
   private long maxScannerResultSize;
 
   /**
@@ -144,11 +144,13 @@ public class HTable implements HTableInterface {
       HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
       HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
     this.maxKeyValueSize = conf.getInt("hbase.client.keyvalue.maxsize", -1);
-    
-    int nrThreads = conf.getInt("hbase.htable.threads.max", getCurrentNrHRS());
-    if (nrThreads == 0) {
-      nrThreads = 1; // is there a better default?
+
+    int nrHRS = getCurrentNrHRS();
+    if (nrHRS == 0) {
+      // No servers running -- set default of 10 threads.
+      nrHRS = 10;
     }
+    int nrThreads = conf.getInt("hbase.htable.threads.max", nrHRS);
 
     // Unfortunately Executors.newCachedThreadPool does not allow us to
     // set the maximum size of the pool, so we have to do it ourselves.
@@ -169,12 +171,13 @@ public class HTable implements HTableInterface {
    * @throws IOException if a remote or network exception occurs
    */
   int getCurrentNrHRS() throws IOException {
-    return HConnectionManager
-      .getClientZooKeeperWatcher(this.configuration)
-      .getZooKeeperWrapper()
-      .getRSDirectoryCount();
+    HBaseAdmin admin = new HBaseAdmin(this.configuration);
+    return admin.getClusterStatus().getServers();
   }
 
+  // For multiput
+  private ExecutorService pool;
+
   /**
    * Tells whether or not a table is enabled or not.
    * @param tableName Name of table to check.
@@ -505,40 +508,6 @@ public class HTable implements HTableInterface {
     );
   }
 
-  /**
-   * Method that does a batch call on Deletes, Gets and Puts.
-   *
-   * @param actions list of Get, Put, Delete objects
-   * @param results Empty Result[], same size as actions. Provides access to partial
-   * results, in case an exception is thrown. A null in the result array means that
-   * the call for that action failed, even after retries
-   * @throws IOException
-   */
-  public synchronized void batch(final List<Row> actions, final Result[] results) throws IOException {
-    connection.processBatch(actions, tableName, pool, results);
-  }
-
-  /**
-   * Method that does a batch call on Deletes, Gets and Puts.
-   * 
-   * @param actions list of Get, Put, Delete objects
-   * @return the results from the actions. A null in the return array means that
-   * the call for that action failed, even after retries
-   * @throws IOException
-   */
-  public synchronized Result[] batch(final List<Row> actions) throws IOException {
-    Result[] results = new Result[actions.size()];
-    connection.processBatch(actions, tableName, pool, results);
-    return results;
-  }
-
-  /**
-   * Deletes the specified cells/row.
-   * 
-   * @param delete The object that specifies what to delete.
-   * @throws IOException if a remote or network exception occurs.
-   * @since 0.20.0
-   */
   public void delete(final Delete delete)
   throws IOException {
     connection.getRegionServerWithRetries(
@@ -551,28 +520,13 @@ public class HTable implements HTableInterface {
     );
   }
 
-  /**
-   * Deletes the specified cells/rows in bulk.
-   * @param deletes List of things to delete. As a side effect, it will be modified:
-   * successful {@link Delete}s are removed. The ordering of the list will not change. 
-   * @throws IOException if a remote or network exception occurs. In that case
-   * the {@code deletes} argument will contain the {@link Delete} instances
-   * that have not be successfully applied.
-   * @since 0.20.1
-   */
   public void delete(final List<Delete> deletes)
   throws IOException {
-    Result[] results = new Result[deletes.size()];
-    connection.processBatch((List) deletes, tableName, pool, results);
-
-    // mutate list so that it is empty for complete success, or contains only failed records
-    // results are returned in the same order as the requests in list
-    // walk the list backwards, so we can remove from list without impacting the indexes of earlier members
-    for (int i = results.length - 1; i>=0; i--) {
-      // if result is not null, it succeeded
-      if (results[i] != null) {
-        deletes.remove(i);
-      }
+    int last = 0;
+    try {
+      last = connection.processBatchOfDeletes(deletes, this.tableName);
+    } finally {
+      deletes.subList(0, last).clear();
     }
   }
 
@@ -601,7 +555,6 @@ public class HTable implements HTableInterface {
     return incrementColumnValue(row, family, qualifier, amount, true);
   }
 
-  @SuppressWarnings({"ThrowableInstanceNeverThrown"})
   public long incrementColumnValue(final byte [] row, final byte [] family,
       final byte [] qualifier, final long amount, final boolean writeToWAL)
   throws IOException {
@@ -675,7 +628,7 @@ public class HTable implements HTableInterface {
           public Boolean call() throws IOException {
             return server.checkAndDelete(
                 location.getRegionInfo().getRegionName(),
-                row, family, qualifier, value, delete) 
+                row, family, qualifier, value, delete)
             ? Boolean.TRUE : Boolean.FALSE;
           }
         }
@@ -704,17 +657,10 @@ public class HTable implements HTableInterface {
     );
   }
 
-  /**
-   * Executes all the buffered {@link Put} operations.
-   * <p>
-   * This method gets called once automatically for every {@link Put} or batch
-   * of {@link Put}s (when {@link #batch(List)} is used) when
-   * {@link #isAutoFlush()} is {@code true}.
-   * @throws IOException if a remote or network exception occurs.
-   */
   public void flushCommits() throws IOException {
     try {
-      connection.processBatchOfPuts(writeBuffer, tableName, pool);
+      connection.processBatchOfPuts(writeBuffer,
+          tableName, pool);
     } finally {
       // the write buffer was adjusted by processBatchOfPuts
       currentWriteBufferSize = 0;
@@ -1153,10 +1099,12 @@ public class HTable implements HTableInterface {
             Thread t = new Thread(group, r,
                                   namePrefix + threadNumber.getAndIncrement(),
                                   0);
-            if (!t.isDaemon())
-                t.setDaemon(true);
-            if (t.getPriority() != Thread.NORM_PRIORITY)
-                t.setPriority(Thread.NORM_PRIORITY);
+            if (!t.isDaemon()) {
+              t.setDaemon(true);
+            }
+            if (t.getPriority() != Thread.NORM_PRIORITY) {
+              t.setPriority(Thread.NORM_PRIORITY);
+            }
             return t;
         }
   }
@@ -1168,9 +1116,10 @@ public class HTable implements HTableInterface {
    * @param tableName name of table to configure.
    * @param enable Set to true to enable region cache prefetch. Or set to
    * false to disable it.
+   * @throws ZooKeeperConnectionException
    */
   public static void setRegionCachePrefetch(final byte[] tableName,
-      boolean enable) {
+      boolean enable) throws ZooKeeperConnectionException {
     HConnectionManager.getConnection(HBaseConfiguration.create()).
     setRegionCachePrefetch(tableName, enable);
   }
@@ -1183,9 +1132,10 @@ public class HTable implements HTableInterface {
    * @param tableName name of table to configure.
    * @param enable Set to true to enable region cache prefetch. Or set to
    * false to disable it.
+   * @throws ZooKeeperConnectionException
    */
   public static void setRegionCachePrefetch(final Configuration conf,
-      final byte[] tableName, boolean enable) {
+      final byte[] tableName, boolean enable) throws ZooKeeperConnectionException {
     HConnectionManager.getConnection(conf).setRegionCachePrefetch(
         tableName, enable);
   }
@@ -1196,9 +1146,10 @@ public class HTable implements HTableInterface {
    * @param tableName name of table to check
    * @return true if table's region cache prefecth is enabled. Otherwise
    * it is disabled.
+   * @throws ZooKeeperConnectionException
    */
   public static boolean getRegionCachePrefetch(final Configuration conf,
-      final byte[] tableName) {
+      final byte[] tableName) throws ZooKeeperConnectionException {
     return HConnectionManager.getConnection(conf).getRegionCachePrefetch(
         tableName);
   }
@@ -1208,8 +1159,9 @@ public class HTable implements HTableInterface {
    * @param tableName name of table to check
    * @return true if table's region cache prefecth is enabled. Otherwise
    * it is disabled.
+   * @throws ZooKeeperConnectionException
    */
-  public static boolean getRegionCachePrefetch(final byte[] tableName) {
+  public static boolean getRegionCachePrefetch(final byte[] tableName) throws ZooKeeperConnectionException {
     return HConnectionManager.getConnection(HBaseConfiguration.create()).
     getRegionCachePrefetch(tableName);
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
index 5f8b9092874..0306cd4fc00 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
@@ -20,6 +20,10 @@
 
 package org.apache.hadoop.hbase.client;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -27,8 +31,6 @@ import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Writables;
 
-import java.io.IOException;
-
 /**
  * Scanner class that contains the <code>.META.</code> table scanning logic
  * and uses a Retryable scanner. Provided visitors will be called
@@ -38,7 +40,6 @@ import java.io.IOException;
  * minor releases.
  */
 public class MetaScanner {
-
   /**
    * Scans the meta table and calls a visitor on each RowResult and uses a empty
    * start row value as table name.
@@ -50,7 +51,7 @@ public class MetaScanner {
   public static void metaScan(Configuration configuration,
       MetaScannerVisitor visitor)
   throws IOException {
-    metaScan(configuration, visitor, HConstants.EMPTY_START_ROW);
+    metaScan(configuration, visitor, null);
   }
 
   /**
@@ -169,6 +170,32 @@ public class MetaScanner {
     } while (Bytes.compareTo(startRow, HConstants.LAST_ROW) != 0);
   }
 
+  /**
+   * Lists all of the regions currently in META.
+   * @return
+   * @throws IOException
+   */
+  public static List<HRegionInfo> listAllRegions(Configuration conf)
+  throws IOException {
+    final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
+    MetaScannerVisitor visitor =
+      new MetaScannerVisitor() {
+        @Override
+        public boolean processRow(Result result) throws IOException {
+          if (result == null || result.isEmpty()) {
+            return true;
+          }
+          HRegionInfo regionInfo = Writables.getHRegionInfo(
+              result.getValue(HConstants.CATALOG_FAMILY,
+                  HConstants.REGIONINFO_QUALIFIER));
+          regions.add(regionInfo);
+          return true;
+        }
+    };
+    metaScan(conf, visitor);
+    return regions;
+  }
+
   /**
    * Visitor class called to process each row of the .META. table
    */
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MultiPut.java b/src/main/java/org/apache/hadoop/hbase/client/MultiPut.java
index 058fe5340f7..8ffd1006f19 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MultiPut.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MultiPut.java
@@ -34,7 +34,6 @@ import java.util.Map;
 import java.util.TreeMap;
 
 /**
- * @deprecated Use MultiAction instead
  * Data type class for putting multiple regions worth of puts in one RPC.
  */
 public class MultiPut implements Writable {
diff --git a/src/main/java/org/apache/hadoop/hbase/client/MultiPutResponse.java b/src/main/java/org/apache/hadoop/hbase/client/MultiPutResponse.java
index 7e0311a04f2..0b7a6c690cc 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/MultiPutResponse.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/MultiPutResponse.java
@@ -30,7 +30,6 @@ import java.util.Map;
 import java.util.TreeMap;
 
 /**
- * @deprecated Replaced by MultiResponse
  * Response class for MultiPut.
  */
 public class MultiPutResponse implements Writable {
diff --git a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
index b7cfc7880ff..89d2abe6d98 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
@@ -31,6 +31,10 @@ public class RetriesExhaustedException extends IOException {
     super(msg);
   }
 
+  public RetriesExhaustedException(final String msg, final IOException e) {
+    super(msg, e);
+  }
+
   /**
    * Create a new RetriesExhaustedException from the list of prior failures.
    * @param serverName name of HRegionServer
diff --git a/src/main/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java
index dbb7f57e29c..61174aa5f25 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java
@@ -21,6 +21,7 @@
 package org.apache.hadoop.hbase.client;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 
 
 /**
@@ -38,8 +39,9 @@ public class ServerConnectionManager extends HConnectionManager {
    * If no current connection exists, create a new connection for that instance
    * @param conf configuration
    * @return HConnection object for the instance specified by the configuration
+   * @throws ZooKeeperConnectionException
    */
-  public static ServerConnection getConnection(Configuration conf) {
+  public static ServerConnection getConnection(Configuration conf) throws ZooKeeperConnectionException {
     return (ServerConnection) HConnectionManager.getConnection(conf);
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
new file mode 100644
index 00000000000..73a2d9557d0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
@@ -0,0 +1,223 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.executor;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Server;
+
+
+/**
+ * Abstract base class for all HBase event handlers. Subclasses should
+ * implement the {@link #process()} method.  Subclasses should also do all
+ * necessary checks up in their constructor if possible -- check table exists,
+ * is disabled, etc. -- so they fail fast rather than later when process is
+ * running.  Do it this way because process be invoked directly but event
+ * handlers are also
+ * run in an executor context -- i.e. asynchronously -- and in this case,
+ * exceptions thrown at process time will not be seen by the invoker, not till
+ * we implement a call-back mechanism so the client can pick them up later.
+ * <p>
+ * Event handlers have an {@link EventType}.
+ * {@link EventType} is a list of ALL handler event types.  We need to keep
+ * a full list in one place -- and as enums is a good shorthand for an
+ * implemenations -- because event handlers can be passed to executors when
+ * they are to be run asynchronously. The
+ * hbase executor, see {@link ExecutorService}, has a switch for passing
+ * event type to executor.
+ * <p>
+ * Event listeners can be installed and will be called pre- and post- process if
+ * this EventHandler is run in a Thread (its a Runnable so if its {@link #run()}
+ * method gets called).  Implement
+ * {@link EventHandlerListener}s, and registering using
+ * {@link #setListener(EventHandlerListener)}.
+ * @see {@link ExecutorService}
+ */
+public abstract class EventHandler implements Runnable, Comparable<Runnable> {
+  private static final Log LOG = LogFactory.getLog(EventHandler.class);
+
+  // type of event this object represents
+  protected EventType eventType;
+
+  protected Server server;
+
+  // sequence id generator for default FIFO ordering of events
+  protected static AtomicLong seqids = new AtomicLong(0);
+
+  // sequence id for this event
+  private final long seqid;
+
+  // Listener to call pre- and post- processing.  May be null.
+  private EventHandlerListener listener;
+
+  /**
+   * This interface provides pre- and post-process hooks for events.
+   */
+  public interface EventHandlerListener {
+    /**
+     * Called before any event is processed
+     * @param The event handler whose process method is about to be called.
+     */
+    public void beforeProcess(EventHandler event);
+    /**
+     * Called after any event is processed
+     * @param The event handler whose process method is about to be called.
+     */
+    public void afterProcess(EventHandler event);
+  }
+
+  /**
+   * List of all HBase event handler types.  Event types are named by a
+   * convention: event type names specify the component from which the event
+   * originated and then where its destined -- e.g. RS2ZK_ prefix means the
+   * event came from a regionserver destined for zookeeper -- and then what
+   * the even is; e.g. REGION_OPENING.
+   * 
+   * <p>We give the enums indices so we can add types later and keep them
+   * grouped together rather than have to add them always to the end as we
+   * would have to if we used raw enum ordinals.
+   */
+  public enum EventType {
+    // Messages originating from RS (NOTE: there is NO direct communication from
+    // RS to Master). These are a result of RS updates into ZK.
+    RS2ZK_REGION_CLOSING      (1),   // RS is in process of closing a region
+    RS2ZK_REGION_CLOSED       (2),   // RS has finished closing a region
+    RS2ZK_REGION_OPENING      (3),   // RS is in process of opening a region
+    RS2ZK_REGION_OPENED       (4),   // RS has finished opening a region
+
+    // Messages originating from Master to RS
+    M2RS_OPEN_REGION          (20),  // Master asking RS to open a region
+    M2RS_OPEN_ROOT            (21),  // Master asking RS to open root
+    M2RS_OPEN_META            (22),  // Master asking RS to open meta
+    M2RS_CLOSE_REGION         (23),  // Master asking RS to close a region
+    M2RS_CLOSE_ROOT           (24),  // Master asking RS to close root
+    M2RS_CLOSE_META           (25),  // Master asking RS to close meta
+
+    // Messages originating from Client to Master
+    C2M_DELETE_TABLE          (40),   // Client asking Master to delete a table
+    C2M_DISABLE_TABLE         (41),   // Client asking Master to disable a table
+    C2M_ENABLE_TABLE          (42),   // Client asking Master to enable a table
+    C2M_MODIFY_TABLE          (43),   // Client asking Master to modify a table
+    C2M_ADD_FAMILY            (44),   // Client asking Master to add family to table
+    C2M_DELETE_FAMILY         (45),   // Client asking Master to delete family of table
+    C2M_MODIFY_FAMILY         (46),   // Client asking Master to modify family of table
+
+    // Updates from master to ZK. This is done by the master and there is
+    // nothing to process by either Master or RS
+    M2ZK_REGION_OFFLINE       (50),  // Master adds this region as offline in ZK
+
+    // Master controlled events to be executed on the master
+    M_SERVER_SHUTDOWN         (70);  // Master is processing shutdown of a RS
+
+    /**
+     * Constructor
+     */
+    EventType(int value) {}
+  }
+
+  /**
+   * Default base class constructor.
+   */
+  public EventHandler(Server server, EventType eventType) {
+    this.server = server;
+    this.eventType = eventType;
+    seqid = seqids.incrementAndGet();
+  }
+
+  public void run() {
+    try {
+      if (getListener() != null) getListener().beforeProcess(this);
+      process();
+      if (getListener() != null) getListener().afterProcess(this);
+    } catch(Throwable t) {
+      LOG.error("Caught throwable while processing event " + eventType, t);
+    }
+  }
+
+  /**
+   * This method is the main processing loop to be implemented by the various
+   * subclasses.
+   * @throws IOException
+   */
+  public abstract void process() throws IOException;
+
+  /**
+   * Return the event type
+   * @return
+   */
+  public EventType getEventType() {
+    return this.eventType;
+  }
+
+  /**
+   * Get the priority level for this handler instance.  This uses natural
+   * ordering so lower numbers are higher priority.
+   * <p>
+   * Lowest priority is Integer.MAX_VALUE.  Highest priority is 0.
+   * <p>
+   * Subclasses should override this method to allow prioritizing handlers.
+   * <p>
+   * Handlers with the same priority are handled in FIFO order.
+   * <p>
+   * @return Integer.MAX_VALUE by default, override to set higher priorities
+   */
+  public int getPriority() {
+    return Integer.MAX_VALUE;
+  }
+
+  /**
+   * @return This events' sequence id.
+   */
+  public long getSeqid() {
+    return this.seqid;
+  }
+
+  /**
+   * Default prioritized runnable comparator which implements a FIFO ordering.
+   * <p>
+   * Subclasses should not override this.  Instead, if they want to implement
+   * priority beyond FIFO, they should override {@link #getPriority()}.
+   */
+  @Override
+  public int compareTo(Runnable o) {
+    EventHandler eh = (EventHandler)o;
+    if(getPriority() != eh.getPriority()) {
+      return (getPriority() < eh.getPriority()) ? -1 : 1;
+    }
+    return (this.seqid < eh.seqid) ? -1 : 1;
+  }
+
+  /**
+   * @return Current listener or null if none set.
+   */
+  public synchronized EventHandlerListener getListener() {
+    return listener;
+  }
+
+  /**
+   * @param listener Listener to call pre- and post- {@link #process()}.
+   */
+  public synchronized void setListener(EventHandlerListener listener) {
+    this.listener = listener;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java b/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
new file mode 100644
index 00000000000..d8a20abb608
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
@@ -0,0 +1,285 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.executor;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * This is a generic executor service. This component abstracts a
+ * threadpool, a queue to which {@link EventHandler.EventType}s can be submitted,
+ * and a <code>Runnable</code> that handles the object that is added to the queue.
+ *
+ * <p>In order to create a new service, create an instance of this class and 
+ * then do: <code>instance.startExecutorService("myService");</code>.  When done
+ * call {@link #shutdown()}.
+ *
+ * <p>In order to use the service created above, call
+ * {@link #submit(EventHandler)}. Register pre- and post- processing listeners
+ * by registering your implementation of {@link EventHandler.EventHandlerListener}
+ * with {@link #registerListener(EventType, EventHandlerListener)}.  Be sure
+ * to deregister your listener when done via {@link #unregisterListener(EventType)}.
+ */
+public class ExecutorService {
+  private static final Log LOG = LogFactory.getLog(ExecutorService.class);
+
+  // hold the all the executors created in a map addressable by their names
+  private final ConcurrentHashMap<String, Executor> executorMap =
+    new ConcurrentHashMap<String, Executor>();
+
+  // listeners that are called before and after an event is processed
+  private ConcurrentHashMap<EventHandler.EventType, EventHandlerListener> eventHandlerListeners =
+    new ConcurrentHashMap<EventHandler.EventType, EventHandlerListener>();
+
+  // Name of the server hosting this executor service.
+  private final String servername;
+
+  /**
+   * The following is a list of all executor types, both those that run in the
+   * master and those that run in the regionserver.
+   */
+  public enum ExecutorType {
+
+    // Master executor services
+    MASTER_CLOSE_REGION        (1),
+    MASTER_OPEN_REGION         (2),
+    MASTER_SERVER_OPERATIONS   (3),
+    MASTER_TABLE_OPERATIONS    (4),
+    MASTER_RS_SHUTDOWN         (5),
+
+    // RegionServer executor services
+    RS_OPEN_REGION             (20),
+    RS_OPEN_ROOT               (21),
+    RS_OPEN_META               (22),
+    RS_CLOSE_REGION            (23),
+    RS_CLOSE_ROOT              (24),
+    RS_CLOSE_META              (25);
+
+    ExecutorType(int value) {}
+
+    /**
+     * @param serverName
+     * @return Conflation of the executor type and the passed servername.
+     */
+    String getExecutorName(String serverName) {
+      return this.toString() + "-" + serverName;
+    }
+  }
+
+  /**
+   * Returns the executor service type (the thread pool instance) for the
+   * passed event handler type.
+   * @param type EventHandler type.
+   */
+  public ExecutorType getExecutorServiceType(final EventHandler.EventType type) {
+    switch(type) {
+      // Master executor services
+
+      case RS2ZK_REGION_CLOSED:
+        return ExecutorType.MASTER_CLOSE_REGION;
+
+      case RS2ZK_REGION_OPENED:
+        return ExecutorType.MASTER_OPEN_REGION;
+
+      case M_SERVER_SHUTDOWN:
+        return ExecutorType.MASTER_SERVER_OPERATIONS;
+
+      case C2M_DELETE_TABLE:
+      case C2M_DISABLE_TABLE:
+      case C2M_ENABLE_TABLE:
+      case C2M_MODIFY_TABLE:
+        return ExecutorType.MASTER_TABLE_OPERATIONS;
+
+      // RegionServer executor services
+
+      case M2RS_OPEN_REGION:
+        return ExecutorType.RS_OPEN_REGION;
+
+      case M2RS_OPEN_ROOT:
+        return ExecutorType.RS_OPEN_ROOT;
+
+      case M2RS_OPEN_META:
+        return ExecutorType.RS_OPEN_META;
+
+      case M2RS_CLOSE_REGION:
+        return ExecutorType.RS_CLOSE_REGION;
+
+      case M2RS_CLOSE_ROOT:
+        return ExecutorType.RS_CLOSE_ROOT;
+
+      case M2RS_CLOSE_META:
+        return ExecutorType.RS_CLOSE_META;
+
+      default:
+        throw new RuntimeException("Unhandled event type " + type);
+    }
+  }
+
+  /**
+   * Default constructor.
+   * @param Name of the hosting server.
+   */
+  public ExecutorService(final String servername) {
+    super();
+    this.servername = servername;
+  }
+
+  /**
+   * Start an executor service with a given name. If there was a service already
+   * started with the same name, this throws a RuntimeException.
+   * @param name Name of the service to start.
+   */
+  void startExecutorService(String name, int maxThreads) {
+    if (this.executorMap.get(name) != null) {
+      throw new RuntimeException("An executor service with the name " + name +
+        " is already running!");
+    }
+    Executor hbes = new Executor(name, maxThreads, this.eventHandlerListeners);
+    if (this.executorMap.putIfAbsent(name, hbes) != null) {
+      throw new RuntimeException("An executor service with the name " + name +
+      " is already running (2)!");
+    }
+    LOG.debug("Starting executor service: " + name);
+  }
+
+  boolean isExecutorServiceRunning(String name) {
+    return this.executorMap.containsKey(name);
+  }
+
+  public void shutdown() {
+    for(Entry<String, Executor> entry: this.executorMap.entrySet()) {
+      List<Runnable> wasRunning =
+        entry.getValue().threadPoolExecutor.shutdownNow();
+      if (!wasRunning.isEmpty()) {
+        LOG.info(entry.getKey() + " had " + wasRunning + " on shutdown");
+      }
+    }
+    this.executorMap.clear();
+  }
+
+  Executor getExecutor(final ExecutorType type) {
+    return getExecutor(type.getExecutorName(this.servername));
+  }
+
+  Executor getExecutor(String name) {
+    Executor executor = this.executorMap.get(name);
+    if (executor == null) {
+      LOG.debug("Executor service [" + name + "] not found in " + this.executorMap);
+    }
+    return executor;
+  }
+
+
+  public void startExecutorService(final ExecutorType type, final int maxThreads) {
+    String name = type.getExecutorName(this.servername);
+    if (isExecutorServiceRunning(name)) {
+      LOG.debug("Executor service " + toString() + " already running on " +
+        this.servername);
+      return;
+    }
+    startExecutorService(name, maxThreads);
+  }
+
+  public void submit(final EventHandler eh) {
+    getExecutor(getExecutorServiceType(eh.getEventType())).submit(eh);
+  }
+
+  /**
+   * Subscribe to updates before and after processing instances of
+   * {@link EventHandler.EventType}.  Currently only one listener per
+   * event type.
+   * @param type Type of event we're registering listener for
+   * @param listener The listener to run.
+   * @return The <code>listener</code> that was passed
+   */
+  public void registerListener(final EventHandler.EventType type,
+      final EventHandlerListener listener) {
+    this.eventHandlerListeners.put(type, listener);
+  }
+
+  /**
+   * Stop receiving updates before and after processing instances of
+   * {@link EventHandler.EventType}
+   * @param type Type of event we're registering listener for
+   * @return The listener we removed or null if we did not remove it.
+   */
+  public EventHandlerListener unregisterListener(final EventHandler.EventType type) {
+    return this.eventHandlerListeners.remove(type);
+  }
+
+  /**
+   * Executor instance.
+   */
+  private static class Executor {
+    // default number of threads in the pool
+    private int corePoolSize = 1;
+    // how long to retain excess threads
+    private long keepAliveTimeInMillis = 1000;
+    // the thread pool executor that services the requests
+    private final ThreadPoolExecutor threadPoolExecutor;
+    // work queue to use - unbounded queue
+    BlockingQueue<Runnable> workQueue = new PriorityBlockingQueue<Runnable>();
+    private final AtomicInteger threadid = new AtomicInteger(0);
+    private final String name;
+    private final Map<EventHandler.EventType, EventHandlerListener> eventHandlerListeners;
+
+    protected Executor(String name, int maxThreads,
+        final Map<EventHandler.EventType, EventHandlerListener> eventHandlerListeners) {
+      this.name = name;
+      this.eventHandlerListeners = eventHandlerListeners;
+      // create the thread pool executor
+      this.threadPoolExecutor = new ThreadPoolExecutor(corePoolSize, maxThreads,
+          keepAliveTimeInMillis, TimeUnit.MILLISECONDS, workQueue);
+      // name the threads for this threadpool
+      ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
+      tfb.setNameFormat(this.name + "-" + this.threadid.incrementAndGet());
+      this.threadPoolExecutor.setThreadFactory(tfb.build());
+    }
+
+    /**
+     * Submit the event to the queue for handling.
+     * @param event
+     */
+    void submit(final EventHandler event) {
+      // If there is a listener for this type, make sure we call the before
+      // and after process methods.
+      EventHandlerListener listener =
+        this.eventHandlerListeners.get(event.getEventType());
+      if (listener != null) {
+        event.setListener(listener);
+      }
+      this.threadPoolExecutor.execute(event);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java b/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java
deleted file mode 100644
index d4f4318afe0..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java
+++ /dev/null
@@ -1,278 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.executor;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.executor.HBaseExecutorService.HBaseExecutorServiceType;
-import org.apache.hadoop.hbase.master.ServerManager;
-
-
-/**
- * Abstract base class for all HBase event handlers. Subclasses should 
- * implement the process() method where the actual handling of the event 
- * happens.
- * 
- * HBaseEventType is a list of ALL events (which also corresponds to messages - 
- * either internal to one component or between components). The event type 
- * names specify the component from which the event originated, and the 
- * component which is supposed to handle it.
- * 
- * Listeners can listen to all the events by implementing the interface 
- * HBaseEventHandlerListener, and by registering themselves as a listener. They 
- * will be called back before and after the process of every event.
- * 
- * TODO: Rename HBaseEvent and HBaseEventType to EventHandler and EventType 
- * after ZK refactor as it currently would clash with EventType from ZK and 
- * make the code very confusing.
- */
-public abstract class HBaseEventHandler implements Runnable
-{
-  private static final Log LOG = LogFactory.getLog(HBaseEventHandler.class);
-  // type of event this object represents
-  protected HBaseEventType eventType = HBaseEventType.NONE;
-  // is this a region server or master?
-  protected boolean isRegionServer;
-  // name of the server - this is needed for naming executors in case of tests 
-  // where region servers may be co-located.
-  protected String serverName;
-  // listeners that are called before and after an event is processed
-  protected static List<HBaseEventHandlerListener> eventHandlerListeners = 
-    Collections.synchronizedList(new ArrayList<HBaseEventHandlerListener>());  
-
-  /**
-   * This interface provides hooks to listen to various events received by the 
-   * queue. A class implementing this can listen to the updates by calling 
-   * registerListener and stop receiving updates by calling unregisterListener
-   */
-  public interface HBaseEventHandlerListener {
-    /**
-     * Called before any event is processed
-     */
-    public void beforeProcess(HBaseEventHandler event);
-    /**
-     * Called after any event is processed
-     */
-    public void afterProcess(HBaseEventHandler event);
-  }
-
-  /**
-   * These are a list of HBase events that can be handled by the various
-   * HBaseExecutorService's. All the events are serialized as byte values.
-   */
-  public enum HBaseEventType {
-    NONE (-1),
-    // Messages originating from RS (NOTE: there is NO direct communication from 
-    // RS to Master). These are a result of RS updates into ZK.
-    RS2ZK_REGION_CLOSING      (1),   // RS is in process of closing a region
-    RS2ZK_REGION_CLOSED       (2),   // RS has finished closing a region
-    RS2ZK_REGION_OPENING      (3),   // RS is in process of opening a region
-    RS2ZK_REGION_OPENED       (4),   // RS has finished opening a region
-    
-    // Updates from master to ZK. This is done by the master and there is 
-    // nothing to process by either Master or RS
-    M2ZK_REGION_OFFLINE       (50);  // Master adds this region as offline in ZK
-    
-    private final byte value;
-    
-    /**
-     * Called by the HMaster. Returns a name of the executor service given an 
-     * event type. Every event type has en entry - if the event should not be 
-     * handled just add the NONE executor.
-     * @return name of the executor service
-     */
-    public HBaseExecutorServiceType getMasterExecutorForEvent() {
-      HBaseExecutorServiceType executorServiceType = null;
-      switch(this) {
-      
-      case RS2ZK_REGION_CLOSING:
-      case RS2ZK_REGION_CLOSED:
-        executorServiceType = HBaseExecutorServiceType.MASTER_CLOSEREGION;
-        break;
-
-      case RS2ZK_REGION_OPENING:
-      case RS2ZK_REGION_OPENED:
-        executorServiceType = HBaseExecutorServiceType.MASTER_OPENREGION;
-        break;
-        
-      case M2ZK_REGION_OFFLINE:
-        executorServiceType = HBaseExecutorServiceType.NONE;
-        break;
-        
-      default:
-        throw new RuntimeException("Unhandled event type in the master.");
-      }
-      
-      return executorServiceType;
-    }
-
-    /**
-     * Called by the RegionServer. Returns a name of the executor service given an 
-     * event type. Every event type has en entry - if the event should not be 
-     * handled just return a null executor name.
-     * @return name of the event service
-     */
-    public static String getRSExecutorForEvent(String serverName) {
-      throw new RuntimeException("Unsupported operation.");
-    }
-    
-    /**
-     * Start the executor service that handles the passed in event type. The 
-     * server that starts these event executor services wants to handle these 
-     * event types.
-     */
-    public void startMasterExecutorService(String serverName) {
-      HBaseExecutorServiceType serviceType = getMasterExecutorForEvent();
-      if(serviceType == HBaseExecutorServiceType.NONE) {
-        throw new RuntimeException("Event type " + toString() + " not handled on master.");
-      }
-      serviceType.startExecutorService(serverName);
-    }
-
-    public static void startRSExecutorService() {
-      
-    }
-
-    HBaseEventType(int intValue) {
-      this.value = (byte)intValue;
-    }
-    
-    public byte getByteValue() {
-      return value;
-    }
-
-    public static HBaseEventType fromByte(byte value) {
-      switch(value) {
-        case  -1: return HBaseEventType.NONE;
-        case  1 : return HBaseEventType.RS2ZK_REGION_CLOSING;
-        case  2 : return HBaseEventType.RS2ZK_REGION_CLOSED;
-        case  3 : return HBaseEventType.RS2ZK_REGION_OPENING;
-        case  4 : return HBaseEventType.RS2ZK_REGION_OPENED;
-        case  50: return HBaseEventType.M2ZK_REGION_OFFLINE;
-
-        default:
-          throw new RuntimeException("Invalid byte value for conversion to HBaseEventType");
-      }
-    }
-  }
-  
-  /**
-   * Default base class constructor.
-   * 
-   * TODO: isRegionServer and serverName will go away once we do the HMaster 
-   * refactor. We will end up passing a ServerStatus which should tell us both 
-   * the name and if it is a RS or master.
-   */
-  public HBaseEventHandler(boolean isRegionServer, String serverName, HBaseEventType eventType) {
-    this.isRegionServer = isRegionServer;
-    this.eventType = eventType;
-    this.serverName = serverName;
-  }
-  
-  /**
-   * This is a wrapper around process, used to update listeners before and after 
-   * events are processed. 
-   */
-  public void run() {
-    // fire all beforeProcess listeners
-    for(HBaseEventHandlerListener listener : eventHandlerListeners) {
-      listener.beforeProcess(this);
-    }
-    
-    // call the main process function
-    try {
-      process();
-    } catch(Throwable t) {
-      LOG.error("Caught throwable while processing event " + eventType, t);
-    }
-
-    // fire all afterProcess listeners
-    for(HBaseEventHandlerListener listener : eventHandlerListeners) {
-      LOG.debug("Firing " + listener.getClass().getName() + 
-                ".afterProcess event listener for event " + eventType);
-      listener.afterProcess(this);
-    }
-  }
-  
-  /**
-   * This method is the main processing loop to be implemented by the various 
-   * subclasses.
-   */
-  public abstract void process();
-  
-  /**
-   * Subscribe to updates before and after processing events
-   */
-  public static void registerListener(HBaseEventHandlerListener listener) {
-    eventHandlerListeners.add(listener);
-  }
-  
-  /**
-   * Stop receiving updates before and after processing events
-   */
-  public static void unregisterListener(HBaseEventHandlerListener listener) {
-    eventHandlerListeners.remove(listener);
-  }
-  
-  public boolean isRegionServer() {
-    return isRegionServer;
-  }
-
-  /**
-   * Return the name for this event type.
-   * @return
-   */
-  public HBaseExecutorServiceType getEventHandlerName() {
-    // TODO: check for isRegionServer here
-    return eventType.getMasterExecutorForEvent();
-  }
-  
-  /**
-   * Return the event type
-   * @return
-   */
-  public HBaseEventType getHBEvent() {
-    return eventType;
-  }
-
-  /**
-   * Submits this event object to the correct executor service. This is causes
-   * this object to get executed by the correct ExecutorService.
-   */
-  public void submit() {
-    HBaseExecutorServiceType serviceType = getEventHandlerName();
-    if(serviceType == null) {
-      throw new RuntimeException("Event " + eventType + " not handled on this server " + serverName);
-    }
-    serviceType.getExecutor(serverName).submit(this);
-  }
-  
-  /**
-   * Executes this event object in the caller's thread. This is a synchronous 
-   * way of executing the event.
-   */
-  public void execute() {
-    this.run();
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/HBaseExecutorService.java b/src/main/java/org/apache/hadoop/hbase/executor/HBaseExecutorService.java
deleted file mode 100644
index b5f8987bb47..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/executor/HBaseExecutorService.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.executor;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * This is a generic HBase executor service. This component abstract a
- * threadpool, a queue to which jobs can be submitted and a Runnable that
- * handles the object that is added to the queue.
- *
- * In order to create a new HBExecutorService, you need to do:
- *   HBExecutorService.startExecutorService("myService");
- *
- * In order to use the service created above, you need to override the
- * HBEventHandler class and create an event type that submits to this service.
- *
- */
-public class HBaseExecutorService
-{
-  private static final Log LOG = LogFactory.getLog(HBaseExecutorService.class);
-  // default number of threads in the pool
-  private int corePoolSize = 1;
-  // max number of threads - maximum concurrency
-  private int maximumPoolSize = 5;
-  // how long to retain excess threads
-  private long keepAliveTimeInMillis = 1000;
-  // the thread pool executor that services the requests
-  ThreadPoolExecutor threadPoolExecutor;
-  // work queue to use - unbounded queue
-  BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<Runnable>();
-  // name for this executor service
-  String name;
-  // hold the all the executors created in a map addressable by their names
-  static Map<String, HBaseExecutorService> executorServicesMap =
-    Collections.synchronizedMap(new HashMap<String, HBaseExecutorService>());
-
-  
-  /**
-   * The following is a list of names for the various executor services in both 
-   * the master and the region server.
-   */
-  public enum HBaseExecutorServiceType {
-    NONE                       (-1),
-    MASTER_CLOSEREGION         (1),
-    MASTER_OPENREGION          (2);
-    
-    private final int value;
-    
-    HBaseExecutorServiceType(int intValue) {
-      this.value = intValue;
-    }
-    
-    public void startExecutorService(String serverName) {
-      // if this is NONE then there is no executor to start
-      if(value == NONE.value) {
-        throw new RuntimeException("Cannot start NONE executor type.");
-      }
-      String name = getExecutorName(serverName);
-      if(HBaseExecutorService.isExecutorServiceRunning(name)) {
-        LOG.debug("Executor service " + toString() + " already running on " + serverName);
-        return;
-      }
-      HBaseExecutorService.startExecutorService(name);
-    }
-    
-    public HBaseExecutorService getExecutor(String serverName) {
-      // if this is NONE then there is no executor
-      if(value == NONE.value) {
-        return null;
-      }
-      return HBaseExecutorService.getExecutorService(getExecutorName(serverName));
-    }
-    
-    public String getExecutorName(String serverName) {
-      // if this is NONE then there is no executor
-      if(value == NONE.value) {
-        return null;
-      }
-      return (this.toString() + "-" + serverName);
-    }
-  }
-
-
-
-  /**
-   * Start an executor service with a given name. If there was a service already
-   * started with the same name, this throws a RuntimeException.
-   * @param name Name of the service to start.
-   */
-  public static void startExecutorService(String name) {
-    if(executorServicesMap.get(name) != null) {
-      throw new RuntimeException("An executor service with the name " + name + " is already running!");
-    }
-    HBaseExecutorService hbes = new HBaseExecutorService(name);
-    executorServicesMap.put(name, hbes);
-    LOG.debug("Starting executor service: " + name);
-  }
-  
-  public static boolean isExecutorServiceRunning(String name) {
-    return (executorServicesMap.containsKey(name));
-  }
-
-  /**
-   * This method is an accessor for all the HBExecutorServices running so far
-   * addressable by name. If there is no such service, then it returns null.
-   */
-  public static HBaseExecutorService getExecutorService(String name) {
-    HBaseExecutorService executor = executorServicesMap.get(name);
-    if(executor == null) {
-      LOG.debug("Executor service [" + name + "] not found.");
-    }
-    return executor;
-  }
-  
-  public static void shutdown() {
-    for(Entry<String, HBaseExecutorService> entry : executorServicesMap.entrySet()) {
-      entry.getValue().threadPoolExecutor.shutdown();
-    }
-    executorServicesMap.clear();
-  }
-
-  protected HBaseExecutorService(String name) {
-    this.name = name;
-    // create the thread pool executor
-    threadPoolExecutor = new ThreadPoolExecutor(
-                                corePoolSize,
-                                maximumPoolSize,
-                                keepAliveTimeInMillis,
-                                TimeUnit.MILLISECONDS,
-                                workQueue
-                                );
-    // name the threads for this threadpool
-    threadPoolExecutor.setThreadFactory(new NamedThreadFactory(name));
-  }
-
-  /**
-   * Submit the event to the queue for handling.
-   * @param event
-   */
-  public void submit(Runnable event) {
-    threadPoolExecutor.execute(event);
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java b/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java
new file mode 100644
index 00000000000..4e33ebad22d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java
@@ -0,0 +1,210 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.executor;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Data serialized into ZooKeeper for region transitions.
+ */
+public class RegionTransitionData implements Writable {
+  /**
+   * Type of transition event (offline, opening, opened, closing, closed).
+   * Required.
+   */
+  private EventType eventType;
+
+  /** Region being transitioned.  Required. */
+  private byte [] regionName;
+
+  /** Server event originated from.  Optional. */
+  private String serverName;
+
+  /** Time the event was created.  Required but automatically set. */
+  private long stamp;
+
+  /**
+   * Writable constructor.  Do not use directly.
+   */
+  public RegionTransitionData() {}
+
+  /**
+   * Construct data for a new region transition event with the specified event
+   * type and region name.
+   *
+   * <p>Used when the server name is not known (the master is setting it).  This
+   * happens during cluster startup or during failure scenarios.  When
+   * processing a failed regionserver, the master assigns the regions from that
+   * server to other servers though the region was never 'closed'.  During
+   * master failover, the new master may have regions stuck in transition
+   * without a destination so may have to set regions offline and generate a new
+   * assignment.
+   *
+   * <p>Since only the master uses this constructor, the type should always be
+   * {@link EventType#M2ZK_REGION_OFFLINE}.
+   *
+   * @param eventType type of event
+   * @param regionName name of region
+   */
+  public RegionTransitionData(EventType eventType, byte [] regionName) {
+    this(eventType, regionName, null);
+  }
+
+  /**
+   * Construct data for a new region transition event with the specified event
+   * type, region name, and server name.
+   *
+   * <p>Used when the server name is known (a regionserver is setting it).
+   *
+   * <p>Valid types for this constructor are {@link EventType#RS2ZK_REGION_CLOSING},
+   * {@link EventType#RS2ZK_REGION_CLOSED}, {@link EventType#RS2ZK_REGION_OPENING},
+   * and {@link EventType#RS2ZK_REGION_OPENED}.
+   *
+   * @param eventType type of event
+   * @param regionName name of region
+   * @param serverName name of server setting data
+   */
+  public RegionTransitionData(EventType eventType, byte [] regionName,
+      String serverName) {
+    this.eventType = eventType;
+    this.stamp = System.currentTimeMillis();
+    this.regionName = regionName;
+    this.serverName = serverName;
+  }
+
+  /**
+   * Gets the type of region transition event.
+   *
+   * <p>One of:
+   * <ul>
+   * <li>{@link EventType#M2ZK_REGION_OFFLINE}
+   * <li>{@link EventType#RS2ZK_REGION_CLOSING}
+   * <li>{@link EventType#RS2ZK_REGION_CLOSED}
+   * <li>{@link EventType#RS2ZK_REGION_OPENING}
+   * <li>{@link EventType#RS2ZK_REGION_OPENED}
+   * </ul>
+   * @return type of region transition event
+   */
+  public EventType getEventType() {
+    return eventType;
+  }
+
+  /**
+   * Gets the name of the region being transitioned.
+   *
+   * <p>Region name is required so this never returns null.
+   * @return region name
+   */
+  public byte [] getRegionName() {
+    return regionName;
+  }
+
+  /**
+   * Gets the server the event originated from.  If null, this event originated
+   * from the master.
+   *
+   * @return server name of originating regionserver, or null if from master
+   */
+  public String getServerName() {
+    return serverName;
+  }
+
+  /**
+   * Gets the timestamp when this event was created.
+   *
+   * @return stamp event was created
+   */
+  public long getStamp() {
+    return stamp;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    // the event type byte
+    eventType = EventType.values()[in.readShort()];
+    // the timestamp
+    stamp = in.readLong();
+    // the encoded name of the region being transitioned
+    regionName = Bytes.readByteArray(in);
+    // remaining fields are optional so prefixed with boolean
+    // the name of the regionserver sending the data
+    if(in.readBoolean()) {
+      serverName = in.readUTF();
+    } else {
+      serverName = null;
+    }
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeShort(eventType.ordinal());
+    out.writeLong(System.currentTimeMillis());
+    Bytes.writeByteArray(out, regionName);
+    // remaining fields are optional so prefixed with boolean
+    out.writeBoolean(serverName != null);
+    if(serverName != null) {
+      out.writeUTF(serverName);
+    }
+  }
+
+  /**
+   * Get the bytes for this instance.  Throws a {@link RuntimeException} if
+   * there is an error deserializing this instance because it represents a code
+   * bug.
+   * @return binary representation of this instance
+   */
+  public byte [] getBytes() {
+    try {
+      return Writables.getBytes(this);
+    } catch(IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Get an instance from bytes.  Throws a {@link RuntimeException} if
+   * there is an error serializing this instance from bytes because it
+   * represents a code bug.
+   * @param bytes binary representation of this instance
+   * @return instance of this class
+   */
+  public static RegionTransitionData fromBytes(byte [] bytes) {
+    try {
+      RegionTransitionData data = new RegionTransitionData();
+      Writables.getWritable(bytes, data);
+      return data;
+    } catch(IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "region=" + Bytes.toString(regionName) + ", server=" + serverName +
+      ", state=" + eventType;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionEventData.java b/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionEventData.java
deleted file mode 100644
index 1b39de9ccb9..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionEventData.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.executor;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
-import org.apache.hadoop.io.Writable;
-
-public class RegionTransitionEventData implements Writable {
-  private HBaseEventType hbEvent;
-  private String rsName;
-  private long timeStamp;
-  private HMsg hmsg;
-  
-  public RegionTransitionEventData() {
-  }
-
-  public RegionTransitionEventData(HBaseEventType hbEvent, String rsName) {
-    this(hbEvent, rsName, null);
-  }
-
-  public RegionTransitionEventData(HBaseEventType hbEvent, String rsName, HMsg hmsg) {
-    this.hbEvent = hbEvent;
-    this.rsName = rsName;
-    this.timeStamp = System.currentTimeMillis();
-    this.hmsg = hmsg;
-  }
-  
-  public HBaseEventType getHbEvent() {
-    return hbEvent;
-  }
-
-  public String getRsName() {
-    return rsName;
-  }
-
-  public long getTimeStamp() {
-    return timeStamp;
-  }
-
-  public HMsg getHmsg() {
-    return hmsg;
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    // the event type byte
-    hbEvent = HBaseEventType.fromByte(in.readByte());
-    // the hostname of the RS sending the data
-    rsName = in.readUTF();
-    // the timestamp
-    timeStamp = in.readLong();
-    if(in.readBoolean()) {
-      // deserialized the HMsg from ZK
-      hmsg = new HMsg();
-      hmsg.readFields(in);
-    }
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeByte(hbEvent.getByteValue());
-    out.writeUTF(rsName);
-    out.writeLong(System.currentTimeMillis());
-    out.writeBoolean((hmsg != null));
-    if(hmsg != null) {
-      hmsg.write(out);
-    }
-  }
-
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
index 5c8a1bea93d..d1df652c9ae 100644
--- a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
+++ b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
@@ -23,11 +23,10 @@ import java.io.DataOutput;
 import java.io.IOException;
 import java.lang.reflect.Array;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.NavigableSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -45,26 +44,37 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.MultiAction;
-import org.apache.hadoop.hbase.client.Action;
-import org.apache.hadoop.hbase.client.MultiResponse;
+import org.apache.hadoop.hbase.client.MultiPut;
+import org.apache.hadoop.hbase.client.MultiPutResponse;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Row;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.MultiPutResponse;
-import org.apache.hadoop.hbase.client.MultiPut;
-import org.apache.hadoop.hbase.filter.*;
-import org.apache.hadoop.hbase.io.HbaseMapWritable;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
+import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.DependentColumnFilter;
+import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.hbase.filter.RowFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.filter.SkipFilter;
+import org.apache.hadoop.hbase.filter.ValueFilter;
+import org.apache.hadoop.hbase.filter.WhileMatchFilter;
+import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.ObjectWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
-import org.apache.hadoop.hbase.util.Bytes;
 
 /**
  * This is a customized version of the polymorphic hadoop
@@ -173,16 +183,10 @@ public class HbaseObjectWritable implements Writable, Configurable {
     addToMap(HLog.Entry[].class, code++);
     addToMap(HLogKey.class, code++);
 
-    // List
     addToMap(List.class, code++);
+
+    addToMap(NavigableSet.class, code++);
     addToMap(ColumnPrefixFilter.class, code++);
-    
-    // Multi
-    addToMap(Row.class, code++);
-    addToMap(Action.class, code++);
-    addToMap(MultiAction.class, code++);
-    addToMap(MultiResponse.class, code++);
-    
   }
 
   private Class<?> declaredClass;
@@ -512,4 +516,4 @@ public class HbaseObjectWritable implements Writable, Configurable {
   public Configuration getConf() {
     return this.conf;
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
index 98731720b5d..134288b7cd3 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
@@ -345,8 +345,9 @@ public class HBaseRPC {
         if (maxAttempts >= 0 && ++reconnectAttempts >= maxAttempts) {
           LOG.info("Server at " + addr + " could not be reached after " +
             reconnectAttempts + " tries, giving up.");
-          throw new RetriesExhaustedException("Failed setting up proxy to " +
-            addr.toString() + " after attempts=" + reconnectAttempts);
+          throw new RetriesExhaustedException("Failed setting up proxy " +
+            protocol + " to " + addr.toString() + " after attempts=" +
+            reconnectAttempts, se);
       }
       } catch(SocketTimeoutException te) {  // namenode is busy
         LOG.info("Problem connecting to server: " + addr);
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
index d4bcbedd27a..10ddd7d858f 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
@@ -76,7 +76,8 @@ public interface HBaseRPCProtocolVersion extends VersionedProtocol {
    * <li>Version 22: HBASE-2209. Added List support to RPC</li>
    * <li>Version 23: HBASE-2066, multi-put.</li>
    * <li>Version 24: HBASE-2473, create table with regions.</li>
+   * <li>Version 25: Added openRegion and Stoppable/Abortable to API.</li>
    * </ul>
    */
-  public static final long versionID = 24L;
+  public static final long versionID = 25L;
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
index bd48a4bb603..e6ed1ff987f 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -19,13 +19,12 @@
  */
 package org.apache.hadoop.hbase.ipc;
 
+import java.io.IOException;
+
 import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.io.Writable;
-
-import java.io.IOException;
+import org.apache.hadoop.hbase.UnknownRegionException;
 
 /**
  * Clients interact with the HMasterInterface to gain access to meta-level
@@ -73,12 +72,10 @@ public interface HMasterInterface extends HBaseRPCProtocolVersion {
   /**
    * Modifies an existing column on the specified table
    * @param tableName table name
-   * @param columnName name of the column to edit
    * @param descriptor new column descriptor
    * @throws IOException e
    */
-  public void modifyColumn(final byte [] tableName, final byte [] columnName,
-    HColumnDescriptor descriptor)
+  public void modifyColumn(final byte [] tableName, HColumnDescriptor descriptor)
   throws IOException;
 
 
@@ -110,12 +107,11 @@ public interface HMasterInterface extends HBaseRPCProtocolVersion {
    * Modify a table's metadata
    *
    * @param tableName table to modify
-   * @param op the operation to do
-   * @param args arguments for operation
+   * @param htd new descriptor for table
    * @throws IOException e
    */
-  public void modifyTable(byte[] tableName, HConstants.Modify op, Writable[] args)
-    throws IOException;
+  public void modifyTable(byte[] tableName, HTableDescriptor htd)
+  throws IOException;
 
   /**
    * Shutdown an HBase cluster.
@@ -123,9 +119,33 @@ public interface HMasterInterface extends HBaseRPCProtocolVersion {
    */
   public void shutdown() throws IOException;
 
+  /**
+   * Stop HBase Master only.
+   * Does not shutdown the cluster.
+   * @throws IOException e
+   */
+  public void stopMaster() throws IOException;
+
   /**
    * Return cluster status.
    * @return status object
    */
   public ClusterStatus getClusterStatus();
-}
+
+
+  /**
+   * Move the region <code>r</code> to <code>dest</code>.
+   * @param encodedRegionName The encoded region name.
+   * @param destServerName The servername of the destination regionserver
+   * @throws UnknownRegionException Thrown if we can't find a region named
+   * <code>encodedRegionName</code>
+   */
+  public void move(final byte [] encodedRegionName, final byte [] destServerName)
+  throws UnknownRegionException;
+
+  /**
+   * @param b If true, enable balancer. If false, disable balancer.
+   * @return Previous balancer value
+   */
+  public boolean balance(final boolean b);
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
index bcb238135a9..9a6df8d3b8f 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
@@ -19,31 +19,31 @@
  */
 package org.apache.hadoop.hbase.ipc;
 
+import java.io.IOException;
+import java.util.List;
+import java.util.NavigableSet;
+
+import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.MultiAction;
-import org.apache.hadoop.hbase.client.MultiResponse;
 import org.apache.hadoop.hbase.client.MultiPut;
 import org.apache.hadoop.hbase.client.MultiPutResponse;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 
-import java.io.IOException;
-import java.util.List;
-
 /**
  * Clients interact with HRegionServers using a handle to the HRegionInterface.
  *
  * <p>NOTE: if you change the interface, you must change the RPC version
  * number in HBaseRPCProtocolVersion
  */
-public interface HRegionInterface extends HBaseRPCProtocolVersion {
+public interface HRegionInterface extends HBaseRPCProtocolVersion, Stoppable, Abortable {
   /**
    * Get metainfo about an HRegion
    *
@@ -69,12 +69,6 @@ public interface HRegionInterface extends HBaseRPCProtocolVersion {
     final byte [] row, final byte [] family)
   throws IOException;
 
-  /**
-   *
-   * @return the regions served by this regionserver
-   */
-  public HRegion [] getOnlineRegionsAsArray();
-
   /**
    * Perform Get operation.
    * @param regionName name of region to get from
@@ -260,11 +254,10 @@ public interface HRegionInterface extends HBaseRPCProtocolVersion {
 
 
   /**
-   * Method used when a master is taking the place of another failed one.
-   * @return All regions assigned on this region server
+   * @return All regions online on this region server
    * @throws IOException e
    */
-  public HRegionInfo[] getRegionsAssignment() throws IOException;
+  public NavigableSet<HRegionInfo> getOnlineRegions();
 
   /**
    * Method used when a master is taking the place of another failed one.
@@ -273,13 +266,6 @@ public interface HRegionInterface extends HBaseRPCProtocolVersion {
    */
   public HServerInfo getHServerInfo() throws IOException;
 
-  /**
-   * Method used for doing multiple actions(Deletes, Gets and Puts) in one call
-   * @param multi
-   * @return MultiResult
-   * @throws IOException
-   */
-  public MultiResponse multi(MultiAction multi) throws IOException;
 
   /**
    * Multi put for putting multiple regions worth of puts at once.
@@ -296,6 +282,60 @@ public interface HRegionInterface extends HBaseRPCProtocolVersion {
   public void bulkLoadHFile(String hfilePath,
       byte[] regionName, byte[] familyName) throws IOException;
 
+  // Master methods
+
+  /**
+   * Opens the specified region.
+   * @param region region to open
+   */
+  public void openRegion(final HRegionInfo region);
+
+  /**
+   * Closes the specified region.
+   * @param region region to close
+   * @return true if closing region, false if not
+   */
+  public boolean closeRegion(final HRegionInfo region)
+  throws NotServingRegionException;
+
+  // Region administrative methods
+
+  /**
+   * Flushes the MemStore of the specified region.
+   * <p>
+   * This method is synchronous.
+   * @param regionInfo region to flush
+   * @throws NotServingRegionException
+   * @throws IOException
+   */
+  void flushRegion(HRegionInfo regionInfo)
+  throws NotServingRegionException, IOException;
+
+  /**
+   * Splits the specified region.
+   * <p>
+   * This method currently flushes the region and then forces a compaction which
+   * will then trigger a split.  The flush is done synchronously but the
+   * compaction is asynchronous.
+   * @param regionInfo region to split
+   * @throws NotServingRegionException
+   * @throws IOException
+   */
+  void splitRegion(HRegionInfo regionInfo)
+  throws NotServingRegionException, IOException;
+
+  /**
+   * Compacts the specified region.  Performs a major compaction if specified.
+   * <p>
+   * This method is asynchronous.
+   * @param regionInfo region to compact
+   * @param major true to force major compaction
+   * @throws NotServingRegionException
+   * @throws IOException
+   */
+  void compactRegion(HRegionInfo regionInfo, boolean major)
+  throws NotServingRegionException, IOException;
+
   /**
    * Replicates the given entries. The guarantee is that the given entries
    * will be durable on the slave cluster if this method returns without
@@ -306,5 +346,4 @@ public interface HRegionInterface extends HBaseRPCProtocolVersion {
    * @throws IOException
    */
   public void replicateLogEntries(HLog.Entry[] entries) throws IOException;
-
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
new file mode 100644
index 00000000000..371d0ea2294
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -0,0 +1,173 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Handles everything on master-side related to master election.
+ *
+ * <p>Listens and responds to ZooKeeper notifications on the master znode,
+ * both <code>nodeCreated</code> and <code>nodeDeleted</code>.
+ *
+ * <p>Contains blocking methods which will hold up backup masters, waiting
+ * for the active master to fail.
+ *
+ * <p>This class is instantiated in the HMaster constructor and the method
+ * {@link #blockUntilBecomingActiveMaster()} is called to wait until becoming
+ * the active master of the cluster.
+ */
+class ActiveMasterManager extends ZooKeeperListener {
+  private static final Log LOG = LogFactory.getLog(ActiveMasterManager.class);
+
+  final AtomicBoolean clusterHasActiveMaster = new AtomicBoolean(false);
+
+  private final HServerAddress address;
+  private final Server master;
+
+  ActiveMasterManager(ZooKeeperWatcher watcher, HServerAddress address,
+      Server master) {
+    super(watcher);
+    this.address = address;
+    this.master = master;
+  }
+
+  @Override
+  public void nodeCreated(String path) {
+    if(path.equals(watcher.masterAddressZNode) && !master.isStopped()) {
+      handleMasterNodeChange();
+    }
+  }
+
+  @Override
+  public void nodeDeleted(String path) {
+    if(path.equals(watcher.masterAddressZNode) && !master.isStopped()) {
+      handleMasterNodeChange();
+    }
+  }
+
+  /**
+   * Handle a change in the master node.  Doesn't matter whether this was called
+   * from a nodeCreated or nodeDeleted event because there are no guarantees
+   * that the current state of the master node matches the event at the time of
+   * our next ZK request.
+   *
+   * <p>Uses the watchAndCheckExists method which watches the master address node
+   * regardless of whether it exists or not.  If it does exist (there is an
+   * active master), it returns true.  Otherwise it returns false.
+   *
+   * <p>A watcher is set which guarantees that this method will get called again if
+   * there is another change in the master node.
+   */
+  private void handleMasterNodeChange() {
+    // Watch the node and check if it exists.
+    try {
+      synchronized(clusterHasActiveMaster) {
+        if(ZKUtil.watchAndCheckExists(watcher, watcher.masterAddressZNode)) {
+          // A master node exists, there is an active master
+          LOG.debug("A master is now available");
+          clusterHasActiveMaster.set(true);
+        } else {
+          // Node is no longer there, cluster does not have an active master
+          LOG.debug("No master available. notifying waiting threads");
+          clusterHasActiveMaster.set(false);
+          // Notify any thread waiting to become the active master
+          clusterHasActiveMaster.notifyAll();
+        }
+      }
+    } catch (KeeperException ke) {
+      master.abort("Received an unexpected KeeperException, aborting", ke);
+    }
+  }
+
+  /**
+   * Block until becoming the active master.
+   *
+   * Method blocks until there is not another active master and our attempt
+   * to become the new active master is successful.
+   *
+   * This also makes sure that we are watching the master znode so will be
+   * notified if another master dies.
+   * @return False if we did not start up this cluster, another
+   * master did, or if a problem (zookeeper, stop flag has been set on this
+   * Master)
+   */
+  boolean blockUntilBecomingActiveMaster() {
+    boolean thisMasterStartedCluster = true;
+    // Try to become the active master, watch if there is another master
+    try {
+      if(ZKUtil.setAddressAndWatch(watcher, watcher.masterAddressZNode,
+          address)) {
+        // We are the master, return
+        clusterHasActiveMaster.set(true);
+        return thisMasterStartedCluster;
+      }
+    } catch (KeeperException ke) {
+      master.abort("Received an unexpected KeeperException, aborting", ke);
+      return false;
+    }
+    // There is another active master, this is not a cluster startup
+    // and we must wait until the active master dies
+    LOG.info("Another master is already the active master, waiting to become " +
+      "the next active master");
+    clusterHasActiveMaster.set(true);
+    thisMasterStartedCluster = false;
+    synchronized(clusterHasActiveMaster) {
+      while(clusterHasActiveMaster.get() && !master.isStopped()) {
+        try {
+          clusterHasActiveMaster.wait();
+        } catch (InterruptedException e) {
+          // We expect to be interrupted when a master dies, will fall out if so
+          LOG.debug("Interrupted waiting for master to die", e);
+        }
+      }
+      if(master.isStopped()) {
+        return thisMasterStartedCluster;
+      }
+      // Try to become active master again now that there is no active master
+      blockUntilBecomingActiveMaster();
+    }
+    return thisMasterStartedCluster;
+  }
+
+  public void stop() {
+    try {
+      // If our address is in ZK, delete it on our way out
+      HServerAddress zkAddress =
+        ZKUtil.getDataAsAddress(watcher, watcher.masterAddressZNode);
+      // TODO: redo this to make it atomic (only added for tests)
+      if(zkAddress != null &&
+          zkAddress.equals(address)) {
+        ZKUtil.deleteNode(watcher, watcher.masterAddressZNode);
+      }
+    } catch (KeeperException e) {
+      watcher.error("Error deleting our own master address node", e);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
new file mode 100644
index 00000000000..701e128b8e1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -0,0 +1,1188 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.catalog.RootLocationEditor;
+import org.apache.hadoop.hbase.client.MetaScanner;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan;
+import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler;
+import org.apache.hadoop.hbase.master.handler.OpenedRegionHandler;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKTableDisable;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.NodeAndData;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.io.Writable;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Manages and performs region assignment.
+ * <p>
+ * Monitors ZooKeeper for events related to regions in transition.
+ * <p>
+ * Handles existing regions in transition during master failover.
+ */
+public class AssignmentManager extends ZooKeeperListener {
+  private static final Log LOG = LogFactory.getLog(AssignmentManager.class);
+
+  protected Server master;
+
+  private ServerManager serverManager;
+
+  private CatalogTracker catalogTracker;
+
+  private TimeoutMonitor timeoutMonitor;
+
+  /** Regions currently in transition. */
+  private final Map<String, RegionState> regionsInTransition =
+    new TreeMap<String, RegionState>();
+
+  /** Plans for region movement. */
+  // TODO: When do plans get cleaned out?  Ever?
+  // Its cleaned on server shutdown processing -- St.Ack
+  private final Map<String, RegionPlan> regionPlans =
+    new TreeMap<String, RegionPlan>();
+
+  /** Set of tables that have been disabled. */
+  private final Set<String> disabledTables =
+    Collections.synchronizedSet(new HashSet<String>());
+
+  /**
+   * Server to regions assignment map.
+   * Contains the set of regions currently assigned to a given server.
+   * This Map and {@link #regions} are tied.  Always update this in tandem
+   * with the other under a lock on {@link #regions}
+   * @see #regions
+   */
+  private final NavigableMap<HServerInfo, List<HRegionInfo>> servers =
+    new TreeMap<HServerInfo, List<HRegionInfo>>();
+
+  /**
+   * Region to server assignment map.
+   * Contains the server a given region is currently assigned to.
+   * This Map and {@link #servers} are tied.  Always update this in tandem
+   * with the other under a lock on {@link #regions}
+   * @see #servers
+   */
+  private final SortedMap<HRegionInfo,HServerInfo> regions =
+    new TreeMap<HRegionInfo,HServerInfo>();
+
+  private final ReentrantLock assignLock = new ReentrantLock();
+
+  private final ExecutorService executorService;
+
+  /**
+   * Constructs a new assignment manager.
+   *
+   * <p>This manager must be started with {@link #start()}.
+   *
+   * @param status master status
+   * @param serverManager
+   * @param catalogTracker
+   * @param service
+   */
+  public AssignmentManager(Server master, ServerManager serverManager,
+      CatalogTracker catalogTracker, final ExecutorService service) {
+    super(master.getZooKeeper());
+    this.master = master;
+    this.serverManager = serverManager;
+    this.catalogTracker = catalogTracker;
+    this.executorService = service;
+    Configuration conf = master.getConfiguration();
+    this.timeoutMonitor = new TimeoutMonitor(
+        conf.getInt("hbase.master.assignment.timeoutmonitor.period", 30000),
+        master,
+        conf.getInt("hbase.master.assignment.timeoutmonitor.timeout", 15000));
+    Threads.setDaemonThreadRunning(timeoutMonitor,
+        master.getServerName() + ".timeoutMonitor");
+  }
+
+  /**
+   * Reset all unassigned znodes.  Called on startup of master.
+   * Call {@link #assignAllUserRegions()} after root and meta have been assigned.
+   * @throws IOException
+   * @throws KeeperException
+   */
+  void cleanoutUnassigned() throws IOException, KeeperException {
+    // Cleanup any existing ZK nodes and start watching
+    ZKAssign.deleteAllNodes(watcher);
+    ZKUtil.listChildrenAndWatchForNewChildren(watcher,
+        watcher.assignmentZNode);
+  }
+
+  /**
+   * Handle failover.  Restore state from META and ZK.  Handle any regions in
+   * transition.
+   * @throws KeeperException
+   * @throws IOException
+   */
+  void processFailover() throws KeeperException, IOException {
+    // Concurrency note: In the below the accesses on regionsInTransition are
+    // outside of a synchronization block where usually all accesses to RIT are
+    // synchronized.  The presumption is that in this case it is safe since this
+    // method is being played by a single thread on startup.
+
+    // Scan META to build list of existing regions, servers, and assignment
+    rebuildUserRegions();
+    // Pickup any disabled tables
+    rebuildDisabledTables();
+    // Check existing regions in transition
+    List<String> nodes = ZKUtil.listChildrenAndWatchForNewChildren(watcher,
+        watcher.assignmentZNode);
+    if(nodes.isEmpty()) {
+      LOG.info("No regions in transition in ZK to process on failover");
+      return;
+    }
+    LOG.info("Failed-over master needs to process " + nodes.size() +
+        " regions in transition");
+    for(String regionName : nodes) {
+      RegionTransitionData data = ZKAssign.getData(watcher, regionName);
+      HRegionInfo regionInfo =
+        MetaReader.getRegion(catalogTracker, data.getRegionName()).getFirst();
+      String encodedName = regionInfo.getEncodedName();
+      switch(data.getEventType()) {
+        case RS2ZK_REGION_CLOSING:
+          // Just insert region into RIT.
+          // If this never updates the timeout will trigger new assignment
+          regionsInTransition.put(encodedName,
+              new RegionState(regionInfo, RegionState.State.CLOSING,
+                  data.getStamp()));
+          break;
+
+        case RS2ZK_REGION_CLOSED:
+          // Region is closed, insert into RIT and handle it
+          regionsInTransition.put(encodedName,
+              new RegionState(regionInfo, RegionState.State.CLOSED,
+                  data.getStamp()));
+          new ClosedRegionHandler(master, this, data, regionInfo).process();
+          break;
+
+        case RS2ZK_REGION_OPENING:
+          // Just insert region into RIT
+          // If this never updates the timeout will trigger new assignment
+          regionsInTransition.put(encodedName,
+              new RegionState(regionInfo, RegionState.State.OPENING,
+                  data.getStamp()));
+          break;
+
+        case RS2ZK_REGION_OPENED:
+          // Region is opened, insert into RIT and handle it
+          regionsInTransition.put(encodedName,
+              new RegionState(regionInfo, RegionState.State.OPENING,
+                  data.getStamp()));
+          new OpenedRegionHandler(master, this, data, regionInfo,
+              serverManager.getServerInfo(data.getServerName())).process();
+          break;
+      }
+    }
+  }
+
+  /**
+   * Handles various states an unassigned node can be in.
+   * <p>
+   * Method is called when a state change is suspected for an unassigned node.
+   * <p>
+   * This deals with skipped transitions (we got a CLOSED but didn't see CLOSING
+   * yet).
+   * @param data
+   */
+  private void handleRegion(RegionTransitionData data) {
+    synchronized(regionsInTransition) {
+      // Verify this is a known server
+      if(!serverManager.isServerOnline(data.getServerName())) {
+        LOG.warn("Attempted to handle region transition for server " +
+          data.getServerName() + " but server is not online");
+        return;
+      }
+      String encodedName = HRegionInfo.encodeRegionName(data.getRegionName());
+      String prettyPrintedRegionName = HRegionInfo.prettyPrint(encodedName);
+      LOG.debug("Handling transition=" + data.getEventType() +
+        ", server=" + data.getServerName() + ", region=" + prettyPrintedRegionName);
+      RegionState regionState = regionsInTransition.get(encodedName);
+      switch(data.getEventType()) {
+
+        case RS2ZK_REGION_CLOSING:
+          // Should see CLOSING after we have asked it to CLOSE or additional
+          // times after already being in state of CLOSING
+          if (regionState == null ||
+              (!regionState.isPendingClose() && !regionState.isClosing())) {
+            LOG.warn("Received CLOSING for region " + prettyPrintedRegionName +
+                " from server " + data.getServerName() + " but region was in " +
+                " the state " + regionState + " and not " +
+                "in expected PENDING_CLOSE or CLOSING states");
+            return;
+          }
+          // Transition to CLOSING (or update stamp if already CLOSING)
+          regionState.update(RegionState.State.CLOSING, data.getStamp());
+          break;
+
+        case RS2ZK_REGION_CLOSED:
+          // Should see CLOSED after CLOSING but possible after PENDING_CLOSE
+          if (regionState == null ||
+              (!regionState.isPendingClose() && !regionState.isClosing())) {
+            LOG.warn("Received CLOSED for region " + prettyPrintedRegionName +
+                " from server " + data.getServerName() + " but region was in " +
+                " the state " + regionState + " and not " +
+                "in expected PENDING_CLOSE or CLOSING states");
+            return;
+          }
+          // Handle CLOSED by assigning elsewhere or stopping if a disable
+          // If we got here all is good.  Need to update RegionState -- else
+          // what follows will fail because not in expected state.
+          regionState.update(RegionState.State.CLOSED, data.getStamp());
+          this.executorService.submit(new ClosedRegionHandler(master,
+            this, data, regionState.getRegion()));
+          break;
+
+        case RS2ZK_REGION_OPENING:
+          // Should see OPENING after we have asked it to OPEN or additional
+          // times after already being in state of OPENING
+          if(regionState == null ||
+              (!regionState.isPendingOpen() && !regionState.isOpening())) {
+            LOG.warn("Received OPENING for region " +
+                prettyPrintedRegionName +
+                " from server " + data.getServerName() + " but region was in " +
+                " the state " + regionState + " and not " +
+                "in expected PENDING_OPEN or OPENING states");
+            return;
+          }
+          // Transition to OPENING (or update stamp if already OPENING)
+          regionState.update(RegionState.State.OPENING, data.getStamp());
+          break;
+
+        case RS2ZK_REGION_OPENED:
+          // Should see OPENED after OPENING but possible after PENDING_OPEN
+          if(regionState == null ||
+              (!regionState.isPendingOpen() && !regionState.isOpening())) {
+            LOG.warn("Received OPENED for region " +
+                prettyPrintedRegionName +
+                " from server " + data.getServerName() + " but region was in " +
+                " the state " + regionState + " and not " +
+                "in expected PENDING_OPEN or OPENING states");
+            return;
+          }
+          // Handle OPENED by removing from transition and deleted zk node
+          this.executorService.submit(
+            new OpenedRegionHandler(master, this, data, regionState.getRegion(),
+              this.serverManager.getServerInfo(data.getServerName())));
+          break;
+      }
+    }
+  }
+
+  // ZooKeeper events
+
+  /**
+   * New unassigned node has been created.
+   *
+   * <p>This happens when an RS begins the OPENING or CLOSING of a region by
+   * creating an unassigned node.
+   *
+   * <p>When this happens we must:
+   * <ol>
+   *   <li>Watch the node for further events</li>
+   *   <li>Read and handle the state in the node</li>
+   * </ol>
+   */
+  @Override
+  public void nodeCreated(String path) {
+    if(path.startsWith(watcher.assignmentZNode)) {
+      synchronized(regionsInTransition) {
+        try {
+          RegionTransitionData data = ZKAssign.getData(watcher, path);
+          if(data == null) {
+            return;
+          }
+          handleRegion(data);
+        } catch (KeeperException e) {
+          master.abort("Unexpected ZK exception reading unassigned node data", e);
+        }
+      }
+    }
+  }
+
+  /**
+   * Existing unassigned node has had data changed.
+   *
+   * <p>This happens when an RS transitions from OFFLINE to OPENING, or between
+   * OPENING/OPENED and CLOSING/CLOSED.
+   *
+   * <p>When this happens we must:
+   * <ol>
+   *   <li>Watch the node for further events</li>
+   *   <li>Read and handle the state in the node</li>
+   * </ol>
+   */
+  @Override
+  public void nodeDataChanged(String path) {
+    if(path.startsWith(watcher.assignmentZNode)) {
+      synchronized(regionsInTransition) {
+        try {
+          RegionTransitionData data = ZKAssign.getData(watcher, path);
+          if(data == null) {
+            return;
+          }
+          handleRegion(data);
+        } catch (KeeperException e) {
+          master.abort("Unexpected ZK exception reading unassigned node data", e);
+        }
+      }
+    }
+  }
+
+  /**
+   * New unassigned node has been created.
+   *
+   * <p>This happens when an RS begins the OPENING or CLOSING of a region by
+   * creating an unassigned node.
+   *
+   * <p>When this happens we must:
+   * <ol>
+   *   <li>Watch the node for further children changed events</li>
+   *   <li>Watch all new children for changed events</li>
+   *   <li>Read all children and handle them</li>
+   * </ol>
+   */
+  @Override
+  public void nodeChildrenChanged(String path) {
+    if(path.equals(watcher.assignmentZNode)) {
+      synchronized(regionsInTransition) {
+        try {
+          List<NodeAndData> newNodes = ZKUtil.watchAndGetNewChildren(watcher,
+              watcher.assignmentZNode);
+          for(NodeAndData newNode : newNodes) {
+            LOG.debug("Handling new unassigned node: " + newNode);
+            handleRegion(RegionTransitionData.fromBytes(newNode.getData()));
+          }
+        } catch(KeeperException e) {
+          master.abort("Unexpected ZK exception reading unassigned children", e);
+        }
+      }
+    }
+  }
+
+  /**
+   * Marks the region as online.  Removes it from regions in transition and
+   * updates the in-memory assignment information.
+   * <p>
+   * Used when a region has been successfully opened on a region server.
+   * @param regionInfo
+   * @param serverInfo
+   */
+  public void regionOnline(HRegionInfo regionInfo, HServerInfo serverInfo) {
+    synchronized(regionsInTransition) {
+      regionsInTransition.remove(regionInfo.getEncodedName());
+      regionsInTransition.notifyAll();
+    }
+    synchronized(regions) {
+      regions.put(regionInfo, serverInfo);
+      addToServers(serverInfo, regionInfo);
+    }
+  }
+
+  /**
+   * Marks the region as offline.  Removes it from regions in transition and
+   * removes in-memory assignment information.
+   * <p>
+   * Used when a region has been closed and should remain closed.
+   * @param regionInfo
+   * @param serverInfo
+   */
+  public void regionOffline(HRegionInfo regionInfo) {
+    synchronized(regionsInTransition) {
+      regionsInTransition.remove(regionInfo.getEncodedName());
+      regionsInTransition.notifyAll();
+    }
+    synchronized(regions) {
+      HServerInfo serverInfo = regions.remove(regionInfo);
+      List<HRegionInfo> serverRegions = servers.get(serverInfo);
+      serverRegions.remove(regionInfo);
+    }
+  }
+
+  /**
+   * Sets the region as offline by removing in-memory assignment information but
+   * retaining transition information.
+   * <p>
+   * Used when a region has been closed but should be reassigned.
+   * @param regionInfo
+   */
+  public void setOffline(HRegionInfo regionInfo) {
+    synchronized(regions) {
+      HServerInfo serverInfo = regions.remove(regionInfo);
+      List<HRegionInfo> serverRegions = servers.get(serverInfo);
+      serverRegions.remove(regionInfo);
+    }
+  }
+
+  // Assignment methods
+
+  /**
+   * Assigns the specified region.
+   * <p>
+   * If a RegionPlan is available with a valid destination then it will be used
+   * to determine what server region is assigned to.  If no RegionPlan is
+   * available, region will be assigned to a random available server.
+   * <p>
+   * Updates the RegionState and sends the OPEN RPC.
+   * <p>
+   * This will only succeed if the region is in transition and in a CLOSED or
+   * OFFLINE state or not in transition (in-memory not zk), and of course, the
+   * chosen server is up and running (It may have just crashed!).  If the
+   * in-memory checks pass, the zk node is forced to OFFLINE before assigning.
+   *
+   * @param regionName server to be assigned
+   */
+  public void assign(HRegionInfo region) {
+    LOG.debug("Starting assignment for region " + region.getRegionNameAsString());
+    // Grab the state of this region and synchronize on it
+    String encodedName = region.getEncodedName();
+    RegionState state;
+    // This assignLock is used bridging the two synchronization blocks.  Once
+    // we've made it into the 'state' synchronization block, then we can let
+    // go of this lock.  There must be a better construct that this -- St.Ack 20100811
+    this.assignLock.lock();
+    try {
+      synchronized(regionsInTransition) {
+        state = regionsInTransition.get(encodedName);
+        if(state == null) {
+          state = new RegionState(region, RegionState.State.OFFLINE);
+          regionsInTransition.put(encodedName, state);
+        }
+      }
+      synchronized(state) {
+        this.assignLock.unlock();
+        assign(state);
+      }
+    } finally {
+      if (this.assignLock.isHeldByCurrentThread()) this.assignLock.unlock();
+    }
+  }
+
+  /**
+   * Caller must hold lock on the passed <code>state</code> object.
+   * @param state 
+   */
+  private void assign(final RegionState state) {
+    if(!state.isClosed() && !state.isOffline()) {
+      LOG.info("Attempting to assign region but it is in transition and in " +
+          "an unexpected state:" + state);
+      return;
+    } else {
+      state.update(RegionState.State.OFFLINE);
+    }
+    try {
+      if(!ZKAssign.createOrForceNodeOffline(master.getZooKeeper(),
+          state.getRegion(), master.getServerName())) {
+        LOG.warn("Attempted to create/force node into OFFLINE state before " +
+            "completing assignment but failed to do so");
+        return;
+      }
+    } catch (KeeperException e) {
+      master.abort("Unexpected ZK exception creating/setting node OFFLINE", e);
+      return;
+    }
+    // Pickup existing plan or make a new one
+    String encodedName = state.getRegion().getEncodedName();
+    RegionPlan plan;
+    synchronized(regionPlans) {
+      plan = regionPlans.get(encodedName);
+      if (plan == null) {
+        LOG.debug("No previous transition plan for " +
+            state.getRegion().getRegionNameAsString() +
+            " so generating a random one from " + serverManager.numServers() +
+            " ( " + serverManager.getOnlineServers().size() + ") available servers");
+        plan = new RegionPlan(state.getRegion(), null,
+          LoadBalancer.randomAssignment(serverManager.getOnlineServersList()));
+        regionPlans.put(encodedName, plan);
+      }
+    }
+    try {
+      // Send OPEN RPC. This can fail if the server on other end is is not up.
+      serverManager.sendRegionOpen(plan.getDestination(), state.getRegion());
+      // Transition RegionState to PENDING_OPEN
+      state.update(RegionState.State.PENDING_OPEN);
+    } catch (Throwable t) {
+      LOG.warn("Failed assignment of " +
+        state.getRegion().getRegionNameAsString() + " to " +
+        plan.getDestination(), t);
+      // Clean out plan we failed execute and one that doesn't look like it'll
+      // succeed anyways; we need a new plan!
+      synchronized(regionPlans) {
+        this.regionPlans.remove(encodedName);
+      }
+    }
+  }
+
+  /**
+   * Unassigns the specified region.
+   * <p>
+   * Updates the RegionState and sends the CLOSE RPC.
+   * <p>
+   * If a RegionPlan is already set, it will remain.  If this is being used
+   * to disable a table, be sure to use {@link #disableTable(String)} to ensure
+   * regions are not onlined after being closed.
+   *
+   * @param regionName server to be unassigned
+   */
+  public void unassign(HRegionInfo region) {
+    LOG.debug("Starting unassignment of region " +
+      region.getRegionNameAsString() + " (offlining)");
+    // Check if this region is currently assigned
+    if (!regions.containsKey(region)) {
+      LOG.debug("Attempted to unassign region " + region.getRegionNameAsString() +
+        " but it is not " +
+        "currently assigned anywhere");
+      return;
+    }
+    String encodedName = region.getEncodedName();
+    // Grab the state of this region and synchronize on it
+    RegionState state;
+    synchronized(regionsInTransition) {
+      state = regionsInTransition.get(encodedName);
+      if(state == null) {
+        state = new RegionState(region, RegionState.State.PENDING_CLOSE);
+        regionsInTransition.put(encodedName, state);
+      } else {
+        LOG.debug("Attempting to unassign region " +
+          region.getRegionNameAsString() + " but it is " +
+          "already in transition (" + state.getState() + ")");
+        return;
+      }
+    }
+    // Send CLOSE RPC
+    try {
+      serverManager.sendRegionClose(regions.get(region), state.getRegion());
+    } catch (NotServingRegionException e) {
+      LOG.warn("Attempted to close region " + region.getRegionNameAsString() +
+        " but got an NSRE", e);
+    }
+  }
+
+  /**
+   * Waits until the specified region has completed assignment.
+   * <p>
+   * If the region is already assigned, returns immediately.  Otherwise, method
+   * blocks until the region is assigned.
+   * @param regionInfo region to wait on assignment for
+   * @throws InterruptedException
+   */
+  public void waitForAssignment(HRegionInfo regionInfo)
+  throws InterruptedException {
+    synchronized(regions) {
+      while(!regions.containsKey(regionInfo)) {
+        regions.wait();
+      }
+    }
+  }
+
+  /**
+   * Assigns the ROOT region.
+   * <p>
+   * Assumes that ROOT is currently closed and is not being actively served by
+   * any RegionServer.
+   * <p>
+   * Forcibly unsets the current root region location in ZooKeeper and assigns
+   * ROOT to a random RegionServer.
+   * @throws KeeperException 
+   */
+  public void assignRoot() throws KeeperException {
+    RootLocationEditor.deleteRootLocation(this.master.getZooKeeper());
+    assign(HRegionInfo.ROOT_REGIONINFO);
+  }
+
+  /**
+   * Assigns the META region.
+   * <p>
+   * Assumes that META is currently closed and is not being actively served by
+   * any RegionServer.
+   * <p>
+   * Forcibly assigns META to a random RegionServer.
+   */
+  public void assignMeta() {
+    // Force assignment to a random server
+    assign(HRegionInfo.FIRST_META_REGIONINFO);
+  }
+
+  /**
+   * Assigns all user regions, if any exist.  Used during cluster startup.
+   * <p>
+   * This is a synchronous call and will return once every region has been
+   * assigned.  If anything fails, an exception is thrown and the cluster
+   * should be shutdown.
+   */
+  public void assignAllUserRegions() throws IOException {
+    // First experiment at synchronous assignment
+    // Simpler because just wait for no regions in transition
+
+    // Scan META for all user regions
+    List<HRegionInfo> allRegions =
+      MetaScanner.listAllRegions(master.getConfiguration());
+    if (allRegions == null || allRegions.isEmpty()) {
+      return;
+    }
+
+    // Get all available servers
+    List<HServerInfo> servers = serverManager.getOnlineServersList();
+
+    LOG.info("Assigning " + allRegions.size() + " across " + servers.size() +
+        " servers");
+
+    // Generate a cluster startup region placement plan
+    Map<HServerInfo,List<HRegionInfo>> bulkPlan =
+      LoadBalancer.bulkAssignment(allRegions, servers);
+
+    // For each server, create OFFLINE nodes and send OPEN RPCs
+    for(Map.Entry<HServerInfo,List<HRegionInfo>> entry : bulkPlan.entrySet()) {
+      HServerInfo server = entry.getKey();
+      List<HRegionInfo> regions = entry.getValue();
+      LOG.debug("Assigning " + regions.size() + " regions to " + server);
+      for(HRegionInfo region : regions) {
+        LOG.debug("Assigning " + region.getRegionNameAsString() + " to " + server);
+        String regionName = region.getEncodedName();
+        RegionPlan plan = new RegionPlan(region, null,server);
+        regionPlans.put(regionName, plan);
+        assign(region);
+      }
+    }
+
+    // Wait for no regions to be in transition
+    try {
+      waitUntilNoRegionsInTransition();
+    } catch (InterruptedException e) {
+      LOG.error("Interrupted waiting for regions to be assigned", e);
+      throw new IOException(e);
+    }
+
+    LOG.info("All user regions have been assigned");
+  }
+
+  private void rebuildUserRegions() throws IOException {
+    Map<HRegionInfo,HServerAddress> allRegions =
+      MetaReader.fullScan(catalogTracker);
+    for(Map.Entry<HRegionInfo,HServerAddress> region : allRegions.entrySet()) {
+      HServerAddress regionLocation = region.getValue();
+      HRegionInfo regionInfo = region.getKey();
+      if(regionLocation == null) {
+        regions.put(regionInfo, null);
+        continue;
+      }
+      HServerInfo serverInfo = serverManager.getHServerInfo(regionLocation);
+      regions.put(regionInfo, serverInfo);
+      addToServers(serverInfo, regionInfo);
+    }
+  }
+
+  /*
+   * Presumes caller has taken care of necessary locking modifying servers Map.
+   * @param hsi
+   * @param hri
+   */
+  private void addToServers(final HServerInfo hsi, final HRegionInfo hri) {
+    List<HRegionInfo> hris = servers.get(hsi);
+    if (hris == null) {
+      hris = new ArrayList<HRegionInfo>();
+      servers.put(hsi, hris);
+    }
+    hris.add(hri);
+  }
+
+  /**
+   * Blocks until there are no regions in transition.  It is possible that there
+   * are regions in transition immediately after this returns but guarantees
+   * that if it returns without an exception that there was a period of time
+   * with no regions in transition from the point-of-view of the in-memory
+   * state of the Master.
+   * @throws InterruptedException
+   */
+  public void waitUntilNoRegionsInTransition() throws InterruptedException {
+    synchronized(regionsInTransition) {
+      while(regionsInTransition.size() > 0) {
+        regionsInTransition.wait();
+      }
+    }
+  }
+
+  /**
+   * @return A copy of the Map of regions currently in transition.
+   */
+  public NavigableMap<String, RegionState> getRegionsInTransition() {
+    return new TreeMap<String, RegionState>(this.regionsInTransition);
+  }
+
+  /**
+   * @return True if regions in transition.
+   */
+  public boolean isRegionsInTransition() {
+    return !this.regionsInTransition.isEmpty();
+  }
+
+  /**
+   * Checks if the specified table has been disabled by the user.
+   * @param tableName
+   * @return
+   */
+  public boolean isTableDisabled(String tableName) {
+    synchronized(disabledTables) {
+      return disabledTables.contains(tableName);
+    }
+  }
+
+  /**
+   * Checks if the table of the specified region has been disabled by the user.
+   * @param regionName
+   * @return
+   */
+  public boolean isTableOfRegionDisabled(byte [] regionName) {
+    return isTableDisabled(Bytes.toString(
+        HRegionInfo.getTableName(regionName)));
+  }
+
+  /**
+   * Sets the specified table to be disabled.
+   * @param tableName table to be disabled
+   */
+  public void disableTable(String tableName) {
+    synchronized(disabledTables) {
+      if(!isTableDisabled(tableName)) {
+        disabledTables.add(tableName);
+        try {
+          ZKTableDisable.disableTable(master.getZooKeeper(), tableName);
+        } catch (KeeperException e) {
+          LOG.warn("ZK error setting table as disabled", e);
+        }
+      }
+    }
+  }
+
+  /**
+   * Unsets the specified table from being disabled.
+   * <p>
+   * This operation only acts on the in-memory
+   * @param tableName table to be undisabled
+   */
+  public void undisableTable(String tableName) {
+    synchronized(disabledTables) {
+      if(isTableDisabled(tableName)) {
+        disabledTables.remove(tableName);
+        try {
+          ZKTableDisable.undisableTable(master.getZooKeeper(), tableName);
+        } catch (KeeperException e) {
+          LOG.warn("ZK error setting table as disabled", e);
+        }
+      }
+    }
+  }
+
+  /**
+   * Rebuild the set of disabled tables from zookeeper.  Used during master
+   * failover.
+   */
+  private void rebuildDisabledTables() {
+    synchronized(disabledTables) {
+      List<String> disabledTables;
+      try {
+        disabledTables = ZKTableDisable.getDisabledTables(master.getZooKeeper());
+      } catch (KeeperException e) {
+        LOG.warn("ZK error getting list of disabled tables", e);
+        return;
+      }
+      if(!disabledTables.isEmpty()) {
+        LOG.info("Rebuilt list of " + disabledTables.size() + " disabled " +
+            "tables from zookeeper");
+        disabledTables.addAll(disabledTables);
+      }
+    }
+  }
+
+  /**
+   * Gets the online regions of the specified table.
+   * @param tableName
+   * @return
+   */
+  public List<HRegionInfo> getRegionsOfTable(byte[] tableName) {
+    List<HRegionInfo> tableRegions = new ArrayList<HRegionInfo>();
+    for(HRegionInfo regionInfo : regions.tailMap(new HRegionInfo(
+        new HTableDescriptor(tableName), null, null)).keySet()) {
+      if(Bytes.equals(regionInfo.getTableDesc().getName(), tableName)) {
+        tableRegions.add(regionInfo);
+      } else {
+        break;
+      }
+    }
+    return tableRegions;
+  }
+
+  /**
+   * Unsets the specified table as disabled (enables it).
+   */
+  public class TimeoutMonitor extends Chore {
+
+    private final int timeout;
+
+    /**
+     * Creates a periodic monitor to check for time outs on region transition
+     * operations.  This will deal with retries if for some reason something
+     * doesn't happen within the specified timeout.
+     * @param period
+   * @param stopper When {@link Stoppable#isStopped()} is true, this thread will
+   * cleanup and exit cleanly.
+     * @param timeout
+     */
+    public TimeoutMonitor(final int period, final Stoppable stopper,
+        final int timeout) {
+      super("AssignmentTimeoutMonitor", period, stopper);
+      this.timeout = timeout;
+    }
+
+    @Override
+    protected void chore() {
+      synchronized (regionsInTransition) {
+        // Iterate all regions in transition checking for time outs
+        long now = System.currentTimeMillis();
+        for (RegionState regionState : regionsInTransition.values()) {
+          if(regionState.getStamp() + timeout <= now) {
+            HRegionInfo regionInfo = regionState.getRegion();
+            LOG.info("Regions in transition timed out:  " + regionState);
+            // Expired!  Do a retry.
+            switch(regionState.getState()) {
+              case OFFLINE:
+              case CLOSED:
+                LOG.info("Region has been OFFLINE or CLOSED for too long, " +
+                    "reassigning " + regionInfo.getRegionNameAsString());
+                assign(regionState.getRegion());
+                break;
+              case PENDING_OPEN:
+              case OPENING:
+                LOG.info("Region has been PENDING_OPEN or OPENING for too " +
+                    "long, reassigning " + regionInfo.getRegionNameAsString());
+                assign(regionState.getRegion());
+                break;
+              case OPEN:
+                LOG.warn("Long-running region in OPEN state?  This should " +
+                    "not happen");
+                break;
+              case PENDING_CLOSE:
+              case CLOSING:
+                LOG.info("Region has been PENDING_CLOSE or CLOSING for too " +
+                    "long, resending close rpc");
+                unassign(regionInfo);
+                break;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Process shutdown server removing any assignments.
+   * @param hsi Server that went down.
+   */
+  public void processServerShutdown(final HServerInfo hsi) {
+    // Clean out any exisiting assignment plans for this server
+    synchronized (this.regionPlans) {
+      for (Iterator <Map.Entry<String, RegionPlan>> i =
+        this.regionPlans.entrySet().iterator(); i.hasNext();) {
+        Map.Entry<String, RegionPlan> e = i.next();
+        if (e.getValue().getDestination().equals(hsi)) {
+          // Use iterator's remove else we'll get CME.fail a
+          i.remove();
+        }
+      }
+    }
+    synchronized (regionsInTransition) {
+      // Iterate all regions in transition checking if were on this server
+      final String serverName = hsi.getServerName();
+      for (Map.Entry<String, RegionState> e: this.regionsInTransition.entrySet()) {
+        if (!e.getKey().equals(serverName)) continue;
+        RegionState regionState = e.getValue();
+        switch(regionState.getState()) {
+          case PENDING_OPEN:
+          case OPENING:
+          case OFFLINE:
+          case CLOSED:
+          case PENDING_CLOSE:
+          case CLOSING:
+            LOG.info("Region " + regionState.getRegion().getRegionNameAsString() +
+              " was in state=" + regionState.getState() + " on shutdown server=" +
+              serverName + ", reassigning");
+            assign(regionState.getRegion());
+            break;
+
+          case OPEN:
+            LOG.warn("Long-running region in OPEN state?  Should not happen");
+            break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Update inmemory structures.
+   * @param hsi Server that reported the split
+   * @param parent Parent region that was split
+   * @param a Daughter region A
+   * @param b Daughter region B
+   */
+  public void handleSplitReport(final HServerInfo hsi, final HRegionInfo parent,
+      final HRegionInfo a, final HRegionInfo b) {
+    synchronized (this.regions) {
+      checkRegion(hsi, parent, true);
+      checkRegion(hsi, a, false);
+      this.regions.put(a, hsi);
+      this.regions.put(b, hsi);
+      removeFromServers(hsi, parent, true);
+      removeFromServers(hsi, a, false);
+      removeFromServers(hsi, b, false);
+      addToServers(hsi, a);
+      addToServers(hsi, b);
+    }
+  }
+
+  /*
+   * Caller must hold locks on regions Map.
+   * @param hsi
+   * @param hri
+   * @param expected
+   */
+  private void checkRegion(final HServerInfo hsi, final HRegionInfo hri,
+      final boolean expected) {
+    HServerInfo serverInfo = regions.remove(hri);
+    if (expected) {
+      if (serverInfo == null) {
+        LOG.info("Region not on a server: " + hri.getRegionNameAsString());
+      }
+    } else {
+      if (serverInfo != null) {
+        LOG.warn("Region present on " + hsi + "; unexpected");
+      }
+    }
+  }
+
+  /*
+   * Caller must hold locks on servers Map.
+   * @param hsi
+   * @param hri
+   * @param expected
+   */
+  private void removeFromServers(final HServerInfo hsi, final HRegionInfo hri,
+      final boolean expected) {
+    List<HRegionInfo> serverRegions = this.servers.get(hsi);
+    boolean removed = serverRegions.remove(hri);
+    if (expected) {
+      if (!removed) {
+        LOG.warn(hri.getRegionNameAsString() + " not found on " + hsi +
+          "; unexpected");
+      }
+    } else {
+      if (removed) {
+        LOG.warn(hri.getRegionNameAsString() + " found on " + hsi +
+        "; unexpected");
+      }
+    }
+  }
+
+  /**
+   * @return A clone of current assignments. Note, this is assignments only.
+   * If a new server has come in and it has no regions, it will not be included
+   * in the returned Map.
+   */
+  Map<HServerInfo, List<HRegionInfo>> getAssignments() {
+    // This is an EXPENSIVE clone.  Cloning though is the safest thing to do.
+    // Can't let out original since it can change and at least the loadbalancer
+    // wants to iterate this exported list.  We need to synchronize on regions
+    // since all access to this.servers is under a lock on this.regions.
+    Map<HServerInfo, List<HRegionInfo>> result = null;
+    synchronized (this.regions) {
+      result = new HashMap<HServerInfo, List<HRegionInfo>>(this.servers.size());
+      for (Map.Entry<HServerInfo, List<HRegionInfo>> e: this.servers.entrySet()) {
+        List<HRegionInfo> shallowCopy = new ArrayList<HRegionInfo>(e.getValue());
+        HServerInfo clone = new HServerInfo(e.getKey());
+        // Set into server load the number of regions this server is carrying
+        // The load balancer calculation needs it at least and its handy.
+        clone.getLoad().setNumberOfRegions(e.getValue().size());
+        result.put(clone, shallowCopy);
+      }
+    }
+    return result;
+  }
+
+  /**
+   * @param encodedRegionName Region encoded name.
+   * @return Null or a {@link Pair} instance that holds the full {@link HRegionInfo}
+   * and the hosting servers {@link HServerInfo}.
+   */
+  Pair<HRegionInfo, HServerInfo> getAssignment(final byte [] encodedRegionName) {
+    String name = Bytes.toString(encodedRegionName);
+    synchronized(this.regions) {
+      for (Map.Entry<HRegionInfo, HServerInfo> e: this.regions.entrySet()) {
+        if (e.getKey().getEncodedName().equals(name)) {
+          return new Pair<HRegionInfo, HServerInfo>(e.getKey(), e.getValue());
+        }
+      }
+    }
+    return null;
+  }
+
+  /**
+   * @param plan Plan to execute.
+   */
+  void balance(final RegionPlan plan) {
+    synchronized (this.regionPlans) {
+      this.regionPlans.put(plan.getRegionName(), plan);
+    }
+    unassign(plan.getRegionInfo());
+  }
+
+  public static class RegionState implements Writable {
+    private HRegionInfo region;
+
+    public enum State {
+      OFFLINE,        // region is in an offline state
+      PENDING_OPEN,   // sent rpc to server to open but has not begun
+      OPENING,        // server has begun to open but not yet done
+      OPEN,           // server opened region and updated meta
+      PENDING_CLOSE,  // sent rpc to server to close but has not begun
+      CLOSING,        // server has begun to close but not yet done
+      CLOSED          // server closed region and updated meta
+    }
+
+    private State state;
+    private long stamp;
+
+    public RegionState() {}
+
+    RegionState(HRegionInfo region, State state) {
+      this(region, state, System.currentTimeMillis());
+    }
+
+    RegionState(HRegionInfo region, State state, long stamp) {
+      this.region = region;
+      this.state = state;
+      this.stamp = stamp;
+    }
+
+    public void update(State state, long stamp) {
+      this.state = state;
+      this.stamp = stamp;
+    }
+
+    public void update(State state) {
+      this.state = state;
+      this.stamp = System.currentTimeMillis();
+    }
+
+    public State getState() {
+      return state;
+    }
+
+    public long getStamp() {
+      return stamp;
+    }
+
+    public HRegionInfo getRegion() {
+      return region;
+    }
+
+    public boolean isClosing() {
+      return state == State.CLOSING;
+    }
+
+    public boolean isClosed() {
+      return state == State.CLOSED;
+    }
+
+    public boolean isPendingClose() {
+      return state == State.PENDING_CLOSE;
+    }
+
+    public boolean isOpening() {
+      return state == State.OPENING;
+    }
+
+    public boolean isOpened() {
+      return state == State.OPEN;
+    }
+
+    public boolean isPendingOpen() {
+      return state == State.PENDING_OPEN;
+    }
+
+    public boolean isOffline() {
+      return state == State.OFFLINE;
+    }
+
+    @Override
+    public String toString() {
+      return region.getRegionNameAsString() + " state=" + state +
+        ", ts=" + stamp;
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+      region = new HRegionInfo();
+      region.readFields(in);
+      state = State.valueOf(in.readUTF());
+      stamp = in.readLong();
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+      region.write(out);
+      out.writeUTF(state.name());
+      out.writeLong(stamp);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java b/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java
deleted file mode 100644
index 2deea4a1d79..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java
+++ /dev/null
@@ -1,620 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hbase.Chore;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.UnknownScannerException;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.ipc.RemoteException;
-
-
-/**
- * Base HRegion scanner class. Holds utilty common to <code>ROOT</code> and
- * <code>META</code> HRegion scanners.
- *
- * <p>How do we know if all regions are assigned? After the initial scan of
- * the <code>ROOT</code> and <code>META</code> regions, all regions known at
- * that time will have been or are in the process of being assigned.</p>
- *
- * <p>When a region is split the region server notifies the master of the
- * split and the new regions are assigned. But suppose the master loses the
- * split message? We need to periodically rescan the <code>ROOT</code> and
- * <code>META</code> regions.
- *    <ul>
- *    <li>If we rescan, any regions that are new but not assigned will have
- *    no server info. Any regions that are not being served by the same
- *    server will get re-assigned.</li>
- *
- *    <li>Thus a periodic rescan of the root region will find any new
- *    <code>META</code> regions where we missed the <code>META</code> split
- *    message or we failed to detect a server death and consequently need to
- *    assign the region to a new server.</li>
- *
- *    <li>if we keep track of all the known <code>META</code> regions, then
- *    we can rescan them periodically. If we do this then we can detect any
- *    regions for which we missed a region split message.</li>
- *    </ul>
- *
- * Thus just keeping track of all the <code>META</code> regions permits
- * periodic rescanning which will detect unassigned regions (new or
- * otherwise) without the need to keep track of every region.</p>
- *
- * <p>So the <code>ROOT</code> region scanner needs to wake up:
- * <ol>
- * <li>when the master receives notification that the <code>ROOT</code>
- * region has been opened.</li>
- * <li>periodically after the first scan</li>
- * </ol>
- *
- * The <code>META</code>  scanner needs to wake up:
- * <ol>
- * <li>when a <code>META</code> region comes on line</li>
- * </li>periodically to rescan the online <code>META</code> regions</li>
- * </ol>
- *
- * <p>A <code>META</code> region is not 'online' until it has been scanned
- * once.
- */
-abstract class BaseScanner extends Chore {
-  static final Log LOG = LogFactory.getLog(BaseScanner.class.getName());
-  // These are names of new columns in a meta region offlined parent row.  They
-  // are added by the metascanner after we verify that split daughter made it
-  // in.  Their value is 'true' if present.
-  private static final byte[] SPLITA_CHECKED =
-    Bytes.toBytes(Bytes.toString(HConstants.SPLITA_QUALIFIER) + "_checked");
-  private static final byte[] SPLITB_CHECKED =
-    Bytes.toBytes(Bytes.toString(HConstants.SPLITB_QUALIFIER) + "_checked");
-  // Make the 'true' Writable once only.
-  private static byte[] TRUE_WRITABLE_AS_BYTES;
-  static {
-    try {
-      TRUE_WRITABLE_AS_BYTES = Writables.getBytes(new BooleanWritable(true));
-    } catch (IOException e) {
-      e.printStackTrace();
-    }
-  }
-  private final boolean rootRegion;
-  protected final HMaster master;
-
-  protected boolean initialScanComplete;
-
-  protected abstract boolean initialScan();
-  protected abstract void maintenanceScan();
-
-  // will use this variable to synchronize and make sure we aren't interrupted
-  // mid-scan
-  final Object scannerLock = new Object();
-
-  BaseScanner(final HMaster master, final boolean rootRegion,
-      final AtomicBoolean stop) {
-    super("Scanner for " + (rootRegion ? "-ROOT-":".META.") + " table",
-        master.getConfiguration().
-        getInt("hbase.master.meta.thread.rescanfrequency", 60 * 1000), stop);
-    this.rootRegion = rootRegion;
-    this.master = master;
-    this.initialScanComplete = false;
-  }
-
-  /** @return true if initial scan completed successfully */
-  public boolean isInitialScanComplete() {
-    return initialScanComplete;
-  }
-
-  @Override
-  protected boolean initialChore() {
-    return initialScan();
-  }
-
-  @Override
-  protected void chore() {
-    maintenanceScan();
-  }
-
-  /**
-   * @param region Region to scan
-   * @throws IOException
-   */
-  protected void scanRegion(final MetaRegion region) throws IOException {
-    HRegionInterface regionServer = null;
-    long scannerId = -1L;
-    LOG.info(Thread.currentThread().getName() + " scanning meta region " +
-      region.toString());
-
-    // Array to hold list of split parents found.  Scan adds to list.  After
-    // scan we go check if parents can be removed and that their daughters
-    // are in place.
-    NavigableMap<HRegionInfo, Result> splitParents =
-      new TreeMap<HRegionInfo, Result>();
-    List<byte []> emptyRows = new ArrayList<byte []>();
-    int rows = 0;
-    try {
-      regionServer =
-        this.master.getServerConnection().getHRegionConnection(region.getServer());
-      Scan s = new Scan().addFamily(HConstants.CATALOG_FAMILY);
-      // Make this scan do a row at a time otherwise, data can be stale.
-      s.setCaching(1);
-      scannerId = regionServer.openScanner(region.getRegionName(), s);
-      while (true) {
-        Result values = regionServer.next(scannerId);
-        if (values == null || values.size() == 0) {
-          break;
-        }
-        HRegionInfo info = master.getHRegionInfo(values.getRow(), values);
-        if (info == null) {
-          emptyRows.add(values.getRow());
-          continue;
-        }
-        String serverAddress = getServerAddress(values);
-        long startCode = getStartCode(values);
-
-        // Note Region has been assigned.
-        checkAssigned(regionServer, region, info, serverAddress, startCode, true);
-        if (isSplitParent(info)) {
-          splitParents.put(info, values);
-        }
-        rows += 1;
-      }
-      if (rootRegion) {
-        this.master.getRegionManager().setNumMetaRegions(rows);
-      }
-    } catch (IOException e) {
-      if (e instanceof RemoteException) {
-        e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
-        if (e instanceof UnknownScannerException) {
-          // Reset scannerId so we do not try closing a scanner the other side
-          // has lost account of: prevents duplicated stack trace out of the
-          // below close in the finally.
-          scannerId = -1L;
-        }
-      }
-      throw e;
-    } finally {
-      try {
-        if (scannerId != -1L && regionServer != null) {
-          regionServer.close(scannerId);
-        }
-      } catch (IOException e) {
-        LOG.error("Closing scanner",
-            RemoteExceptionHandler.checkIOException(e));
-      }
-    }
-
-    // Scan is finished.
-
-    // First clean up any meta region rows which had null HRegionInfos
-    if (emptyRows.size() > 0) {
-      LOG.warn("Found " + emptyRows.size() + " rows with empty HRegionInfo " +
-        "while scanning meta region " + Bytes.toString(region.getRegionName()));
-      this.master.deleteEmptyMetaRows(regionServer, region.getRegionName(),
-          emptyRows);
-    }
-
-    // Take a look at split parents to see if any we can clean up any and to
-    // make sure that daughter regions are in place.
-    if (splitParents.size() > 0) {
-      for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
-        HRegionInfo hri = e.getKey();
-        cleanupAndVerifySplits(region.getRegionName(), regionServer,
-          hri, e.getValue());
-      }
-    }
-    LOG.info(Thread.currentThread().getName() + " scan of " + rows +
-      " row(s) of meta region " + region.toString() + " complete");
-  }
-
-  /*
-   * @param r
-   * @return Empty String or server address found in <code>r</code>
-   */
-  static String getServerAddress(final Result r) {
-    final byte[] val = r.getValue(HConstants.CATALOG_FAMILY,
-        HConstants.SERVER_QUALIFIER);
-    return val == null || val.length <= 0 ? "" : Bytes.toString(val);
-  }
-
-  /*
-   * @param r
-   * @return Return 0L or server startcode found in <code>r</code>
-   */
-  static long getStartCode(final Result r) {
-    final byte[] val = r.getValue(HConstants.CATALOG_FAMILY,
-        HConstants.STARTCODE_QUALIFIER);
-    return val == null || val.length <= 0 ? 0L : Bytes.toLong(val);
-  }
-
-  /*
-   * @param info Region to check.
-   * @return True if this is a split parent.
-   */
-  private boolean isSplitParent(final HRegionInfo info) {
-    if (!info.isSplit()) {
-      return false;
-    }
-    if (!info.isOffline()) {
-      LOG.warn("Region is split but not offline: " +
-        info.getRegionNameAsString());
-    }
-    return true;
-  }
-
-  /*
-   * If daughters no longer hold reference to the parents, delete the parent.
-   * If the parent is lone without daughter splits AND there are references in
-   * the filesystem, then a daughters was not added to .META. -- must have been
-   * a crash before their addition.  Add them here.
-   * @param metaRegionName Meta region name: e.g. .META.,,1
-   * @param server HRegionInterface of meta server to talk to
-   * @param parent HRegionInfo of split offlined parent
-   * @param rowContent Content of <code>parent</code> row in
-   * <code>metaRegionName</code>
-   * @return True if we removed <code>parent</code> from meta table and from
-   * the filesystem.
-   * @throws IOException
-   */
-  private boolean cleanupAndVerifySplits(final byte [] metaRegionName,
-    final HRegionInterface srvr, final HRegionInfo parent,
-    Result rowContent)
-  throws IOException {
-    boolean result = false;
-    // Run checks on each daughter split.
-    boolean hasReferencesA = checkDaughter(metaRegionName, srvr,
-      parent, rowContent, HConstants.SPLITA_QUALIFIER);
-    boolean hasReferencesB = checkDaughter(metaRegionName, srvr,
-        parent, rowContent, HConstants.SPLITB_QUALIFIER);
-    if (!hasReferencesA && !hasReferencesB) {
-      LOG.info("Deleting region " + parent.getRegionNameAsString() +
-        " because daughter splits no longer hold references");
-      HRegion.deleteRegion(this.master.getFileSystem(),
-        this.master.getRootDir(), parent);
-      HRegion.removeRegionFromMETA(srvr, metaRegionName,
-        parent.getRegionName());
-      result = true;
-    }
-    return result;
-  }
-
-
-  /*
-   * See if the passed daughter has references in the filesystem to the parent
-   * and if not, remove the note of daughter region in the parent row: its
-   * column info:splitA or info:splitB.  Also make sure that daughter row is
-   * present in the .META. and mark the parent row when confirmed so we don't
-   * keep checking.  The mark will be info:splitA_checked and its value will be
-   * a true BooleanWritable.
-   * @param metaRegionName
-   * @param srvr
-   * @param parent
-   * @param rowContent
-   * @param qualifier
-   * @return True if this daughter still has references to the parent.
-   * @throws IOException
-   */
-  private boolean checkDaughter(final byte [] metaRegionName,
-    final HRegionInterface srvr, final HRegionInfo parent,
-    final Result rowContent, final byte [] qualifier)
-  throws IOException {
-    HRegionInfo hri = getDaughterRegionInfo(rowContent, qualifier);
-    boolean references = hasReferences(metaRegionName, srvr, parent, rowContent,
-        hri, qualifier);
-    // Return if no references.
-    if (!references) return references;
-    if (!verifyDaughterRowPresent(rowContent, qualifier, srvr, metaRegionName,
-        hri, parent)) {
-      // If we got here, then the parent row does not yet have the
-      // "daughter row verified present" marker present. Add it.
-      addDaughterRowChecked(metaRegionName, srvr, parent.getRegionName(), hri,
-        qualifier);
-    }
-    return references;
-  }
-
-  /*
-   * Check the daughter of parent is present in meta table.  If not there,
-   * add it.
-   * @param rowContent
-   * @param daughter
-   * @param srvr
-   * @param metaRegionName
-   * @param daughterHRI
-   * @throws IOException
-   * @return True, if parent row has marker for "daughter row verified present"
-   * else, false (and will do fixup adding daughter if daughter not present).
-   */
-  private boolean verifyDaughterRowPresent(final Result rowContent,
-      final byte [] daughter, final HRegionInterface srvr,
-      final byte [] metaRegionName,
-      final HRegionInfo daughterHRI, final HRegionInfo parent)
-  throws IOException {
-    // See if the 'checked' column is in parent. If so, we're done.
-    boolean present = getDaughterRowChecked(rowContent, daughter);
-    if (present) return present;
-    // Parent is not carrying the splitA_checked/splitB_checked so this must
-    // be the first time through here checking splitA/splitB are in metatable.
-    byte [] daughterRowKey = daughterHRI.getRegionName();
-    Get g = new Get(daughterRowKey);
-    g.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-    Result r = srvr.get(metaRegionName, g);
-    if (r == null || r.isEmpty()) {
-      // Daughter row not present.  Fixup kicks in.  Insert it.
-      LOG.warn("Fixup broke split: Add missing split daughter to meta," +
-       " daughter=" + daughterHRI.toString() + ", parent=" + parent.toString());
-      Put p = new Put(daughterRowKey);
-      p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
-        Writables.getBytes(daughterHRI));
-      srvr.put(metaRegionName, p);
-    }
-    return present;
-  }
-
-  /*
-   * Add to parent a marker that we verified the daughter exists.
-   * @param metaRegionName
-   * @param srvr
-   * @param parent
-   * @param split
-   * @param daughter
-   * @throws IOException
-   */
-  private void addDaughterRowChecked(final byte [] metaRegionName,
-    final HRegionInterface srvr, final byte [] parent,
-    final HRegionInfo split, final byte [] daughter)
-  throws IOException {
-    Put p = new Put(parent);
-    p.add(HConstants.CATALOG_FAMILY, getNameOfVerifiedDaughterColumn(daughter),
-      TRUE_WRITABLE_AS_BYTES);
-    srvr.put(metaRegionName, p);
-  }
-
-  /*
-   * @param rowContent
-   * @param which
-   * @return True if the daughter row has already been verified present in
-   * metatable.
-   * @throws IOException
-   */
-  private boolean getDaughterRowChecked(final Result rowContent,
-    final byte[] which)
-  throws IOException {
-    final byte[] b = rowContent.getValue(HConstants.CATALOG_FAMILY,
-      getNameOfVerifiedDaughterColumn(which));
-    BooleanWritable bw = null;
-    if (b != null && b.length > 0) {
-      bw = (BooleanWritable)Writables.getWritable(b, new BooleanWritable());
-    }
-    return bw == null? false: bw.get();
-  }
-
-  /*
-   * @param daughter
-   * @return Returns splitA_checked or splitB_checked dependent on what
-   * <code>daughter</code> is.
-   */
-  private static byte [] getNameOfVerifiedDaughterColumn(final byte [] daughter) {
-    return (Bytes.equals(HConstants.SPLITA_QUALIFIER, daughter)
-            ? SPLITA_CHECKED : SPLITB_CHECKED);
-  }
-
-  /*
-   * Get daughter HRegionInfo out of parent info:splitA/info:splitB columns.
-   * @param rowContent
-   * @param which Whether "info:splitA" or "info:splitB" column
-   * @return Deserialized content of the info:splitA or info:splitB as a
-   * HRegionInfo
-   * @throws IOException
-   */
-  private HRegionInfo getDaughterRegionInfo(final Result rowContent,
-    final byte [] which)
-  throws IOException {
-    return Writables.getHRegionInfoOrNull(
-        rowContent.getValue(HConstants.CATALOG_FAMILY, which));
-  }
-
-  /*
-   * Remove mention of daughter from parent row.
-   * parent row.
-   * @param metaRegionName
-   * @param srvr
-   * @param parent
-   * @param split
-   * @param qualifier
-   * @throws IOException
-   */
-  private void removeDaughterFromParent(final byte [] metaRegionName,
-    final HRegionInterface srvr, final HRegionInfo parent,
-    final HRegionInfo split, final byte [] qualifier)
-  throws IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug(split.getRegionNameAsString() +
-        " no longer has references to " + parent.getRegionNameAsString());
-    }
-    Delete delete = new Delete(parent.getRegionName());
-    delete.deleteColumns(HConstants.CATALOG_FAMILY, qualifier);
-    srvr.delete(metaRegionName, delete);
-  }
-
-  /*
-   * Checks if a daughter region -- either splitA or splitB -- still holds
-   * references to parent.  If not, removes reference to the split from
-   * the parent meta region row so we don't check it any more.
-   * @param metaRegionName Name of meta region to look in.
-   * @param srvr Where region resides.
-   * @param parent Parent region name.
-   * @param rowContent Keyed content of the parent row in meta region.
-   * @param split Which column family.
-   * @param qualifier Which of the daughters to look at, splitA or splitB.
-   * @return True if still has references to parent.
-   * @throws IOException
-   */
-  private boolean hasReferences(final byte [] metaRegionName,
-    final HRegionInterface srvr, final HRegionInfo parent,
-    Result rowContent, final HRegionInfo split, byte [] qualifier)
-  throws IOException {
-    boolean result = false;
-    if (split == null) {
-      return result;
-    }
-    Path tabledir =
-      new Path(this.master.getRootDir(), split.getTableDesc().getNameAsString());
-    for (HColumnDescriptor family: split.getTableDesc().getFamilies()) {
-      Path p = Store.getStoreHomedir(tabledir, split.getEncodedName(),
-        family.getName());
-      if (!this.master.getFileSystem().exists(p)) continue;
-      // Look for reference files.  Call listStatus with an anonymous
-      // instance of PathFilter.
-      FileStatus [] ps =
-        this.master.getFileSystem().listStatus(p, new PathFilter () {
-            public boolean accept(Path path) {
-              return StoreFile.isReference(path);
-            }
-          }
-      );
-
-      if (ps != null && ps.length > 0) {
-        result = true;
-        break;
-      }
-    }
-    if (!result) {
-      removeDaughterFromParent(metaRegionName, srvr, parent, split, qualifier);
-    }
-    return result;
-  }
-
-  /*
-   * Check the passed region is assigned.  If not, add to unassigned.
-   * @param regionServer
-   * @param meta
-   * @param info
-   * @param hostnameAndPort hostname ':' port as it comes out of .META.
-   * @param startCode
-   * @param checkTwice should we check twice before adding a region
-   * to unassigned pool.              
-   * @throws IOException
-   */
-  protected void checkAssigned(final HRegionInterface regionServer,
-    final MetaRegion meta, HRegionInfo info,
-    final String hostnameAndPort, final long startCode, boolean checkTwice)
-  throws IOException {
-    boolean tryAgain = false;
-    String serverName = null;
-    String sa = hostnameAndPort;
-    long sc = startCode;
-    if (sa == null || sa.length() <= 0) {
-      // Scans are sloppy.  They cache a row internally so may have data that
-      // is a little stale.  Make sure that for sure this serverAddress is null.
-      // We are trying to avoid double-assignments.  See hbase-1784.
-      Get g = new Get(info.getRegionName());
-      g.addFamily(HConstants.CATALOG_FAMILY);
-      Result r = regionServer.get(meta.getRegionName(), g);
-      if (r != null && !r.isEmpty()) {
-        sa = getServerAddress(r);
-        sc = getStartCode(r);
-        info = master.getHRegionInfo(r.getRow(), r);
-      }
-    }
-    if (sa != null && sa.length() > 0) {
-      serverName = HServerInfo.getServerName(sa, sc);
-    }
-    HServerInfo storedInfo = null;
-    synchronized (this.master.getRegionManager()) {
-      /* We don't assign regions that are offline, in transition or were on
-       * a dead server. Regions that were on a dead server will get reassigned
-       * by ProcessServerShutdown
-       */
-      if (info == null || info.isOffline() ||
-        this.master.getRegionManager().regionIsInTransition(info.getRegionNameAsString()) ||
-          (serverName != null && this.master.getServerManager().isDead(serverName))) {
-        return;
-      }
-      if (serverName != null) {
-        storedInfo = this.master.getServerManager().getServerInfo(serverName);
-      }
-
-      // If we can't find the HServerInfo, then add it to the list of
-      //  unassigned regions.
-      if (storedInfo == null) {
-        if (checkTwice) {
-          tryAgain = true;
-        } else {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Current assignment of " + info.getRegionNameAsString() +
-              " is not valid; " + " serverAddress=" + sa +
-              ", startCode=" + sc + " unknown.");
-          }
-          // Now get the region assigned
-          this.master.getRegionManager().setUnassigned(info, true);
-        }
-      }
-    }
-    if (tryAgain) {
-      // The current assignment is invalid. But we need to try again.
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Current assignment of " + info.getRegionNameAsString() +
-          " is not valid; " + " serverAddress=" + sa +
-          ", startCode=" + sc + " unknown; checking once more!");
-      }
-      // passing null for hostNameAndPort will force the function
-      // to reget the assignment from META and protect against
-      // double assignment race conditions (HBASE-2755).
-      checkAssigned(regionServer, meta, info, null, 0, false);
-    }
-  }
-
-  /**
-   * Interrupt thread regardless of what it's doing
-   */
-  public void interruptAndStop() {
-    synchronized(scannerLock){
-      if (isAlive()) {
-        super.interrupt();
-        LOG.info("Interrupted");
-      }
-    }
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ChangeTableState.java b/src/main/java/org/apache/hadoop/hbase/master/ChangeTableState.java
deleted file mode 100644
index c26bb562abf..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/ChangeTableState.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.util.Writables;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * Instantiated to enable or disable a table
- */
-class ChangeTableState extends TableOperation {
-  private final Log LOG = LogFactory.getLog(this.getClass());
-  private boolean online;
-  // Do in order.
-  protected final Map<String, HashSet<HRegionInfo>> servedRegions =
-    new TreeMap<String, HashSet<HRegionInfo>>();
-  protected long lockid;
-
-  ChangeTableState(final HMaster master, final byte [] tableName,
-    final boolean onLine)
-  throws IOException {
-    super(master, tableName);
-    this.online = onLine;
-  }
-
-  @Override
-  protected void processScanItem(String serverName, HRegionInfo info) {
-    if (isBeingServed(serverName)) {
-      HashSet<HRegionInfo> regions = this.servedRegions.get(serverName);
-      if (regions == null) {
-        regions = new HashSet<HRegionInfo>();
-      }
-      regions.add(info);
-      this.servedRegions.put(serverName, regions);
-    }
-  }
-
-  @Override
-  protected void postProcessMeta(MetaRegion m, HRegionInterface server)
-  throws IOException {
-    // Process regions not being served
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Processing unserved regions");
-    }
-    for (HRegionInfo i: this.unservedRegions) {
-      if (i.isOffline() && i.isSplit()) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Skipping region " + i.toString() +
-            " because it is offline and split");
-        }
-        continue;
-      }
-
-      if(!this.online && this.master.getRegionManager().
-          isPendingOpen(i.getRegionNameAsString())) {
-        LOG.debug("Skipping region " + i.toString() +
-          " because it is pending open, will tell it to close later");
-        continue;
-      }
-
-      // If it's already offline then don't set it a second/third time, skip
-      // Same for online, don't set again if already online
-      if (!(i.isOffline() && !online) && !(!i.isOffline() && online)) {
-        // Update meta table
-        Put put = updateRegionInfo(i);
-        server.put(m.getRegionName(), put);
-        Delete delete = new Delete(i.getRegionName());
-        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
-        server.delete(m.getRegionName(), delete);
-      }
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Removed server and startcode from row and set online=" +
-          this.online + ": " + i.getRegionNameAsString());
-      }
-      synchronized (master.getRegionManager()) {
-        if (this.online) {
-          // Bring offline regions on-line
-          if (!this.master.getRegionManager().regionIsOpening(i.getRegionNameAsString())) {
-            this.master.getRegionManager().setUnassigned(i, false);
-          }
-        } else {
-          // Prevent region from getting assigned.
-          this.master.getRegionManager().removeRegion(i);
-        }
-      }
-    }
-
-    // Process regions currently being served
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Processing regions currently being served");
-    }
-    synchronized (this.master.getRegionManager()) {
-      for (Map.Entry<String, HashSet<HRegionInfo>> e:
-          this.servedRegions.entrySet()) {
-        String serverName = e.getKey();
-        if (this.online) {
-          LOG.debug("Already online");
-          continue;                             // Already being served
-        }
-
-        // Cause regions being served to be taken off-line and disabled
-        for (HRegionInfo i: e.getValue()) {
-          // The scan we did could be totally staled, get the freshest data
-          Get get = new Get(i.getRegionName());
-          get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-          Result values = server.get(m.getRegionName(), get);
-          String serverAddress = BaseScanner.getServerAddress(values);
-          // If this region is unassigned, skip!
-          if(serverAddress.length() == 0) {
-            continue;
-          }
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Adding region " + i.getRegionNameAsString() +
-              " to setClosing list");
-          }
-          // this marks the regions to be closed
-          this.master.getRegionManager().setClosing(serverName, i, true);
-        }
-      }
-    }
-    this.servedRegions.clear();
-  }
-
-  protected Put updateRegionInfo(final HRegionInfo i)
-  throws IOException {
-    i.setOffline(!online);
-    Put put = new Put(i.getRegionName());
-    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(i));
-    return put;
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ColumnOperation.java b/src/main/java/org/apache/hadoop/hbase/master/ColumnOperation.java
deleted file mode 100644
index c8206ac8c13..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/ColumnOperation.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.TableNotDisabledException;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.util.Writables;
-
-import java.io.IOException;
-
-abstract class ColumnOperation extends TableOperation {
-  private final Log LOG = LogFactory.getLog(this.getClass());
-
-  protected ColumnOperation(final HMaster master, final byte [] tableName)
-  throws IOException {
-    super(master, tableName);
-  }
-
-  @Override
-  protected void processScanItem(String serverName, final HRegionInfo info)
-  throws IOException {
-    if (isEnabled(info)) {
-      throw new TableNotDisabledException(tableName);
-    }
-  }
-
-  protected void updateRegionInfo(HRegionInterface server, byte [] regionName,
-    HRegionInfo i) throws IOException {
-    Put put = new Put(i.getRegionName());
-    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(i));
-    server.put(regionName, put);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("updated columns in row: " + i.getRegionNameAsString());
-    }
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java b/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
new file mode 100644
index 00000000000..9085355c6df
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
@@ -0,0 +1,133 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.hbase.HServerInfo;
+
+/**
+ * Class to hold dead servers list and utility querying dead server list.
+ */
+public class DeadServer implements Set<String> {
+  /**
+   * Set of known dead servers.  On znode expiration, servers are added here.
+   * This is needed in case of a network partitioning where the server's lease
+   * expires, but the server is still running. After the network is healed,
+   * and it's server logs are recovered, it will be told to call server startup
+   * because by then, its regions have probably been reassigned.
+   */
+  private final Set<String> deadServers = new HashSet<String>();
+
+
+  /**
+   * @param serverName
+   * @return true if server is dead
+   */
+  public boolean isDeadServer(final String serverName) {
+    return isDeadServer(serverName, false);
+  }
+
+  /**
+   * @param serverName Servername as either <code>host:port</code> or
+   * <code>host,port,startcode</code>.
+   * @param hostAndPortOnly True if <code>serverName</code> is host and
+   * port only (<code>host:port</code>) and if so, then we do a prefix compare
+   * (ignoring start codes) looking for dead server.
+   * @return true if server is dead
+   */
+  boolean isDeadServer(final String serverName, final boolean hostAndPortOnly) {
+    return HServerInfo.isServer(this, serverName, hostAndPortOnly);
+  }
+
+  public synchronized Set<String> clone() {
+    Set<String> clone = new HashSet<String>(this.deadServers.size());
+    clone.addAll(this.deadServers);
+    return clone;
+  }
+
+  public synchronized int size() {
+    return deadServers.size();
+  }
+
+  public synchronized boolean isEmpty() {
+    return deadServers.isEmpty();
+  }
+
+  public synchronized boolean contains(Object o) {
+    return deadServers.contains(o);
+  }
+
+  public Iterator<String> iterator() {
+    return this.deadServers.iterator();
+  }
+
+  public synchronized Object[] toArray() {
+    return deadServers.toArray();
+  }
+
+  public synchronized <T> T[] toArray(T[] a) {
+    return deadServers.toArray(a);
+  }
+
+  public synchronized boolean add(String e) {
+    return deadServers.add(e);
+  }
+
+  public synchronized boolean remove(Object o) {
+    return deadServers.remove(o);
+  }
+
+  public synchronized boolean containsAll(Collection<?> c) {
+    return deadServers.containsAll(c);
+  }
+
+  public synchronized boolean addAll(Collection<? extends String> c) {
+    return deadServers.addAll(c);
+  }
+
+  public synchronized boolean retainAll(Collection<?> c) {
+    return deadServers.retainAll(c);
+  }
+
+  public synchronized boolean removeAll(Collection<?> c) {
+    return deadServers.removeAll(c);
+  }
+
+  public synchronized void clear() {
+    throw new NotImplementedException();
+  }
+
+  public synchronized boolean equals(Object o) {
+    return deadServers.equals(o);
+  }
+
+  public synchronized int hashCode() {
+    return deadServers.hashCode();
+  }
+
+  public synchronized String toString() {
+    return this.deadServers.toString();
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/DeleteColumn.java b/src/main/java/org/apache/hadoop/hbase/master/DeleteColumn.java
deleted file mode 100644
index 1d586680839..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/DeleteColumn.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.regionserver.Store;
-
-import java.io.IOException;
-
-/** Instantiated to remove a column family from a table */
-class DeleteColumn extends ColumnOperation {
-  private final byte [] columnName;
-
-  DeleteColumn(final HMaster master, final byte [] tableName,
-    final byte [] columnName)
-  throws IOException {
-    super(master, tableName);
-    this.columnName = columnName;
-  }
-
-  @Override
-  protected void postProcessMeta(MetaRegion m, HRegionInterface server)
-  throws IOException {
-    for (HRegionInfo i: unservedRegions) {
-      i.getTableDesc().removeFamily(columnName);
-      updateRegionInfo(server, m.getRegionName(), i);
-      // Delete the directories used by the column
-      Path tabledir =
-        new Path(this.master.getRootDir(), i.getTableDesc().getNameAsString());
-      this.master.getFileSystem().
-        delete(Store.getStoreHomedir(tabledir, i.getEncodedName(),
-        this.columnName), true);
-    }
-  }
-}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 47353040723..ca6ac6aa13d 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -24,18 +24,12 @@ import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.RuntimeMXBean;
 import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
@@ -44,66 +38,69 @@ import org.apache.commons.cli.ParseException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Chore;
 import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HMsg;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.HServerLoad;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.LocalHBaseCluster;
 import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.MiniZooKeeperCluster;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.TableNotDisabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.UnknownRegionException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.MetaScanner;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.ServerConnection;
 import org.apache.hadoop.hbase.client.ServerConnectionManager;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
-import org.apache.hadoop.hbase.executor.HBaseExecutorService;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.HBaseRPCProtocolVersion;
 import org.apache.hadoop.hbase.ipc.HBaseServer;
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan;
+import org.apache.hadoop.hbase.master.handler.DeleteTableHandler;
+import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
+import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
+import org.apache.hadoop.hbase.master.handler.ModifyTableHandler;
+import org.apache.hadoop.hbase.master.handler.TableAddFamilyHandler;
+import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
+import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.InfoServer;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Sleeper;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.VersionInfo;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.net.DNS;
-import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.EventType;
-import org.apache.zookeeper.Watcher.Event.KeeperState;
-
-import com.google.common.collect.Lists;
 
 /**
  * HMaster is the "master server" for HBase. An HBase cluster has one active
@@ -111,253 +108,257 @@ import com.google.common.collect.Lists;
  * run the cluster.  All others park themselves in their constructor until
  * master or cluster shutdown or until the active master loses its lease in
  * zookeeper.  Thereafter, all running master jostle to take over master role.
+ * 
+ * <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In
+ * this case it will tell all regionservers to go down and then wait on them
+ * all reporting in that they are down.  This master will then shut itself down.
+ * 
+ * <p>You can also shutdown just this master.  Call {@link #stopMaster()}.
+ * 
  * @see HMasterInterface
  * @see HMasterRegionInterface
  * @see Watcher
  */
-public class HMaster extends Thread implements HMasterInterface,
-    HMasterRegionInterface, Watcher {
+public class HMaster extends Thread
+implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
+  private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
+
   // MASTER is name of the webapp and the attribute name used stuffing this
   //instance into web context.
   public static final String MASTER = "master";
-  private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
-
-  // We start out with closed flag on.  Its set to off after construction.
-  // Use AtomicBoolean rather than plain boolean because we want other threads
-  // able to set shutdown flag.  Using AtomicBoolean can pass a reference
-  // rather than have them have to know about the hosting Master class.
-  final AtomicBoolean closed = new AtomicBoolean(true);
-  // TODO: Is this separate flag necessary?
-  private final AtomicBoolean shutdownRequested = new AtomicBoolean(false);
 
+  // The configuration for the Master
   private final Configuration conf;
-  private final Path rootdir;
+  // server for the web ui
   private InfoServer infoServer;
-  private final int threadWakeFrequency;
-  private final int numRetries;
-
-  // Metrics is set when we call run.
-  private final MasterMetrics metrics;
-
-  final Lock splitLogLock = new ReentrantLock();
 
   // Our zk client.
-  private ZooKeeperWrapper zooKeeperWrapper;
-  // Watcher for master address and for cluster shutdown.
-  private final ZKMasterAddressWatcher zkMasterAddressWatcher;
-  // A Sleeper that sleeps for threadWakeFrequency; sleep if nothing todo.
-  private final Sleeper sleeper;
-  // Keep around for convenience.
-  private final FileSystem fs;
-  // Is the fileystem ok?
-  private volatile boolean fsOk = true;
-  // The Path to the old logs dir
-  private final Path oldLogDir;
+  private ZooKeeperWatcher zooKeeper;
+  // Manager and zk listener for master election
+  private ActiveMasterManager activeMasterManager;
+  // Region server tracker
+  private RegionServerTracker regionServerTracker;
 
+  // RPC server for the HMaster
   private final HBaseServer rpcServer;
+  // Address of the HMaster
   private final HServerAddress address;
+  // file system manager for the master FS operations
+  private final MasterFileSystem fileSystemManager;
 
   private final ServerConnection connection;
+  // server manager to deal with region server info
   private final ServerManager serverManager;
-  private final RegionManager regionManager;
 
-  private long lastFragmentationQuery = -1L;
-  private Map<String, Integer> fragmentation = null;
-  private final RegionServerOperationQueue regionServerOperationQueue;
-  
+  // manager of assignment nodes in zookeeper
+  final AssignmentManager assignmentManager;
+  // manager of catalog regions
+  private final CatalogTracker catalogTracker;
+  // Cluster status zk tracker and local setter
+  private ClusterStatusTracker clusterStatusTracker;
+
   // True if this is the master that started the cluster.
-  boolean isClusterStartup;
+  boolean clusterStarter;
+
+  // This flag is for stopping this Master instance.
+  private boolean stopped = false;
+  // Set on abort -- usually failure of our zk session
+  private volatile boolean abort = false;
+
+  // Instance of the hbase executor service.
+  ExecutorService executorService;
+
+  private LoadBalancer balancer = new LoadBalancer();
+  private Chore balancerChore;
+  private volatile boolean balance = true;
 
   /**
-   * Constructor
-   * @param conf configuration
-   * @throws IOException
+   * Initializes the HMaster. The steps are as follows:
+   *
+   * <ol>
+   * <li>Initialize HMaster RPC and address
+   * <li>Connect to ZooKeeper and figure out if this is a fresh cluster start or
+   *     a failed over master
+   * <li>Block until becoming active master
+   * <li>Initialize master components - server manager, region manager,
+   *     region server queue, file system manager, etc
+   * </ol>
+   * @throws InterruptedException 
    */
-  public HMaster(Configuration conf) throws IOException {
+  public HMaster(final Configuration conf)
+  throws IOException, KeeperException, InterruptedException {
     this.conf = conf;
-    
-    // Figure out if this is a fresh cluster start. This is done by checking the 
-    // number of RS ephemeral nodes. RS ephemeral nodes are created only after 
-    // the primary master has written the address to ZK. So this has to be done 
-    // before we race to write our address to zookeeper.
-    zooKeeperWrapper = ZooKeeperWrapper.createInstance(conf, HMaster.class.getName());
-    isClusterStartup = (zooKeeperWrapper.scanRSDirectory().size() == 0);
-    
-    // Get my address and create an rpc server instance.  The rpc-server port
-    // can be ephemeral...ensure we have the correct info
+    /*
+     * 1. Determine address and initialize RPC server (but do not start).
+     * The RPC server ports can be ephemeral.
+     */
     HServerAddress a = new HServerAddress(getMyAddress(this.conf));
-    this.rpcServer = HBaseRPC.getServer(this, a.getBindAddress(),
-      a.getPort(), conf.getInt("hbase.regionserver.handler.count", 10),
-      false, conf);
-    this.address = new HServerAddress(this.rpcServer.getListenerAddress());
+    int numHandlers = conf.getInt("hbase.regionserver.handler.count", 10);
+    this.rpcServer = HBaseRPC.getServer(this, a.getBindAddress(), a.getPort(),
+      numHandlers, false, conf);
+    this.address = new HServerAddress(rpcServer.getListenerAddress());
 
-    this.numRetries =  conf.getInt("hbase.client.retries.number", 2);
-    this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
-        10 * 1000);
+    // set the thread name now we have an address
+    setName(MASTER + "-" + this.address);
 
-    this.sleeper = new Sleeper(this.threadWakeFrequency, this.closed);
-    this.connection = ServerConnectionManager.getConnection(conf);
-
-    // hack! Maps DFSClient => Master for logs.  HDFS made this 
+    // Hack! Maps DFSClient => Master for logs.  HDFS made this 
     // config param for task trackers, but we can piggyback off of it.
     if (this.conf.get("mapred.task.id") == null) {
       this.conf.set("mapred.task.id", "hb_m_" + this.address.toString());
     }
 
-    // Set filesystem to be that of this.rootdir else we get complaints about
-    // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
-    // default localfs.  Presumption is that rootdir is fully-qualified before
-    // we get to here with appropriate fs scheme.
-    this.rootdir = FSUtils.getRootDir(this.conf);
-    // Cover both bases, the old way of setting default fs and the new.
-    // We're supposed to run on 0.20 and 0.21 anyways.
-    this.conf.set("fs.default.name", this.rootdir.toString());
-    this.conf.set("fs.defaultFS", this.rootdir.toString());
-    this.fs = FileSystem.get(this.conf);
-    checkRootDir(this.rootdir, this.conf, this.fs);
+    /*
+     * 2. Determine if this is a fresh cluster startup or failed over master.
+     * This is done by checking for the existence of any ephemeral
+     * RegionServer nodes in ZooKeeper.  These nodes are created by RSs on
+     * their initialization but only after they find the primary master.  As
+     * long as this check is done before we write our address into ZK, this
+     * will work.  Note that multiple masters could find this to be true on
+     * startup (none have become active master yet), which is why there is an
+     * additional check if this master does not become primary on its first attempt.
+     */
+    this.zooKeeper =
+      new ZooKeeperWatcher(conf, MASTER + "-" + getMasterAddress(), this);
+    this.clusterStarter = 0 ==
+      ZKUtil.getNumberOfChildren(zooKeeper, zooKeeper.rsZNode);
 
-    // Make sure the region servers can archive their old logs
-    this.oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
-    if(!this.fs.exists(this.oldLogDir)) {
-      this.fs.mkdirs(this.oldLogDir);
-    }
+    /*
+     * 3. Block on becoming the active master.
+     * We race with other masters to write our address into ZooKeeper.  If we
+     * succeed, we are the primary/active master and finish initialization.
+     *
+     * If we do not succeed, there is another active master and we should
+     * now wait until it dies to try and become the next active master.  If we
+     * do not succeed on our first attempt, this is no longer a cluster startup.
+     */
+    activeMasterManager = new ActiveMasterManager(zooKeeper, address, this);
+    zooKeeper.registerListener(activeMasterManager);
 
-    // Get our zookeeper wrapper and then try to write our address to zookeeper.
-    // We'll succeed if we are only  master or if we win the race when many
-    // masters.  Otherwise we park here inside in writeAddressToZooKeeper.
-    // TODO: Bring up the UI to redirect to active Master.
-    zooKeeperWrapper.registerListener(this);
-    this.zkMasterAddressWatcher =
-      new ZKMasterAddressWatcher(this.zooKeeperWrapper, this.shutdownRequested);
-    zooKeeperWrapper.registerListener(zkMasterAddressWatcher);
+    // Wait here until we are the active master
+    clusterStarter = activeMasterManager.blockUntilBecomingActiveMaster();
 
-    // if we're a backup master, stall until a primary to writes his address
-    if(conf.getBoolean(HConstants.MASTER_TYPE_BACKUP, HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
-      // this will only be a minute or so while the cluster starts up,
-      // so don't worry about setting watches on the parent znode
-      while (!zooKeeperWrapper.masterAddressExists()) {
-        try {
-          LOG.debug("Waiting for master address ZNode to be written " +
-            "(Also watching cluster state node)");
-          Thread.sleep(conf.getInt("zookeeper.session.timeout", 60 * 1000));
-        } catch (InterruptedException e) {
-          // interrupted = user wants to kill us.  Don't continue
-          throw new IOException("Interrupted waiting for master address");
-        }
-      }
-    }
-    this.zkMasterAddressWatcher.writeAddressToZooKeeper(this.address, true);
-    this.regionServerOperationQueue =
-      new RegionServerOperationQueue(this.conf, this.closed);
+    /**
+     * 4. We are active master now... go initialize components we need to run.
+     */
+    // TODO: Do this using Dependency Injection, using PicoContainer or Spring.
+    this.fileSystemManager = new MasterFileSystem(this);
+    this.connection = ServerConnectionManager.getConnection(conf);
+    this.executorService = new ExecutorService(getServerName());
 
-    serverManager = new ServerManager(this);
+    this.serverManager = new ServerManager(this, this);
 
-    
-    // Start the unassigned watcher - which will create the unassigned region 
-    // in ZK. This is needed before RegionManager() constructor tries to assign 
-    // the root region.
-    ZKUnassignedWatcher.start(this.conf, this);
-    // start the "close region" executor service
-    HBaseEventType.RS2ZK_REGION_CLOSED.startMasterExecutorService(address.toString());
-    // start the "open region" executor service
-    HBaseEventType.RS2ZK_REGION_OPENED.startMasterExecutorService(address.toString());
+    this.catalogTracker = new CatalogTracker(this.zooKeeper, this.connection,
+      this, conf.getInt("hbase.master.catalog.timeout", -1));
+    this.catalogTracker.start();
 
-    
-    // start the region manager
-    regionManager = new RegionManager(this);
+    this.assignmentManager = new AssignmentManager(this, serverManager,
+      this.catalogTracker, this.executorService);
+    zooKeeper.registerListener(assignmentManager);
 
-    setName(MASTER);
-    this.metrics = new MasterMetrics(MASTER);
-    // We're almost open for business
-    this.closed.set(false);
-    LOG.info("HMaster initialized on " + this.address.toString());
+    this.regionServerTracker = new RegionServerTracker(zooKeeper, this,
+      this.serverManager);
+    regionServerTracker.start();
+
+    // Set the cluster as up.
+    this.clusterStatusTracker = new ClusterStatusTracker(getZooKeeper(), this);
+    this.clusterStatusTracker.setClusterUp();
+    this.clusterStatusTracker.start();
+
+    LOG.info("Server active/primary master; " + this.address +
+      "; clusterStarter=" + this.clusterStarter);
   }
-  
+
   /**
-   * Returns true if this master process was responsible for starting the 
-   * cluster.
+   * Main processing loop for the HMaster.
+   * 1. Handle both fresh cluster start as well as failed over initialization of
+   *    the HMaster.
+   * 2. Start the necessary services
+   * 3. Reassign the root region
+   * 4. The master is no longer closed - set "closed" to false
    */
-  public boolean isClusterStartup() {
-    return isClusterStartup;
-  }
-  
-  public void resetClusterStartup() {
-    isClusterStartup = false;
-  }
-  
-  public HServerAddress getHServerAddress() {
-    return address;
-  }
-
-  /*
-   * Get the rootdir.  Make sure its wholesome and exists before returning.
-   * @param rd
-   * @param conf
-   * @param fs
-   * @return hbase.rootdir (after checks for existence and bootstrapping if
-   * needed populating the directory with necessary bootup files).
-   * @throws IOException
-   */
-  private static Path checkRootDir(final Path rd, final Configuration c,
-    final FileSystem fs)
-  throws IOException {
-    // If FS is in safe mode wait till out of it.
-    FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
-        10 * 1000));
-    // Filesystem is good. Go ahead and check for hbase.rootdir.
-    if (!fs.exists(rd)) {
-      fs.mkdirs(rd);
-      FSUtils.setVersion(fs, rd);
-    } else {
-      FSUtils.checkVersion(fs, rd, true);
-    }
-    // Make sure the root region directory exists!
-    if (!FSUtils.rootRegionExists(fs, rd)) {
-      bootstrap(rd, c);
-    }
-    return rd;
-  }
-
-  private static void bootstrap(final Path rd, final Configuration c)
-  throws IOException {
-    LOG.info("BOOTSTRAP: creating ROOT and first META regions");
+  @Override
+  public void run() {
     try {
-      // Bootstrapping, make sure blockcache is off.  Else, one will be
-      // created here in bootstap and it'll need to be cleaned up.  Better to
-      // not make it in first place.  Turn off block caching for bootstrap.
-      // Enable after.
-      HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
-      setInfoFamilyCaching(rootHRI, false);
-      HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
-      setInfoFamilyCaching(metaHRI, false);
-      HRegion root = HRegion.createHRegion(rootHRI, rd, c);
-      HRegion meta = HRegion.createHRegion(metaHRI, rd, c);
-      setInfoFamilyCaching(rootHRI, true);
-      setInfoFamilyCaching(metaHRI, true);
-      // Add first region from the META table to the ROOT region.
-      HRegion.addRegionToMETA(root, meta);
-      root.close();
-      root.getLog().closeAndDelete();
-      meta.close();
-      meta.getLog().closeAndDelete();
-    } catch (IOException e) {
-      e = RemoteExceptionHandler.checkIOException(e);
-      LOG.error("bootstrap", e);
-      throw e;
+      // start up all service threads.
+      startServiceThreads();
+      // wait for minimum number of region servers to be up
+      this.serverManager.waitForMinServers();
+      // start assignment of user regions, startup or failure
+      if (this.clusterStarter) {
+        clusterStarterInitializations(this.fileSystemManager,
+            this.serverManager, this.catalogTracker, this.assignmentManager);
+      } else {
+        // Process existing unassigned nodes in ZK, read all regions from META,
+        // rebuild in-memory state.
+        this.assignmentManager.processFailover();
+      }
+      // Check if we should stop every second.
+      Sleeper sleeper = new Sleeper(1000, this);
+      while (!this.stopped  && !this.abort) {
+        sleeper.sleep();
+      }
+    } catch (Throwable t) {
+      abort("Unhandled exception. Starting shutdown.", t);
     }
-  }
 
-  /*
-   * @param hri Set all family block caching to <code>b</code>
-   * @param b
-   */
-  private static void setInfoFamilyCaching(final HRegionInfo hri, final boolean b) {
-    for (HColumnDescriptor hcd: hri.getTableDesc().families.values()) {
-      if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
-        hcd.setBlockCacheEnabled(b);
-        hcd.setInMemory(b);
+    // Wait for all the remaining region servers to report in IFF we were
+    // running a cluster shutdown AND we were NOT aborting.
+    if (!this.abort && this.serverManager.isClusterShutdown()) {
+      this.serverManager.letRegionServersShutdown();
+    }
+
+    // Clean up and close up shop
+    if (this.infoServer != null) {
+      LOG.info("Stopping infoServer");
+      try {
+        this.infoServer.stop();
+      } catch (Exception ex) {
+        ex.printStackTrace();
       }
     }
+    this.rpcServer.stop();
+    if (this.balancerChore != null) this.balancerChore.interrupt();
+    this.activeMasterManager.stop();
+    this.zooKeeper.close();
+    this.executorService.shutdown();
+    LOG.info("HMaster main thread exiting");
+  }
+
+  /*
+   * Initializations we need to do if we are cluster starter.
+   * @param starter
+   * @param mfs
+   * @throws IOException 
+   */
+  private static void clusterStarterInitializations(final MasterFileSystem mfs,
+    final ServerManager sm, final CatalogTracker ct, final AssignmentManager am)
+  throws IOException, InterruptedException, KeeperException {
+      // This master is starting the cluster (its not a preexisting cluster
+      // that this master is joining).
+      // Initialize the filesystem, which does the following:
+      //   - Creates the root hbase directory in the FS if DNE
+      //   - If fresh start, create first ROOT and META regions (bootstrap)
+      //   - Checks the FS to make sure the root directory is readable
+      //   - Creates the archive directory for logs
+      mfs.initialize();
+      // Do any log splitting necessary
+      // TODO: Should do this in background rather than block master startup
+      // TODO: Do we want to do this before/while/after RSs check in?
+      //       It seems that this method looks at active RSs but happens
+      //       concurrently with when we expect them to be checking in
+      mfs.splitLogAfterStartup(sm.getOnlineServers());
+      // Clean out current state of unassigned
+      am.cleanoutUnassigned();
+      // assign the root region
+      am.assignRoot();
+      ct.waitForRoot();
+      // assign the meta region
+      am.assignMeta();
+      ct.waitForMeta();
+      // above check waits for general meta availability but this does not
+      // guarantee that the transition has completed
+      am.waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO);
+      am.assignAllUserRegions();
   }
 
   /*
@@ -374,24 +375,6 @@ public class HMaster extends Thread implements HMasterInterface,
     return s;
   }
 
-  /**
-   * Checks to see if the file system is still accessible.
-   * If not, sets closed
-   * @return false if file system is not available
-   */
-  protected boolean checkFileSystem() {
-    if (this.fsOk) {
-      try {
-        FSUtils.checkFileSystemAvailable(this.fs);
-      } catch (IOException e) {
-        LOG.fatal("Shutting down HBase cluster: file system not available", e);
-        this.closed.set(true);
-        this.fsOk = false;
-      }
-    }
-    return this.fsOk;
-  }
-
   /** @return HServerAddress of the master server */
   public HServerAddress getMasterAddress() {
     return this.address;
@@ -406,267 +389,32 @@ public class HMaster extends Thread implements HMasterInterface,
     return this.infoServer;
   }
 
-  /**
-   * @return HBase root dir.
-   * @throws IOException
-   */
-  public Path getRootDir() {
-    return this.rootdir;
-  }
-
-  public int getNumRetries() {
-    return this.numRetries;
-  }
-
-  /**
-   * @return Server metrics
-   */
-  public MasterMetrics getMetrics() {
-    return this.metrics;
-  }
-
-  /**
-   * @return Return configuration being used by this server.
-   */
+  @Override
   public Configuration getConfiguration() {
     return this.conf;
   }
 
+  @Override
   public ServerManager getServerManager() {
     return this.serverManager;
   }
 
-  public RegionManager getRegionManager() {
-    return this.regionManager;
+  @Override
+  public ExecutorService getExecutorService() {
+    return this.executorService;
   }
 
-  int getThreadWakeFrequency() {
-    return this.threadWakeFrequency;
-  }
-
-  FileSystem getFileSystem() {
-    return this.fs;
-  }
-
-  AtomicBoolean getShutdownRequested() {
-    return this.shutdownRequested;
-  }
-
-  AtomicBoolean getClosed() {
-    return this.closed;
-  }
-
-  boolean isClosed() {
-    return this.closed.get();
-  }
-
-  ServerConnection getServerConnection() {
-    return this.connection;
+  @Override
+  public MasterFileSystem getMasterFileSystem() {
+    return this.fileSystemManager;
   }
 
   /**
-   * Get the ZK wrapper object
+   * Get the ZK wrapper object - needed by master_jsp.java
    * @return the zookeeper wrapper
    */
-  public ZooKeeperWrapper getZooKeeperWrapper() {
-    return this.zooKeeperWrapper;
-  }
-
-  // These methods are so don't have to pollute RegionManager with ServerManager.
-  SortedMap<HServerLoad, Set<String>> getLoadToServers() {
-    return this.serverManager.getLoadToServers();
-  }
-
-  int numServers() {
-    return this.serverManager.numServers();
-  }
-
-  public double getAverageLoad() {
-    return this.serverManager.getAverageLoad();
-  }
-
-  public RegionServerOperationQueue getRegionServerOperationQueue () {
-    return this.regionServerOperationQueue;
-  }
-
-  /**
-   * Get the directory where old logs go
-   * @return the dir
-   */
-  public Path getOldLogDir() {
-    return this.oldLogDir;
-  }
-
-  /**
-   * Add to the passed <code>m</code> servers that are loaded less than
-   * <code>l</code>.
-   * @param l
-   * @param m
-   */
-  void getLightServers(final HServerLoad l,
-      SortedMap<HServerLoad, Set<String>> m) {
-    this.serverManager.getLightServers(l, m);
-  }
-
-  /** Main processing loop */
-  @Override
-  public void run() {
-    joinCluster();
-    startServiceThreads();
-    /* Main processing loop */
-    try {
-      FINISHED: while (!this.closed.get()) {
-        // check if we should be shutting down
-        if (this.shutdownRequested.get()) {
-          // The region servers won't all exit until we stop scanning the
-          // meta regions
-          this.regionManager.stopScanners();
-          if (this.serverManager.numServers() == 0) {
-            startShutdown();
-            break;
-          } else {
-            LOG.debug("Waiting on " +
-              this.serverManager.getServersToServerInfo().keySet().toString());
-          }
-        }
-        switch (this.regionServerOperationQueue.process()) {
-        case FAILED:
-            // If FAILED op processing, bad. Exit.
-          break FINISHED;
-        case REQUEUED_BUT_PROBLEM:
-          if (!checkFileSystem())
-              // If bad filesystem, exit.
-            break FINISHED;
-          default:
-            // Continue run loop if conditions are PROCESSED, NOOP, REQUEUED
-          break;
-        }
-      }
-    } catch (Throwable t) {
-      LOG.fatal("Unhandled exception. Starting shutdown.", t);
-      this.closed.set(true);
-    }
-
-    // Wait for all the remaining region servers to report in.
-    this.serverManager.letRegionServersShutdown();
-
-    /*
-     * Clean up and close up shop
-     */
-    if (this.infoServer != null) {
-      LOG.info("Stopping infoServer");
-      try {
-        this.infoServer.stop();
-      } catch (Exception ex) {
-        ex.printStackTrace();
-      }
-    }
-    this.rpcServer.stop();
-    this.regionManager.stop();
-    this.zooKeeperWrapper.close();
-    HBaseExecutorService.shutdown();
-    LOG.info("HMaster main thread exiting");
-  }
-
-  /*
-   * Joins cluster.  Checks to see if this instance of HBase is fresh or the
-   * master was started following a failover. In the second case, it inspects
-   * the region server directory and gets their regions assignment.
-   */
-  private void joinCluster()  {
-      LOG.debug("Checking cluster state...");
-      HServerAddress rootLocation =
-        this.zooKeeperWrapper.readRootRegionLocation();
-      List<HServerAddress> addresses = this.zooKeeperWrapper.scanRSDirectory();
-      // Check if this is a fresh start of the cluster
-      if (addresses.isEmpty()) {
-        LOG.debug("Master fresh start, proceeding with normal startup");
-        splitLogAfterStartup();
-        return;
-      }
-      // Failover case.
-      LOG.info("Master failover, ZK inspection begins...");
-      boolean isRootRegionAssigned = false;
-      Map <byte[], HRegionInfo> assignedRegions =
-        new HashMap<byte[], HRegionInfo>();
-      // We must:
-      // - contact every region server to add them to the regionservers list
-      // - get their current regions assignment
-      // TODO: Run in parallel?
-      for (HServerAddress address : addresses) {
-        HRegionInfo[] regions = null;
-        try {
-          HRegionInterface hri =
-            this.connection.getHRegionConnection(address, false);
-          HServerInfo info = hri.getHServerInfo();
-          LOG.debug("Inspection found server " + info.getServerName());
-          this.serverManager.recordNewServer(info, true);
-          regions = hri.getRegionsAssignment();
-        } catch (IOException e) {
-          LOG.error("Failed contacting " + address.toString(), e);
-          continue;
-        }
-        for (HRegionInfo r: regions) {
-          if (r.isRootRegion()) {
-            this.connection.setRootRegionLocation(new HRegionLocation(r, rootLocation));
-            this.regionManager.setRootRegionLocation(rootLocation);
-            // Undo the unassign work in the RegionManager constructor
-            this.regionManager.removeRegion(r);
-            isRootRegionAssigned = true;
-          } else if (r.isMetaRegion()) {
-            MetaRegion m = new MetaRegion(new HServerAddress(address), r);
-            this.regionManager.addMetaRegionToScan(m);
-          }
-          assignedRegions.put(r.getRegionName(), r);
-        }
-      }
-      LOG.info("Inspection found " + assignedRegions.size() + " regions, " +
-        (isRootRegionAssigned ? "with -ROOT-" : "but -ROOT- was MIA"));
-      splitLogAfterStartup();
-  }
-
-  /*
-   * Inspect the log directory to recover any log file without
-   * ad active region server.
-   */
-  private void splitLogAfterStartup() {
-    Path logsDirPath =
-      new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
-    try {
-      if (!this.fs.exists(logsDirPath)) return;
-    } catch (IOException e) {
-      throw new RuntimeException("Could exists for " + logsDirPath, e);
-    }
-    FileStatus[] logFolders;
-    try {
-      logFolders = this.fs.listStatus(logsDirPath);
-    } catch (IOException e) {
-      throw new RuntimeException("Failed listing " + logsDirPath.toString(), e);
-    }
-    if (logFolders == null || logFolders.length == 0) {
-      LOG.debug("No log files to split, proceeding...");
-      return;
-    }
-    for (FileStatus status : logFolders) {
-      String serverName = status.getPath().getName();
-      LOG.info("Found log folder : " + serverName);
-      if(this.serverManager.getServerInfo(serverName) == null) {
-        LOG.info("Log folder doesn't belong " +
-          "to a known region server, splitting");
-        this.splitLogLock.lock();
-        Path logDir =
-          new Path(this.rootdir, HLog.getHLogDirectoryName(serverName));
-        try {
-          HLog.splitLog(this.rootdir, logDir, oldLogDir, this.fs, getConfiguration());
-        } catch (IOException e) {
-          LOG.error("Failed splitting " + logDir.toString(), e);
-        } finally {
-          this.splitLogLock.unlock();
-        }
-      } else {
-        LOG.info("Log folder belongs to an existing region server");
-      }
-    }
+  public ZooKeeperWatcher getZooKeeperWatcher() {
+    return this.zooKeeper;
   }
 
   /*
@@ -678,7 +426,16 @@ public class HMaster extends Thread implements HMasterInterface,
    */
   private void startServiceThreads() {
     try {
-      this.regionManager.start();
+      // Start the executor service pools
+      this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
+        conf.getInt("hbase.master.executor.openregion.threads", 5));
+      this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,
+        conf.getInt("hbase.master.executor.closeregion.threads", 5));
+      this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
+        conf.getInt("hbase.master.executor.serverops.threads", 5));
+      this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS,
+        conf.getInt("hbase.master.executor.tableops.threads", 5));
+
       // Put up info server.
       int port = this.conf.getInt("hbase.master.info.port", 60010);
       if (port >= 0) {
@@ -687,7 +444,9 @@ public class HMaster extends Thread implements HMasterInterface,
         this.infoServer.setAttribute(MASTER, this);
         this.infoServer.start();
       }
-      // Start the server so everything else is running before we start
+      this.balancerChore = getAndStartBalancerChore(this);
+
+      // Start the server last so everything else is running before we start
       // receiving requests.
       this.rpcServer.start();
       if (LOG.isDebugEnabled()) {
@@ -702,19 +461,22 @@ public class HMaster extends Thread implements HMasterInterface,
         }
       }
       // Something happened during startup. Shut things down.
-      this.closed.set(true);
-      LOG.error("Failed startup", e);
+      abort("Failed startup", e);
     }
   }
 
-  /*
-   * Start shutting down the master
-   */
-  void startShutdown() {
-    this.closed.set(true);
-    this.regionManager.stopScanners();
-    this.regionServerOperationQueue.shutdown();
-    this.serverManager.notifyServers();
+  private static Chore getAndStartBalancerChore(final HMaster master) {
+    String name = master.getServerName() + "-balancerChore";
+    int period = master.getConfiguration().getInt("hbase.balancer.period", 600000);
+    // Start up the load balancer chore
+    Chore chore = new Chore(name, period, master) {
+      @Override
+      protected void chore() {
+        master.balance();
+      }
+    };
+    Threads.setDaemonThreadRunning(chore, name);
+    return chore;
   }
 
   public MapWritable regionServerStartup(final HServerInfo serverInfo)
@@ -722,6 +484,9 @@ public class HMaster extends Thread implements HMasterInterface,
     // Set the ip into the passed in serverInfo.  Its ip is more than likely
     // not the ip that the master sees here.  See at end of this method where
     // we pass it back to the regionserver by setting "hbase.regionserver.address"
+    // Everafter, the HSI combination 'server name' is what uniquely identifies
+    // the incoming RegionServer.  No more DNS meddling of this little messing
+    // belose.
     String rsAddress = HBaseServer.getRemoteAddress();
     serverInfo.setServerAddress(new HServerAddress(rsAddress,
       serverInfo.getServerAddress().getPort()));
@@ -747,6 +512,7 @@ public class HMaster extends Thread implements HMasterInterface,
     return mw;
   }
 
+  @Override
   public HMsg [] regionServerReport(HServerInfo serverInfo, HMsg msgs[],
     HRegionInfo[] mostLoadedRegions)
   throws IOException {
@@ -767,16 +533,69 @@ public class HMaster extends Thread implements HMasterInterface,
   }
 
   public boolean isMasterRunning() {
-    return !this.closed.get();
+    return !isStopped();
   }
 
-  public void shutdown() {
-    LOG.info("Cluster shutdown requested. Starting to quiesce servers");
-    this.shutdownRequested.set(true);
-    this.zooKeeperWrapper.setClusterState(false);
+  /**
+   * Run the balancer.
+   * @return True if balancer ran, false otherwise.
+   */
+  public boolean balance() {
+    // If balance not true, don't run balancer.
+    if (!this.balance) return false;
+    synchronized (this.balancer) {
+      // Only allow one balance run at at time.
+      if (this.assignmentManager.isRegionsInTransition()) {
+        LOG.debug("Not running balancer because regions in transition: " +
+          this.assignmentManager.getRegionsInTransition());
+        return false;
+      }
+      Map<HServerInfo, List<HRegionInfo>> assignments =
+        this.assignmentManager.getAssignments();
+      // Returned Map from AM does not include mention of servers w/o assignments.
+      for (Map.Entry<String, HServerInfo> e:
+          this.serverManager.getOnlineServers().entrySet()) {
+        HServerInfo hsi = e.getValue();
+        if (!assignments.containsKey(hsi)) {
+          assignments.put(hsi, new ArrayList<HRegionInfo>());
+        }
+      }
+      List<RegionPlan> plans = this.balancer.balanceCluster(assignments);
+      if (plans != null && !plans.isEmpty()) {
+        for (RegionPlan plan: plans) {
+          this.assignmentManager.balance(plan);
+        }
+      }
+    }
+    return true;
+  }
+
+  @Override
+  public boolean balance(final boolean b) {
+    boolean oldValue = this.balance;
+    this.balance = b;
+    LOG.info("Balance=" + b);
+    return oldValue;
+  }
+
+  @Override
+  public void move(final byte[] encodedRegionName, final byte[] destServerName)
+  throws UnknownRegionException {
+    Pair<HRegionInfo, HServerInfo> p =
+      this.assignmentManager.getAssignment(encodedRegionName);
+    if (p == null) throw new UnknownRegionException(Bytes.toString(encodedRegionName));
+    HServerInfo dest = this.serverManager.getServerInfo(new String(destServerName));
+    RegionPlan rp = new RegionPlan(p.getFirst(), p.getSecond(), dest);
+    this.assignmentManager.balance(rp);
   }
 
   public void createTable(HTableDescriptor desc, byte [][] splitKeys)
+  throws IOException {
+    createTable(desc, splitKeys, false);
+  }
+
+  public void createTable(HTableDescriptor desc, byte [][] splitKeys,
+      boolean sync)
   throws IOException {
     if (!isMasterRunning()) {
       throw new MasterNotRunningException();
@@ -795,148 +614,90 @@ public class HMaster extends Thread implements HMasterInterface,
         startKey = endKey;
       }
     }
-    for (int tries = 0; tries < this.numRetries; tries++) {
-      try {
-        // We can not create a table unless meta regions have already been
-        // assigned and scanned.
-        if (!this.regionManager.areAllMetaRegionsOnline()) {
-          throw new NotAllMetaRegionsOnlineException();
+    int timeout = conf.getInt("hbase.client.catalog.timeout", 10000);
+    // Need META availability to create a table
+    try {
+      if(catalogTracker.waitForMeta(timeout) == null) {
+        throw new NotAllMetaRegionsOnlineException();
+      }
+    } catch (InterruptedException e) {
+      LOG.warn("Interrupted waiting for meta availability", e);
+      throw new IOException(e);
+    }
+    createTable(newRegions, sync);
+  }
+
+  private synchronized void createTable(final HRegionInfo [] newRegions,
+      boolean sync)
+  throws IOException {
+    String tableName = newRegions[0].getTableDesc().getNameAsString();
+    if(MetaReader.tableExists(catalogTracker, tableName)) {
+      throw new TableExistsException(tableName);
+    }
+    for(HRegionInfo newRegion : newRegions) {
+      // 1. Create HRegion
+      HRegion region = HRegion.createHRegion(newRegion,
+          fileSystemManager.getRootDir(), conf);
+
+      // 2. Insert into META
+      MetaEditor.addRegionToMeta(catalogTracker, region.getRegionInfo());
+
+      // 3. Close the new region to flush to disk.  Close log file too.
+      region.close();
+      region.getLog().closeAndDelete();
+
+      // 4. Trigger immediate assignment of this region
+      assignmentManager.assign(region.getRegionInfo());
+    }
+
+    // 5. If sync, wait for assignment of regions
+    if(sync) {
+      LOG.debug("Waiting for " + newRegions.length + " region(s) to be " +
+          "assigned before returning");
+      for(HRegionInfo regionInfo : newRegions) {
+        try {
+          assignmentManager.waitForAssignment(regionInfo);
+        } catch (InterruptedException e) {
+          LOG.info("Interrupted waiting for region to be assigned during " +
+              "create table call");
+          return;
         }
-        if (!this.serverManager.canAssignUserRegions()) {
-          throw new IOException("not enough servers to create table yet");
-        }
-        createTable(newRegions);
-        LOG.info("created table " + desc.getNameAsString());
-        break;
-      } catch (TableExistsException e) {
-        throw e;
-      } catch (IOException e) {
-        if (tries == this.numRetries - 1) {
-          throw RemoteExceptionHandler.checkIOException(e);
-        }
-        this.sleeper.sleep();
       }
     }
   }
 
-  private synchronized void createTable(final HRegionInfo [] newRegions)
-  throws IOException {
-    String tableName = newRegions[0].getTableDesc().getNameAsString();
-    // 1. Check to see if table already exists. Get meta region where
-    // table would sit should it exist. Open scanner on it. If a region
-    // for the table we want to create already exists, then table already
-    // created. Throw already-exists exception.
-    MetaRegion m = regionManager.getFirstMetaRegionForRegion(newRegions[0]);
-    byte [] metaRegionName = m.getRegionName();
-    HRegionInterface srvr = this.connection.getHRegionConnection(m.getServer());
-    byte[] firstRowInTable = Bytes.toBytes(tableName + ",,");
-    Scan scan = new Scan(firstRowInTable);
-    scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-    long scannerid = srvr.openScanner(metaRegionName, scan);
-    try {
-      Result data = srvr.next(scannerid);
-      if (data != null && data.size() > 0) {
-        HRegionInfo info = Writables.getHRegionInfo(
-          data.getValue(HConstants.CATALOG_FAMILY,
-              HConstants.REGIONINFO_QUALIFIER));
-        if (info.getTableDesc().getNameAsString().equals(tableName)) {
-          // A region for this table already exists. Ergo table exists.
-          throw new TableExistsException(tableName);
-        }
-      }
-    } finally {
-      srvr.close(scannerid);
-    }
-    for(HRegionInfo newRegion : newRegions) {
-      regionManager.createRegion(newRegion, srvr, metaRegionName);
-    }
+  private static boolean isCatalogTable(final byte [] tableName) {
+    return Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME) ||
+           Bytes.equals(tableName, HConstants.META_TABLE_NAME);
   }
 
   public void deleteTable(final byte [] tableName) throws IOException {
-    if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
-      throw new IOException("Can't delete root table");
-    }
-    new TableDelete(this, tableName).process();
-    LOG.info("deleted table: " + Bytes.toString(tableName));
+    new DeleteTableHandler(tableName, this, this).process();
   }
 
   public void addColumn(byte [] tableName, HColumnDescriptor column)
   throws IOException {
-    new AddColumn(this, tableName, column).process();
+    new TableAddFamilyHandler(tableName, column, this, this).process();
   }
 
-  public void modifyColumn(byte [] tableName, byte [] columnName,
-    HColumnDescriptor descriptor)
+  public void modifyColumn(byte [] tableName, HColumnDescriptor descriptor)
   throws IOException {
-    new ModifyColumn(this, tableName, columnName, descriptor).process();
+    new TableModifyFamilyHandler(tableName, descriptor, this, this).process();
   }
 
   public void deleteColumn(final byte [] tableName, final byte [] c)
   throws IOException {
-    new DeleteColumn(this, tableName, KeyValue.parseColumn(c)[0]).process();
+    new TableDeleteFamilyHandler(tableName, c, this, this).process();
   }
 
   public void enableTable(final byte [] tableName) throws IOException {
-    if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
-      throw new IOException("Can't enable root table");
-    }
-    new ChangeTableState(this, tableName, true).process();
+    new EnableTableHandler(this, tableName, catalogTracker, assignmentManager)
+      .process();
   }
 
   public void disableTable(final byte [] tableName) throws IOException {
-    if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
-      throw new IOException("Can't disable root table");
-    }
-    new ChangeTableState(this, tableName, false).process();
-  }
-
-  /**
-   * Get a list of the regions for a given table. The pairs may have
-   * null for their second element in the case that they are not
-   * currently deployed.
-   * TODO: Redo so this method does not duplicate code with subsequent methods.
-   */
-  List<Pair<HRegionInfo,HServerAddress>> getTableRegions(
-      final byte [] tableName)
-  throws IOException {
-    final ArrayList<Pair<HRegionInfo, HServerAddress>> result =
-      Lists.newArrayList();
-    MetaScannerVisitor visitor =
-      new MetaScannerVisitor() {
-        @Override
-        public boolean processRow(Result data) throws IOException {
-          if (data == null || data.size() <= 0)
-            return true;
-          Pair<HRegionInfo, HServerAddress> pair =
-            metaRowToRegionPair(data);
-          if (pair == null) return false;
-          if (!Bytes.equals(pair.getFirst().getTableDesc().getName(),
-                tableName)) {
-            return false;
-          }
-          result.add(pair);
-          return true;
-        }
-    };
-
-    MetaScanner.metaScan(conf, visitor, tableName); 
-    return result;
-  }
-  
-  private Pair<HRegionInfo, HServerAddress> metaRowToRegionPair(
-      Result data) throws IOException {
-    HRegionInfo info = Writables.getHRegionInfo(
-        data.getValue(HConstants.CATALOG_FAMILY,
-            HConstants.REGIONINFO_QUALIFIER));
-    final byte[] value = data.getValue(HConstants.CATALOG_FAMILY,
-        HConstants.SERVER_QUALIFIER);
-    if (value != null && value.length > 0) {
-      HServerAddress server = new HServerAddress(Bytes.toString(value));
-      return new Pair<HRegionInfo,HServerAddress>(info, server);
-    } else {
-      //undeployed
-      return new Pair<HRegionInfo, HServerAddress>(info, null);
-    }    
+    new DisableTableHandler(this, tableName, catalogTracker, assignmentManager)
+      .process();
   }
 
   /**
@@ -950,16 +711,19 @@ public class HMaster extends Thread implements HMasterInterface,
   throws IOException {
     final AtomicReference<Pair<HRegionInfo, HServerAddress>> result =
       new AtomicReference<Pair<HRegionInfo, HServerAddress>>(null);
-    
+
     MetaScannerVisitor visitor =
       new MetaScannerVisitor() {
         @Override
         public boolean processRow(Result data) throws IOException {
-          if (data == null || data.size() <= 0)
+          if (data == null || data.size() <= 0) {
             return true;
+          }
           Pair<HRegionInfo, HServerAddress> pair =
-            metaRowToRegionPair(data);
-          if (pair == null) return false;
+            MetaReader.metaRowToRegionPair(data);
+          if (pair == null) {
+            return false;
+          }
           if (!Bytes.equals(pair.getFirst().getTableDesc().getName(),
                 tableName)) {
             return false;
@@ -972,137 +736,25 @@ public class HMaster extends Thread implements HMasterInterface,
     MetaScanner.metaScan(conf, visitor, tableName, rowKey, 1);
     return result.get();
   }
-  
-  Pair<HRegionInfo,HServerAddress> getTableRegionFromName(
-      final byte [] regionName)
+
+  @Override
+  public void modifyTable(final byte[] tableName, HTableDescriptor htd)
   throws IOException {
-    byte [] tableName = HRegionInfo.parseRegionName(regionName)[0];
-    
-    Set<MetaRegion> regions = regionManager.getMetaRegionsForTable(tableName);
-    for (MetaRegion m: regions) {
-      byte [] metaRegionName = m.getRegionName();
-      HRegionInterface srvr = connection.getHRegionConnection(m.getServer());
-      Get get = new Get(regionName);
-      get.addColumn(HConstants.CATALOG_FAMILY,
-          HConstants.REGIONINFO_QUALIFIER);
-      get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-      Result data = srvr.get(metaRegionName, get);
-      if(data == null || data.size() <= 0) continue;
-      return metaRowToRegionPair(data);
+    this.executorService.submit(new ModifyTableHandler(tableName, htd, this, this));
+  }
+
+  @Override
+  public void checkTableModifiable(final byte [] tableName)
+  throws IOException {
+    String tableNameStr = Bytes.toString(tableName);
+    if (isCatalogTable(tableName)) {
+      throw new IOException("Can't modify catalog tables");
     }
-    return null;
-  }
-
-  /**
-   * Get row from meta table.
-   * @param row
-   * @param family
-   * @return Result
-   * @throws IOException
-   */
-  protected Result getFromMETA(final byte [] row, final byte [] family)
-  throws IOException {
-    MetaRegion meta = this.regionManager.getMetaRegionForRow(row);
-    HRegionInterface srvr = getMETAServer(meta);
-    Get get = new Get(row);
-    get.addFamily(family);
-    return srvr.get(meta.getRegionName(), get);
-  }
-
-  /*
-   * @param meta
-   * @return Server connection to <code>meta</code> .META. region.
-   * @throws IOException
-   */
-  private HRegionInterface getMETAServer(final MetaRegion meta)
-  throws IOException {
-    return this.connection.getHRegionConnection(meta.getServer());
-  }
-
-  public void modifyTable(final byte[] tableName, HConstants.Modify op,
-      Writable[] args)
-  throws IOException {
-    switch (op) {
-    case TABLE_SET_HTD:
-      if (args == null || args.length < 1 ||
-          !(args[0] instanceof HTableDescriptor))
-        throw new IOException("SET_HTD request requires an HTableDescriptor");
-      HTableDescriptor htd = (HTableDescriptor) args[0];
-      LOG.info("modifyTable(SET_HTD): " + htd);
-      new ModifyTableMeta(this, tableName, htd).process();
-      break;
-
-    case TABLE_SPLIT:
-    case TABLE_COMPACT:
-    case TABLE_MAJOR_COMPACT:
-    case TABLE_FLUSH:
-      if (args != null && args.length > 0) {
-        if (!(args[0] instanceof ImmutableBytesWritable))
-          throw new IOException(
-            "request argument must be ImmutableBytesWritable");
-        Pair<HRegionInfo,HServerAddress> pair = null;
-        if(tableName == null) {
-          byte [] regionName = ((ImmutableBytesWritable)args[0]).get();
-          pair = getTableRegionFromName(regionName);
-        } else {
-          byte [] rowKey = ((ImmutableBytesWritable)args[0]).get();
-          pair = getTableRegionForRow(tableName, rowKey);
-        }
-        LOG.info("About to " + op.toString() + " on " + Bytes.toString(tableName) + " and pair is " + pair);
-        if (pair != null && pair.getSecond() != null) {
-          this.regionManager.startAction(pair.getFirst().getRegionName(),
-            pair.getFirst(), pair.getSecond(), op);
-        }
-      } else {
-        for (Pair<HRegionInfo,HServerAddress> pair: getTableRegions(tableName)) {
-          if (pair.getSecond() == null) continue; // undeployed
-          this.regionManager.startAction(pair.getFirst().getRegionName(),
-            pair.getFirst(), pair.getSecond(), op);
-        }
-      }
-      break;
-
-    case CLOSE_REGION:
-      if (args == null || args.length < 1 || args.length > 2) {
-        throw new IOException("Requires at least a region name; " +
-          "or cannot have more than region name and servername");
-      }
-      // Arguments are regionname and an optional server name.
-      byte [] regionname = ((ImmutableBytesWritable)args[0]).get();
-      LOG.debug("Attempting to close region: " + Bytes.toStringBinary(regionname));
-      String hostnameAndPort = null;
-      if (args.length == 2) {
-        hostnameAndPort = Bytes.toString(((ImmutableBytesWritable)args[1]).get());
-      }
-      // Need hri
-      Result rr = getFromMETA(regionname, HConstants.CATALOG_FAMILY);
-      HRegionInfo hri = getHRegionInfo(rr.getRow(), rr);
-      if (hostnameAndPort == null) {
-        // Get server from the .META. if it wasn't passed as argument
-        hostnameAndPort =
-          Bytes.toString(rr.getValue(HConstants.CATALOG_FAMILY,
-              HConstants.SERVER_QUALIFIER));
-      }
-      // Take region out of the intransistions in case it got stuck there doing
-      // an open or whatever.
-      this.regionManager.clearFromInTransition(regionname);
-      // If hostnameAndPort is still null, then none, exit.
-      if (hostnameAndPort == null) break;
-      long startCode =
-        Bytes.toLong(rr.getValue(HConstants.CATALOG_FAMILY,
-            HConstants.STARTCODE_QUALIFIER));
-      String name = HServerInfo.getServerName(hostnameAndPort, startCode);
-      LOG.info("Marking " + hri.getRegionNameAsString() +
-        " as closing on " + name + "; cleaning SERVER + STARTCODE; " +
-          "master will tell regionserver to close region on next heartbeat");
-      this.regionManager.setClosing(name, hri, hri.isOffline());
-      MetaRegion meta = this.regionManager.getMetaRegionForRow(regionname);
-      HRegionInterface srvr = getMETAServer(meta);
-      HRegion.cleanRegionInMETA(srvr, meta.getRegionName(), hri);
-      break;
-
-    default:
-      throw new IOException("unsupported modifyTable op " + op);
+    if (!MetaReader.tableExists(getCatalogTracker(), tableNameStr)) {
+      throw new TableNotFoundException(tableNameStr);
+    }
+    if (!getAssignmentManager().isTableDisabled(Bytes.toString(tableName))) {
+      throw new TableNotDisabledException(tableName);
     }
   }
 
@@ -1112,122 +764,78 @@ public class HMaster extends Thread implements HMasterInterface,
   public ClusterStatus getClusterStatus() {
     ClusterStatus status = new ClusterStatus();
     status.setHBaseVersion(VersionInfo.getVersion());
-    status.setServerInfo(serverManager.getServersToServerInfo().values());
+    status.setServerInfo(serverManager.getOnlineServers().values());
     status.setDeadServers(serverManager.getDeadServers());
-    status.setRegionsInTransition(this.regionManager.getRegionsInTransition());
+    status.setRegionsInTransition(assignmentManager.getRegionsInTransition());
     return status;
   }
 
-  // TODO ryan rework this function
-  /*
-   * Get HRegionInfo from passed META map of row values.
-   * Returns null if none found (and logs fact that expected COL_REGIONINFO
-   * was missing).  Utility method used by scanners of META tables.
-   * @param row name of the row
-   * @param map Map to do lookup in.
-   * @return Null or found HRegionInfo.
-   * @throws IOException
-   */
-  HRegionInfo getHRegionInfo(final byte [] row, final Result res)
-  throws IOException {
-    byte[] regioninfo = res.getValue(HConstants.CATALOG_FAMILY,
-        HConstants.REGIONINFO_QUALIFIER);
-    if (regioninfo == null) {
-      StringBuilder sb =  new StringBuilder();
-      NavigableMap<byte[], byte[]> infoMap =
-        res.getFamilyMap(HConstants.CATALOG_FAMILY);
-      for (byte [] e: infoMap.keySet()) {
-        if (sb.length() > 0) {
-          sb.append(", ");
-        }
-        sb.append(Bytes.toString(HConstants.CATALOG_FAMILY) + ":"
-            + Bytes.toString(e));
-      }
-      LOG.warn(Bytes.toString(HConstants.CATALOG_FAMILY) + ":" +
-          Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
-          + " is empty for row: " + Bytes.toString(row) + "; has keys: "
-          + sb.toString());
-      return null;
-    }
-    return Writables.getHRegionInfo(regioninfo);
-  }
-
-  /*
-   * When we find rows in a meta region that has an empty HRegionInfo, we
-   * clean them up here.
-   *
-   * @param s connection to server serving meta region
-   * @param metaRegionName name of the meta region we scanned
-   * @param emptyRows the row keys that had empty HRegionInfos
-   */
-  protected void deleteEmptyMetaRows(HRegionInterface s,
-      byte [] metaRegionName,
-      List<byte []> emptyRows) {
-    for (byte [] regionName: emptyRows) {
-      try {
-        HRegion.removeRegionFromMETA(s, metaRegionName, regionName);
-        LOG.warn("Removed region: " + Bytes.toString(regionName) +
-          " from meta region: " +
-          Bytes.toString(metaRegionName) + " because HRegionInfo was empty");
-      } catch (IOException e) {
-        LOG.error("deleting region: " + Bytes.toString(regionName) +
-            " from meta region: " + Bytes.toString(metaRegionName), e);
-      }
-    }
-  }
-
-  /**
-   * @see org.apache.zookeeper.Watcher#process(org.apache.zookeeper.WatchedEvent)
-   */
-  @Override
-  public void process(WatchedEvent event) {
-    LOG.debug("Event " + event.getType() + 
-              " with state " + event.getState() +  
-              " with path " + event.getPath());
-    // Master should kill itself if its session expired or if its
-    // znode was deleted manually (usually for testing purposes)
-    if(event.getState() == KeeperState.Expired ||
-      (event.getType().equals(EventType.NodeDeleted) &&
-        event.getPath().equals(this.zooKeeperWrapper.getMasterElectionZNode())) &&
-        !shutdownRequested.get()) {
-
-      LOG.info("Master lost its znode, trying to get a new one");
-
-      // Can we still be the master? If not, goodbye
-
-      zooKeeperWrapper.close();
-      try {
-        zooKeeperWrapper =
-            ZooKeeperWrapper.createInstance(conf, HMaster.class.getName());
-        zooKeeperWrapper.registerListener(this);
-        this.zkMasterAddressWatcher.setZookeeper(zooKeeperWrapper);
-        if(!this.zkMasterAddressWatcher.
-            writeAddressToZooKeeper(this.address,false)) {
-          throw new Exception("Another Master is currently active");
-        }
-
-        // we are a failed over master, reset the fact that we started the 
-        // cluster
-        resetClusterStartup();
-        // Verify the cluster to see if anything happened while we were away
-        joinCluster();
-      } catch (Exception e) {
-        LOG.error("Killing master because of", e);
-        System.exit(1);
-      }
-    }
-  }
-
   private static void printUsageAndExit() {
     System.err.println("Usage: Master [opts] start|stop");
     System.err.println(" start  Start Master. If local mode, start Master and RegionServer in same JVM");
     System.err.println(" stop   Start cluster shutdown; Master signals RegionServer shutdown");
     System.err.println(" where [opts] are:");
     System.err.println("   --minServers=<servers>    Minimum RegionServers needed to host user tables.");
-    System.err.println("   -D opt=<value>            Override HBase configuration settings.");
     System.exit(0);
   }
 
+  @Override
+  public void abort(final String msg, final Throwable t) {
+    if (t != null) LOG.fatal(msg, t);
+    else LOG.fatal(msg);
+    this.abort = true;
+  }
+
+  @Override
+  public ZooKeeperWatcher getZooKeeper() {
+    return zooKeeper;
+  }
+
+  @Override
+  public String getServerName() {
+    return address.toString();
+  }
+
+  @Override
+  public CatalogTracker getCatalogTracker() {
+    return catalogTracker;
+  }
+
+  @Override
+  public AssignmentManager getAssignmentManager() {
+    return this.assignmentManager;
+  }
+
+  @Override
+  public void shutdown() {
+    this.serverManager.shutdownCluster();
+    try {
+      this.clusterStatusTracker.setClusterDown();
+    } catch (KeeperException e) {
+      LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
+    }
+  }
+
+  @Override
+  public void stopMaster() {
+    stop("Stopped by " + Thread.currentThread().getName());
+  }
+
+  @Override
+  public void stop(final String why) {
+    LOG.info(why);
+    this.stopped = true;
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.stopped;
+  }
+
+  public void assignRegion(HRegionInfo hri) {
+    assignmentManager.assign(hri);
+  }
+
   /**
    * Utility for constructing an instance of the passed HMaster class.
    * @param masterClass
@@ -1240,10 +848,16 @@ public class HMaster extends Thread implements HMasterInterface,
       Constructor<? extends HMaster> c =
         masterClass.getConstructor(Configuration.class);
       return c.newInstance(conf);
+    } catch (InvocationTargetException ite) {
+      Throwable target = ite.getTargetException() != null?
+        ite.getTargetException(): ite;
+      if (target.getCause() != null) target = target.getCause();
+      throw new RuntimeException("Failed construction of Master: " +
+        masterClass.toString(), target);
     } catch (Exception e) {
-      throw new RuntimeException("Failed construction of " +
-        "Master: " + masterClass.toString() +
-        ((e.getCause() != null)? e.getCause().getMessage(): ""), e);
+      throw new RuntimeException("Failed construction of Master: " +
+        masterClass.toString() + ((e.getCause() != null)?
+          e.getCause().getMessage(): ""), e);
     }
   }
 
@@ -1253,7 +867,8 @@ public class HMaster extends Thread implements HMasterInterface,
   static class LocalHMaster extends HMaster {
     private MiniZooKeeperCluster zkcluster = null;
 
-    public LocalHMaster(Configuration conf) throws IOException {
+    public LocalHMaster(Configuration conf)
+    throws IOException, KeeperException, InterruptedException {
       super(conf);
     }
 
@@ -1348,7 +963,7 @@ public class HMaster extends Thread implements HMasterInterface,
             cluster.startup();
           } else {
             HMaster master = constructMaster(masterClass, conf);
-            if (master.shutdownRequested.get()) {
+            if (master.isStopped()) {
               LOG.info("Won't bring the Master up as a shutdown is requested");
               return;
             }
@@ -1365,6 +980,9 @@ public class HMaster extends Thread implements HMasterInterface,
         } catch (MasterNotRunningException e) {
           LOG.error("Master not running");
           System.exit(0);
+        } catch (ZooKeeperConnectionException e) {
+          LOG.error("ZooKeeper not available");
+          System.exit(0);
         }
         try {
           adm.shutdown();
@@ -1382,22 +1000,12 @@ public class HMaster extends Thread implements HMasterInterface,
     }
   }
 
-  public Map<String, Integer> getTableFragmentation() throws IOException {
-    long now = System.currentTimeMillis();
-    // only check every two minutes by default
-    int check = this.conf.getInt("hbase.master.fragmentation.check.frequency", 2 * 60 * 1000);
-    if (lastFragmentationQuery == -1 || now - lastFragmentationQuery > check) {
-      fragmentation = FSUtils.getTableFragmentation(this);
-      lastFragmentationQuery = now;
-    }
-    return fragmentation;
-  }
-
   /**
    * Main program
    * @param args
+   * @throws IOException 
    */
-  public static void main(String [] args) {
+  public static void main(String [] args) throws IOException {
     doMain(args, HMaster.class);
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java b/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
new file mode 100644
index 00000000000..50a852236d9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
@@ -0,0 +1,593 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Makes decisions about the placement and movement of Regions across
+ * RegionServers.
+ *
+ * <p>Cluster-wide load balancing will occur only when there are no regions in
+ * transition and according to a fixed period of a time using {@link #balanceCluster(Map)}.
+ *
+ * <p>Inline region placement with {@link #immediateAssignment} can be used when
+ * the Master needs to handle closed regions that it currently does not have
+ * a destination set for.  This can happen during master failover.
+ *
+ * <p>On cluster startup, {@link #bulkAssignment} can be used to determine
+ * locations for all Regions in a cluster.
+ * 
+ * <p>This classes produces plans for the {@link AssignmentManager} to execute.
+ */
+public class LoadBalancer {
+  private static final Log LOG = LogFactory.getLog(LoadBalancer.class);
+  private static final Random rand = new Random();
+
+  /**
+   * Generate a global load balancing plan according to the specified map of
+   * server information to the most loaded regions of each server.
+   *
+   * The load balancing invariant is that all servers are within 1 region of the
+   * average number of regions per server.  If the average is an integer number,
+   * all servers will be balanced to the average.  Otherwise, all servers will
+   * have either floor(average) or ceiling(average) regions.
+   *
+   * The algorithm is currently implemented as such:
+   *
+   * <ol>
+   * <li>Determine the two valid numbers of regions each server should have,
+   *     <b>MIN</b>=floor(average) and <b>MAX</b>=ceiling(average).
+   *
+   * <li>Iterate down the most loaded servers, shedding regions from each so
+   *     each server hosts exactly <b>MAX</b> regions.  Stop once you reach a
+   *     server that already has &lt;= <b>MAX</b> regions.
+   *
+   * <li>Iterate down the least loaded servers, assigning regions so each server
+   *     has exactly </b>MIN</b> regions.  Stop once you reach a server that
+   *     already has &gt;= <b>MIN</b> regions.
+   *
+   *     Regions being assigned to underloaded servers are those that were shed
+   *     in the previous step.  It is possible that there were not enough
+   *     regions shed to fill each underloaded server to <b>MIN</b>.  If so we
+   *     end up with a number of regions required to do so, <b>neededRegions</b>.
+   *
+   *     It is also possible that we were able fill each underloaded but ended
+   *     up with regions that were unassigned from overloaded servers but that
+   *     still do not have assignment.
+   *
+   *     If neither of these conditions hold (no regions needed to fill the
+   *     underloaded servers, no regions leftover from overloaded servers),
+   *     we are done and return.  Otherwise we handle these cases below.
+   *
+   * <li>If <b>neededRegions</b> is non-zero (still have underloaded servers),
+   *     we iterate the most loaded servers again, shedding a single server from
+   *     each (this brings them from having <b>MAX</b> regions to having
+   *     <b>MIN</b> regions).
+   *
+   * <li>We now definitely have more regions that need assignment, either from
+   *     the previous step or from the original shedding from overloaded servers.
+   *
+   *     Iterate the least loaded servers filling each to <b>MIN</b>.
+   *
+   * <li>If we still have more regions that need assignment, again iterate the
+   *     least loaded servers, this time giving each one (filling them to
+   *     </b>MAX</b>) until we run out.
+   *
+   * <li>All servers will now either host <b>MIN</b> or <b>MAX</b> regions.
+   *
+   *     In addition, any server hosting &gt;= <b>MAX</b> regions is guaranteed
+   *     to end up with <b>MAX</b> regions at the end of the balancing.  This
+   *     ensures the minimal number of regions possible are moved.
+   * </ol>
+   *
+   * TODO: We can at-most reassign the number of regions away from a particular
+   *       server to be how many they report as most loaded.
+   *       Should we just keep all assignment in memory?  Any objections?
+   *       Does this mean we need HeapSize on HMaster?  Or just careful monitor?
+   *       (current thinking is we will hold all assignments in memory)
+   *
+   * @param serverInfo map of regionservers and their load/region information to
+   *                   a list of their most loaded regions
+   * @return a list of regions to be moved, including source and destination,
+   *         or null if cluster is already balanced
+   */
+  public List<RegionPlan> balanceCluster(
+      Map<HServerInfo,List<HRegionInfo>> clusterState) {
+    LOG.debug("Running load balancer");
+
+    long startTime = System.currentTimeMillis();
+
+    // Make a map sorted by load and count regions
+    TreeMap<HServerInfo,List<HRegionInfo>> serversByLoad =
+      new TreeMap<HServerInfo,List<HRegionInfo>>(
+          new HServerInfo.LoadComparator());
+    int numServers = clusterState.size();
+    int numRegions = 0;
+    // Iterate so we can count regions as we build the map
+    for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
+      clusterState.entrySet()) {
+      server.getKey().getLoad().setNumberOfRegions(server.getValue().size());
+      numRegions += server.getKey().getLoad().getNumberOfRegions();
+      serversByLoad.put(server.getKey(), server.getValue());
+    }
+
+    // Check if we even need to do any load balancing
+    float average = (float)numRegions / numServers; // for logging
+    int min = numRegions / numServers;
+    int max = numRegions % numServers == 0 ? min : min + 1;
+    if(serversByLoad.lastKey().getLoad().getNumberOfRegions() <= max &&
+       serversByLoad.firstKey().getLoad().getNumberOfRegions() >= min) {
+      // Skipped because no server outside (min,max) range
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Skipping load balancing.  servers=" + numServers + " " +
+            "regions=" + numRegions + " average=" + average + " " +
+            "mostloaded=" + serversByLoad.lastKey().getLoad().getNumberOfRegions() +
+            " leastloaded=" + serversByLoad.lastKey().getLoad().getNumberOfRegions());
+      }
+      return null;
+    }
+
+    // Balance the cluster
+    // TODO: Look at data block locality or a more complex load to do this
+    List<RegionPlan> regionsToMove = new ArrayList<RegionPlan>();
+    int regionidx = 0; // track the index in above list for setting destination
+
+    // Walk down most loaded, pruning each to the max
+    int serversOverloaded = 0;
+    Map<HServerInfo,BalanceInfo> serverBalanceInfo =
+      new TreeMap<HServerInfo,BalanceInfo>();
+    for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
+      serversByLoad.descendingMap().entrySet()) {
+      HServerInfo serverInfo = server.getKey();
+      int regionCount = serverInfo.getLoad().getNumberOfRegions();
+      if(regionCount <= max) {
+        serverBalanceInfo.put(serverInfo, new BalanceInfo(0, 0));
+        break;
+      }
+      serversOverloaded++;
+      List<HRegionInfo> regions = server.getValue();
+      int numToOffload = Math.min(regionCount - max, regions.size());
+      int numTaken = 0;
+      for (HRegionInfo hri: regions) {
+        // Don't rebalance meta regions.
+        if (hri.isMetaRegion()) continue;
+        regionsToMove.add(new RegionPlan(hri, serverInfo, null));
+        numTaken++;
+        if (numTaken >= numToOffload) break;
+      }
+      serverBalanceInfo.put(serverInfo,
+          new BalanceInfo(numToOffload, (-1)*numTaken));
+    }
+
+    // Walk down least loaded, filling each to the min
+    int serversUnderloaded = 0; // number of servers that get new regions
+    int neededRegions = 0; // number of regions needed to bring all up to min
+    for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
+      serversByLoad.entrySet()) {
+      int regionCount = server.getKey().getLoad().getNumberOfRegions();
+      if(regionCount >= min) {
+        break;
+      }
+      serversUnderloaded++;
+      int numToTake = min - regionCount;
+      int numTaken = 0;
+      while(numTaken < numToTake && regionidx < regionsToMove.size()) {
+        regionsToMove.get(regionidx).setDestination(server.getKey());
+        numTaken++;
+        regionidx++;
+      }
+      serverBalanceInfo.put(server.getKey(), new BalanceInfo(0, numTaken));
+      // If we still want to take some, increment needed
+      if(numTaken < numToTake) {
+        neededRegions += (numToTake - numTaken);
+      }
+    }
+
+    // If none needed to fill all to min and none left to drain all to max,
+    // we are done
+    if(neededRegions == 0 && regionidx == regionsToMove.size()) {
+      long endTime = System.currentTimeMillis();
+      LOG.info("Calculated a load balance in " + (endTime-startTime) + "ms. " +
+          "Moving " + regionsToMove.size() + " regions off of " +
+          serversOverloaded + " overloaded servers onto " +
+          serversUnderloaded + " less loaded servers");
+      return regionsToMove;
+    }
+
+    // Need to do a second pass.
+    // Either more regions to assign out or servers that are still underloaded
+
+    // If we need more to fill min, grab one from each most loaded until enough
+    if(neededRegions != 0) {
+      // Walk down most loaded, grabbing one from each until we get enough
+      for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
+        serversByLoad.descendingMap().entrySet()) {
+        BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey());
+        int idx =
+          balanceInfo == null ? 0 : balanceInfo.getNextRegionForUnload();
+        HRegionInfo region = server.getValue().get(idx);
+        if (region.isMetaRegion()) continue; // Don't move meta regions.
+        regionsToMove.add(new RegionPlan(region, server.getKey(), null));
+        if(--neededRegions == 0) {
+          // No more regions needed, done shedding
+          break;
+        }
+      }
+    }
+
+    // Now we have a set of regions that must be all assigned out
+    // Assign each underloaded up to the min, then if leftovers, assign to max
+
+    // Walk down least loaded, assigning to each to fill up to min
+    for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
+      serversByLoad.entrySet()) {
+      int regionCount = server.getKey().getLoad().getNumberOfRegions();
+      BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey());
+      if(balanceInfo != null) {
+        regionCount += balanceInfo.getNumRegionsAdded();
+      }
+      if(regionCount >= min) {
+        break;
+      }
+      int numToTake = min - regionCount;
+      int numTaken = 0;
+      while(numTaken < numToTake && regionidx < regionsToMove.size()) {
+        regionsToMove.get(regionidx).setDestination(server.getKey());
+        numTaken++;
+        regionidx++;
+      }
+    }
+
+    // If we still have regions to dish out, assign underloaded to max
+    if(regionidx != regionsToMove.size()) {
+      for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
+        serversByLoad.entrySet()) {
+        int regionCount = server.getKey().getLoad().getNumberOfRegions();
+        if(regionCount >= max) {
+          break;
+        }
+        regionsToMove.get(regionidx).setDestination(server.getKey());
+        regionidx++;
+        if(regionidx == regionsToMove.size()) {
+          break;
+        }
+      }
+    }
+
+    long endTime = System.currentTimeMillis();
+
+    assert(regionidx == regionsToMove.size());
+    assert(neededRegions == 0);
+
+    // All done!
+    LOG.info("Calculated a load balance in " + (endTime-startTime) + "ms. " +
+        "Moving " + regionsToMove.size() + " regions off of " +
+        serversOverloaded + " overloaded servers onto " +
+        serversUnderloaded + " less loaded servers");
+
+    return regionsToMove;
+  }
+
+  /**
+   * Stores additional per-server information about the regions added/removed
+   * during the run of the balancing algorithm.
+   *
+   * For servers that receive additional regions, we are not updating the number
+   * of regions in HServerInfo once we decide to reassign regions to a server,
+   * but we need this information later in the algorithm.  This is stored in
+   * <b>numRegionsAdded</b>.
+   *
+   * For servers that shed regions, we need to track which regions we have
+   * already shed.  <b>nextRegionForUnload</b> contains the index in the list
+   * of regions on the server that is the next to be shed.
+   */
+  private static class BalanceInfo {
+
+    private final int nextRegionForUnload;
+    private final int numRegionsAdded;
+
+    public BalanceInfo(int nextRegionForUnload, int numRegionsAdded) {
+      this.nextRegionForUnload = nextRegionForUnload;
+      this.numRegionsAdded = numRegionsAdded;
+    }
+
+    public int getNextRegionForUnload() {
+      return nextRegionForUnload;
+    }
+
+    public int getNumRegionsAdded() {
+      return numRegionsAdded;
+    }
+  }
+
+  /**
+   * Generates a bulk assignment plan to be used on cluster startup.
+   *
+   * Takes a list of all the regions and all the servers in the cluster and
+   * returns a map of each server to the regions that it should be assigned.
+   *
+   * Currently implemented as a round-robin assignment.  Same invariant as
+   * load balancing, all servers holding floor(avg) or ceiling(avg).
+   *
+   * TODO: Use block locations from HDFS to place regions with their blocks
+   *
+   * @param regions all regions
+   * @param servers all servers
+   * @return map of server to the regions it should take, or null if no
+   *         assignment is possible (ie. no regions or no servers)
+   */
+  public static Map<HServerInfo,List<HRegionInfo>> bulkAssignment(
+      List<HRegionInfo> regions, List<HServerInfo> servers) {
+    if(regions.size() == 0 || servers.size() == 0) {
+      return null;
+    }
+    Map<HServerInfo,List<HRegionInfo>> assignments =
+      new TreeMap<HServerInfo,List<HRegionInfo>>();
+    int numRegions = regions.size();
+    int numServers = servers.size();
+    int max = (int)Math.ceil((float)numRegions/numServers);
+    int serverIdx = 0;
+    for(HServerInfo server : servers) {
+      List<HRegionInfo> serverRegions = new ArrayList<HRegionInfo>(max);
+      for(int i=serverIdx;i<regions.size();i+=numServers) {
+        serverRegions.add(regions.get(i));
+      }
+      assignments.put(server, serverRegions);
+      serverIdx++;
+    }
+    return assignments;
+  }
+
+  /**
+   * Find the block locations for all of the files for the specified region.
+   *
+   * Returns an ordered list of hosts that are hosting the blocks for this
+   * region.  The weight of each host is the sum of the block lengths of all
+   * files on that host, so the first host in the list is the server which
+   * holds the most bytes of the given region's HFiles.
+   *
+   * TODO: Make this work.  Need to figure out how to match hadoop's hostnames
+   *       given for block locations with our HServerAddress.
+   * TODO: Use the right directory for the region
+   * TODO: Use getFileBlockLocations on the files not the directory
+   *
+   * @param fs the filesystem
+   * @param region region
+   * @return ordered list of hosts holding blocks of the specified region
+   * @throws IOException if any filesystem errors
+   */
+  private List<String> getTopBlockLocations(FileSystem fs, HRegionInfo region)
+  throws IOException {
+    String encodedName = region.getEncodedName();
+    Path path = new Path("/hbase/table/" + encodedName);
+    FileStatus status = fs.getFileStatus(path);
+    BlockLocation [] blockLocations =
+      fs.getFileBlockLocations(status, 0, status.getLen());
+    Map<HostAndWeight,HostAndWeight> hostWeights =
+      new TreeMap<HostAndWeight,HostAndWeight>(new HostAndWeight.HostComparator());
+    for(BlockLocation bl : blockLocations) {
+      String [] hosts = bl.getHosts();
+      long len = bl.getLength();
+      for(String host : hosts) {
+        HostAndWeight haw = hostWeights.get(host);
+        if(haw == null) {
+          haw = new HostAndWeight(host, len);
+          hostWeights.put(haw, haw);
+        } else {
+          haw.addWeight(len);
+        }
+      }
+    }
+    NavigableSet<HostAndWeight> orderedHosts = new TreeSet<HostAndWeight>(
+        new HostAndWeight.WeightComparator());
+    orderedHosts.addAll(hostWeights.values());
+    List<String> topHosts = new ArrayList<String>(orderedHosts.size());
+    for(HostAndWeight haw : orderedHosts.descendingSet()) {
+      topHosts.add(haw.getHost());
+    }
+    return topHosts;
+  }
+
+  /**
+   * Stores the hostname and weight for that hostname.
+   *
+   * This is used when determining the physical locations of the blocks making
+   * up a region.
+   *
+   * To make a prioritized list of the hosts holding the most data of a region,
+   * this class is used to count the total weight for each host.  The weight is
+   * currently just the size of the file.
+   */
+  private static class HostAndWeight {
+
+    private final String host;
+    private long weight;
+
+    public HostAndWeight(String host, long weight) {
+      this.host = host;
+      this.weight = weight;
+    }
+
+    public void addWeight(long weight) {
+      this.weight += weight;
+    }
+
+    public String getHost() {
+      return host;
+    }
+
+    public long getWeight() {
+      return weight;
+    }
+
+    private static class HostComparator implements Comparator<HostAndWeight> {
+      @Override
+      public int compare(HostAndWeight l, HostAndWeight r) {
+        return l.getHost().compareTo(r.getHost());
+      }
+    }
+
+    private static class WeightComparator implements Comparator<HostAndWeight> {
+      @Override
+      public int compare(HostAndWeight l, HostAndWeight r) {
+        if(l.getWeight() == r.getWeight()) {
+          return l.getHost().compareTo(r.getHost());
+        }
+        return l.getWeight() < r.getWeight() ? -1 : 1;
+      }
+    }
+  }
+
+  /**
+   * Generates an immediate assignment plan to be used by a new master for
+   * regions in transition that do not have an already known destination.
+   *
+   * Takes a list of regions that need immediate assignment and a list of
+   * all available servers.  Returns a map of regions to the server they
+   * should be assigned to.
+   *
+   * This method will return quickly and does not do any intelligent
+   * balancing.  The goal is to make a fast decision not the best decision
+   * possible.
+   *
+   * Currently this is random.
+   *
+   * @param regions
+   * @param servers
+   * @return map of regions to the server it should be assigned to
+   */
+  public static Map<HRegionInfo,HServerInfo> immediateAssignment(
+      List<HRegionInfo> regions, List<HServerInfo> servers) {
+    Map<HRegionInfo,HServerInfo> assignments =
+      new TreeMap<HRegionInfo,HServerInfo>();
+    for(HRegionInfo region : regions) {
+      assignments.put(region, servers.get(rand.nextInt(servers.size())));
+    }
+    return assignments;
+  }
+
+  public static HServerInfo randomAssignment(List<HServerInfo> servers) {
+    if (servers == null || servers.isEmpty()) {
+      LOG.warn("Wanted to do random assignment but no servers to assign to");
+      return null;
+    }
+    return servers.get(rand.nextInt(servers.size()));
+  }
+
+  /**
+   * Stores the plan for the move of an individual region.
+   *
+   * Contains info for the region being moved, info for the server the region
+   * should be moved from, and info for the server the region should be moved
+   * to.
+   *
+   * The comparable implementation of this class compares only the region
+   * information and not the source/dest server info.
+   */
+  public static class RegionPlan implements Comparable<RegionPlan> {
+    private final HRegionInfo hri;
+    private final HServerInfo source;
+    private HServerInfo dest;
+
+    
+
+    /**
+     * Instantiate a plan for a region move, moving the specified region from
+     * the specified source server to the specified destination server.
+     *
+     * Destination server can be instantiated as null and later set
+     * with {@link #setDestination(HServerInfo)}.
+     *
+     * @param hri region to be moved
+     * @param source regionserver region should be moved from
+     * @param dest regionserver region should be moved to
+     */
+    public RegionPlan(final HRegionInfo hri, HServerInfo source, HServerInfo dest) {
+      this.hri = hri;
+      this.source = source;
+      this.dest = dest;
+    }
+
+    /**
+     * Set the destination server for the plan for this region.
+     */
+    public void setDestination(HServerInfo dest) {
+      this.dest = dest;
+    }
+
+    /**
+     * Get the source server for the plan for this region.
+     * @return server info for source
+     */
+    public HServerInfo getSource() {
+      return source;
+    }
+
+    /**
+     * Get the destination server for the plan for this region.
+     * @return server info for destination
+     */
+    public HServerInfo getDestination() {
+      return dest;
+    }
+
+    /**
+     * Get the encoded region name for the region this plan is for.
+     * @return Encoded region name
+     */
+    public String getRegionName() {
+      return this.hri.getEncodedName();
+    }
+ 
+    public HRegionInfo getRegionInfo() {
+      return this.hri;
+    }
+
+    /**
+     * Compare the region info.
+     * @param o region plan you are comparing against
+     */
+    @Override
+    public int compareTo(RegionPlan o) {
+      return getRegionName().compareTo(o.getRegionName());
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/LogsCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java
similarity index 93%
rename from src/main/java/org/apache/hadoop/hbase/master/LogsCleaner.java
rename to src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java
index 9d1a8b8f57b..2a4bc8282d4 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/LogsCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java
@@ -19,6 +19,10 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -26,23 +30,17 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Chore;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 /**
- * This Chore, everytime it runs, will clear the logs in the old logs folder
+ * This Chore, everytime it runs, will clear the wal logs in the old logs folder
  * that are deletable for each log cleaner in the chain, in order to limit the
  * number of deletes it sends, will only delete maximum 20 in a single run.
  */
-public class LogsCleaner extends Chore {
-
-  static final Log LOG = LogFactory.getLog(LogsCleaner.class.getName());
+public class LogCleaner extends Chore {
+  static final Log LOG = LogFactory.getLog(LogCleaner.class.getName());
 
   // Max number we can delete on every chore, this is to make sure we don't
   // issue thousands of delete commands around the same time
@@ -55,12 +53,12 @@ public class LogsCleaner extends Chore {
   /**
    *
    * @param p the period of time to sleep between each run
-   * @param s the stopper boolean
+   * @param s the stopper
    * @param conf configuration to use
    * @param fs handle to the FS
    * @param oldLogDir the path to the archived logs
    */
-  public LogsCleaner(final int p, final AtomicBoolean s,
+  public LogCleaner(final int p, final Stoppable s,
                         Configuration conf, FileSystem fs,
                         Path oldLogDir) {
     super("LogsCleaner", p, s);
@@ -154,4 +152,4 @@ public class LogsCleaner extends Chore {
       LOG.warn("Error while cleaning the logs", e);
     }
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java
index 1c338310104..a66fe0fecdb 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java
@@ -45,5 +45,4 @@ public interface LogCleanerDelegate extends Configurable {
    * @return true if the log is deletable, false if not
    */
   public boolean isLogDeletable(Path filePath);
-}
-
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
new file mode 100644
index 00000000000..498650fd2aa
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -0,0 +1,279 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * This class abstracts a bunch of operations the HMaster needs to interact with
+ * the underlying file system, including splitting log files, checking file
+ * system status, etc.
+ */
+public class MasterFileSystem {
+  private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
+  // HBase configuration
+  Configuration conf;
+  // master status
+  Server master;
+  // Keep around for convenience.
+  private final FileSystem fs;
+  // Is the fileystem ok?
+  private volatile boolean fsOk = true;
+  // The Path to the old logs dir
+  private final Path oldLogDir;
+  // root hbase directory on the FS
+  private final Path rootdir;
+  // create the split log lock
+  final Lock splitLogLock = new ReentrantLock();
+
+  public MasterFileSystem(Server master) throws IOException {
+    this.conf = master.getConfiguration();
+    this.master = master;
+    // Set filesystem to be that of this.rootdir else we get complaints about
+    // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
+    // default localfs.  Presumption is that rootdir is fully-qualified before
+    // we get to here with appropriate fs scheme.
+    this.rootdir = FSUtils.getRootDir(conf);
+    // Cover both bases, the old way of setting default fs and the new.
+    // We're supposed to run on 0.20 and 0.21 anyways.
+    conf.set("fs.default.name", this.rootdir.toString());
+    conf.set("fs.defaultFS", this.rootdir.toString());
+    // setup the filesystem variable
+    this.fs = FileSystem.get(conf);
+    // set up the archived logs path
+    this.oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
+  }
+
+  /**
+   * <ol>
+   * <li>Check if the root region exists and is readable, if not create it</li>
+   * <li>Create a log archive directory for RS to put archived logs</li>
+   * </ol>
+   */
+  public void initialize() throws IOException {
+    // check if the root directory exists
+    checkRootDir(this.rootdir, conf, this.fs);
+
+    // Make sure the region servers can archive their old logs
+    if(!this.fs.exists(this.oldLogDir)) {
+      this.fs.mkdirs(this.oldLogDir);
+    }
+  }
+
+  public FileSystem getFileSystem() {
+    return this.fs;
+  }
+
+  /**
+   * Get the directory where old logs go
+   * @return the dir
+   */
+  public Path getOldLogDir() {
+    return this.oldLogDir;
+  }
+
+  /**
+   * Checks to see if the file system is still accessible.
+   * If not, sets closed
+   * @return false if file system is not available
+   */
+  public boolean checkFileSystem() {
+    if (this.fsOk) {
+      try {
+        FSUtils.checkFileSystemAvailable(this.fs);
+      } catch (IOException e) {
+        master.abort("Shutting down HBase cluster: file system not available", e);
+        this.fsOk = false;
+      }
+    }
+    return this.fsOk;
+  }
+
+  /**
+   * @return HBase root dir.
+   * @throws IOException
+   */
+  public Path getRootDir() {
+    return this.rootdir;
+  }
+
+  /**
+   * Inspect the log directory to recover any log file without
+   * an active region server.
+   * @param onlineServers Map of online servers keyed by
+   * {@link HServerInfo#getServerName()}
+   */
+  void splitLogAfterStartup(final Map<String, HServerInfo> onlineServers) {
+    Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
+    try {
+      if (!this.fs.exists(logsDirPath)) {
+        return;
+      }
+    } catch (IOException e) {
+      throw new RuntimeException("Failed exists test on " + logsDirPath, e);
+    }
+    FileStatus[] logFolders;
+    try {
+      logFolders = this.fs.listStatus(logsDirPath);
+    } catch (IOException e) {
+      throw new RuntimeException("Failed listing " + logsDirPath.toString(), e);
+    }
+    if (logFolders == null || logFolders.length == 0) {
+      LOG.debug("No log files to split, proceeding...");
+      return;
+    }
+    for (FileStatus status : logFolders) {
+      String serverName = status.getPath().getName();
+      LOG.info("Found log folder : " + serverName);
+      if(onlineServers.get(serverName) == null) {
+        LOG.info("Log folder doesn't belong " +
+          "to a known region server, splitting");
+        splitLog(serverName);
+      } else {
+        LOG.info("Log folder belongs to an existing region server");
+      }
+    }
+  }
+
+  public void splitLog(final String serverName) {
+    this.splitLogLock.lock();
+    Path logDir = new Path(this.rootdir, HLog.getHLogDirectoryName(serverName));
+    try {
+      HLog.splitLog(this.rootdir, logDir, oldLogDir, this.fs, conf);
+    } catch (IOException e) {
+      LOG.error("Failed splitting " + logDir.toString(), e);
+    } finally {
+      this.splitLogLock.unlock();
+    }
+  }
+
+  /**
+   * Get the rootdir.  Make sure its wholesome and exists before returning.
+   * @param rd
+   * @param conf
+   * @param fs
+   * @return hbase.rootdir (after checks for existence and bootstrapping if
+   * needed populating the directory with necessary bootup files).
+   * @throws IOException
+   */
+  private static Path checkRootDir(final Path rd, final Configuration c,
+    final FileSystem fs)
+  throws IOException {
+    // If FS is in safe mode wait till out of it.
+    FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+        10 * 1000));
+    // Filesystem is good. Go ahead and check for hbase.rootdir.
+    if (!fs.exists(rd)) {
+      fs.mkdirs(rd);
+      FSUtils.setVersion(fs, rd);
+    } else {
+      FSUtils.checkVersion(fs, rd, true);
+    }
+    // Make sure the root region directory exists!
+    if (!FSUtils.rootRegionExists(fs, rd)) {
+      bootstrap(rd, c);
+    }
+    return rd;
+  }
+
+  private static void bootstrap(final Path rd, final Configuration c)
+  throws IOException {
+    LOG.info("BOOTSTRAP: creating ROOT and first META regions");
+    try {
+      // Bootstrapping, make sure blockcache is off.  Else, one will be
+      // created here in bootstap and it'll need to be cleaned up.  Better to
+      // not make it in first place.  Turn off block caching for bootstrap.
+      // Enable after.
+      HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
+      setInfoFamilyCaching(rootHRI, false);
+      HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
+      setInfoFamilyCaching(metaHRI, false);
+      HRegion root = HRegion.createHRegion(rootHRI, rd, c);
+      HRegion meta = HRegion.createHRegion(metaHRI, rd, c);
+      setInfoFamilyCaching(rootHRI, true);
+      setInfoFamilyCaching(metaHRI, true);
+      // Add first region from the META table to the ROOT region.
+      HRegion.addRegionToMETA(root, meta);
+      root.close();
+      root.getLog().closeAndDelete();
+      meta.close();
+      meta.getLog().closeAndDelete();
+    } catch (IOException e) {
+      e = RemoteExceptionHandler.checkIOException(e);
+      LOG.error("bootstrap", e);
+      throw e;
+    }
+  }
+
+  /**
+   * @param hri Set all family block caching to <code>b</code>
+   * @param b
+   */
+  private static void setInfoFamilyCaching(final HRegionInfo hri, final boolean b) {
+    for (HColumnDescriptor hcd: hri.getTableDesc().families.values()) {
+      if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
+        hcd.setBlockCacheEnabled(b);
+        hcd.setInMemory(b);
+      }
+    }
+  }
+
+  public void deleteRegion(HRegionInfo region) throws IOException {
+    fs.delete(HRegion.getRegionDir(rootdir, region), true);
+  }
+
+  public void deleteTable(byte[] tableName) throws IOException {
+    fs.delete(new Path(rootdir, Bytes.toString(tableName)), true);
+  }
+
+  public void updateRegionInfo(HRegionInfo region) {
+    // TODO implement this.  i think this is currently broken in trunk i don't
+    //      see this getting updated.
+    //      @see HRegion.checkRegioninfoOnFilesystem()
+  }
+
+  public void deleteFamily(HRegionInfo region, byte[] familyName)
+  throws IOException {
+    fs.delete(Store.getStoreHomedir(
+        new Path(rootdir, region.getTableDesc().getNameAsString()),
+        region.getEncodedName(), familyName), true);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
new file mode 100644
index 00000000000..593254ba38e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -0,0 +1,59 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.TableNotDisabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+
+/**
+ * Services Master supplies
+ */
+public interface MasterServices {
+  /**
+   * @return Master's instance of the {@link AssignmentManager}
+   */
+  public AssignmentManager getAssignmentManager();
+
+  /**
+   * @return Master's filesystem {@link MasterFileSystem} utility class.
+   */
+  public MasterFileSystem getMasterFileSystem();
+
+  /**
+   * @return Master's {@link ServerManager} instance.
+   */
+  public ServerManager getServerManager();
+
+  /**
+   * @return Master's instance of {@link ExecutorService}
+   */
+  public ExecutorService getExecutorService();
+
+  /**
+   * Check table is modifiable; i.e. exists and is offline.
+   * @param tableName Name of table to check.
+   * @throws TableNotDisabledException
+   * @throws TableNotFoundException 
+   */
+  public void checkTableModifiable(final byte [] tableName) throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MetaRegion.java b/src/main/java/org/apache/hadoop/hbase/master/MetaRegion.java
deleted file mode 100644
index 8d9a2db9993..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/MetaRegion.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.util.Bytes;
-
-
-/** Describes a meta region and its server */
-public class MetaRegion implements Comparable<MetaRegion> {
-  private final HServerAddress server;
-  private HRegionInfo regionInfo;
-
-  MetaRegion(final HServerAddress server, HRegionInfo regionInfo) {
-    if (server == null) {
-      throw new IllegalArgumentException("server cannot be null");
-    }
-    this.server = server;
-    if (regionInfo == null) {
-      throw new IllegalArgumentException("regionInfo cannot be null");
-    }
-    this.regionInfo = regionInfo;
-  }
-
-  @Override
-  public String toString() {
-    return "{server: " + this.server.toString() + ", regionname: " +
-        regionInfo.getRegionNameAsString() + ", startKey: <" +
-        Bytes.toString(regionInfo.getStartKey()) + ">}";
-  }
-
-  /** @return the regionName */
-  public byte [] getRegionName() {
-    return regionInfo.getRegionName();
-  }
-
-  /** @return the server */
-  public HServerAddress getServer() {
-    return server;
-  }
-
-  /** @return the startKey */
-  public byte [] getStartKey() {
-    return regionInfo.getStartKey();
-  }
-
-
-  /** @return the endKey */
-  public byte [] getEndKey() {
-    return regionInfo.getEndKey();
-  }
-
-
-  public HRegionInfo getRegionInfo() {
-    return regionInfo;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    return o instanceof MetaRegion && this.compareTo((MetaRegion)o) == 0;
-  }
-
-  @Override
-  public int hashCode() {
-    return regionInfo.hashCode();
-  }
-
-  // Comparable
-
-  public int compareTo(MetaRegion other) {
-    int cmp = regionInfo.compareTo(other.regionInfo);
-    if(cmp == 0) {
-      // Might be on different host?
-      cmp = this.server.compareTo(other.server);
-    }
-    return cmp;
-  }
-}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/master/MetaScanner.java
deleted file mode 100644
index e6434ba8e1a..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/MetaScanner.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * MetaScanner <code>META</code> table.
- *
- * When a <code>META</code> server comes on line, a MetaRegion object is
- * queued up by regionServerReport() and this thread wakes up.
- *
- * It's important to do this work in a separate thread, or else the blocking
- * action would prevent other work from getting done.
- */
-class MetaScanner extends BaseScanner {
-  /** Initial work for the meta scanner is queued up here */
-  private volatile BlockingQueue<MetaRegion> metaRegionsToScan =
-    new LinkedBlockingQueue<MetaRegion>();
-
-  private final List<MetaRegion> metaRegionsToRescan =
-    new ArrayList<MetaRegion>();
-
-  /**
-   * Constructor
-   *
-   * @param master
-   */
-  public MetaScanner(HMaster master) {
-    super(master, false, master.getShutdownRequested());
-  }
-
-  // Don't retry if we get an error while scanning. Errors are most often
-  // caused by the server going away. Wait until next rescan interval when
-  // things should be back to normal.
-  private boolean scanOneMetaRegion(MetaRegion region) {
-    while (!this.master.isClosed() &&
-        !this.master.getRegionManager().isInitialRootScanComplete() &&
-        this.master.getRegionManager().getRootRegionLocation() == null) {
-      sleep();
-    }
-    if (this.master.isClosed()) {
-      return false;
-    }
-
-    try {
-      // Don't interrupt us while we're working
-      synchronized (scannerLock) {
-        scanRegion(region);
-        this.master.getRegionManager().putMetaRegionOnline(region);
-      }
-    } catch (IOException e) {
-      e = RemoteExceptionHandler.checkIOException(e);
-      LOG.warn("Scan one META region: " + region.toString(), e);
-      // The region may have moved (TestRegionServerAbort, etc.).  If
-      // so, either it won't be in the onlineMetaRegions list or its host
-      // address has changed and the containsValue will fail. If not
-      // found, best thing to do here is probably return.
-      if (!this.master.getRegionManager().isMetaRegionOnline(region.getStartKey())) {
-        LOG.debug("Scanned region is no longer in map of online " +
-        "regions or its value has changed");
-        return false;
-      }
-      // Make sure the file system is still available
-      this.master.checkFileSystem();
-    } catch (Exception e) {
-      // If for some reason we get some other kind of exception,
-      // at least log it rather than go out silently.
-      LOG.error("Unexpected exception", e);
-    }
-    return true;
-  }
-
-  @Override
-  protected boolean initialScan() {
-    MetaRegion region = null;
-    while (!this.master.isClosed() &&
-        (region == null && metaRegionsToScan.size() > 0) &&
-          !metaRegionsScanned()) {
-      try {
-        region = metaRegionsToScan.poll(this.master.getThreadWakeFrequency(),
-          TimeUnit.MILLISECONDS);
-      } catch (InterruptedException e) {
-        // continue
-      }
-      if (region == null && metaRegionsToRescan.size() != 0) {
-        region = metaRegionsToRescan.remove(0);
-      }
-      if (region != null) {
-        if (!scanOneMetaRegion(region)) {
-          metaRegionsToRescan.add(region);
-        }
-      }
-    }
-    initialScanComplete = true;
-    return true;
-  }
-
-  @Override
-  protected void maintenanceScan() {
-    List<MetaRegion> regions =
-      this.master.getRegionManager().getListOfOnlineMetaRegions();
-    int regionCount = 0;
-    for (MetaRegion r: regions) {
-      scanOneMetaRegion(r);
-      regionCount++;
-    }
-    LOG.info("All " + regionCount + " .META. region(s) scanned");
-    metaRegionsScanned();
-  }
-
-  /*
-   * Called by the meta scanner when it has completed scanning all meta
-   * regions. This wakes up any threads that were waiting for this to happen.
-   * @param totalRows Total rows scanned.
-   * @param regionCount Count of regions in  .META. table.
-   * @return False if number of meta regions matches count of online regions.
-   */
-  private synchronized boolean metaRegionsScanned() {
-    if (!this.master.getRegionManager().isInitialRootScanComplete() ||
-        this.master.getRegionManager().numMetaRegions() !=
-          this.master.getRegionManager().numOnlineMetaRegions()) {
-      return false;
-    }
-    notifyAll();
-    return true;
-  }
-
-  /**
-   * Other threads call this method to wait until all the meta regions have
-   * been scanned.
-   */
-  synchronized boolean waitForMetaRegionsOrClose() {
-    while (!this.master.isClosed()) {
-      synchronized (master.getRegionManager()) {
-        if (this.master.getRegionManager().isInitialRootScanComplete() &&
-            this.master.getRegionManager().numMetaRegions() ==
-              this.master.getRegionManager().numOnlineMetaRegions()) {
-          break;
-        }
-      }
-      try {
-        wait(this.master.getThreadWakeFrequency());
-      } catch (InterruptedException e) {
-        // continue
-      }
-    }
-    return this.master.isClosed();
-  }
-
-  /**
-   * Add another meta region to scan to the queue.
-   */
-  void addMetaRegionToScan(MetaRegion m) {
-    metaRegionsToScan.add(m);
-  }
-}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ModifyColumn.java b/src/main/java/org/apache/hadoop/hbase/master/ModifyColumn.java
deleted file mode 100644
index 2099444ceb5..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/ModifyColumn.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-
-/** Instantiated to modify an existing column family on a table */
-class ModifyColumn extends ColumnOperation {
-  private final HColumnDescriptor descriptor;
-  private final byte [] columnName;
-
-  ModifyColumn(final HMaster master, final byte [] tableName,
-    final byte [] columnName, HColumnDescriptor descriptor)
-  throws IOException {
-    super(master, tableName);
-    this.descriptor = descriptor;
-    this.columnName = columnName;
-  }
-
-  @Override
-  protected void postProcessMeta(MetaRegion m, HRegionInterface server)
-  throws IOException {
-    for (HRegionInfo i: unservedRegions) {
-      if (i.getTableDesc().hasFamily(columnName)) {
-        i.getTableDesc().addFamily(descriptor);
-        updateRegionInfo(server, m.getRegionName(), i);
-      } else { // otherwise, we have an error.
-        throw new InvalidColumnNameException("Column family '" +
-          Bytes.toString(columnName) +
-          "' doesn't exist, so cannot be modified.");
-      }
-    }
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ModifyTableMeta.java b/src/main/java/org/apache/hadoop/hbase/master/ModifyTableMeta.java
deleted file mode 100644
index c985da9912f..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/ModifyTableMeta.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableNotDisabledException;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Writables;
-
-import java.io.IOException;
-
-/** Instantiated to modify table descriptor metadata */
-class ModifyTableMeta extends TableOperation {
-
-  private static Log LOG = LogFactory.getLog(ModifyTableMeta.class);
-
-  private HTableDescriptor desc;
-
-  ModifyTableMeta(final HMaster master, final byte [] tableName,
-    HTableDescriptor desc)
-  throws IOException {
-    super(master, tableName);
-    this.desc = desc;
-    LOG.debug("modifying " + Bytes.toString(tableName) + ": " +
-        desc.toString());
-  }
-
-  protected void updateRegionInfo(HRegionInterface server, byte [] regionName,
-    HRegionInfo i)
-  throws IOException {
-    Put put = new Put(i.getRegionName());
-    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(i));
-    server.put(regionName, put);
-    LOG.debug("updated HTableDescriptor for region " + i.getRegionNameAsString());
-  }
-
-  @Override
-  protected void processScanItem(String serverName,
-      final HRegionInfo info) throws IOException {
-    if (isEnabled(info)) {
-      throw new TableNotDisabledException(Bytes.toString(tableName));
-    }
-  }
-
-  @Override
-  protected void postProcessMeta(MetaRegion m, HRegionInterface server)
-  throws IOException {
-    for (HRegionInfo i: unservedRegions) {
-      i.setTableDesc(desc);
-      updateRegionInfo(server, m.getRegionName(), i);
-    }
-    // kick off a meta scan right away
-    master.getRegionManager().metaScannerThread.triggerNow();
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java b/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java
deleted file mode 100644
index b94e8871b70..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-
-import java.io.IOException;
-
-/**
- * ProcessRegionClose is the way we do post-processing on a closed region. We
- * only spawn one of these asynchronous tasks when the region needs to be
- * either offlined or deleted. We used to create one of these tasks whenever
- * a region was closed, but since closing a region that isn't being offlined
- * or deleted doesn't actually require post processing, it's no longer
- * necessary.
- */
-public class ProcessRegionClose extends ProcessRegionStatusChange {
-  protected final boolean offlineRegion;
-  protected final boolean reassignRegion;
-
-  /**
-  * @param master
-  * @param regionInfo Region to operate on
-  * @param offlineRegion if true, set the region to offline in meta
-  * @param reassignRegion if true, region is to be reassigned
-  */
-  public ProcessRegionClose(HMaster master, HRegionInfo regionInfo,
-      boolean offlineRegion, boolean reassignRegion) {
-
-   super(master, regionInfo);
-   this.offlineRegion = offlineRegion;
-   this.reassignRegion = reassignRegion;
-  }
-
-  @Override
-  public String toString() {
-    return "ProcessRegionClose of " + this.regionInfo.getRegionNameAsString() +
-      ", " + this.offlineRegion + ", reassign: " + this.reassignRegion;
-  }
-
-  @Override
-  protected boolean process() throws IOException {
-    if (!metaRegionAvailable()) {
-      // We can't proceed unless the meta region we are going to update
-      // is online. metaRegionAvailable() has put this operation on the
-      // delayedToDoQueue, so return true so the operation is not put
-      // back on the toDoQueue
-      return true;
-    }
-    Boolean result = null;
-    if (offlineRegion || reassignRegion) {
-      result =
-        new RetryableMetaOperation<Boolean>(getMetaRegion(), this.master) {
-          public Boolean call() throws IOException {
-
-
-            // We can't proceed unless the meta region we are going to update
-            // is online. metaRegionAvailable() will put this operation on the
-            // delayedToDoQueue, so return true so the operation is not put
-            // back on the toDoQueue
-
-            if (metaRegionAvailable()) {
-              if(offlineRegion) {
-                // offline the region in meta and then remove it from the
-                // set of regions in transition
-                HRegion.offlineRegionInMETA(server, metaRegionName,
-                    regionInfo);
-                master.getRegionManager().removeRegion(regionInfo);
-                LOG.info("region closed: " + regionInfo.getRegionNameAsString());
-              } else {
-                // we are reassigning the region eventually, so set it unassigned
-                // and remove the server info
-                HRegion.cleanRegionInMETA(server, metaRegionName,
-                    regionInfo);
-                master.getRegionManager().setUnassigned(regionInfo, false);
-                LOG.info("region set as unassigned: " + regionInfo.getRegionNameAsString());
-              }
-            }
-            return true;
-          }
-        }.doWithRetries();
-        result = result == null ? true : result;
-
-    } else {
-      LOG.info("Region was neither offlined, or asked to be reassigned, what gives: " +
-      regionInfo.getRegionNameAsString());
-    }
-
-    return result == null ? true : result;
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java b/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
deleted file mode 100644
index 89e9915cb1c..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-
-import java.io.IOException;
-
-/**
- * ProcessRegionOpen is instantiated when a region server reports that it is
- * serving a region. This applies to all meta and user regions except the
- * root region which is handled specially.
- */
-public class ProcessRegionOpen extends ProcessRegionStatusChange {
-  protected final HServerInfo serverInfo;
-
-  /**
-   * @param master
-   * @param info
-   * @param regionInfo
-   */
-  public ProcessRegionOpen(HMaster master, HServerInfo info,
-      HRegionInfo regionInfo) {
-    super(master, regionInfo);
-    if (info == null) {
-      throw new NullPointerException("HServerInfo cannot be null; " +
-        "hbase-958 debugging");
-    }
-    this.serverInfo = info;
-  }
-
-  @Override
-  public String toString() {
-    return "PendingOpenOperation from " + serverInfo.getServerName();
-  }
-
-  @Override
-  protected boolean process() throws IOException {
-    // TODO: The below check is way too convoluted!!!
-    if (!metaRegionAvailable()) {
-      // We can't proceed unless the meta region we are going to update
-      // is online. metaRegionAvailable() has put this operation on the
-      // delayedToDoQueue, so return true so the operation is not put
-      // back on the toDoQueue
-      return true;
-    }
-    HRegionInterface server =
-        master.getServerConnection().getHRegionConnection(getMetaRegion().getServer());
-    LOG.info(regionInfo.getRegionNameAsString() + " open on " +
-      serverInfo.getServerName());
-
-    // Register the newly-available Region's location.
-    Put p = new Put(regionInfo.getRegionName());
-    p.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
-      Bytes.toBytes(serverInfo.getHostnamePort()));
-    p.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
-      Bytes.toBytes(serverInfo.getStartCode()));
-    server.put(metaRegionName, p);
-    LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
-      " in region " + Bytes.toString(metaRegionName) + " with startcode=" +
-      serverInfo.getStartCode() + ", server=" + serverInfo.getHostnamePort());
-    synchronized (master.getRegionManager()) {
-      if (isMetaTable) {
-        // It's a meta region.
-        MetaRegion m =
-            new MetaRegion(new HServerAddress(serverInfo.getServerAddress()),
-                regionInfo);
-        if (!master.getRegionManager().isInitialMetaScanComplete()) {
-          // Put it on the queue to be scanned for the first time.
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Adding " + m.toString() + " to regions to scan");
-          }
-          master.getRegionManager().addMetaRegionToScan(m);
-        } else {
-          // Add it to the online meta regions
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Adding to onlineMetaRegions: " + m.toString());
-          }
-          master.getRegionManager().putMetaRegionOnline(m);
-          // Interrupting the Meta Scanner sleep so that it can
-          // process regions right away
-          master.getRegionManager().metaScannerThread.triggerNow();
-        }
-      }
-      // If updated successfully, remove from pending list if the state
-      // is consistent. For example, a disable could be called before the
-      // synchronization.
-      if(master.getRegionManager().
-          isOfflined(regionInfo.getRegionNameAsString())) {
-        LOG.warn("We opened a region while it was asked to be closed.");
-      } else {
-        master.getRegionManager().removeRegion(regionInfo);
-      }
-      ZooKeeperWrapper zkWrapper =
-          ZooKeeperWrapper.getInstance(master.getConfiguration(),
-              HMaster.class.getName());
-      zkWrapper.deleteUnassignedRegion(regionInfo.getEncodedName());
-      return true;
-    }
-  }
-
-  @Override
-  protected int getPriority() {
-    return 0; // highest priority
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java b/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java
deleted file mode 100644
index b55c4f5c65b..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.HRegionInfo;
-
-/**
- * Abstract class that performs common operations for
- * @see ProcessRegionClose and @see ProcessRegionOpen
- */
-abstract class ProcessRegionStatusChange extends RegionServerOperation {
-  protected final boolean isMetaTable;
-  protected final HRegionInfo regionInfo;
-  @SuppressWarnings({"FieldCanBeLocal"})
-  private volatile MetaRegion metaRegion = null;
-  protected volatile byte[] metaRegionName = null;
-
-  /**
-   * @param master the master
-   * @param regionInfo region info
-   */
-  public ProcessRegionStatusChange(HMaster master, HRegionInfo regionInfo) {
-    super(master);
-    this.regionInfo = regionInfo;
-    this.isMetaTable = regionInfo.isMetaTable();
-  }
-
-  protected boolean metaRegionAvailable() {
-    boolean available = true;
-    if (isMetaTable) {
-      // This operation is for the meta table
-      if (!rootAvailable()) {
-        requeue();
-        // But we can't proceed unless the root region is available
-        available = false;
-      }
-    } else {
-      if (!master.getRegionManager().isInitialRootScanComplete() ||
-          !metaTableAvailable()) {
-        // The root region has not been scanned or the meta table is not
-        // available so we can't proceed.
-        // Put the operation on the delayedToDoQueue
-        requeue();
-        available = false;
-      }
-    }
-    return available;
-  }
-
-  protected MetaRegion getMetaRegion() {
-    if (isMetaTable) {
-      this.metaRegionName = HRegionInfo.ROOT_REGIONINFO.getRegionName();
-      this.metaRegion = new MetaRegion(master.getRegionManager().getRootRegionLocation(),
-          HRegionInfo.ROOT_REGIONINFO);
-    } else {
-      this.metaRegion =
-        master.getRegionManager().getFirstMetaRegionForRegion(regionInfo);
-      if (this.metaRegion != null) {
-        this.metaRegionName = this.metaRegion.getRegionName();
-      }
-    }
-    return this.metaRegion;
-  }
-  
-  public HRegionInfo getRegionInfo() {
-    return regionInfo;
-  }
-}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java b/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
deleted file mode 100644
index 2c1a1d85de7..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.master.RegionManager.RegionState;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Instantiated when a server's lease has expired, meaning it has crashed.
- * The region server's log file needs to be split up for each region it was
- * serving, and the regions need to get reassigned.
- */
-class ProcessServerShutdown extends RegionServerOperation {
-  // Server name made of the concatenation of hostname, port and startcode
-  // formatted as <code>&lt;hostname> ',' &lt;port> ',' &lt;startcode></code>
-  private final String deadServer;
-  private boolean isRootServer;
-  private List<MetaRegion> metaRegions;
-
-  private Path rsLogDir;
-  private boolean logSplit;
-  private boolean rootRescanned;
-  private HServerAddress deadServerAddress;
-
-  private static class ToDoEntry {
-    boolean regionOffline;
-    final HRegionInfo info;
-
-    ToDoEntry(final HRegionInfo info) {
-      this.regionOffline = false;
-      this.info = info;
-    }
-  }
-
-  /**
-   * @param master
-   * @param serverInfo
-   */
-  public ProcessServerShutdown(HMaster master, HServerInfo serverInfo) {
-    super(master);
-    this.deadServer = serverInfo.getServerName();
-    this.deadServerAddress = serverInfo.getServerAddress();
-    this.logSplit = false;
-    this.rootRescanned = false;
-    this.rsLogDir =
-      new Path(master.getRootDir(), HLog.getHLogDirectoryName(serverInfo));
-
-    // check to see if I am responsible for either ROOT or any of the META tables.
-
-    // TODO Why do we do this now instead of at processing time?
-    closeMetaRegions();
-  }
-
-  private void closeMetaRegions() {
-    this.isRootServer =
-      this.master.getRegionManager().isRootServer(this.deadServerAddress) ||
-      this.master.getRegionManager().isRootInTransitionOnThisServer(deadServer);
-    if (this.isRootServer) {
-      this.master.getRegionManager().unsetRootRegion();
-    }
-    List<byte[]> metaStarts =
-      this.master.getRegionManager().listMetaRegionsForServer(deadServerAddress);
-
-    this.metaRegions = new ArrayList<MetaRegion>();
-    for (byte [] startKey: metaStarts) {
-      MetaRegion r = master.getRegionManager().offlineMetaRegionWithStartKey(startKey);
-      this.metaRegions.add(r);
-    }
-
-    //HBASE-1928: Check whether this server has been transitioning the META table
-    HRegionInfo metaServerRegionInfo = master.getRegionManager().getMetaServerRegionInfo (deadServer);
-    if (metaServerRegionInfo != null) {
-      metaRegions.add (new MetaRegion (deadServerAddress, metaServerRegionInfo));
-    }
-  }
-
-  /**
-   * @return Name of server we are processing.
-   */
-  public HServerAddress getDeadServerAddress() {
-    return this.deadServerAddress;
-  }
-
-  private void closeRegionsInTransition() {
-    Map<String, RegionState> inTransition =
-      master.getRegionManager().getRegionsInTransitionOnServer(deadServer);
-    for (Map.Entry<String, RegionState> entry : inTransition.entrySet()) {
-      String regionName = entry.getKey();
-      RegionState state = entry.getValue();
-
-      LOG.info("Region " + regionName + " was in transition " +
-          state + " on dead server " + deadServer + " - marking unassigned");
-      master.getRegionManager().setUnassigned(state.getRegionInfo(), true);
-    }
-  }
-
-  @Override
-  public String toString() {
-    return "ProcessServerShutdown of " + this.deadServer;
-  }
-
-  /** Finds regions that the dead region server was serving
-   */
-  protected void scanMetaRegion(HRegionInterface server, long scannerId,
-    byte [] regionName)
-  throws IOException {
-    List<ToDoEntry> toDoList = new ArrayList<ToDoEntry>();
-    Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
-    List<byte []> emptyRows = new ArrayList<byte []>();
-    try {
-      while (true) {
-        Result values = null;
-        try {
-          values = server.next(scannerId);
-        } catch (IOException e) {
-          LOG.error("Shutdown scanning of meta region",
-            RemoteExceptionHandler.checkIOException(e));
-          break;
-        }
-        if (values == null || values.size() == 0) {
-          break;
-        }
-        byte [] row = values.getRow();
-        // Check server name.  If null, skip (We used to consider it was on
-        // shutdown server but that would mean that we'd reassign regions that
-        // were already out being assigned, ones that were product of a split
-        // that happened while the shutdown was being processed).
-        String serverAddress = BaseScanner.getServerAddress(values);
-        long startCode = BaseScanner.getStartCode(values);
-
-        String serverName = null;
-        if (serverAddress != null && serverAddress.length() > 0) {
-          serverName = HServerInfo.getServerName(serverAddress, startCode);
-        }
-        if (serverName == null || !deadServer.equals(serverName)) {
-          // This isn't the server you're looking for - move along
-          continue;
-        }
-
-        if (LOG.isDebugEnabled() && row != null) {
-          LOG.debug("Shutdown scanner for " + serverName + " processing " +
-            Bytes.toString(row));
-        }
-
-        HRegionInfo info = master.getHRegionInfo(row, values);
-        if (info == null) {
-          emptyRows.add(row);
-          continue;
-        }
-
-        synchronized (master.getRegionManager()) {
-          if (info.isMetaTable()) {
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("removing meta region " +
-                  Bytes.toString(info.getRegionName()) +
-              " from online meta regions");
-            }
-            master.getRegionManager().offlineMetaRegionWithStartKey(info.getStartKey());
-          }
-
-          ToDoEntry todo = new ToDoEntry(info);
-          toDoList.add(todo);
-
-          if (master.getRegionManager().isOfflined(info.getRegionNameAsString()) ||
-              info.isOffline()) {
-            master.getRegionManager().removeRegion(info);
-            // Mark region offline
-            if (!info.isOffline()) {
-              todo.regionOffline = true;
-            }
-          } else {
-            if (!info.isOffline() && !info.isSplit()) {
-              // Get region reassigned
-              regions.add(info);
-            }
-          }
-        }
-      }
-    } finally {
-      if (scannerId != -1L) {
-        try {
-          server.close(scannerId);
-        } catch (IOException e) {
-          LOG.error("Closing scanner",
-            RemoteExceptionHandler.checkIOException(e));
-        }
-      }
-    }
-
-    // Scan complete. Remove any rows which had empty HRegionInfos
-
-    if (emptyRows.size() > 0) {
-      LOG.warn("Found " + emptyRows.size() +
-        " rows with empty HRegionInfo while scanning meta region " +
-        Bytes.toString(regionName));
-      master.deleteEmptyMetaRows(server, regionName, emptyRows);
-    }
-    // Update server in root/meta entries
-    for (ToDoEntry e: toDoList) {
-      if (e.regionOffline) {
-        HRegion.offlineRegionInMETA(server, regionName, e.info);
-      }
-    }
-
-    // Get regions reassigned
-    for (HRegionInfo info: regions) {
-      master.getRegionManager().setUnassigned(info, true);
-    }
-  }
-
-  private class ScanRootRegion extends RetryableMetaOperation<Boolean> {
-    ScanRootRegion(MetaRegion m, HMaster master) {
-      super(m, master);
-    }
-
-    public Boolean call() throws IOException {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Process server shutdown scanning root region on " +
-            master.getRegionManager().getRootRegionLocation().getBindAddress());
-      }
-      Scan scan = new Scan();
-      scan.addFamily(HConstants.CATALOG_FAMILY);
-      long scannerId = server.openScanner(
-          HRegionInfo.ROOT_REGIONINFO.getRegionName(), scan);
-      scanMetaRegion(server, scannerId,
-          HRegionInfo.ROOT_REGIONINFO.getRegionName());
-      return true;
-    }
-  }
-
-  private class ScanMetaRegions extends RetryableMetaOperation<Boolean> {
-    ScanMetaRegions(MetaRegion m, HMaster master) {
-      super(m, master);
-    }
-
-    public Boolean call() throws IOException {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("process server shutdown scanning " +
-          Bytes.toString(m.getRegionName()) + " on " + m.getServer());
-      }
-      Scan scan = new Scan();
-      scan.addFamily(HConstants.CATALOG_FAMILY);
-      long scannerId = server.openScanner(
-          m.getRegionName(), scan);
-      scanMetaRegion(server, scannerId, m.getRegionName());
-      return true;
-    }
-  }
-
-  @Override
-  protected boolean process() throws IOException {
-    LOG.info("Process shutdown of server " + this.deadServer +
-      ": logSplit: " + logSplit + ", rootRescanned: " + rootRescanned +
-      ", numberOfMetaRegions: " + master.getRegionManager().numMetaRegions() +
-      ", onlineMetaRegions.size(): " +
-      master.getRegionManager().numOnlineMetaRegions());
-    if (!logSplit) {
-      // Process the old log file
-      if (this.master.getFileSystem().exists(rsLogDir)) {
-        if (!master.splitLogLock.tryLock()) {
-          return false;
-        }
-        try {
-          HLog.splitLog(master.getRootDir(), rsLogDir,
-              this.master.getOldLogDir(), this.master.getFileSystem(),
-            this.master.getConfiguration());
-        } finally {
-          master.splitLogLock.unlock();
-        }
-      }
-      logSplit = true;
-    }
-    LOG.info("Log split complete, meta reassignment and scanning:");
-    if (this.isRootServer) {
-      LOG.info("ProcessServerShutdown reassigning ROOT region");
-      master.getRegionManager().reassignRootRegion();
-      isRootServer = false;  // prevent double reassignment... heh.
-    }
-
-    for (MetaRegion metaRegion : metaRegions) {
-      LOG.info("ProcessServerShutdown setting to unassigned: " + metaRegion.toString());
-      master.getRegionManager().setUnassigned(metaRegion.getRegionInfo(), true);
-    }
-    // one the meta regions are online, "forget" about them.  Since there are explicit
-    // checks below to make sure meta/root are online, this is likely to occur.
-    metaRegions.clear();
-
-    if (!rootAvailable()) {
-      // Return true so that worker does not put this request back on the
-      // toDoQueue.
-      // rootAvailable() has already put it on the delayedToDoQueue
-      return true;
-    }
-
-    if (!rootRescanned) {
-      // Scan the ROOT region
-      Boolean result = new ScanRootRegion(
-          new MetaRegion(master.getRegionManager().getRootRegionLocation(),
-              HRegionInfo.ROOT_REGIONINFO), this.master).doWithRetries();
-      if (result == null) {
-        // Master is closing - give up
-        return true;
-      }
-
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Process server shutdown scanning root region on " +
-          master.getRegionManager().getRootRegionLocation().getBindAddress() +
-          " finished " + Thread.currentThread().getName());
-      }
-      rootRescanned = true;
-    }
-
-    if (!metaTableAvailable()) {
-      // We can't proceed because not all meta regions are online.
-      // metaAvailable() has put this request on the delayedToDoQueue
-      // Return true so that worker does not put this on the toDoQueue
-      return true;
-    }
-
-    List<MetaRegion> regions = master.getRegionManager().getListOfOnlineMetaRegions();
-    for (MetaRegion r: regions) {
-      Boolean result = new ScanMetaRegions(r, this.master).doWithRetries();
-      if (result == null) {
-        break;
-      }
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("process server shutdown finished scanning " +
-          Bytes.toString(r.getRegionName()) + " on " + r.getServer());
-      }
-    }
-
-    closeRegionsInTransition();
-    this.master.getServerManager().removeDeadServer(deadServer);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Removed " + deadServer + " from deadservers Map");
-    }
-    return true;
-  }
-
-  @Override
-  protected int getPriority() {
-    return 2; // high but not highest priority
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java b/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
deleted file mode 100644
index 6c6b98e9e16..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
+++ /dev/null
@@ -1,1690 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.HServerLoad;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-
-/**
- * Class to manage assigning regions to servers, state of root and meta, etc.
- */
-public class RegionManager {
-  protected static final Log LOG = LogFactory.getLog(RegionManager.class);
-
-  private AtomicReference<HServerAddress> rootRegionLocation =
-    new AtomicReference<HServerAddress>(null);
-
-  private final RootScanner rootScannerThread;
-  final MetaScanner metaScannerThread;
-
-  /** Set by root scanner to indicate the number of meta regions */
-  private final AtomicInteger numberOfMetaRegions = new AtomicInteger();
-
-  /** These are the online meta regions */
-  private final NavigableMap<byte [], MetaRegion> onlineMetaRegions =
-    new ConcurrentSkipListMap<byte [], MetaRegion>(Bytes.BYTES_COMPARATOR);
-
-  private static final byte[] OVERLOADED = Bytes.toBytes("Overloaded");
-
-  private static final byte [] META_REGION_PREFIX = Bytes.toBytes(".META.,");
-
-  /**
-   * Map of region name to RegionState for regions that are in transition such as
-   *
-   * unassigned -> pendingOpen -> open
-   * closing -> pendingClose -> closed; if (closed && !offline) -> unassigned
-   *
-   * At the end of a transition, removeRegion is used to remove the region from
-   * the map (since it is no longer in transition)
-   *
-   * Note: Needs to be SortedMap so we can specify a comparator
-   *
-   * @see RegionState inner-class below
-   */
-   final SortedMap<String, RegionState> regionsInTransition =
-    Collections.synchronizedSortedMap(new TreeMap<String, RegionState>());
-   
-   // regions in transition are also recorded in ZK using the zk wrapper
-   final ZooKeeperWrapper zkWrapper;
-
-  // How many regions to assign a server at a time.
-  private final int maxAssignInOneGo;
-
-  final HMaster master;
-  private final LoadBalancer loadBalancer;
-
-  /** Set of regions to split. */
-  private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>>
-    regionsToSplit = Collections.synchronizedSortedMap(
-        new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
-        (Bytes.BYTES_COMPARATOR));
-  /** Set of regions to compact. */
-  private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>>
-    regionsToCompact = Collections.synchronizedSortedMap(
-        new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
-        (Bytes.BYTES_COMPARATOR));
-  /** Set of regions to major compact. */
-  private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>>
-    regionsToMajorCompact = Collections.synchronizedSortedMap(
-        new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
-        (Bytes.BYTES_COMPARATOR));
-  /** Set of regions to flush. */
-  private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>>
-    regionsToFlush = Collections.synchronizedSortedMap(
-        new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
-        (Bytes.BYTES_COMPARATOR));
-  private final int zooKeeperNumRetries;
-  private final int zooKeeperPause;
-
-  RegionManager(HMaster master) throws IOException {
-    Configuration conf = master.getConfiguration();
-
-    this.master = master;
-    this.zkWrapper =
-        ZooKeeperWrapper.getInstance(conf, HMaster.class.getName());
-    this.maxAssignInOneGo = conf.getInt("hbase.regions.percheckin", 10);
-    this.loadBalancer = new LoadBalancer(conf);
-
-    // The root region
-    rootScannerThread = new RootScanner(master);
-
-    // Scans the meta table
-    metaScannerThread = new MetaScanner(master);
-
-    zooKeeperNumRetries = conf.getInt(HConstants.ZOOKEEPER_RETRIES,
-        HConstants.DEFAULT_ZOOKEEPER_RETRIES);
-    zooKeeperPause = conf.getInt(HConstants.ZOOKEEPER_PAUSE,
-        HConstants.DEFAULT_ZOOKEEPER_PAUSE);
-
-    reassignRootRegion();
-  }
-
-  void start() {
-    Threads.setDaemonThreadRunning(rootScannerThread,
-      "RegionManager.rootScanner");
-    Threads.setDaemonThreadRunning(metaScannerThread,
-      "RegionManager.metaScanner");
-  }
-
-  void unsetRootRegion() {
-    synchronized (regionsInTransition) {
-      rootRegionLocation.set(null);
-      regionsInTransition.remove(
-          HRegionInfo.ROOT_REGIONINFO.getRegionNameAsString());
-      LOG.info("-ROOT- region unset (but not set to be reassigned)");
-    }
-  }
-
-  void reassignRootRegion() {
-    unsetRootRegion();
-    if (!master.getShutdownRequested().get()) {
-      synchronized (regionsInTransition) {
-        String regionName = HRegionInfo.ROOT_REGIONINFO.getRegionNameAsString();
-        byte[] data = null;
-        try {
-          data = Writables.getBytes(new RegionTransitionEventData(HBaseEventType.M2ZK_REGION_OFFLINE, HMaster.MASTER));
-        } catch (IOException e) {
-          LOG.error("Error creating event data for " + HBaseEventType.M2ZK_REGION_OFFLINE, e);
-        }
-        zkWrapper.createOrUpdateUnassignedRegion(
-            HRegionInfo.ROOT_REGIONINFO.getEncodedName(), data);
-        LOG.debug("Created UNASSIGNED zNode " + regionName + " in state " + HBaseEventType.M2ZK_REGION_OFFLINE);
-        RegionState s = new RegionState(HRegionInfo.ROOT_REGIONINFO, RegionState.State.UNASSIGNED);
-        regionsInTransition.put(regionName, s);
-        LOG.info("ROOT inserted into regionsInTransition");
-      }
-    }
-  }
-
-  /*
-   * Assigns regions to region servers attempting to balance the load across
-   * all region servers. Note that no synchronization is necessary as the caller
-   * (ServerManager.processMsgs) already owns the monitor for the RegionManager.
-   *
-   * @param info
-   * @param mostLoadedRegions
-   * @param returnMsgs
-   */
-  void assignRegions(HServerInfo info, HRegionInfo[] mostLoadedRegions,
-      ArrayList<HMsg> returnMsgs) {
-    HServerLoad thisServersLoad = info.getLoad();
-    boolean isSingleServer = this.master.numServers() == 1;
-
-    // figure out what regions need to be assigned and aren't currently being
-    // worked on elsewhere.
-    Set<RegionState> regionsToAssign =
-      regionsAwaitingAssignment(info.getServerAddress(), isSingleServer);
-    if (regionsToAssign.size() == 0) {
-      // There are no regions waiting to be assigned.
-      this.loadBalancer.loadBalancing(info, mostLoadedRegions, returnMsgs);
-    } else {
-      // if there's only one server, just give it all the regions
-      if (isSingleServer) {
-        assignRegionsToOneServer(regionsToAssign, info, returnMsgs);
-      } else {
-        // otherwise, give this server a few regions taking into account the
-        // load of all the other servers.
-        assignRegionsToMultipleServers(thisServersLoad, regionsToAssign,
-            info, returnMsgs);
-      }
-    }
-  }
-
-  /*
-   * Make region assignments taking into account multiple servers' loads.
-   *
-   * Note that no synchronization is needed while we iterate over
-   * regionsInTransition because this method is only called by assignRegions
-   * whose caller owns the monitor for RegionManager
-   *
-   * TODO: This code is unintelligible.  REWRITE. Add TESTS! St.Ack 09/30/2009
-   * @param thisServersLoad
-   * @param regionsToAssign
-   * @param info
-   * @param returnMsgs
-   */
-  private void assignRegionsToMultipleServers(final HServerLoad thisServersLoad,
-    final Set<RegionState> regionsToAssign, final HServerInfo info,
-    final ArrayList<HMsg> returnMsgs) {
-    boolean isMetaAssign = false;
-    for (RegionState s : regionsToAssign) {
-      if (s.getRegionInfo().isMetaRegion())
-        isMetaAssign = true;
-    }
-    int nRegionsToAssign = regionsToAssign.size();
-    int otherServersRegionsCount =
-      regionsToGiveOtherServers(nRegionsToAssign, thisServersLoad);
-    nRegionsToAssign -= otherServersRegionsCount;
-    if (nRegionsToAssign > 0 || isMetaAssign) {
-      LOG.debug("Assigning for " + info + ": total nregions to assign=" +
-        nRegionsToAssign + ", regions to give other servers than this=" +
-        otherServersRegionsCount + ", isMetaAssign=" + isMetaAssign);
-
-      // See how many we can assign before this server becomes more heavily
-      // loaded than the next most heavily loaded server.
-      HServerLoad heavierLoad = new HServerLoad();
-      int nservers = computeNextHeaviestLoad(thisServersLoad, heavierLoad);
-      int nregions = 0;
-      // Advance past any less-loaded servers
-      for (HServerLoad load = new HServerLoad(thisServersLoad);
-      load.compareTo(heavierLoad) <= 0 && nregions < nRegionsToAssign;
-      load.setNumberOfRegions(load.getNumberOfRegions() + 1), nregions++) {
-        // continue;
-      }
-      if (nregions < nRegionsToAssign) {
-        // There are some more heavily loaded servers
-        // but we can't assign all the regions to this server.
-        if (nservers > 0) {
-          // There are other servers that can share the load.
-          // Split regions that need assignment across the servers.
-          nregions = (int) Math.ceil((1.0 * nRegionsToAssign)/(1.0 * nservers));
-        } else {
-          // No other servers with same load.
-          // Split regions over all available servers
-          nregions = (int) Math.ceil((1.0 * nRegionsToAssign)/
-              (1.0 * master.getServerManager().numServers()));
-        }
-      } else {
-        // Assign all regions to this server
-        nregions = nRegionsToAssign;
-      }
-      LOG.debug("Assigning " + info + " " + nregions + " regions");
-      assignRegions(regionsToAssign, nregions, info, returnMsgs);
-    }
-  }
-
-  /*
-   * Assign <code>nregions</code> regions.
-   * @param regionsToAssign
-   * @param nregions
-   * @param info
-   * @param returnMsgs
-   */
-  private void assignRegions(final Set<RegionState> regionsToAssign,
-      final int nregions, final HServerInfo info,
-      final ArrayList<HMsg> returnMsgs) {
-    int count = nregions;
-    if (count > this.maxAssignInOneGo) {
-      count = this.maxAssignInOneGo;
-    }
-    for (RegionState s: regionsToAssign) {
-      doRegionAssignment(s, info, returnMsgs);
-      if (--count <= 0) {
-        break;
-      }
-    }
-  }
-
-  /*
-   * Assign all to the only server. An unlikely case but still possible.
-   *
-   * Note that no synchronization is needed on regionsInTransition while
-   * iterating on it because the only caller is assignRegions whose caller owns
-   * the monitor for RegionManager
-   *
-   * @param regionsToAssign
-   * @param serverName
-   * @param returnMsgs
-   */
-  private void assignRegionsToOneServer(final Set<RegionState> regionsToAssign,
-      final HServerInfo info, final ArrayList<HMsg> returnMsgs) {
-    for (RegionState s: regionsToAssign) {
-      doRegionAssignment(s, info, returnMsgs);
-    }
-  }
-
-  /*
-   * Do single region assignment.
-   * @param rs
-   * @param sinfo
-   * @param returnMsgs
-   */
-  private void doRegionAssignment(final RegionState rs,
-      final HServerInfo sinfo, final ArrayList<HMsg> returnMsgs) {
-    String regionName = rs.getRegionInfo().getRegionNameAsString();
-    LOG.info("Assigning region " + regionName + " to " + sinfo.getServerName());
-    rs.setPendingOpen(sinfo.getServerName());
-    synchronized (this.regionsInTransition) {
-      byte[] data = null;
-      try {
-        data = Writables.getBytes(new RegionTransitionEventData(HBaseEventType.M2ZK_REGION_OFFLINE, HMaster.MASTER));
-      } catch (IOException e) {
-        LOG.error("Error creating event data for " + HBaseEventType.M2ZK_REGION_OFFLINE, e);
-      }
-      zkWrapper.createOrUpdateUnassignedRegion(
-          rs.getRegionInfo().getEncodedName(), data);
-      LOG.debug("Created UNASSIGNED zNode " + regionName + " in state " + HBaseEventType.M2ZK_REGION_OFFLINE);
-      this.regionsInTransition.put(regionName, rs);
-    }
-
-    returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_OPEN, rs.getRegionInfo()));
-  }
-
-  /*
-   * @param nRegionsToAssign
-   * @param thisServersLoad
-   * @return How many regions should go to servers other than this one; i.e.
-   * more lightly loaded servers
-   */
-  private int regionsToGiveOtherServers(final int numUnassignedRegions,
-    final HServerLoad thisServersLoad) {
-    SortedMap<HServerLoad, Set<String>> lightServers =
-      new TreeMap<HServerLoad, Set<String>>();
-    this.master.getLightServers(thisServersLoad, lightServers);
-    // Examine the list of servers that are more lightly loaded than this one.
-    // Pretend that we will assign regions to these more lightly loaded servers
-    // until they reach load equal with ours. Then, see how many regions are left
-    // unassigned. That is how many regions we should assign to this server.
-    int nRegions = 0;
-    for (Map.Entry<HServerLoad, Set<String>> e: lightServers.entrySet()) {
-      HServerLoad lightLoad = new HServerLoad(e.getKey());
-      do {
-        lightLoad.setNumberOfRegions(lightLoad.getNumberOfRegions() + 1);
-        nRegions += 1;
-      } while (lightLoad.compareTo(thisServersLoad) <= 0
-          && nRegions < numUnassignedRegions);
-      nRegions *= e.getValue().size();
-      if (nRegions >= numUnassignedRegions) {
-        break;
-      }
-    }
-    return nRegions;
-  }
-
-  /*
-   * Get the set of regions that should be assignable in this pass.
-   *
-   * Note that no synchronization on regionsInTransition is needed because the
-   * only caller (assignRegions, whose caller is ServerManager.processMsgs) owns
-   * the monitor for RegionManager
-   */
-  private Set<RegionState> regionsAwaitingAssignment(HServerAddress addr,
-                                                     boolean isSingleServer) {
-    // set of regions we want to assign to this server
-    Set<RegionState> regionsToAssign = new HashSet<RegionState>();
-
-    boolean isMetaServer = isMetaServer(addr);
-    RegionState rootState = null;
-    // Handle if root is unassigned... only assign root if root is offline.
-    synchronized (this.regionsInTransition) {
-      rootState = regionsInTransition.get(HRegionInfo.ROOT_REGIONINFO.getRegionNameAsString());
-    }
-    if (rootState != null && rootState.isUnassigned()) {
-      // make sure root isnt assigned here first.
-      // if so return 'empty list'
-      // by definition there is no way this could be a ROOT region (since it's
-      // unassigned) so just make sure it isn't hosting META regions (unless
-      // it's the only server left).
-      if (!isMetaServer || isSingleServer) {
-        regionsToAssign.add(rootState);
-      }
-      return regionsToAssign;
-    }
-
-    // Look over the set of regions that aren't currently assigned to
-    // determine which we should assign to this server.
-    boolean reassigningMetas = numberOfMetaRegions.get() != onlineMetaRegions.size();
-    boolean isMetaOrRoot = isMetaServer || isRootServer(addr);
-    if (reassigningMetas && isMetaOrRoot && !isSingleServer) {
-      return regionsToAssign; // dont assign anything to this server.
-    }
-    synchronized (this.regionsInTransition) {
-      for (RegionState s: regionsInTransition.values()) {
-        HRegionInfo i = s.getRegionInfo();
-        if (i == null) {
-          continue;
-        }
-        if (reassigningMetas &&
-            !i.isMetaRegion()) {
-          // Can't assign user regions until all meta regions have been assigned
-          // and are on-line
-          continue;
-        }
-        if (!i.isMetaRegion() &&
-            !master.getServerManager().canAssignUserRegions()) {
-          LOG.debug("user region " + i.getRegionNameAsString() +
-            " is in transition but not enough servers yet");
-          continue;
-        }
-        if (s.isUnassigned()) {
-          regionsToAssign.add(s);
-        }
-      }
-    }
-    return regionsToAssign;
-  }
-
-  /*
-   * Figure out the load that is next highest amongst all regionservers. Also,
-   * return how many servers exist at that load.
-   */
-  private int computeNextHeaviestLoad(HServerLoad referenceLoad,
-    HServerLoad heavierLoad) {
-
-    SortedMap<HServerLoad, Set<String>> heavyServers =
-      new TreeMap<HServerLoad, Set<String>>();
-    synchronized (master.getLoadToServers()) {
-      heavyServers.putAll(
-        master.getLoadToServers().tailMap(referenceLoad));
-    }
-    int nservers = 0;
-    for (Map.Entry<HServerLoad, Set<String>> e : heavyServers.entrySet()) {
-      Set<String> servers = e.getValue();
-      nservers += servers.size();
-      if (e.getKey().compareTo(referenceLoad) == 0) {
-        // This is the load factor of the server we are considering
-        nservers -= 1;
-        continue;
-      }
-
-      // If we get here, we are at the first load entry that is a
-      // heavier load than the server we are considering
-      heavierLoad.setNumberOfRequests(e.getKey().getNumberOfRequests());
-      heavierLoad.setNumberOfRegions(e.getKey().getNumberOfRegions());
-      break;
-    }
-    return nservers;
-  }
-
-  /*
-   * The server checking in right now is overloaded. We will tell it to close
-   * some or all of its most loaded regions, allowing it to reduce its load.
-   * The closed regions will then get picked up by other underloaded machines.
-   *
-   * Note that no synchronization is needed because the only caller
-   * (assignRegions) whose caller owns the monitor for RegionManager
-   */
-  void unassignSomeRegions(final HServerInfo info,
-      int numRegionsToClose, final HRegionInfo[] mostLoadedRegions,
-      ArrayList<HMsg> returnMsgs) {
-    LOG.debug("Unassigning " + numRegionsToClose + " regions from " +
-      info.getServerName());
-    int regionIdx = 0;
-    int regionsClosed = 0;
-    int skipped = 0;
-    while (regionsClosed < numRegionsToClose &&
-        regionIdx < mostLoadedRegions.length) {
-      HRegionInfo currentRegion = mostLoadedRegions[regionIdx];
-      regionIdx++;
-      // skip the region if it's meta or root
-      if (currentRegion.isRootRegion() || currentRegion.isMetaTable()) {
-        continue;
-      }
-      final String regionName = currentRegion.getRegionNameAsString();
-      if (regionIsInTransition(regionName)) {
-        skipped++;
-        continue;
-      }
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Going to close region " + regionName);
-      }
-      // make a message to close the region
-      returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_CLOSE, currentRegion,
-        OVERLOADED));
-      // mark the region as closing
-      setClosing(info.getServerName(), currentRegion, false);
-      setPendingClose(regionName);
-      // increment the count of regions we've marked
-      regionsClosed++;
-    }
-    LOG.info("Skipped assigning " + skipped + " region(s) to " +
-      info.getServerName() + "because already in transition");
-  }
-
-  /*
-   * PathFilter that accepts hbase tables only.
-   */
-  static class TableDirFilter implements PathFilter {
-    public boolean accept(final Path path) {
-      // skip the region servers' log dirs && version file
-      // HBASE-1112 want to separate the log dirs from table's data dirs by a
-      // special character.
-      final String pathname = path.getName();
-      return (!pathname.equals(HConstants.HREGION_LOGDIR_NAME)
-              && !pathname.equals(HConstants.VERSION_FILE_NAME));
-    }
-
-  }
-
-  /*
-   * PathFilter that accepts all but compaction.dir names.
-   */
-  static class RegionDirFilter implements PathFilter {
-    public boolean accept(Path path) {
-      return !path.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME);
-    }
-  }
-
-  /**
-   * @return the rough number of the regions on fs
-   * Note: this method simply counts the regions on fs by accumulating all the dirs
-   * in each table dir (${HBASE_ROOT}/$TABLE) and skipping logfiles, compaction dirs.
-   * @throws IOException
-   */
-  public int countRegionsOnFS() throws IOException {
-    int regions = 0;
-    FileStatus [] tableDirs =
-      this.master.getFileSystem().listStatus(this.master.getRootDir(), new TableDirFilter());
-    FileStatus[] regionDirs;
-    RegionDirFilter rdf = new RegionDirFilter();
-    for(FileStatus tabledir : tableDirs) {
-      if(tabledir.isDir()) {
-        regionDirs = this.master.getFileSystem().listStatus(tabledir.getPath(), rdf);
-        regions += regionDirs.length;
-      }
-    }
-    return regions;
-  }
-
-  /**
-   * @return Read-only map of online regions.
-   */
-  public Map<byte [], MetaRegion> getOnlineMetaRegions() {
-    synchronized (onlineMetaRegions) {
-      return Collections.unmodifiableMap(onlineMetaRegions);
-    }
-  }
-
-  public boolean metaRegionsInTransition() {
-    synchronized (onlineMetaRegions) {
-      for (MetaRegion metaRegion : onlineMetaRegions.values()) {
-        String regionName = Bytes.toString(metaRegion.getRegionName());
-        if (regionIsInTransition(regionName)) {
-          return true;
-        }
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Return a map of the regions in transition on a server.
-   * Returned map entries are region name -> RegionState
-   */
-  Map<String, RegionState> getRegionsInTransitionOnServer(String serverName) {
-    Map<String, RegionState> ret = new HashMap<String, RegionState>();
-    synchronized (regionsInTransition) {
-      for (Map.Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
-        RegionState rs = entry.getValue();
-        if (serverName.equals(rs.getServerName())) {
-          ret.put(entry.getKey(), rs);
-        }
-      }
-    }
-    return ret;
-  }
-
-  /**
-   * Stop the root and meta scanners so that the region servers serving meta
-   * regions can shut down.
-   */
-  public void stopScanners() {
-    this.rootScannerThread.interruptAndStop();
-    this.metaScannerThread.interruptAndStop();
-  }
-
-  /** Stop the region assigner */
-  public void stop() {
-    try {
-      if (rootScannerThread.isAlive()) {
-        rootScannerThread.join();       // Wait for the root scanner to finish.
-      }
-    } catch (Exception iex) {
-      LOG.warn("root scanner", iex);
-    }
-    try {
-      if (metaScannerThread.isAlive()) {
-        metaScannerThread.join();       // Wait for meta scanner to finish.
-      }
-    } catch(Exception iex) {
-      LOG.warn("meta scanner", iex);
-    }
-    master.getZooKeeperWrapper().clearRSDirectory();
-    master.getZooKeeperWrapper().close();
-  }
-
-  /**
-   * Block until meta regions are online or we're shutting down.
-   * @return true if we found meta regions, false if we're closing.
-   */
-  public boolean areAllMetaRegionsOnline() {
-    synchronized (onlineMetaRegions) {
-      return (rootRegionLocation.get() != null &&
-          numberOfMetaRegions.get() == onlineMetaRegions.size());
-    }
-  }
-
-  /**
-   * Search our map of online meta regions to find the first meta region that
-   * should contain a pointer to <i>newRegion</i>.
-   * @param newRegion
-   * @return MetaRegion where the newRegion should live
-   */
-  public MetaRegion getFirstMetaRegionForRegion(HRegionInfo newRegion) {
-    synchronized (onlineMetaRegions) {
-      if (onlineMetaRegions.size() == 0) {
-        return null;
-      } else if (onlineMetaRegions.size() == 1) {
-        return onlineMetaRegions.get(onlineMetaRegions.firstKey());
-      } else {
-        if (onlineMetaRegions.containsKey(newRegion.getRegionName())) {
-          return onlineMetaRegions.get(newRegion.getRegionName());
-        }
-        return onlineMetaRegions.get(onlineMetaRegions.headMap(
-            newRegion.getRegionName()).lastKey());
-      }
-    }
-  }
-
-  /**
-   * Get a set of all the meta regions that contain info about a given table.
-   * @param tableName Table you need to know all the meta regions for
-   * @return set of MetaRegion objects that contain the table
-   * @throws NotAllMetaRegionsOnlineException
-   */
-  public Set<MetaRegion> getMetaRegionsForTable(byte [] tableName)
-  throws NotAllMetaRegionsOnlineException {
-    byte [] firstMetaRegion = null;
-    Set<MetaRegion> metaRegions = new HashSet<MetaRegion>();
-    if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
-      if (rootRegionLocation.get() == null) {
-        throw new NotAllMetaRegionsOnlineException(
-            Bytes.toString(HConstants.ROOT_TABLE_NAME));
-      }
-      metaRegions.add(new MetaRegion(rootRegionLocation.get(),
-          HRegionInfo.ROOT_REGIONINFO));
-    } else {
-      if (!areAllMetaRegionsOnline()) {
-        throw new NotAllMetaRegionsOnlineException();
-      }
-      synchronized (onlineMetaRegions) {
-        if (onlineMetaRegions.size() == 1) {
-          firstMetaRegion = onlineMetaRegions.firstKey();
-        } else if (onlineMetaRegions.containsKey(tableName)) {
-          firstMetaRegion = tableName;
-        } else {
-          firstMetaRegion = onlineMetaRegions.headMap(tableName).lastKey();
-        }
-        metaRegions.addAll(onlineMetaRegions.tailMap(firstMetaRegion).values());
-      }
-    }
-    return metaRegions;
-  }
-
-  /**
-   * Get metaregion that would host passed in row.
-   * @param row Row need to know all the meta regions for
-   * @return MetaRegion for passed row.
-   * @throws NotAllMetaRegionsOnlineException
-   */
-  public MetaRegion getMetaRegionForRow(final byte [] row)
-  throws NotAllMetaRegionsOnlineException {
-    if (!areAllMetaRegionsOnline()) {
-      throw new NotAllMetaRegionsOnlineException();
-    }
-    // Row might be in -ROOT- table.  If so, return -ROOT- region.
-    int prefixlen = META_REGION_PREFIX.length;
-    if (row.length > prefixlen &&
-     Bytes.compareTo(META_REGION_PREFIX, 0, prefixlen, row, 0, prefixlen) == 0) {
-    	return new MetaRegion(this.master.getRegionManager().getRootRegionLocation(),
-    	  HRegionInfo.ROOT_REGIONINFO);
-    }
-    return this.onlineMetaRegions.floorEntry(row).getValue();
-  }
-
-  /**
-   * Create a new HRegion, put a row for it into META (or ROOT), and mark the
-   * new region unassigned so that it will get assigned to a region server.
-   * @param newRegion HRegionInfo for the region to create
-   * @param server server hosting the META (or ROOT) region where the new
-   * region needs to be noted
-   * @param metaRegionName name of the meta region where new region is to be
-   * written
-   * @throws IOException
-   */
-  public void createRegion(HRegionInfo newRegion, HRegionInterface server,
-      byte [] metaRegionName)
-  throws IOException {
-    // 2. Create the HRegion
-    HRegion region = HRegion.createHRegion(newRegion, this.master.getRootDir(),
-      master.getConfiguration());
-
-    // 3. Insert into meta
-    HRegionInfo info = region.getRegionInfo();
-    byte [] regionName = region.getRegionName();
-
-    Put put = new Put(regionName);
-    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
-        Writables.getBytes(info));
-    server.put(metaRegionName, put);
-
-    // 4. Close the new region to flush it to disk.  Close its log file too.
-    region.close();
-    region.getLog().closeAndDelete();
-
-    // 5. Get it assigned to a server
-    setUnassigned(info, true);
-  }
-
-  /**
-   * Set a MetaRegion as online.
-   * @param metaRegion
-   */
-  public void putMetaRegionOnline(MetaRegion metaRegion) {
-    onlineMetaRegions.put(metaRegion.getStartKey(), metaRegion);
-  }
-
-  /**
-   * Get a list of online MetaRegions
-   * @return list of MetaRegion objects
-   */
-  public List<MetaRegion> getListOfOnlineMetaRegions() {
-    List<MetaRegion> regions;
-    synchronized(onlineMetaRegions) {
-      regions = new ArrayList<MetaRegion>(onlineMetaRegions.values());
-    }
-    return regions;
-  }
-
-  /**
-   * Count of online meta regions
-   * @return count of online meta regions
-   */
-  public int numOnlineMetaRegions() {
-    return onlineMetaRegions.size();
-  }
-
-  /**
-   * Check if a meta region is online by its name
-   * @param startKey name of the meta region to check
-   * @return true if the region is online, false otherwise
-   */
-  public boolean isMetaRegionOnline(byte [] startKey) {
-    return onlineMetaRegions.containsKey(startKey);
-  }
-
-  /**
-   * Set an online MetaRegion offline - remove it from the map.
-   * @param startKey Startkey to use finding region to remove.
-   * @return the MetaRegion that was taken offline.
-   */
-  public MetaRegion offlineMetaRegionWithStartKey(byte [] startKey) {
-    LOG.info("META region whose startkey is " + Bytes.toString(startKey) +
-      " removed from onlineMetaRegions");
-    return onlineMetaRegions.remove(startKey);
-  }
-
-  public boolean isRootServer(HServerAddress server) {
-    return this.master.getRegionManager().getRootRegionLocation() != null &&
-      server.equals(master.getRegionManager().getRootRegionLocation());
-  }
-
-  /**
-   * Returns the list of byte[] start-keys for any .META. regions hosted
-   * on the indicated server.
-   *
-   * @param server server address
-   * @return list of meta region start-keys.
-   */
-  public List<byte[]> listMetaRegionsForServer(HServerAddress server) {
-    List<byte[]> metas = new ArrayList<byte[]>();
-    for ( MetaRegion region : onlineMetaRegions.values() ) {
-      if (server.equals(region.getServer())) {
-        metas.add(region.getStartKey());
-      }
-    }
-    return metas;
-  }
-
-  /**
-   * Does this server have any META regions open on it, or any meta
-   * regions being assigned to it?
-   *
-   * @param server Server IP:port
-   * @return true if server has meta region assigned
-   */
-  public boolean isMetaServer(HServerAddress server) {
-    for ( MetaRegion region : onlineMetaRegions.values() ) {
-      if (server.equals(region.getServer())) {
-        return true;
-      }
-    }
-
-    // This might be expensive, but we need to make sure we dont
-    // get double assignment to the same regionserver.
-    synchronized(regionsInTransition) {
-      for (RegionState s : regionsInTransition.values()) {
-        if (s.getRegionInfo().isMetaRegion()
-            && !s.isUnassigned()
-            && s.getServerName() != null
-            && s.getServerName().equals(server.toString())) {
-          // TODO this code appears to be entirely broken, since
-          // server.toString() has no start code, but s.getServerName()
-          // does!
-          LOG.fatal("I DONT BELIEVE YOU WILL EVER SEE THIS!");
-          // Has an outstanding meta region to be assigned.
-          return true;
-        }
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Is this server assigned to transition the ROOT table. HBASE-1928
-   *
-   * @param server Server
-   * @return true if server is transitioning the ROOT table
-   */
-  public boolean isRootInTransitionOnThisServer(final String server) {
-    synchronized (this.regionsInTransition) {
-      for (RegionState s : regionsInTransition.values()) {
-        if (s.getRegionInfo().isRootRegion()
-            && !s.isUnassigned()
-            && s.getServerName() != null
-            && s.getServerName().equals(server)) {
-          // Has an outstanding root region to be assigned.
-          return true;
-        }
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Is this server assigned to transition a META table. HBASE-1928
-   *
-   * @param server Server
-   * @return if this server was transitioning a META table then a not null HRegionInfo pointing to it
-   */
-  public HRegionInfo getMetaServerRegionInfo(final String server) {
-    synchronized (this.regionsInTransition) {
-      for (RegionState s : regionsInTransition.values()) {
-        if (s.getRegionInfo().isMetaRegion()
-            && !s.isUnassigned()
-            && s.getServerName() != null
-            && s.getServerName().equals(server)) {
-          // Has an outstanding meta region to be assigned.
-          return s.getRegionInfo();
-        }
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Call to take this metaserver offline for immediate reassignment.  Used only
-   * when we know a region has shut down cleanly.
-   *
-   * A meta server is a server that hosts either -ROOT- or any .META. regions.
-   *
-   * If you are considering a unclean shutdown potentially, use ProcessServerShutdown which
-   * calls other methods to immediately unassign root/meta but delay the reassign until the
-   * log has been split.
-   *
-   * @param server the server that went down
-   * @return true if this was in fact a meta server, false if it did not carry meta regions.
-   */
-  public synchronized boolean offlineMetaServer(HServerAddress server) {
-    boolean hasMeta = false;
-
-    // check to see if ROOT and/or .META. are on this server, reassign them.
-    // use master.getRootRegionLocation.
-    if (master.getRegionManager().getRootRegionLocation() != null &&
-        server.equals(master.getRegionManager().getRootRegionLocation())) {
-      LOG.info("Offlined ROOT server: " + server);
-      reassignRootRegion();
-      hasMeta = true;
-    }
-    // AND
-    for ( MetaRegion region : onlineMetaRegions.values() ) {
-      if (server.equals(region.getServer())) {
-        LOG.info("Offlining META region: " + region);
-        offlineMetaRegionWithStartKey(region.getStartKey());
-        // Set for reassignment.
-        setUnassigned(region.getRegionInfo(), true);
-        hasMeta = true;
-      }
-    }
-    return hasMeta;
-  }
-
-  /**
-   * Remove a region from the region state map.
-   *
-   * @param info
-   */
-  public void removeRegion(HRegionInfo info) {
-    synchronized (this.regionsInTransition) {
-      this.regionsInTransition.remove(info.getRegionNameAsString());
-    }
-  }
-
-  /**
-   * @param regionName
-   * @return true if the named region is in a transition state
-   */
-  public boolean regionIsInTransition(String regionName) {
-    synchronized (this.regionsInTransition) {
-      return regionsInTransition.containsKey(regionName);
-    }
-  }
-
-  /**
-   * @param regionName
-   * @return true if the region is unassigned, pendingOpen or open
-   */
-  public boolean regionIsOpening(String regionName) {
-    synchronized (this.regionsInTransition) {
-      RegionState state = regionsInTransition.get(regionName);
-      if (state != null) {
-        return state.isOpening();
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Set a region to unassigned
-   * @param info Region to set unassigned
-   * @param force if true mark region unassigned whatever its current state
-   */
-  public void setUnassigned(HRegionInfo info, boolean force) {
-    RegionState s = null;
-    synchronized(this.regionsInTransition) {
-      s = regionsInTransition.get(info.getRegionNameAsString());
-      if (s == null) {
-        byte[] data = null;
-        try {
-          data = Writables.getBytes(new RegionTransitionEventData(HBaseEventType.M2ZK_REGION_OFFLINE, HMaster.MASTER));
-        } catch (IOException e) {
-          // TODO: Review what we should do here.  If Writables work this
-          //       should never happen
-          LOG.error("Error creating event data for " + HBaseEventType.M2ZK_REGION_OFFLINE, e);
-        }
-        zkWrapper.createOrUpdateUnassignedRegion(info.getEncodedName(), data);          
-        LOG.debug("Created/updated UNASSIGNED zNode " + info.getRegionNameAsString() + 
-                  " in state " + HBaseEventType.M2ZK_REGION_OFFLINE);
-        s = new RegionState(info, RegionState.State.UNASSIGNED);
-        regionsInTransition.put(info.getRegionNameAsString(), s);
-      }
-    }
-    if (force || (!s.isPendingOpen() && !s.isOpen())) {
-      s.setUnassigned();
-    }
-  }
-
-  /**
-   * Check if a region is on the unassigned list
-   * @param info HRegionInfo to check for
-   * @return true if on the unassigned list, false if it isn't. Note that this
-   * means a region could not be on the unassigned list AND not be assigned, if
-   * it happens to be between states.
-   */
-  public boolean isUnassigned(HRegionInfo info) {
-    synchronized (regionsInTransition) {
-      RegionState s = regionsInTransition.get(info.getRegionNameAsString());
-      if (s != null) {
-        return s.isUnassigned();
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Check if a region has been assigned and we're waiting for a response from
-   * the region server.
-   *
-   * @param regionName name of the region
-   * @return true if open, false otherwise
-   */
-  public boolean isPendingOpen(String regionName) {
-    synchronized (regionsInTransition) {
-      RegionState s = regionsInTransition.get(regionName);
-      if (s != null) {
-        return s.isPendingOpen();
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Region has been assigned to a server and the server has told us it is open
-   * @param regionName
-   */
-  public void setOpen(String regionName) {
-    synchronized (regionsInTransition) {
-      RegionState s = regionsInTransition.get(regionName);
-      if (s != null) {
-        s.setOpen();
-      }
-    }
-  }
-
-  /**
-   * @param regionName
-   * @return true if region is marked to be offlined.
-   */
-  public boolean isOfflined(String regionName) {
-    synchronized (regionsInTransition) {
-      RegionState s = regionsInTransition.get(regionName);
-      if (s != null) {
-        return s.isOfflined();
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Mark a region as closing
-   * @param serverName
-   * @param regionInfo
-   * @param setOffline
-   */
-  public void setClosing(String serverName, final HRegionInfo regionInfo,
-      final boolean setOffline) {
-    synchronized (this.regionsInTransition) {
-      RegionState s =
-        this.regionsInTransition.get(regionInfo.getRegionNameAsString());
-      if (s == null) {
-        s = new RegionState(regionInfo, RegionState.State.CLOSING);
-      }
-      // If region was asked to open before getting here, we could be taking
-      // the wrong server name
-      if(s.isPendingOpen()) {
-        serverName = s.getServerName();
-      }
-      s.setClosing(serverName, setOffline);
-      this.regionsInTransition.put(regionInfo.getRegionNameAsString(), s);
-    }
-  }
-
-  /**
-   * Remove the map of region names to region infos waiting to be offlined for a
-   * given server
-   *
-   * @param serverName
-   * @return set of infos to close
-   */
-  public Set<HRegionInfo> getMarkedToClose(String serverName) {
-    Set<HRegionInfo> result = new HashSet<HRegionInfo>();
-    synchronized (regionsInTransition) {
-      for (RegionState s: regionsInTransition.values()) {
-        if (s.isClosing() && !s.isPendingClose() && !s.isClosed() &&
-            s.getServerName().compareTo(serverName) == 0) {
-          result.add(s.getRegionInfo());
-        }
-      }
-    }
-    return result;
-  }
-
-  /**
-   * Called when we have told a region server to close the region
-   *
-   * @param regionName
-   */
-  public void setPendingClose(String regionName) {
-    synchronized (regionsInTransition) {
-      RegionState s = regionsInTransition.get(regionName);
-      if (s != null) {
-        s.setPendingClose();
-      }
-    }
-  }
-
-  /**
-   * @param regionName
-   */
-  public void setClosed(String regionName) {
-    synchronized (regionsInTransition) {
-      RegionState s = regionsInTransition.get(regionName);
-      if (s != null) {
-        s.setClosed();
-      }
-    }
-  }
-  /**
-   * Add a meta region to the scan queue
-   * @param m MetaRegion that needs to get scanned
-   */
-  public void addMetaRegionToScan(MetaRegion m) {
-    metaScannerThread.addMetaRegionToScan(m);
-  }
-
-  /**
-   * Check if the initial root scan has been completed.
-   * @return true if scan completed, false otherwise
-   */
-  public boolean isInitialRootScanComplete() {
-    return rootScannerThread.isInitialScanComplete();
-  }
-
-  /**
-   * Check if the initial meta scan has been completed.
-   * @return true if meta completed, false otherwise
-   */
-  public boolean isInitialMetaScanComplete() {
-    return metaScannerThread.isInitialScanComplete();
-  }
-
-  /**
-   * Get the root region location.
-   * @return HServerAddress describing root region server.
-   */
-  public HServerAddress getRootRegionLocation() {
-    return rootRegionLocation.get();
-  }
-
-  /**
-   * Block until either the root region location is available or we're shutting
-   * down.
-   */
-  public void waitForRootRegionLocation() {
-    synchronized (rootRegionLocation) {
-      while (!master.getShutdownRequested().get() &&
-          !master.isClosed() && rootRegionLocation.get() == null) {
-        // rootRegionLocation will be filled in when we get an 'open region'
-        // regionServerReport message from the HRegionServer that has been
-        // allocated the ROOT region below.
-        try {
-          // Cycle rather than hold here in case master is closed meantime.
-          rootRegionLocation.wait(this.master.getThreadWakeFrequency());
-        } catch (InterruptedException e) {
-          // continue
-        }
-      }
-    }
-  }
-
-  /**
-   * Return the number of meta regions.
-   * @return number of meta regions
-   */
-  public int numMetaRegions() {
-    return numberOfMetaRegions.get();
-  }
-
-  /**
-   * Bump the count of meta regions up one
-   */
-  public void incrementNumMetaRegions() {
-    numberOfMetaRegions.incrementAndGet();
-  }
-
-  private long getPauseTime(int tries) {
-    int attempt = tries;
-    if (attempt >= HConstants.RETRY_BACKOFF.length) {
-      attempt = HConstants.RETRY_BACKOFF.length - 1;
-    }
-    return this.zooKeeperPause * HConstants.RETRY_BACKOFF[attempt];
-  }
-
-  private void sleep(int attempt) {
-    try {
-      Thread.sleep(getPauseTime(attempt));
-    } catch (InterruptedException e) {
-      // continue
-    }
-  }
-
-  private void writeRootRegionLocationToZooKeeper(HServerAddress address) {
-    for (int attempt = 0; attempt < zooKeeperNumRetries; ++attempt) {
-      if (master.getZooKeeperWrapper().writeRootRegionLocation(address)) {
-        return;
-      }
-
-      sleep(attempt);
-    }
-
-    LOG.error("Failed to write root region location to ZooKeeper after " +
-              zooKeeperNumRetries + " retries, shutting down");
-
-    this.master.shutdown();
-  }
-
-  /**
-   * Set the root region location.
-   * @param address Address of the region server where the root lives
-   */
-  public void setRootRegionLocation(HServerAddress address) {
-    writeRootRegionLocationToZooKeeper(address);
-    synchronized (rootRegionLocation) {
-      // the root region has been assigned, remove it from transition in ZK
-      zkWrapper.deleteUnassignedRegion(HRegionInfo.ROOT_REGIONINFO.getEncodedName());
-      rootRegionLocation.set(new HServerAddress(address));
-      rootRegionLocation.notifyAll();
-    }
-  }
-
-  /**
-   * Set the number of meta regions.
-   * @param num Number of meta regions
-   */
-  public void setNumMetaRegions(int num) {
-    numberOfMetaRegions.set(num);
-  }
-
-  /**
-   * @param regionName
-   * @param info
-   * @param server
-   * @param op
-   */
-  public void startAction(byte[] regionName, HRegionInfo info,
-      HServerAddress server, HConstants.Modify op) {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Adding operation " + op + " from tasklist");
-    }
-    switch (op) {
-      case TABLE_SPLIT:
-        startAction(regionName, info, server, this.regionsToSplit);
-        break;
-      case TABLE_COMPACT:
-        startAction(regionName, info, server, this.regionsToCompact);
-        break;
-      case TABLE_MAJOR_COMPACT:
-        startAction(regionName, info, server, this.regionsToMajorCompact);
-        break;
-      case TABLE_FLUSH:
-        startAction(regionName, info, server, this.regionsToFlush);
-        break;
-      default:
-        throw new IllegalArgumentException("illegal table action " + op);
-    }
-  }
-
-  private void startAction(final byte[] regionName, final HRegionInfo info,
-      final HServerAddress server,
-      final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>> map) {
-    map.put(regionName, new Pair<HRegionInfo,HServerAddress>(info, server));
-  }
-
-  /**
-   * @param regionName
-   * @param op
-   */
-  public void endAction(byte[] regionName, HConstants.Modify op) {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Removing operation " + op + " from tasklist");
-    }
-    switch (op) {
-    case TABLE_SPLIT:
-      this.regionsToSplit.remove(regionName);
-      break;
-    case TABLE_COMPACT:
-      this.regionsToCompact.remove(regionName);
-      break;
-    case TABLE_MAJOR_COMPACT:
-      this.regionsToMajorCompact.remove(regionName);
-      break;
-    case TABLE_FLUSH:
-      this.regionsToFlush.remove(regionName);
-      break;
-    default:
-      throw new IllegalArgumentException("illegal table action " + op);
-    }
-  }
-
-  /**
-   * @param regionName
-   */
-  public void endActions(byte[] regionName) {
-    regionsToSplit.remove(regionName);
-    regionsToCompact.remove(regionName);
-  }
-
-  /**
-   * Send messages to the given region server asking it to split any
-   * regions in 'regionsToSplit', etc.
-   * @param serverInfo
-   * @param returnMsgs
-   */
-  public void applyActions(HServerInfo serverInfo, ArrayList<HMsg> returnMsgs) {
-    applyActions(serverInfo, returnMsgs, this.regionsToCompact,
-        HMsg.Type.MSG_REGION_COMPACT);
-    applyActions(serverInfo, returnMsgs, this.regionsToSplit,
-      HMsg.Type.MSG_REGION_SPLIT);
-    applyActions(serverInfo, returnMsgs, this.regionsToFlush,
-        HMsg.Type.MSG_REGION_FLUSH);
-    applyActions(serverInfo, returnMsgs, this.regionsToMajorCompact,
-        HMsg.Type.MSG_REGION_MAJOR_COMPACT);
-  }
-
-  private void applyActions(final HServerInfo serverInfo,
-      final ArrayList<HMsg> returnMsgs,
-      final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>> map,
-      final HMsg.Type msg) {
-    HServerAddress addr = serverInfo.getServerAddress();
-    synchronized (map) {
-      Iterator<Pair<HRegionInfo, HServerAddress>> i = map.values().iterator();
-      while (i.hasNext()) {
-        Pair<HRegionInfo,HServerAddress> pair = i.next();
-        if (addr.equals(pair.getSecond())) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Sending " + msg + " " + pair.getFirst() + " to " + addr);
-          }
-          returnMsgs.add(new HMsg(msg, pair.getFirst()));
-          i.remove();
-        }
-      }
-    }
-  }
-
-  /**
-   * Class to balance region servers load.
-   * It keeps Region Servers load in slop range by unassigning Regions
-   * from most loaded servers.
-   *
-   * Equilibrium is reached when load of all serves are in slop range
-   * [avgLoadMinusSlop, avgLoadPlusSlop], where
-   *  avgLoadPlusSlop = Math.ceil(avgLoad * (1 + this.slop)), and
-   *  avgLoadMinusSlop = Math.floor(avgLoad * (1 - this.slop)) - 1.
-   */
-  private class LoadBalancer {
-    private float slop;                 // hbase.regions.slop
-    private final int maxRegToClose;    // hbase.regions.close.max
-
-    LoadBalancer(Configuration conf) {
-      this.slop = conf.getFloat("hbase.regions.slop", (float)0.3);
-      if (this.slop <= 0) this.slop = 1;
-      //maxRegToClose to constrain balance closing per one iteration
-      // -1 to turn off
-      // TODO: change default in HBASE-862, need a suggestion
-      this.maxRegToClose = conf.getInt("hbase.regions.close.max", -1);
-    }
-
-    /**
-     * Balance server load by unassigning some regions.
-     *
-     * @param info - server info
-     * @param mostLoadedRegions - array of most loaded regions
-     * @param returnMsgs - array of return massages
-     */
-    void loadBalancing(HServerInfo info, HRegionInfo[] mostLoadedRegions,
-        ArrayList<HMsg> returnMsgs) {
-      HServerLoad servLoad = info.getLoad();
-      double avg = master.getAverageLoad();
-
-      // nothing to balance if server load not more then average load
-      if(servLoad.getLoad() <= Math.ceil(avg) || avg <= 2.0) {
-        return;
-      }
-
-      // check if current server is overloaded
-      int numRegionsToClose = balanceFromOverloaded(info.getServerName(),
-        servLoad, avg);
-
-      // check if we can unload server by low loaded servers
-      if(numRegionsToClose <= 0) {
-        numRegionsToClose = balanceToLowloaded(info.getServerName(), servLoad,
-            avg);
-      }
-
-      if(maxRegToClose > 0) {
-        numRegionsToClose = Math.min(numRegionsToClose, maxRegToClose);
-      }
-
-      if(numRegionsToClose > 0) {
-        unassignSomeRegions(info, numRegionsToClose, mostLoadedRegions,
-            returnMsgs);
-      }
-    }
-
-    /*
-     * Check if server load is not overloaded (with load > avgLoadPlusSlop).
-     * @return number of regions to unassign.
-     */
-    private int balanceFromOverloaded(final String serverName,
-        HServerLoad srvLoad, double avgLoad) {
-      int avgLoadPlusSlop = (int)Math.ceil(avgLoad * (1 + this.slop));
-      int numSrvRegs = srvLoad.getNumberOfRegions();
-      if (numSrvRegs > avgLoadPlusSlop) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Server " + serverName + " is carrying more than its fair " +
-            "share of regions: " +
-            "load=" + numSrvRegs + ", avg=" + avgLoad + ", slop=" + this.slop);
-        }
-        return numSrvRegs - (int)Math.ceil(avgLoad);
-      }
-      return 0;
-    }
-
-    /*
-     * Check if server is most loaded and can be unloaded to
-     * low loaded servers (with load < avgLoadMinusSlop).
-     * @return number of regions to unassign.
-     */
-    private int balanceToLowloaded(String srvName, HServerLoad srvLoad,
-        double avgLoad) {
-
-      SortedMap<HServerLoad, Set<String>> loadToServers =
-        master.getLoadToServers();
-      // check if server most loaded
-      if (!loadToServers.get(loadToServers.lastKey()).contains(srvName))
-        return 0;
-
-      // this server is most loaded, we will try to unload it by lowest
-      // loaded servers
-      int avgLoadMinusSlop = (int)Math.floor(avgLoad * (1 - this.slop)) - 1;
-      int lowestLoad = loadToServers.firstKey().getNumberOfRegions();
-
-      if(lowestLoad >= avgLoadMinusSlop)
-        return 0; // there is no low loaded servers
-
-      int lowSrvCount = loadToServers.get(loadToServers.firstKey()).size();
-      int numRegionsToClose = 0;
-
-      int numSrvRegs = srvLoad.getNumberOfRegions();
-      int numMoveToLowLoaded = (avgLoadMinusSlop - lowestLoad) * lowSrvCount;
-      numRegionsToClose = numSrvRegs - (int)Math.ceil(avgLoad);
-      numRegionsToClose = Math.min(numRegionsToClose, numMoveToLowLoaded);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Server(s) are carrying only " + lowestLoad + " regions. " +
-          "Server " + srvName + " is most loaded (" + numSrvRegs +
-          "). Shedding " + numRegionsToClose + " regions to pass to " +
-          " least loaded (numMoveToLowLoaded=" + numMoveToLowLoaded +")");
-      }
-      return numRegionsToClose;
-    }
-  }
-
-  /**
-   * @return Snapshot of regionsintransition as a sorted Map.
-   */
-  NavigableMap<String, String> getRegionsInTransition() {
-    NavigableMap<String, String> result = new TreeMap<String, String>();
-    synchronized (this.regionsInTransition) {
-      if (this.regionsInTransition.isEmpty()) return result;
-      for (Map.Entry<String, RegionState> e: this.regionsInTransition.entrySet()) {
-        result.put(e.getKey(), e.getValue().toString());
-      }
-    }
-    return result;
-  }
-
-  /**
-   * @param regionname Name to clear from regions in transistion.
-   * @return True if we removed an element for the passed regionname.
-   */
-  boolean clearFromInTransition(final byte [] regionname) {
-    boolean result = false;
-    synchronized (this.regionsInTransition) {
-      if (this.regionsInTransition.isEmpty()) return result;
-      for (Map.Entry<String, RegionState> e: this.regionsInTransition.entrySet()) {
-        if (Bytes.equals(regionname, e.getValue().getRegionName())) {
-          this.regionsInTransition.remove(e.getKey());
-          LOG.debug("Removed " + e.getKey() + ", " + e.getValue());
-          result = true;
-          break;
-        }
-      }
-    }
-    return result;
-  }
-
-  /*
-   * State of a Region as it transitions from closed to open, etc.  See
-   * note on regionsInTransition data member above for listing of state
-   * transitions.
-   */
-  static class RegionState implements Comparable<RegionState> {
-    private final HRegionInfo regionInfo;
-
-    enum State {
-      UNASSIGNED, // awaiting a server to be assigned
-      PENDING_OPEN, // told a server to open, hasn't opened yet
-      OPEN, // has been opened on RS, but not yet marked in META/ROOT
-      CLOSING, // a msg has been enqueued to close ths region, but not delivered to RS yet
-      PENDING_CLOSE, // msg has been delivered to RS to close this region
-      CLOSED // region has been closed but not yet marked in meta
-
-    }
-
-    private State state;
-
-    private boolean isOfflined;
-
-    /* Set when region is assigned or closing */
-    private String serverName = null;
-
-    /* Constructor */
-    RegionState(HRegionInfo info, State state) {
-      this.regionInfo = info;
-      this.state = state;
-    }
-
-    synchronized HRegionInfo getRegionInfo() {
-      return this.regionInfo;
-    }
-
-    synchronized byte [] getRegionName() {
-      return this.regionInfo.getRegionName();
-    }
-
-    /*
-     * @return Server this region was assigned to
-     */
-    synchronized String getServerName() {
-      return this.serverName;
-    }
-
-    /*
-     * @return true if the region is being opened
-     */
-    synchronized boolean isOpening() {
-      return state == State.UNASSIGNED ||
-        state == State.PENDING_OPEN ||
-        state == State.OPEN;
-    }
-
-    /*
-     * @return true if region is unassigned
-     */
-    synchronized boolean isUnassigned() {
-      return state == State.UNASSIGNED;
-    }
-
-    /*
-     * Note: callers of this method (reassignRootRegion,
-     * regionsAwaitingAssignment, setUnassigned) ensure that this method is not
-     * called unless it is safe to do so.
-     */
-    synchronized void setUnassigned() {
-      state = State.UNASSIGNED;
-      this.serverName = null;
-    }
-
-    synchronized boolean isPendingOpen() {
-      return state == State.PENDING_OPEN;
-    }
-
-    /*
-     * @param serverName Server region was assigned to.
-     */
-    synchronized void setPendingOpen(final String serverName) {
-      if (state != State.UNASSIGNED) {
-        LOG.warn("Cannot assign a region that is not currently unassigned. " +
-          "FIX!! State: " + toString());
-      }
-      state = State.PENDING_OPEN;
-      this.serverName = serverName;
-    }
-
-    synchronized boolean isOpen() {
-      return state == State.OPEN;
-    }
-
-    synchronized void setOpen() {
-      if (state != State.PENDING_OPEN) {
-        LOG.warn("Cannot set a region as open if it has not been pending. " +
-          "FIX!! State: " + toString());
-      }
-      state = State.OPEN;
-    }
-
-    synchronized boolean isClosing() {
-      return state == State.CLOSING;
-    }
-
-    synchronized void setClosing(String serverName, boolean setOffline) {
-      state = State.CLOSING;
-      this.serverName = serverName;
-      this.isOfflined = setOffline;
-    }
-
-    synchronized boolean isPendingClose() {
-      return state == State.PENDING_CLOSE;
-    }
-
-    synchronized void setPendingClose() {
-      if (state != State.CLOSING) {
-        LOG.warn("Cannot set a region as pending close if it has not been " +
-          "closing.  FIX!! State: " + toString());
-      }
-      state = State.PENDING_CLOSE;
-    }
-
-    synchronized boolean isClosed() {
-      return state == State.CLOSED;
-    }
-
-    synchronized void setClosed() {
-      if (state != State.PENDING_CLOSE &&
-          state != State.PENDING_OPEN &&
-          state != State.CLOSING) {
-        throw new IllegalStateException(
-            "Cannot set a region to be closed if it was not already marked as" +
-            " pending close, pending open or closing. State: " + this);
-      }
-      state = State.CLOSED;
-    }
-
-    synchronized boolean isOfflined() {
-      return (state == State.CLOSING ||
-        state == State.PENDING_CLOSE) && isOfflined;
-    }
-
-    @Override
-    public synchronized String toString() {
-      return ("name=" + Bytes.toString(getRegionName()) +
-          ", state=" + this.state);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-      if (this == o) {
-        return true;
-      }
-      if (o == null || getClass() != o.getClass()) {
-        return false;
-      }
-      return this.compareTo((RegionState) o) == 0;
-    }
-
-    @Override
-    public int hashCode() {
-      return Bytes.toString(getRegionName()).hashCode();
-    }
-
-    public int compareTo(RegionState o) {
-      if (o == null) {
-        return 1;
-      }
-      return Bytes.compareTo(getRegionName(), o.getRegionName());
-    }
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java b/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java
deleted file mode 100644
index affd304255b..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import java.io.IOException;
-import java.util.concurrent.Delayed;
-import java.util.concurrent.TimeUnit;
-
-abstract class RegionServerOperation implements Delayed {
-  protected static final Log LOG =
-    LogFactory.getLog(RegionServerOperation.class.getName());
-
-  private long expire;
-  protected final HMaster master;
-  /* How long we stay on queue.
-   */
-  private int delay;
-
-  protected RegionServerOperation(HMaster master) {
-    this.master = master;
-    this.delay = this.master.getConfiguration().
-      getInt("hbase.server.thread.wakefrequency", 10 * 1000);
-    // Set the future time at which we expect to be released from the
-    // DelayQueue we're inserted in on lease expiration.
-    resetExpiration();
-  }
-
-  /**
-   * Call before putting this back on the delay queue.
-   * @return When we will expire next.
-   */
-  long resetExpiration() {
-    // Set the future time at which we expect to be released from the
-    // DelayQueue we're inserted in on lease expiration.
-    this.expire = System.currentTimeMillis() + this.delay;
-    return this.expire;
-  }
-
-  public long getDelay(TimeUnit unit) {
-    return unit.convert(this.expire - System.currentTimeMillis(),
-      TimeUnit.MILLISECONDS);
-  }
-
-  void setDelay(final int d) {
-    this.delay = d;
-  }
-
-  public int compareTo(Delayed o) {
-    return Long.valueOf(getDelay(TimeUnit.MILLISECONDS)
-        - o.getDelay(TimeUnit.MILLISECONDS)).intValue();
-  }
-
-  protected void requeue() {
-    this.master.getRegionServerOperationQueue().putOnDelayQueue(this);
-  }
-
-  private long whenToExpire() {
-    return System.currentTimeMillis() + this.delay;
-  }
-
-  protected boolean rootAvailable() {
-    boolean available = true;
-    if (this.master.getRegionManager().getRootRegionLocation() == null) {
-      available = false;
-      requeue();
-    }
-    return available;
-  }
-
-  protected boolean metaTableAvailable() {
-    boolean available = true;
-    if ((master.getRegionManager().numMetaRegions() !=
-      master.getRegionManager().numOnlineMetaRegions()) ||
-      master.getRegionManager().metaRegionsInTransition()) {
-      // We can't proceed because not all of the meta regions are online.
-      // We can't block either because that would prevent the meta region
-      // online message from being processed. In order to prevent spinning
-      // in the run queue, put this request on the delay queue to give
-      // other threads the opportunity to get the meta regions on-line.
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("numberOfMetaRegions: " +
-            master.getRegionManager().numMetaRegions() +
-            ", onlineMetaRegions.size(): " +
-            master.getRegionManager().numOnlineMetaRegions());
-        LOG.debug("Requeuing because not all meta regions are online");
-      }
-      available = false;
-      requeue();
-    }
-    return available;
-  }
-
-  public int compareTo(RegionServerOperation other) {
-    return getPriority() - other.getPriority();
-  }
-
-  // the Priority of this operation, 0 is lowest priority
-  protected int getPriority() {
-    return Integer.MAX_VALUE;
-  }
-
-  protected abstract boolean process() throws IOException;
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationListener.java b/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationListener.java
deleted file mode 100644
index d221110b17b..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationListener.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.HServerInfo;
-
-/**
- * Listener for regionserver events in master.
- * @see HMaster#registerRegionServerOperationListener(RegionServerOperationListener)
- * @see HMaster#unregisterRegionServerOperationListener(RegionServerOperationListener)
- */
-public interface RegionServerOperationListener {
-  /**
-   * Called for each message passed the master.  Most of the messages that come
-   * in here will go on to become {@link #process(RegionServerOperation)}s but
-   * others like {@linke HMsg.Type#MSG_REPORT_PROCESS_OPEN} go no further;
-   * only in here can you see them come in.
-   * @param serverInfo Server we got the message from.
-   * @param incomingMsg The message received.
-   * @return True to continue processing, false to skip.
-   */
-  public boolean process(final HServerInfo serverInfo,
-      final HMsg incomingMsg);
-
-  /**
-   * Called before processing <code>op</code>
-   * @param op
-   * @return True if we are to proceed w/ processing.
-   * @exception IOException
-   */
-  public boolean process(final RegionServerOperation op) throws IOException;
-
-  /**
-   * Called after <code>op</code> has been processed.
-   * @param op The operation that just completed.
-   */
-  public void processed(final RegionServerOperation op);
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java b/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java
deleted file mode 100644
index d1283706161..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CopyOnWriteArraySet;
-import java.util.concurrent.DelayQueue;
-import java.util.concurrent.PriorityBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.util.Sleeper;
-import org.apache.hadoop.ipc.RemoteException;
-
-/**
- * Keeps up the queue of {@link RegionServerOperation}s.
- * Has both live queue and a temporary put-aside queue; if processing of the
- * live todo queue fails for some reason, we'll add the item back on the delay
- * queue for retry later.  Call {@link #shutdown()} to effect a cleanup of
- * queues when done.  Listen to this queue by registering
- * {@link RegionServerOperationListener}s.
- * @see #registerRegionServerOperationListener(RegionServerOperationListener)
- * @see #unregisterRegionServerOperationListener(RegionServerOperationListener)
- */
-public class RegionServerOperationQueue {
-  // TODO: Build up the junit test of this class.
-  private final Log LOG = LogFactory.getLog(this.getClass());
-
-  /**
-   * Enums returned by {@link RegionServerOperationQueue#process()};
-   */
-  public static enum ProcessingResultCode {
-    /**
-     * Operation was processed successfully.
-     */
-    PROCESSED,
-    /**
-     * Nothing to do.
-     */
-    NOOP,
-    /**
-     * Operation was put-aside for now.  Will be retried later.
-     */
-    REQUEUED,
-    /**
-     * Failed processing of the operation.
-     */
-    FAILED,
-    /**
-     * Operation was requeued but we failed its processing for some reason
-     * (Bad filesystem?).
-     */
-    REQUEUED_BUT_PROBLEM
-  };
-
-  /*
-   * Do not put items directly on this queue. Use {@link #putOnDelayQueue(RegionServerOperation)}.
-   * It makes sure the expiration on the RegionServerOperation added is updated.
-   */
-  private final DelayQueue<RegionServerOperation> delayedToDoQueue =
-    new DelayQueue<RegionServerOperation>();
-  private final BlockingQueue<RegionServerOperation> toDoQueue =
-    new PriorityBlockingQueue<RegionServerOperation>();
-  private final Set<RegionServerOperationListener> listeners =
-    new CopyOnWriteArraySet<RegionServerOperationListener>();
-  private final int threadWakeFrequency;
-  private final AtomicBoolean closed;
-  private final Sleeper sleeper;
-
-  RegionServerOperationQueue(final Configuration c, final AtomicBoolean closed) {
-    this.threadWakeFrequency = c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
-    this.closed = closed;
-    this.sleeper = new Sleeper(this.threadWakeFrequency, this.closed);
-  }
-
-  public void put(final RegionServerOperation op) {
-    try {
-      this.toDoQueue.put(op);
-    } catch (InterruptedException e) {
-      LOG.warn("Insertion into todo queue interrupted; putting on delay queue", e);
-      putOnDelayQueue(op);
-    }
-  }
-
-  /**
-   * Try to get an operation off of the queue and process it.
-   * @return {@link ProcessingResultCode#PROCESSED},
-   * {@link ProcessingResultCode#REQUEUED},
-   * {@link ProcessingResultCode#REQUEUED_BUT_PROBLEM}
-   */
-  public synchronized ProcessingResultCode process() {
-    RegionServerOperation op = null;
-    // Only process the delayed queue if root region is online.  If offline,
-    // the operation to put it online is probably in the toDoQueue.  Process
-    // it first.
-    if (toDoQueue.isEmpty()) {
-      op = delayedToDoQueue.poll();
-    }
-    if (op == null) {
-      try {
-        op = toDoQueue.poll(threadWakeFrequency, TimeUnit.MILLISECONDS);
-      } catch (InterruptedException e) {
-        LOG.debug("Interrupted", e);
-      }
-    }
-
-    // At this point, if there's still no todo operation, or we're supposed to
-    // be closed, return.
-    if (op == null || closed.get()) {
-      return ProcessingResultCode.NOOP;
-    }
-
-    try {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Processing todo: " + op.toString());
-      }
-      if (!process(op)) {
-        // Add it back on the queue.
-        putOnDelayQueue(op);
-      } else if (op.process()) {
-        processed(op);
-      } else {
-        // Operation would have blocked because not all meta regions are
-        // online. This could cause a deadlock, because this thread is waiting
-        // for the missing meta region(s) to come back online, but since it
-        // is waiting, it cannot process the meta region online operation it
-        // is waiting for. So put this operation back on the queue for now.
-        if (toDoQueue.size() == 0) {
-          // The queue is currently empty so wait for a while to see if what
-          // we need comes in first
-          this.sleeper.sleep();
-        }
-        try {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Put " + op.toString() + " back on queue");
-          }
-          toDoQueue.put(op);
-        } catch (InterruptedException e) {
-          throw new RuntimeException(
-            "Putting into toDoQueue was interrupted.", e);
-        }
-      }
-    } catch (Exception ex) {
-      // There was an exception performing the operation.
-      if (ex instanceof RemoteException) {
-        try {
-          ex = RemoteExceptionHandler.decodeRemoteException(
-            (RemoteException)ex);
-        } catch (IOException e) {
-          ex = e;
-          LOG.warn("main processing loop: " + op.toString(), e);
-        }
-      }
-      LOG.warn("Failed processing: " + op.toString() +
-        "; putting onto delayed todo queue", ex);
-      putOnDelayQueue(op);
-      return ProcessingResultCode.REQUEUED_BUT_PROBLEM;
-    }
-    return ProcessingResultCode.REQUEUED;
-  }
-
-  void putOnDelayQueue(final RegionServerOperation op) {
-    op.resetExpiration();
-    this.delayedToDoQueue.put(op);
-  }
-
-  /**
-   * Clean up the queues.
-   */
-  public synchronized void shutdown() {
-    this.toDoQueue.clear();
-    this.delayedToDoQueue.clear();
-  }
-
-  /**
-   * @param l Register this listener of RegionServerOperation events.
-   */
-  public void registerRegionServerOperationListener(final RegionServerOperationListener l) {
-    this.listeners.add(l);
-  }
-
-  /**
-   * @param l Unregister this listener for RegionServerOperation events.
-   * @return True if this listener was registered.
-   */
-  public boolean unregisterRegionServerOperationListener(final RegionServerOperationListener l) {
-    return this.listeners.remove(l);
-  }
-
-  /*
-   * Tell listeners that we processed a RegionServerOperation.
-   * @param op Operation to tell the world about.
-   */
-  private void processed(final RegionServerOperation op) {
-    if (this.listeners.isEmpty()) return;
-    for (RegionServerOperationListener listener: this.listeners) {
-      listener.processed(op);
-    }
-  }
-
-  /**
-   * Called for each message passed the master.  Most of the messages that come
-   * in here will go on to become {@link #process(RegionServerOperation)}s but
-   * others like {@linke HMsg.Type#MSG_REPORT_PROCESS_OPEN} go no further;
-   * only in here can you see them come in.
-   * @param serverInfo Server we got the message from.
-   * @param incomingMsg The message received.
-   * @return True to continue processing, false to skip.
-   */
-  boolean process(final HServerInfo serverInfo,
-      final HMsg incomingMsg) {
-    if (this.listeners.isEmpty()) return true;
-    for (RegionServerOperationListener listener: this.listeners) {
-      if (!listener.process(serverInfo, incomingMsg)) return false;
-    }
-    return true;
-  }
-
-  /*
-   * Tell listeners that we processed a RegionServerOperation.
-   * @param op Operation to tell the world about.
-   */
-  private boolean process(final RegionServerOperation op) throws IOException {
-    if (this.listeners.isEmpty()) return true;
-    for (RegionServerOperationListener listener: this.listeners) {
-      if (!listener.process(op)) return false;
-    }
-    return true;
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java b/src/main/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java
deleted file mode 100644
index 7b667855c0a..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.master;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.TableNotDisabledException;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Sleeper;
-import org.apache.hadoop.ipc.RemoteException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Callable;
-
-/**
- * Uses Callable pattern so that operations against meta regions do not need
- * to duplicate retry logic.
- */
-abstract class RetryableMetaOperation<T> implements Callable<T> {
-  protected final Log LOG = LogFactory.getLog(this.getClass());
-  protected final Sleeper sleeper;
-  protected final MetaRegion m;
-  protected final HMaster master;
-
-  protected HRegionInterface server;
-
-  protected RetryableMetaOperation(MetaRegion m, HMaster master) {
-    this.m = m;
-    this.master = master;
-    this.sleeper = new Sleeper(this.master.getThreadWakeFrequency(),
-      this.master.getClosed());
-  }
-
-  protected T doWithRetries()
-  throws IOException, RuntimeException {
-    List<IOException> exceptions = new ArrayList<IOException>();
-    for (int tries = 0; tries < this.master.getNumRetries(); tries++) {
-      if (this.master.isClosed()) {
-        return null;
-      }
-      try {
-        this.server =
-          this.master.getServerConnection().getHRegionConnection(m.getServer());
-        return this.call();
-      } catch (IOException e) {
-        if (e instanceof TableNotFoundException ||
-            e instanceof TableNotDisabledException ||
-            e instanceof InvalidColumnNameException) {
-          throw e;
-        }
-        if (e instanceof RemoteException) {
-          e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
-        }
-        if (tries == this.master.getNumRetries() - 1) {
-          if (LOG.isDebugEnabled()) {
-            StringBuilder message = new StringBuilder(
-                "Trying to contact region server for regionName '" +
-                Bytes.toString(m.getRegionName()) + "', but failed after " +
-                (tries + 1) + " attempts.\n");
-            int i = 1;
-            for (IOException e2 : exceptions) {
-              message.append("Exception " + i + ":\n" + e2);
-            }
-            LOG.debug(message);
-          }
-          this.master.checkFileSystem();
-          throw e;
-        }
-        if (LOG.isDebugEnabled()) {
-          exceptions.add(e);
-        }
-      } catch (Exception e) {
-        LOG.debug("Exception in RetryableMetaOperation: ", e);
-        throw new RuntimeException(e);
-      }
-      this.sleeper.sleep();
-    }
-    return null;
-  }
-}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/RootScanner.java b/src/main/java/org/apache/hadoop/hbase/master/RootScanner.java
deleted file mode 100644
index 7547928f265..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/RootScanner.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-
-import java.io.IOException;
-
-/** Scanner for the <code>ROOT</code> HRegion. */
-class RootScanner extends BaseScanner {
-  /**
-   * Constructor
-   * @param master
-   */
-  public RootScanner(HMaster master) {
-    super(master, true, master.getShutdownRequested());
-  }
-
-  /**
-   * Don't retry if we get an error while scanning. Errors are most often
-   *
-   * caused by the server going away. Wait until next rescan interval when
-   * things should be back to normal.
-   * @return True if successfully scanned.
-   */
-  private boolean scanRoot() {
-    master.getRegionManager().waitForRootRegionLocation();
-    if (master.isClosed()) {
-      return false;
-    }
-
-    try {
-      // Don't interrupt us while we're working
-      synchronized(scannerLock) {
-        if (master.getRegionManager().getRootRegionLocation() != null) {
-          scanRegion(new MetaRegion(master.getRegionManager().getRootRegionLocation(),
-            HRegionInfo.ROOT_REGIONINFO));
-        }
-      }
-    } catch (IOException e) {
-      e = RemoteExceptionHandler.checkIOException(e);
-      LOG.warn("Scan ROOT region", e);
-      // Make sure the file system is still available
-      master.checkFileSystem();
-    } catch (Exception e) {
-      // If for some reason we get some other kind of exception,
-      // at least log it rather than go out silently.
-      LOG.error("Unexpected exception", e);
-    }
-    return true;
-  }
-
-  @Override
-  protected boolean initialScan() {
-    this.initialScanComplete = scanRoot();
-    return initialScanComplete;
-  }
-
-  @Override
-  protected void maintenanceScan() {
-    scanRoot();
-  }
-}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 558ff10424c..8901b45bc77 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -19,144 +19,127 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Chore;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HMsg;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.PleaseHoldException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.YouAreDeadException;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ServerConnection;
+import org.apache.hadoop.hbase.client.ServerConnectionManager;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.master.RegionManager.RegionState;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException;
 import org.apache.hadoop.hbase.util.Threads;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.EventType;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * The ServerManager class manages info about region servers - HServerInfo,
  * load numbers, dying servers, etc.
+ * <p>
+ * Maintains lists of online and dead servers.  Processes the startups,
+ * shutdowns, and deaths of region servers.
+ * <p>
+ * Servers are distinguished in two different ways.  A given server has a
+ * location, specified by hostname and port, and of which there can only be one
+ * online at any given time.  A server instance is specified by the location
+ * (hostname and port) as well as the startcode (timestamp from when the server
+ * was started).  This is used to differentiate a restarted instance of a given
+ * server from the original instance.
  */
 public class ServerManager {
-  private static final Log LOG =
-    LogFactory.getLog(ServerManager.class.getName());
+  private static final Log LOG = LogFactory.getLog(ServerManager.class);
 
-  private final AtomicInteger quiescedServers = new AtomicInteger(0);
+  // Set if we are to shutdown the cluster.
+  private volatile boolean clusterShutdown = false;
 
-  // The map of known server names to server info
-  private final Map<String, HServerInfo> serversToServerInfo =
+  /** The map of known server names to server info */
+  private final Map<String, HServerInfo> onlineServers =
     new ConcurrentHashMap<String, HServerInfo>();
 
-  /*
-   * Set of known dead servers.  On znode expiration, servers are added here.
-   * This is needed in case of a network partitioning where the server's lease
-   * expires, but the server is still running. After the network is healed,
-   * and it's server logs are recovered, it will be told to call server startup
-   * because by then, its regions have probably been reassigned.
+  // TODO: This is strange to have two maps but HSI above is used on both sides
+  /**
+   * Map from full server-instance name to the RPC connection for this server.
    */
-  private final Set<String> deadServers =
-    Collections.synchronizedSet(new HashSet<String>());
+  private final Map<String, HRegionInterface> serverConnections =
+    new HashMap<String, HRegionInterface>();
 
-  // SortedMap server load -> Set of server names
-  private final SortedMap<HServerLoad, Set<String>> loadToServers =
-    Collections.synchronizedSortedMap(new TreeMap<HServerLoad, Set<String>>());
-  // Map of server names -> server load
-  private final Map<String, HServerLoad> serversToLoad =
-    new ConcurrentHashMap<String, HServerLoad>();
-
-  private HMaster master;
-
-  /* The regionserver will not be assigned or asked close regions if it
-   * is currently opening >= this many regions.
-   */
-  private final int nobalancingCount;
+  private final Server master;
+  private final MasterServices services;
 
   private final ServerMonitor serverMonitorThread;
 
   private int minimumServerCount;
 
-  private final LogsCleaner logCleaner;
+  private final LogCleaner logCleaner;
 
-  /*
+  // Reporting to track master metrics.
+  private final MasterMetrics metrics;
+
+  private final DeadServer deadservers = new DeadServer();
+
+  /**
    * Dumps into log current stats on dead servers and number of servers
    * TODO: Make this a metric; dump metrics into log.
    */
   class ServerMonitor extends Chore {
-    ServerMonitor(final int period, final AtomicBoolean stop) {
-      super("ServerMonitor", period, stop);
+    ServerMonitor(final int period, final Stoppable stopper) {
+      super("ServerMonitor", period, stopper);
     }
 
     @Override
     protected void chore() {
-      int numServers = serversToServerInfo.size();
-      int numDeadServers = deadServers.size();
+      int numServers = numServers();
+      int numDeadServers = deadservers.size();
       double averageLoad = getAverageLoad();
-      String deadServersList = null;
-      if (numDeadServers > 0) {
-        StringBuilder sb = new StringBuilder("Dead Server [");
-        boolean first = true;
-        synchronized (deadServers) {
-          for (String server: deadServers) {
-            if (!first) {
-              sb.append(",  ");
-              first = false;
-            }
-            sb.append(server);
-          }
-        }
-        sb.append("]");
-        deadServersList = sb.toString();
-      }
+      String deadServersList = deadservers.toString();
       LOG.info(numServers + " region servers, " + numDeadServers +
         " dead, average load " + averageLoad +
-        (deadServersList != null? deadServers: ""));
+        ((deadServersList != null && deadServersList.length() > 0)?
+          deadServersList: ""));
     }
   }
 
   /**
    * Constructor.
    * @param master
+   * @param services
    */
-  public ServerManager(HMaster master) {
+  public ServerManager(final Server master, final MasterServices services) {
     this.master = master;
+    this.services = services;
     Configuration c = master.getConfiguration();
-    this.nobalancingCount = c.getInt("hbase.regions.nobalancing.count", 4);
     int metaRescanInterval = c.getInt("hbase.master.meta.thread.rescanfrequency",
       60 * 1000);
-    this.minimumServerCount = c.getInt("hbase.regions.server.count.min", 0);
-    this.serverMonitorThread = new ServerMonitor(metaRescanInterval,
-      this.master.getShutdownRequested());
+    this.minimumServerCount = c.getInt("hbase.regions.server.count.min", 1);
+    this.metrics = new MasterMetrics(master.getServerName());
+    this.serverMonitorThread = new ServerMonitor(metaRescanInterval, master);
     String n = Thread.currentThread().getName();
     Threads.setDaemonThreadRunning(this.serverMonitorThread,
       n + ".serverMonitor");
-    this.logCleaner = new LogsCleaner(
+    this.logCleaner = new LogCleaner(
       c.getInt("hbase.master.meta.thread.rescanfrequency",60 * 1000),
-        this.master.getShutdownRequested(), c,
-        master.getFileSystem(), master.getOldLogDir());
+      master, c, this.services.getMasterFileSystem().getFileSystem(),
+      this.services.getMasterFileSystem().getOldLogDir());
     Threads.setDaemonThreadRunning(logCleaner,
       n + ".oldLogCleaner");
-
   }
 
   /**
@@ -175,7 +158,8 @@ public class ServerManager {
     // for processing by ProcessServerShutdown.
     HServerInfo info = new HServerInfo(serverInfo);
     String hostAndPort = info.getServerAddress().toString();
-    HServerInfo existingServer = haveServerWithSameHostAndPortAlready(info.getHostnamePort());
+    HServerInfo existingServer =
+      haveServerWithSameHostAndPortAlready(info.getHostnamePort());
     if (existingServer != null) {
       String message = "Server start rejected; we already have " + hostAndPort +
         " registered; existingServer=" + existingServer + ", newServer=" + info;
@@ -192,8 +176,8 @@ public class ServerManager {
   }
 
   private HServerInfo haveServerWithSameHostAndPortAlready(final String hostnamePort) {
-    synchronized (this.serversToServerInfo) {
-      for (Map.Entry<String, HServerInfo> e: this.serversToServerInfo.entrySet()) {
+    synchronized (this.onlineServers) {
+      for (Map.Entry<String, HServerInfo> e: this.onlineServers.entrySet()) {
         if (e.getValue().getHostnamePort().equals(hostnamePort)) {
           return e.getValue();
         }
@@ -202,7 +186,7 @@ public class ServerManager {
     return null;
   }
 
-  /*
+  /**
    * If this server is on the dead list, reject it with a LeaseStillHeldException
    * @param serverName Server name formatted as host_port_startcode.
    * @param what START or REPORT
@@ -210,7 +194,7 @@ public class ServerManager {
    */
   private void checkIsDead(final String serverName, final String what)
   throws YouAreDeadException {
-    if (!isDead(serverName)) return;
+    if (!this.deadservers.isDeadServer(serverName)) return;
     String message = "Server " + what + " rejected; currently processing " +
       serverName + " as dead server";
     LOG.debug(message);
@@ -222,7 +206,7 @@ public class ServerManager {
    * @param info The region server informations
    */
   public void recordNewServer(HServerInfo info) {
-    recordNewServer(info, false);
+    recordNewServer(info, false, null);
   }
 
   /**
@@ -231,23 +215,18 @@ public class ServerManager {
    * @param useInfoLoad True if the load from the info should be used
    *                    like under a master failover
    */
-  void recordNewServer(HServerInfo info, boolean useInfoLoad) {
+  void recordNewServer(HServerInfo info, boolean useInfoLoad,
+      HRegionInterface hri) {
     HServerLoad load = useInfoLoad ? info.getLoad() : new HServerLoad();
     String serverName = info.getServerName();
     info.setLoad(load);
-    // We must set this watcher here because it can be set on a fresh start
-    // or on a failover
-    Watcher watcher = new ServerExpirer(new HServerInfo(info));
-    this.master.getZooKeeperWrapper().updateRSLocationGetWatch(info, watcher);
-    this.serversToServerInfo.put(serverName, info);
-    this.serversToLoad.put(serverName, load);
-    synchronized (this.loadToServers) {
-      Set<String> servers = this.loadToServers.get(load);
-      if (servers == null) {
-        servers = new HashSet<String>();
-      }
-      servers.add(serverName);
-      this.loadToServers.put(load, servers);
+    // TODO: Why did we update the RS location ourself?  Shouldn't RS do this?
+    // masterStatus.getZooKeeper().updateRSLocationGetWatch(info, watcher);
+    onlineServers.put(serverName, info);
+    if(hri == null) {
+      serverConnections.remove(serverName);
+    } else {
+      serverConnections.put(serverName, hri);
     }
   }
 
@@ -265,130 +244,76 @@ public class ServerManager {
    * @throws IOException
    */
   HMsg [] regionServerReport(final HServerInfo serverInfo,
-    final HMsg msgs[], final HRegionInfo[] mostLoadedRegions)
+    final HMsg [] msgs, final HRegionInfo[] mostLoadedRegions)
   throws IOException {
+    // Be careful. This method does returns in the middle.
     HServerInfo info = new HServerInfo(serverInfo);
-    checkIsDead(info.getServerName(), "REPORT");
-    if (msgs.length > 0) {
-      if (msgs[0].isType(HMsg.Type.MSG_REPORT_EXITING)) {
-        processRegionServerExit(info, msgs);
-        return HMsg.EMPTY_HMSG_ARRAY;
-      } else if (msgs[0].isType(HMsg.Type.MSG_REPORT_QUIESCED)) {
-        LOG.info("Region server " + info.getServerName() + " quiesced");
-        this.quiescedServers.incrementAndGet();
-      }
-    }
-    if (this.master.getShutdownRequested().get()) {
-      if (quiescedServers.get() >= serversToServerInfo.size()) {
-        // If the only servers we know about are meta servers, then we can
-        // proceed with shutdown
-        LOG.info("All user tables quiesced. Proceeding with shutdown");
-        this.master.startShutdown();
-      }
-      if (!this.master.isClosed()) {
-        if (msgs.length > 0 &&
-            msgs[0].isType(HMsg.Type.MSG_REPORT_QUIESCED)) {
-          // Server is already quiesced, but we aren't ready to shut down
-          // return empty response
-          return HMsg.EMPTY_HMSG_ARRAY;
-        }
-        // Tell the server to stop serving any user regions
-        return new HMsg [] {HMsg.REGIONSERVER_QUIESCE};
-      }
-    }
-    if (this.master.isClosed()) {
-      // Tell server to shut down if we are shutting down.  This should
-      // happen after check of MSG_REPORT_EXITING above, since region server
-      // will send us one of these messages after it gets MSG_REGIONSERVER_STOP
-      return new HMsg [] {HMsg.REGIONSERVER_STOP};
-    }
 
-    HServerInfo storedInfo = this.serversToServerInfo.get(info.getServerName());
+    // Check if dead.  If it is, it'll get a 'You Are Dead!' exception.
+    checkIsDead(info.getServerName(), "REPORT");
+
+    // If we don't know this server, tell it shutdown.
+    HServerInfo storedInfo = this.onlineServers.get(info.getServerName());
     if (storedInfo == null) {
       LOG.warn("Received report from unknown server -- telling it " +
-        "to " + HMsg.REGIONSERVER_STOP + ": " + info.getServerName());
-      // The HBaseMaster may have been restarted.
-      // Tell the RegionServer to abort!
-      return new HMsg[] {HMsg.REGIONSERVER_STOP};
-    } else if (storedInfo.getStartCode() != info.getStartCode()) {
+        "to " + HMsg.Type.STOP_REGIONSERVER + ": " + info.getServerName());
+      return HMsg.STOP_REGIONSERVER_ARRAY;
+    }
+
+    // Check startcodes
+    if (raceThatShouldNotHappenAnymore(storedInfo, info)) {
+      return HMsg.STOP_REGIONSERVER_ARRAY;
+    }
+
+    for (HMsg msg: msgs) {
+      LOG.info("Received " + msg);
+      switch (msg.getType()) {
+      case REGION_SPLIT:
+        this.services.getAssignmentManager().handleSplitReport(serverInfo,
+            msg.getRegionInfo(), msg.getDaughterA(), msg.getDaughterB());
+        break;
+
+        default:
+          LOG.error("Unhandled msg type " + msg);
+      }
+    }
+
+    HMsg [] reply = null;
+    int numservers = numServers();
+    if (this.clusterShutdown) {
+      if (numservers <= 2) {
+        // Shutdown needs to be staggered; the meta regions need to close last
+        // in case they need to be updated during the close melee.  If <= 2
+        // servers left, then these are the two that were carrying root and meta
+        // most likely (TODO: This presumes unsplittable meta -- FIX). Tell
+        // these servers can shutdown now too.
+        reply = HMsg.STOP_REGIONSERVER_ARRAY;
+      }
+    }
+    return processRegionServerAllsWell(info, mostLoadedRegions, reply);
+  }
+
+  private boolean raceThatShouldNotHappenAnymore(final HServerInfo storedInfo,
+      final HServerInfo reportedInfo) {
+    if (storedInfo.getStartCode() != reportedInfo.getStartCode()) {
+      // TODO: I don't think this possible any more.  We check startcodes when
+      // server comes in on regionServerStartup -- St.Ack
       // This state is reachable if:
-      //
       // 1) RegionServer A started
-      // 2) RegionServer B started on the same machine, then
-      //    clobbered A in regionServerStartup.
+      // 2) RegionServer B started on the same machine, then clobbered A in regionServerStartup.
       // 3) RegionServer A returns, expecting to work as usual.
-      //
       // The answer is to ask A to shut down for good.
-
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("region server race condition detected: " +
-            info.getServerName());
+      LOG.warn("Race condition detected: " + reportedInfo.getServerName());
+      synchronized (this.onlineServers) {
+        removeServerInfo(reportedInfo.getServerName());
+        notifyOnlineServers();
       }
-
-      synchronized (this.serversToServerInfo) {
-        removeServerInfo(info.getServerName());
-        notifyServers();
-      }
-
-      return new HMsg[] {HMsg.REGIONSERVER_STOP};
-    } else {
-      return processRegionServerAllsWell(info, mostLoadedRegions, msgs);
+      return true;
     }
+    return false;
   }
 
-  /*
-   * Region server is exiting with a clean shutdown.
-   *
-   * In this case, the server sends MSG_REPORT_EXITING in msgs[0] followed by
-   * a MSG_REPORT_CLOSE for each region it was serving.
-   * @param serverInfo
-   * @param msgs
-   */
-  private void processRegionServerExit(HServerInfo serverInfo, HMsg[] msgs) {
-    synchronized (this.serversToServerInfo) {
-      // This method removes ROOT/META from the list and marks them to be
-      // reassigned in addition to other housework.
-      if (removeServerInfo(serverInfo.getServerName())) {
-        // Only process the exit message if the server still has registered info.
-        // Otherwise we could end up processing the server exit twice.
-        LOG.info("Region server " + serverInfo.getServerName() +
-          ": MSG_REPORT_EXITING");
-        // Get all the regions the server was serving reassigned
-        // (if we are not shutting down).
-        if (!master.closed.get()) {
-          for (int i = 1; i < msgs.length; i++) {
-            LOG.info("Processing " + msgs[i] + " from " +
-              serverInfo.getServerName());
-            assert msgs[i].getType() == HMsg.Type.MSG_REGION_CLOSE;
-            HRegionInfo info = msgs[i].getRegionInfo();
-            // Meta/root region offlining is handed in removeServerInfo above.
-            if (!info.isMetaRegion()) {
-              synchronized (master.getRegionManager()) {
-                if (!master.getRegionManager().isOfflined(info.getRegionNameAsString())) {
-                  master.getRegionManager().setUnassigned(info, true);
-                } else {
-                  master.getRegionManager().removeRegion(info);
-                }
-              }
-            }
-          }
-        }
-        // There should not be any regions in transition for this server - the
-        // server should finish transitions itself before closing
-        Map<String, RegionState> inTransition = master.getRegionManager()
-            .getRegionsInTransitionOnServer(serverInfo.getServerName());
-        for (Map.Entry<String, RegionState> entry : inTransition.entrySet()) {
-          LOG.warn("Region server " + serverInfo.getServerName()
-              + " shut down with region " + entry.getKey() + " in transition "
-              + "state " + entry.getValue());
-          master.getRegionManager().setUnassigned(entry.getValue().getRegionInfo(),
-              true);
-        }
-      }
-    }
-  }
-
-  /*
+  /**
    *  RegionServer is checking in, no exceptional circumstances
    * @param serverInfo
    * @param mostLoadedRegions
@@ -400,314 +325,25 @@ public class ServerManager {
       final HRegionInfo[] mostLoadedRegions, HMsg[] msgs)
   throws IOException {
     // Refresh the info object and the load information
-    this.serversToServerInfo.put(serverInfo.getServerName(), serverInfo);
-    HServerLoad load = this.serversToLoad.get(serverInfo.getServerName());
-    if (load != null) {
-      this.master.getMetrics().incrementRequests(load.getNumberOfRequests());
-      if (!load.equals(serverInfo.getLoad())) {
-        updateLoadToServers(serverInfo.getServerName(), load);
-      }
+    this.onlineServers.put(serverInfo.getServerName(), serverInfo);
+    HServerLoad load = serverInfo.getLoad();
+    if (load != null && this.metrics != null) {
+      this.metrics.incrementRequests(load.getNumberOfRequests());
     }
-
-    // Set the current load information
-    load = serverInfo.getLoad();
-    this.serversToLoad.put(serverInfo.getServerName(), load);
-    synchronized (loadToServers) {
-      Set<String> servers = this.loadToServers.get(load);
-      if (servers == null) {
-        servers = new HashSet<String>();
-      }
-      servers.add(serverInfo.getServerName());
-      this.loadToServers.put(load, servers);
-    }
-
-    // Next, process messages for this server
-    return processMsgs(serverInfo, mostLoadedRegions, msgs);
+    // No more piggyback messages on heartbeats for other stuff
+    return msgs;
   }
 
-  /*
-   * Process all the incoming messages from a server that's contacted us.
-   * Note that we never need to update the server's load information because
-   * that has already been done in regionServerReport.
-   * @param serverInfo
-   * @param mostLoadedRegions
-   * @param incomingMsgs
-   * @return
+  /**
+   * @param serverName
+   * @return True if we removed server from the list.
    */
-  private HMsg[] processMsgs(HServerInfo serverInfo,
-      HRegionInfo[] mostLoadedRegions, HMsg incomingMsgs[]) {
-    ArrayList<HMsg> returnMsgs = new ArrayList<HMsg>();
-    if (serverInfo.getServerAddress() == null) {
-      throw new NullPointerException("Server address cannot be null; " +
-        "hbase-958 debugging");
-    }
-    // Get reports on what the RegionServer did.
-    // Be careful that in message processors we don't throw exceptions that
-    // break the switch below because then we might drop messages on the floor.
-    int openingCount = 0;
-    for (int i = 0; i < incomingMsgs.length; i++) {
-      HRegionInfo region = incomingMsgs[i].getRegionInfo();
-      LOG.info("Processing " + incomingMsgs[i] + " from " +
-        serverInfo.getServerName() + "; " + (i + 1) + " of " +
-        incomingMsgs.length);
-      if (!this.master.getRegionServerOperationQueue().
-          process(serverInfo, incomingMsgs[i])) {
-        continue;
-      }
-      switch (incomingMsgs[i].getType()) {
-        case MSG_REPORT_PROCESS_OPEN:
-          openingCount++;
-          break;
-
-        case MSG_REPORT_OPEN:
-          processRegionOpen(serverInfo, region, returnMsgs);
-          break;
-
-        case MSG_REPORT_CLOSE:
-          processRegionClose(region);
-          break;
-
-        case MSG_REPORT_SPLIT:
-          processSplitRegion(region, incomingMsgs[++i].getRegionInfo(),
-            incomingMsgs[++i].getRegionInfo());
-          break;
-
-        case MSG_REPORT_SPLIT_INCLUDES_DAUGHTERS:
-          processSplitRegion(region, incomingMsgs[i].getDaughterA(),
-            incomingMsgs[i].getDaughterB());
-          break;
-
-        default:
-          LOG.warn("Impossible state during message processing. Instruction: " +
-            incomingMsgs[i].getType());
-      }
-    }
-
-    synchronized (this.master.getRegionManager()) {
-      // Tell the region server to close regions that we have marked for closing.
-      for (HRegionInfo i:
-        this.master.getRegionManager().getMarkedToClose(serverInfo.getServerName())) {
-        returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_CLOSE, i));
-        // Transition the region from toClose to closing state
-        this.master.getRegionManager().setPendingClose(i.getRegionNameAsString());
-      }
-
-      // Figure out what the RegionServer ought to do, and write back.
-
-      // Should we tell it close regions because its overloaded?  If its
-      // currently opening regions, leave it alone till all are open.
-      if (openingCount < this.nobalancingCount) {
-        this.master.getRegionManager().assignRegions(serverInfo, mostLoadedRegions,
-          returnMsgs);
-      }
-
-      // Send any pending table actions.
-      this.master.getRegionManager().applyActions(serverInfo, returnMsgs);
-    }
-    return returnMsgs.toArray(new HMsg[returnMsgs.size()]);
-  }
-
-  /*
-   * A region has split.
-   *
-   * @param region
-   * @param splitA
-   * @param splitB
-   * @param returnMsgs
-   */
-  private void processSplitRegion(HRegionInfo region, HRegionInfo a, HRegionInfo b) {
-    synchronized (master.getRegionManager()) {
-      // Cancel any actions pending for the affected region.
-      // This prevents the master from sending a SPLIT message if the table
-      // has already split by the region server.
-      this.master.getRegionManager().endActions(region.getRegionName());
-      assignSplitDaughter(a);
-      assignSplitDaughter(b);
-      if (region.isMetaTable()) {
-        // A meta region has split.
-        this. master.getRegionManager().offlineMetaRegionWithStartKey(region.getStartKey());
-        this.master.getRegionManager().incrementNumMetaRegions();
-      }
-    }
-  }
-
-  /*
-   * Assign new daughter-of-a-split UNLESS its already been assigned.
-   * It could have been assigned already in rare case where there was a large
-   * gap between insertion of the daughter region into .META. by the
-   * splitting regionserver and receipt of the split message in master (See
-   * HBASE-1784).
-   * @param hri Region to assign.
-   */
-  private void assignSplitDaughter(final HRegionInfo hri) {
-    MetaRegion mr =
-      this.master.getRegionManager().getFirstMetaRegionForRegion(hri);
-    Get g = new Get(hri.getRegionName());
-    g.addFamily(HConstants.CATALOG_FAMILY);
-    try {
-      HRegionInterface server =
-        this.master.getServerConnection().getHRegionConnection(mr.getServer());
-      Result r = server.get(mr.getRegionName(), g);
-      // If size > 3 -- presume regioninfo, startcode and server -- then presume
-      // that this daughter already assigned and return.
-      if (r.size() >= 3) return;
-    } catch (IOException e) {
-      LOG.warn("Failed get on " + HConstants.CATALOG_FAMILY_STR +
-        "; possible double-assignment?", e);
-    }
-    this.master.getRegionManager().setUnassigned(hri, false);
-  }
-
-  /*
-   * Region server is reporting that a region is now opened
-   * @param serverInfo
-   * @param region
-   * @param returnMsgs
-   */
-  public void processRegionOpen(HServerInfo serverInfo,
-      HRegionInfo region, ArrayList<HMsg> returnMsgs) {
-    boolean duplicateAssignment = false;
-    synchronized (master.getRegionManager()) {
-      if (!this.master.getRegionManager().isUnassigned(region) &&
-          !this.master.getRegionManager().isPendingOpen(region.getRegionNameAsString())) {
-        if (region.isRootRegion()) {
-          // Root region
-          HServerAddress rootServer =
-            this.master.getRegionManager().getRootRegionLocation();
-          if (rootServer != null) {
-            if (rootServer.compareTo(serverInfo.getServerAddress()) == 0) {
-              // A duplicate open report from the correct server
-              return;
-            }
-            // We received an open report on the root region, but it is
-            // assigned to a different server
-            duplicateAssignment = true;
-          }
-        } else {
-          // Not root region. If it is not a pending region, then we are
-          // going to treat it as a duplicate assignment, although we can't
-          // tell for certain that's the case.
-          if (this.master.getRegionManager().isPendingOpen(
-              region.getRegionNameAsString())) {
-            // A duplicate report from the correct server
-            return;
-          }
-          duplicateAssignment = true;
-        }
-      }
-
-      if (duplicateAssignment) {
-        LOG.warn("region server " + serverInfo.getServerAddress().toString() +
-          " should not have opened region " + Bytes.toString(region.getRegionName()));
-
-        // This Region should not have been opened.
-        // Ask the server to shut it down, but don't report it as closed.
-        // Otherwise the HMaster will think the Region was closed on purpose,
-        // and then try to reopen it elsewhere; that's not what we want.
-        returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_CLOSE_WITHOUT_REPORT,
-          region, "Duplicate assignment".getBytes()));
-      } else {
-        if (region.isRootRegion()) {
-          // it was assigned, and it's not a duplicate assignment, so take it out
-          // of the unassigned list.
-          this.master.getRegionManager().removeRegion(region);
-
-          // Store the Root Region location (in memory)
-          HServerAddress rootServer = serverInfo.getServerAddress();
-          this.master.getServerConnection().setRootRegionLocation(
-            new HRegionLocation(region, rootServer));
-          this.master.getRegionManager().setRootRegionLocation(rootServer);
-        } else {
-          // Note that the table has been assigned and is waiting for the
-          // meta table to be updated.
-          this.master.getRegionManager().setOpen(region.getRegionNameAsString());
-          RegionServerOperation op =
-            new ProcessRegionOpen(master, serverInfo, region);
-          this.master.getRegionServerOperationQueue().put(op);
-        }
-      }
-    }
-  }
-
-  /*
-   * @param region
-   * @throws Exception
-   */
-  public void processRegionClose(HRegionInfo region) {
-    synchronized (this.master.getRegionManager()) {
-      if (region.isRootRegion()) {
-        // Root region
-        this.master.getRegionManager().unsetRootRegion();
-        if (region.isOffline()) {
-          // Can't proceed without root region. Shutdown.
-          LOG.fatal("root region is marked offline");
-          this.master.shutdown();
-          return;
-        }
-
-      } else if (region.isMetaTable()) {
-        // Region is part of the meta table. Remove it from onlineMetaRegions
-        this.master.getRegionManager().offlineMetaRegionWithStartKey(region.getStartKey());
-      }
-
-      boolean offlineRegion =
-        this.master.getRegionManager().isOfflined(region.getRegionNameAsString());
-      boolean reassignRegion = !region.isOffline() && !offlineRegion;
-
-      // NOTE: If the region was just being closed and not offlined, we cannot
-      //       mark the region unassignedRegions as that changes the ordering of
-      //       the messages we've received. In this case, a close could be
-      //       processed before an open resulting in the master not agreeing on
-      //       the region's state.
-      this.master.getRegionManager().setClosed(region.getRegionNameAsString());
-      RegionServerOperation op =
-        new ProcessRegionClose(master, region, offlineRegion, reassignRegion);
-      this.master.getRegionServerOperationQueue().put(op);
-    }
-  }
-
-  /** Update a server load information because it's shutting down*/
   private boolean removeServerInfo(final String serverName) {
-    boolean infoUpdated = false;
-    HServerInfo info = this.serversToServerInfo.remove(serverName);
-    // Only update load information once.
-    // This method can be called a couple of times during shutdown.
+    HServerInfo info = this.onlineServers.remove(serverName);
     if (info != null) {
-      LOG.info("Removing server's info " + serverName);
-      this.master.getRegionManager().offlineMetaServer(info.getServerAddress());
-
-      //HBASE-1928: Check whether this server has been transitioning the ROOT table
-      if (this.master.getRegionManager().isRootInTransitionOnThisServer(serverName)) {
-         this.master.getRegionManager().unsetRootRegion();
-         this.master.getRegionManager().reassignRootRegion();
-      }
-
-      //HBASE-1928: Check whether this server has been transitioning the META table
-      HRegionInfo metaServerRegionInfo = this.master.getRegionManager().getMetaServerRegionInfo (serverName);
-      if (metaServerRegionInfo != null) {
-         this.master.getRegionManager().setUnassigned(metaServerRegionInfo, true);
-      }
-
-      infoUpdated = true;
-      // update load information
-      updateLoadToServers(serverName, this.serversToLoad.remove(serverName));
-    }
-    return infoUpdated;
-  }
-
-  private void updateLoadToServers(final String serverName,
-      final HServerLoad load) {
-    if (load == null) return;
-    synchronized (this.loadToServers) {
-      Set<String> servers = this.loadToServers.get(load);
-      if (servers != null) {
-        servers.remove(serverName);
-        if (servers.size() > 0)
-          this.loadToServers.put(load, servers);
-        else
-          this.loadToServers.remove(load);
-      }
+      return true;
     }
+    return false;
   }
 
   /**
@@ -720,19 +356,22 @@ public class ServerManager {
     int totalLoad = 0;
     int numServers = 0;
     double averageLoad = 0.0;
-    synchronized (serversToLoad) {
-      numServers = serversToLoad.size();
-      for (HServerLoad load : serversToLoad.values()) {
-        totalLoad += load.getNumberOfRegions();
-      }
-      averageLoad = (double)totalLoad / (double)numServers;
+    for (HServerInfo hsi : onlineServers.values()) {
+        numServers++;
+        totalLoad += hsi.getLoad().getNumberOfRegions();
     }
+    averageLoad = (double)totalLoad / (double)numServers;
     return averageLoad;
   }
 
   /** @return the number of active servers */
   public int numServers() {
-    return this.serversToServerInfo.size();
+    int num = -1;
+    // This synchronized seems gratuitous.
+    synchronized (this.onlineServers) {
+      num = this.onlineServers.size();
+    }
+    return num;
   }
 
   /**
@@ -740,57 +379,43 @@ public class ServerManager {
    * @return HServerInfo for the given server address
    */
   public HServerInfo getServerInfo(String name) {
-    return this.serversToServerInfo.get(name);
+    return this.onlineServers.get(name);
   }
 
   /**
-   * @return Read-only map of servers to serverinfo.
+   * @return Read-only map of servers to serverinfo
    */
-  public Map<String, HServerInfo> getServersToServerInfo() {
-    synchronized (this.serversToServerInfo) {
-      return Collections.unmodifiableMap(this.serversToServerInfo);
+  public Map<String, HServerInfo> getOnlineServers() {
+    // Presumption is that iterating the returned Map is OK.
+    synchronized (this.onlineServers) {
+      return Collections.unmodifiableMap(this.onlineServers);
     }
   }
 
+  public Set<String> getDeadServers() {
+    return this.deadservers.clone();
+  }
+
   /**
    * @param hsa
    * @return The HServerInfo whose HServerAddress is <code>hsa</code> or null
    * if nothing found.
    */
   public HServerInfo getHServerInfo(final HServerAddress hsa) {
-    synchronized(this.serversToServerInfo) {
+    synchronized(this.onlineServers) {
       // TODO: This is primitive.  Do a better search.
-      for (Map.Entry<String, HServerInfo> e: this.serversToServerInfo.entrySet()) {
-        if (e.getValue().getServerAddress().equals(hsa)) return e.getValue();
+      for (Map.Entry<String, HServerInfo> e: this.onlineServers.entrySet()) {
+        if (e.getValue().getServerAddress().equals(hsa)) {
+          return e.getValue();
+        }
       }
     }
     return null;
   }
 
-  /**
-   * @return Read-only map of servers to load.
-   */
-  public Map<String, HServerLoad> getServersToLoad() {
-    synchronized (this.serversToLoad) {
-      return Collections.unmodifiableMap(serversToLoad);
-    }
-  }
-
-  /**
-   * @return Read-only map of load to servers.
-   */
-  public SortedMap<HServerLoad, Set<String>> getLoadToServers() {
-    synchronized (this.loadToServers) {
-      return Collections.unmodifiableSortedMap(this.loadToServers);
-    }
-  }
-
-  /**
-   * Wakes up threads waiting on serversToServerInfo
-   */
-  public void notifyServers() {
-    synchronized (this.serversToServerInfo) {
-      this.serversToServerInfo.notifyAll();
+  private void notifyOnlineServers() {
+    synchronized (this.onlineServers) {
+      this.onlineServers.notifyAll();
     }
   }
 
@@ -801,17 +426,12 @@ public class ServerManager {
    * a MSG_REGIONSERVER_STOP.
    */
   void letRegionServersShutdown() {
-    if (!master.checkFileSystem()) {
-      // Forget waiting for the region servers if the file system has gone
-      // away. Just exit as quickly as possible.
-      return;
-    }
-    synchronized (serversToServerInfo) {
-      while (serversToServerInfo.size() > 0) {
+    synchronized (onlineServers) {
+      while (onlineServers.size() > 0) {
         LOG.info("Waiting on following regionserver(s) to go down " +
-          this.serversToServerInfo.values());
+          this.onlineServers.values());
         try {
-          this.serversToServerInfo.wait(500);
+          this.onlineServers.wait(500);
         } catch (InterruptedException e) {
           // continue
         }
@@ -819,108 +439,43 @@ public class ServerManager {
     }
   }
 
-  /** Watcher triggered when a RS znode is deleted */
-  private class ServerExpirer implements Watcher {
-    private HServerInfo server;
-
-    ServerExpirer(final HServerInfo hsi) {
-      this.server = hsi;
-    }
-
-    public void process(WatchedEvent event) {
-      if (!event.getType().equals(EventType.NodeDeleted)) {
-        LOG.warn("Unexpected event=" + event);
-        return;
-      }
-      LOG.info(this.server.getServerName() + " znode expired");
-      expireServer(this.server);
-    }
-  }
-
   /*
    * Expire the passed server.  Add it to list of deadservers and queue a
    * shutdown processing.
    */
-  private synchronized void expireServer(final HServerInfo hsi) {
+  public synchronized void expireServer(final HServerInfo hsi) {
     // First check a server to expire.  ServerName is of the form:
     // <hostname> , <port> , <startcode>
     String serverName = hsi.getServerName();
-    HServerInfo info = this.serversToServerInfo.get(serverName);
+    HServerInfo info = this.onlineServers.get(serverName);
     if (info == null) {
-      LOG.warn("No HServerInfo for " + serverName);
+      LOG.warn("Received expiration of " + hsi.getServerName() +
+        " but server is not currently online");
       return;
     }
-    if (this.deadServers.contains(serverName)) {
-      LOG.warn("Already processing shutdown of " + serverName);
+    if (this.deadservers.contains(serverName)) {
+      // TODO: Can this happen?  It shouldn't be online in this case?
+      LOG.warn("Received expiration of " + hsi.getServerName() +
+          " but server shutdown is already in progress");
       return;
     }
     // Remove the server from the known servers lists and update load info
-    this.serversToServerInfo.remove(serverName);
-    HServerLoad load = this.serversToLoad.remove(serverName);
-    if (load != null) {
-      synchronized (this.loadToServers) {
-        Set<String> servers = this.loadToServers.get(load);
-        if (servers != null) {
-          servers.remove(serverName);
-          if (servers.isEmpty()) this.loadToServers.remove(load);
-        }
+    this.onlineServers.remove(serverName);
+    this.serverConnections.remove(serverName);
+    // If cluster is going down, yes, servers are going to be expiring; don't
+    // process as a dead server
+    if (this.clusterShutdown) {
+      LOG.info("Cluster shutdown set; " + hsi.getServerName() +
+        " expired; onlineServers=" + this.onlineServers.size());
+      if (this.onlineServers.isEmpty()) {
+        master.stop("Cluster shutdown set; onlineServer=0");
       }
+      return;
     }
-    // Add to dead servers and queue a shutdown processing.
+    this.services.getExecutorService().submit(new ServerShutdownHandler(this.master,
+        this.services, deadservers, info));
     LOG.debug("Added=" + serverName +
-      " to dead servers, added shutdown processing operation");
-    this.deadServers.add(serverName);
-    this.master.getRegionServerOperationQueue().
-      put(new ProcessServerShutdown(master, info));
-  }
-
-  /**
-   * @param serverName
-   */
-  void removeDeadServer(String serverName) {
-    this.deadServers.remove(serverName);
-  }
-
-  /**
-   * @param serverName
-   * @return true if server is dead
-   */
-  public boolean isDead(final String serverName) {
-    return isDead(serverName, false);
-  }
-
-  /**
-   * @param serverName Servername as either <code>host:port</code> or
-   * <code>host,port,startcode</code>.
-   * @param hostAndPortOnly True if <code>serverName</code> is host and
-   * port only (<code>host:port</code>) and if so, then we do a prefix compare
-   * (ignoring start codes) looking for dead server.
-   * @return true if server is dead
-   */
-  boolean isDead(final String serverName, final boolean hostAndPortOnly) {
-    return isDead(this.deadServers, serverName, hostAndPortOnly);
-  }
-
-  static boolean isDead(final Set<String> deadServers,
-      final String serverName, final boolean hostAndPortOnly) {
-    return HServerInfo.isServer(deadServers, serverName, hostAndPortOnly);
-  }
-
-  Set<String> getDeadServers() {
-    return this.deadServers;
-  }
-
-  /**
-   * Add to the passed <code>m</code> servers that are loaded less than
-   * <code>l</code>.
-   * @param l
-   * @param m
-   */
-  void getLightServers(final HServerLoad l,
-      SortedMap<HServerLoad, Set<String>> m) {
-    synchronized (this.loadToServers) {
-      m.putAll(this.loadToServers.headMap(l));
-    }
+      " to dead servers, submitted shutdown handler to be executed");
   }
 
   public boolean canAssignUserRegions() {
@@ -933,4 +488,96 @@ public class ServerManager {
   public void setMinimumServerCount(int minimumServerCount) {
     this.minimumServerCount = minimumServerCount;
   }
-}
+
+  // RPC methods to region servers
+
+  /**
+   * Sends an OPEN RPC to the specified server to open the specified region.
+   * <p>
+   * Open should not fail but can if server just crashed.
+   * <p>
+   * @param server server to open a region
+   * @param regionName region to open
+   */
+  public void sendRegionOpen(HServerInfo server, HRegionInfo region) {
+    HRegionInterface hri = getServerConnection(server);
+    if(hri == null) {
+      LOG.warn("Attempting to send OPEN RPC to server " + server.getServerName()
+          + " failed because no RPC connection found to this server");
+      return;
+    }
+    hri.openRegion(region);
+  }
+
+  /**
+   * Sends an CLOSE RPC to the specified server to close the specified region.
+   * <p>
+   * A region server could reject the close request because it either does not
+   * have the specified region or the region is being split.
+   * @param server server to open a region
+   * @param regionName region to open
+   * @return true if server acknowledged close, false if not
+   * @throws NotServingRegionException
+   */
+  public void sendRegionClose(HServerInfo server, HRegionInfo region)
+  throws NotServingRegionException {
+    HRegionInterface hri = getServerConnection(server);
+    if(hri == null) {
+      LOG.warn("Attempting to send CLOSE RPC to server " + server.getServerName()
+          + " failed because no RPC connection found to this server");
+      return;
+    }
+    hri.closeRegion(region);
+  }
+
+  private HRegionInterface getServerConnection(HServerInfo info) {
+    try {
+      ServerConnection connection =
+        ServerConnectionManager.getConnection(this.master.getConfiguration());
+      HRegionInterface hri = serverConnections.get(info.getServerName());
+      if(hri == null) {
+        LOG.info("new connection");
+        hri = connection.getHRegionConnection(info.getServerAddress(), false);
+        serverConnections.put(info.getServerName(), hri);
+      }
+      return hri;
+    } catch (IOException e) {
+      LOG.error("Error connecting to region server", e);
+      throw new RuntimeException("Fatal error connection to RS", e);
+    }
+  }
+
+  /**
+   * Waits for the minimum number of servers to be running.
+   */
+  public void waitForMinServers() {
+    while(numServers() < minimumServerCount) {
+//        !masterStatus.getShutdownRequested().get()) {
+      LOG.info("Waiting for enough servers to check in.  Currently have " +
+          numServers() + " but need at least " + minimumServerCount);
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        LOG.warn("Got interrupted waiting for servers to check in, looping");
+      }
+    }
+  }
+
+  public List<HServerInfo> getOnlineServersList() {
+    // TODO: optimize the load balancer call so we don't need to make a new list
+    return new ArrayList<HServerInfo>(onlineServers.values());
+  }
+
+  public boolean isServerOnline(String serverName) {
+    return onlineServers.containsKey(serverName);
+  }
+
+  public void shutdownCluster() {
+    LOG.info("Cluster shutdown requested");
+    this.clusterShutdown = true;
+  }
+
+  public boolean isClusterShutdown() {
+    return this.clusterShutdown;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/TableDelete.java b/src/main/java/org/apache/hadoop/hbase/master/TableDelete.java
deleted file mode 100644
index 1153e6283f2..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/TableDelete.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.TableNotDisabledException;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-
-/**
- * Instantiated to delete a table. Table must be offline.
- */
-class TableDelete extends TableOperation {
-  private final Log LOG = LogFactory.getLog(this.getClass());
-
-  TableDelete(final HMaster master, final byte [] tableName) throws IOException {
-    super(master, tableName);
-  }
-
-  @Override
-  protected void processScanItem(String serverName,
-      final HRegionInfo info) throws IOException {
-    if (isEnabled(info)) {
-      LOG.debug("Region still enabled: " + info.toString());
-      throw new TableNotDisabledException(tableName);
-    }
-  }
-
-  @Override
-  protected void postProcessMeta(MetaRegion m, HRegionInterface server)
-  throws IOException {
-    for (HRegionInfo i: unservedRegions) {
-      if (!Bytes.equals(this.tableName, i.getTableDesc().getName())) {
-        // Don't delete regions that are not from our table.
-        continue;
-      }
-      // Delete the region
-      try {
-        HRegion.removeRegionFromMETA(server, m.getRegionName(), i.getRegionName());
-        HRegion.deleteRegion(this.master.getFileSystem(),
-          this.master.getRootDir(), i);
-
-      } catch (IOException e) {
-        LOG.error("failed to delete region " + Bytes.toString(i.getRegionName()),
-          RemoteExceptionHandler.checkIOException(e));
-      }
-    }
-
-    // delete the table's folder from fs.
-    this.master.getFileSystem().delete(new Path(this.master.getRootDir(),
-      Bytes.toString(this.tableName)), true);
-  }
-}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/TableOperation.java b/src/main/java/org/apache/hadoop/hbase/master/TableOperation.java
deleted file mode 100644
index 928d60775aa..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/TableOperation.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * Abstract base class for operations that need to examine all HRegionInfo
- * objects in a table. (For a table, operate on each of its rows
- * in .META.).
- */
-abstract class TableOperation {
-  private final Set<MetaRegion> metaRegions;
-  protected final byte [] tableName;
-  // Do regions in order.
-  protected final Set<HRegionInfo> unservedRegions = new TreeSet<HRegionInfo>();
-  protected HMaster master;
-
-  protected TableOperation(final HMaster master, final byte [] tableName)
-  throws IOException {
-    this.master = master;
-    if (!this.master.isMasterRunning()) {
-      throw new MasterNotRunningException();
-    }
-    // add the delimiters.
-    // TODO maybe check if this is necessary?
-    this.tableName = tableName;
-
-    // Don't wait for META table to come on line if we're enabling it
-    if (!Bytes.equals(HConstants.META_TABLE_NAME, this.tableName)) {
-      // We can not access any meta region if they have not already been
-      // assigned and scanned.
-      if (master.getRegionManager().metaScannerThread.waitForMetaRegionsOrClose()) {
-        // We're shutting down. Forget it.
-        throw new MasterNotRunningException();
-      }
-    }
-    this.metaRegions = master.getRegionManager().getMetaRegionsForTable(tableName);
-  }
-
-  private class ProcessTableOperation extends RetryableMetaOperation<Boolean> {
-    ProcessTableOperation(MetaRegion m, HMaster master) {
-      super(m, master);
-    }
-
-    public Boolean call() throws IOException {
-      boolean tableExists = false;
-
-      // Open a scanner on the meta region
-      byte [] tableNameMetaStart =
-        Bytes.toBytes(Bytes.toString(tableName) + ",,");
-      final Scan scan = new Scan(tableNameMetaStart)
-        .addFamily(HConstants.CATALOG_FAMILY);
-      long scannerId = this.server.openScanner(m.getRegionName(), scan);
-      int rows = this.master.getConfiguration().
-        getInt("hbase.meta.scanner.caching", 100);
-      scan.setCaching(rows);
-      List<byte []> emptyRows = new ArrayList<byte []>();
-      try {
-        while (true) {
-          Result values = this.server.next(scannerId);
-          if (values == null || values.isEmpty()) {
-            break;
-          }
-          HRegionInfo info = this.master.getHRegionInfo(values.getRow(), values);
-          if (info == null) {
-            emptyRows.add(values.getRow());
-            LOG.error(Bytes.toString(HConstants.CATALOG_FAMILY) + ":"
-                + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
-                + " not found on "
-                + Bytes.toStringBinary(values.getRow()));
-            continue;
-          }
-          final String serverAddress = BaseScanner.getServerAddress(values);
-          String serverName = null;
-          if (serverAddress != null && serverAddress.length() > 0) {
-            long startCode = BaseScanner.getStartCode(values);
-            serverName = HServerInfo.getServerName(serverAddress, startCode);
-          }
-          if (Bytes.compareTo(info.getTableDesc().getName(), tableName) > 0) {
-            break; // Beyond any more entries for this table
-          }
-
-          tableExists = true;
-          if (!isBeingServed(serverName) || !isEnabled(info)) {
-            unservedRegions.add(info);
-          }
-          processScanItem(serverName, info);
-        }
-      } finally {
-        if (scannerId != -1L) {
-          try {
-            this.server.close(scannerId);
-          } catch (IOException e) {
-            e = RemoteExceptionHandler.checkIOException(e);
-            LOG.error("closing scanner", e);
-          }
-        }
-        scannerId = -1L;
-      }
-
-      // Get rid of any rows that have a null HRegionInfo
-
-      if (emptyRows.size() > 0) {
-        LOG.warn("Found " + emptyRows.size() +
-            " rows with empty HRegionInfo while scanning meta region " +
-            Bytes.toString(m.getRegionName()));
-        master.deleteEmptyMetaRows(server, m.getRegionName(), emptyRows);
-      }
-
-      if (!tableExists) {
-        throw new TableNotFoundException(Bytes.toString(tableName));
-      }
-
-      postProcessMeta(m, server);
-      unservedRegions.clear();
-      return Boolean.TRUE;
-    }
-  }
-
-  void process() throws IOException {
-    // Prevent meta scanner from running
-    synchronized(master.getRegionManager().metaScannerThread.scannerLock) {
-      for (MetaRegion m: metaRegions) {
-        new ProcessTableOperation(m, master).doWithRetries();
-      }
-    }
-  }
-
-  protected boolean isBeingServed(String serverName) {
-    boolean result = false;
-    if (serverName != null && serverName.length() > 0) {
-      HServerInfo s = master.getServerManager().getServerInfo(serverName);
-      result = s != null;
-    }
-    return result;
-  }
-
-  protected boolean isEnabled(HRegionInfo info) {
-    return !info.isOffline();
-  }
-
-  protected abstract void processScanItem(String serverName, HRegionInfo info)
-  throws IOException;
-
-  protected abstract void postProcessMeta(MetaRegion m,
-    HRegionInterface server) throws IOException;
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java b/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java
deleted file mode 100644
index 7b008193f94..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.EventType;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-
-/**
- * ZooKeeper watcher for the master address.  Also watches the cluster state
- * flag so will shutdown this master if cluster has been shutdown.
- * <p>Used by the Master.  Waits on the master address ZNode delete event.  When
- * multiple masters are brought up, they race to become master by writing their
- * address to ZooKeeper. Whoever wins becomes the master, and the rest wait for
- * that ephemeral node in ZooKeeper to evaporate (meaning the master went down),
- * at which point they try to write their own address to become the new master.
- */
-class ZKMasterAddressWatcher implements Watcher {
-  private static final Log LOG = LogFactory.getLog(ZKMasterAddressWatcher.class);
-
-  private ZooKeeperWrapper zookeeper;
-  private final AtomicBoolean requestShutdown;
-
-  /**
-   * Create this watcher using passed ZooKeeperWrapper instance.
-   * @param zk ZooKeeper
-   * @param flag Flag to set to request shutdown.
-   */
-  ZKMasterAddressWatcher(final ZooKeeperWrapper zk, final AtomicBoolean flag) {
-    this.requestShutdown = flag;
-    this.zookeeper = zk;
-  }
-
-  /**
-   * @see org.apache.zookeeper.Watcher#process(org.apache.zookeeper.WatchedEvent)
-   */
-  @Override
-  public synchronized void process (WatchedEvent event) {
-    EventType type = event.getType();
-    LOG.debug(("Got event " + type + " with path " + event.getPath()));
-    if (type.equals(EventType.NodeDeleted)) {
-      if (event.getPath().equals(this.zookeeper.clusterStateZNode)) {
-        LOG.info("Cluster shutdown while waiting, shutting down" +
-          " this master.");
-        this.requestShutdown.set(true);
-      } else {
-        LOG.debug("Master address ZNode deleted, notifying waiting masters");
-        notifyAll();
-      }
-    } else if(type.equals(EventType.NodeCreated) &&
-        event.getPath().equals(this.zookeeper.clusterStateZNode)) {
-      LOG.debug("Resetting watch on cluster state node.");
-      this.zookeeper.setClusterStateWatch();
-    }
-  }
-
-  /**
-   * Wait for master address to be available. This sets a watch in ZooKeeper and
-   * blocks until the master address ZNode gets deleted.
-   */
-  public synchronized void waitForMasterAddressAvailability() {
-    while (zookeeper.readMasterAddress(this) != null) {
-      try {
-        LOG.debug("Waiting for master address ZNode to be deleted " +
-          "(Also watching cluster state node)");
-        this.zookeeper.setClusterStateWatch();
-        wait();
-      } catch (InterruptedException e) {
-      }
-    }
-  }
-
-  /**
-   * Write address to zookeeper.  Parks here until we successfully write our
-   * address (or until cluster shutdown).
-   * @param address Address whose format is HServerAddress.toString
-   */
-  boolean writeAddressToZooKeeper(
-      final HServerAddress address, boolean retry) {
-    do {
-      waitForMasterAddressAvailability();
-      // Check if we need to shutdown instead of taking control
-      if (this.requestShutdown.get()) {
-        LOG.debug("Won't start Master because cluster is shuting down");
-        return false;
-      }
-      if(this.zookeeper.writeMasterAddress(address)) {
-        this.zookeeper.setClusterState(true);
-        this.zookeeper.setClusterStateWatch();
-        // Watch our own node
-        this.zookeeper.readMasterAddress(this);
-        return true;
-      }
-    } while(retry);
-    return false;
-  }
-
-  /**
-   * Reset the ZK in case a new connection is required
-   * @param zookeeper new instance
-   */
-  public void setZookeeper(ZooKeeperWrapper zookeeper) {
-    this.zookeeper = zookeeper;
-  }
-}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java b/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java
deleted file mode 100644
index acd23d11458..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
-import org.apache.hadoop.hbase.master.handler.MasterCloseRegionHandler;
-import org.apache.hadoop.hbase.master.handler.MasterOpenRegionHandler;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper.ZNodePathAndData;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.EventType;
-
-/**
- * Watches the UNASSIGNED znode in ZK for the master, and handles all events 
- * relating to region transitions.
- */
-public class ZKUnassignedWatcher implements Watcher {
-  private static final Log LOG = LogFactory.getLog(ZKUnassignedWatcher.class);
-
-  private ZooKeeperWrapper zkWrapper;
-  String serverName;
-  ServerManager serverManager;
-
-  public static void start(Configuration conf, HMaster master) 
-  throws IOException {
-    new ZKUnassignedWatcher(conf, master);
-    LOG.debug("Started ZKUnassigned watcher");
-  }
-
-  public ZKUnassignedWatcher(Configuration conf, HMaster master) 
-  throws IOException {
-    this.serverName = master.getHServerAddress().toString();
-    this.serverManager = master.getServerManager();
-    zkWrapper = ZooKeeperWrapper.getInstance(conf, HMaster.class.getName());
-    String unassignedZNode = zkWrapper.getRegionInTransitionZNode();
-    
-    // If the UNASSIGNED ZNode exists and this is a fresh cluster start, then 
-    // delete it.
-    if(master.isClusterStartup() && zkWrapper.exists(unassignedZNode, false)) {
-      LOG.info("Cluster start, but found " + unassignedZNode + ", deleting it.");
-      try {
-        zkWrapper.deleteZNode(unassignedZNode, true);
-      } catch (KeeperException e) {
-        LOG.error("Could not delete znode " + unassignedZNode, e);
-        throw new IOException(e);
-      } catch (InterruptedException e) {
-        LOG.error("Could not delete znode " + unassignedZNode, e);
-        throw new IOException(e);
-      }
-    }
-    
-    // If the UNASSIGNED ZNode does not exist, create it.
-    zkWrapper.createZNodeIfNotExists(unassignedZNode);
-    
-    // TODO: get the outstanding changes in UNASSIGNED
-
-    // Set a watch on Zookeeper's UNASSIGNED node if it exists.
-    zkWrapper.registerListener(this);
-  }
-
-  /**
-   * This is the processing loop that gets triggered from the ZooKeeperWrapper.
-   * This zookeeper events process function dies the following:
-   *   - WATCHES the following events: NodeCreated, NodeDataChanged, NodeChildrenChanged
-   *   - IGNORES the following events: None, NodeDeleted
-   */
-  @Override
-  public synchronized void process(WatchedEvent event) {
-    EventType type = event.getType();
-    LOG.debug("ZK-EVENT-PROCESS: Got zkEvent " + type +
-              " state:" + event.getState() +
-              " path:" + event.getPath());
-
-    // Handle the ignored events
-    if(type.equals(EventType.None)       ||
-       type.equals(EventType.NodeDeleted)) {
-      return;
-    }
-
-    // check if the path is for the UNASSIGNED directory we care about
-    if(event.getPath() == null ||
-       !event.getPath().startsWith(zkWrapper.getZNodePathForHBase(
-           zkWrapper.getRegionInTransitionZNode()))) {
-      return;
-    }
-
-    try
-    {
-      /*
-       * If a node is created in the UNASSIGNED directory in zookeeper, then:
-       *   1. watch its updates (this is an unassigned region).
-       *   2. read to see what its state is and handle as needed (state may have
-       *      changed before we started watching it)
-       */
-      if(type.equals(EventType.NodeCreated)) {
-        zkWrapper.watchZNode(event.getPath());
-        handleRegionStateInZK(event.getPath());
-      }
-      /*
-       * Data on some node has changed. Read to see what the state is and handle
-       * as needed.
-       */
-      else if(type.equals(EventType.NodeDataChanged)) {
-        handleRegionStateInZK(event.getPath());
-      }
-      /*
-       * If there were some nodes created then watch those nodes
-       */
-      else if(type.equals(EventType.NodeChildrenChanged)) {
-        List<ZNodePathAndData> newZNodes =
-            zkWrapper.watchAndGetNewChildren(event.getPath());
-        for(ZNodePathAndData zNodePathAndData : newZNodes) {
-          LOG.debug("Handling updates for znode: " + zNodePathAndData.getzNodePath());
-          handleRegionStateInZK(zNodePathAndData.getzNodePath(),
-              zNodePathAndData.getData());
-        }
-      }
-    }
-    catch (IOException e)
-    {
-      LOG.error("Could not process event from ZooKeeper", e);
-    }
-  }
-
-  /**
-   * Read the state of a node in ZK, and do the needful. We want to do the
-   * following:
-   *   1. If region's state is updated as CLOSED, invoke the ClosedRegionHandler.
-   *   2. If region's state is updated as OPENED, invoke the OpenRegionHandler.
-   * @param zNodePath
-   * @throws IOException
-   */
-  private void handleRegionStateInZK(String zNodePath) throws IOException {
-    byte[] data = zkWrapper.readZNode(zNodePath, null);
-    handleRegionStateInZK(zNodePath, data);
-  }
-  
-  private void handleRegionStateInZK(String zNodePath, byte[] data) {
-    // a null value is set when a node is created, we don't need to handle this
-    if(data == null) {
-      return;
-    }
-    String rgnInTransitNode = zkWrapper.getRegionInTransitionZNode();
-    String region = zNodePath.substring(
-        zNodePath.indexOf(rgnInTransitNode) + rgnInTransitNode.length() + 1);
-    HBaseEventType rsEvent = HBaseEventType.fromByte(data[0]);
-    LOG.debug("Got event type [ " + rsEvent + " ] for region " + region);
-
-    // if the node was CLOSED then handle it
-    if(rsEvent == HBaseEventType.RS2ZK_REGION_CLOSED) {
-      new MasterCloseRegionHandler(rsEvent, serverManager, serverName, region, data).submit();
-    }
-    // if the region was OPENED then handle that
-    else if(rsEvent == HBaseEventType.RS2ZK_REGION_OPENED || 
-            rsEvent == HBaseEventType.RS2ZK_REGION_OPENING) {
-      new MasterOpenRegionHandler(rsEvent, serverManager, serverName, region, data).submit();
-    }
-  }
-}
-
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
new file mode 100644
index 00000000000..12f8f24a70b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
@@ -0,0 +1,114 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Handles CLOSED region event on Master.
+ * <p>
+ * If table is being disabled, deletes ZK unassigned node and removes from
+ * regions in transition.
+ * <p>
+ * Otherwise, assigns the region to another server.
+ */
+public class ClosedRegionHandler extends EventHandler implements TotesHRegionInfo {
+  private static final Log LOG = LogFactory.getLog(ClosedRegionHandler.class);
+
+  private final AssignmentManager assignmentManager;
+  private final RegionTransitionData data;
+  private final HRegionInfo regionInfo;
+
+  private final ClosedPriority priority;
+
+  private enum ClosedPriority {
+    ROOT (1),
+    META (2),
+    USER (3);
+
+    private final int value;
+    ClosedPriority(int value) {
+      this.value = value;
+    }
+    public int getValue() {
+      return value;
+    }
+  };
+
+  public ClosedRegionHandler(Server server,
+      AssignmentManager assignmentManager, RegionTransitionData data,
+      HRegionInfo regionInfo) {
+    super(server, EventType.RS2ZK_REGION_CLOSED);
+    this.assignmentManager = assignmentManager;
+    this.data = data;
+    this.regionInfo = regionInfo;
+    if(regionInfo.isRootRegion()) {
+      priority = ClosedPriority.ROOT;
+    } else if(regionInfo.isMetaRegion()) {
+      priority = ClosedPriority.META;
+    } else {
+      priority = ClosedPriority.USER;
+    }
+  }
+
+  @Override
+  public int getPriority() {
+    return priority.getValue();
+  }
+
+  @Override
+  public HRegionInfo getHRegionInfo() {
+    return this.regionInfo;
+  }
+
+  @Override
+  public void process() {
+    LOG.debug("Handling CLOSED event");
+    // Check if this table is being disabled or not
+    if (assignmentManager.isTableOfRegionDisabled(regionInfo.getRegionName())) {
+      // Disabling so should not be reassigned, just delete the CLOSED node
+      LOG.debug("Table being disabled so deleting ZK node and removing from " +
+          "regions in transition, skipping assignment");
+      try {
+        ZKAssign.deleteClosedNode(server.getZooKeeper(),
+            regionInfo.getEncodedName());
+      } catch (KeeperException.NoNodeException nne) {
+        LOG.warn("Tried to delete closed node for " + data + " but it does " +
+            "not exist");
+        return;
+      } catch (KeeperException e) {
+        server.abort("Error deleting CLOSED node in ZK", e);
+      }
+      assignmentManager.regionOffline(regionInfo);
+      return;
+    }
+    // ZK Node is in CLOSED state, assign it.
+    assignmentManager.setOffline(regionInfo);
+    assignmentManager.assign(regionInfo);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
new file mode 100644
index 00000000000..e20f15cf256
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
@@ -0,0 +1,53 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.master.MasterServices;
+
+public class DeleteTableHandler extends TableEventHandler {
+  private static final Log LOG = LogFactory.getLog(DeleteTableHandler.class);
+
+  public DeleteTableHandler(byte [] tableName, Server server,
+      final MasterServices masterServices) throws IOException {
+    super(EventType.C2M_DELETE_TABLE, tableName, server, masterServices);
+  }
+
+  @Override
+  protected void handleTableOperation(List<HRegionInfo> regions)
+  throws IOException {
+    for(HRegionInfo region : regions) {
+      LOG.debug("Deleting region " + region + " from META and FS");
+      // Remove region from META
+      MetaEditor.deleteRegion(this.server.getCatalogTracker(), region);
+      // Delete region from FS
+      this.masterServices.getMasterFileSystem().deleteRegion(region);
+    }
+    // Delete table from FS
+    this.masterServices.getMasterFileSystem().deleteTable(tableName);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
new file mode 100644
index 00000000000..c3778e167ad
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
@@ -0,0 +1,83 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.util.Bytes;
+
+
+public class DisableTableHandler extends EventHandler {
+  private static final Log LOG = LogFactory.getLog(DisableTableHandler.class);
+
+  private final byte [] tableName;
+  private final String tableNameStr;
+  private final AssignmentManager assignmentManager;
+
+  public DisableTableHandler(Server server, byte [] tableName,
+      CatalogTracker catalogTracker, AssignmentManager assignmentManager)
+  throws TableNotFoundException, IOException {
+    super(server, EventType.C2M_DISABLE_TABLE);
+    this.tableName = tableName;
+    this.tableNameStr = Bytes.toString(this.tableName);
+    this.assignmentManager = assignmentManager;
+    // Check if table exists
+    // TODO: do we want to keep this in-memory as well?  i guess this is
+    //       part of old master rewrite, schema to zk to check for table
+    //       existence and such
+    if(!MetaReader.tableExists(catalogTracker, this.tableNameStr)) {
+      throw new TableNotFoundException(Bytes.toString(tableName));
+    }
+  }
+
+  @Override
+  public void process() {
+    try {
+      LOG.info("Attemping to disable the table " + this.tableNameStr);
+      handleDisableTable();
+    } catch (IOException e) {
+      LOG.error("Error trying to disable the table " + this.tableNameStr, e);
+    }
+  }
+
+  private void handleDisableTable() throws IOException {
+    // Set the table as disabled so it doesn't get re-onlined
+    assignmentManager.disableTable(this.tableNameStr);
+    // Get the online regions of this table.
+    // TODO: What if region splitting at the time we get this listing?
+    // TODO: Remove offline flag from HRI
+    // TODO: Confirm we have parallel closing going on.
+    List<HRegionInfo> regions = assignmentManager.getRegionsOfTable(tableName);
+    // Unassign the online regions
+    for(HRegionInfo region : regions) {
+      assignmentManager.unassign(region);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
new file mode 100644
index 00000000000..0fc1cc3cd4a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
@@ -0,0 +1,79 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.util.Bytes;
+
+
+public class EnableTableHandler extends EventHandler {
+  private static final Log LOG = LogFactory.getLog(EnableTableHandler.class);
+
+  private final byte [] tableName;
+  private final String tableNameStr;
+  private final AssignmentManager assignmentManager;
+  private final CatalogTracker ct;
+
+  public EnableTableHandler(Server server, byte [] tableName,
+      CatalogTracker catalogTracker, AssignmentManager assignmentManager)
+  throws TableNotFoundException, IOException {
+    super(server, EventType.C2M_ENABLE_TABLE);
+    this.tableName = tableName;
+    this.tableNameStr = Bytes.toString(tableName);
+    this.ct = catalogTracker;
+    this.assignmentManager = assignmentManager;
+    // Check if table exists
+    if(!MetaReader.tableExists(catalogTracker, this.tableNameStr)) {
+      throw new TableNotFoundException(Bytes.toString(tableName));
+    }
+  }
+
+  @Override
+  public void process() {
+    try {
+      LOG.info("Attemping to enable the table " + this.tableNameStr);
+      handleEnableTable();
+    } catch (IOException e) {
+      LOG.error("Error trying to enable the table " + this.tableNameStr, e);
+    }
+  }
+
+  private void handleEnableTable() throws IOException {
+    // Get the regions of this table
+    List<HRegionInfo> regions = MetaReader.getTableRegions(this.ct, tableName);
+    // Set the table as disabled so it doesn't get re-onlined
+    assignmentManager.undisableTable(this.tableNameStr);
+    // Verify all regions of table are disabled
+    for (HRegionInfo region : regions) {
+      assignmentManager.assign(region);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java
deleted file mode 100644
index 4c850c3dd4c..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.handler;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler;
-import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.ServerManager;
-import org.apache.hadoop.hbase.util.Writables;
-
-/**
- * This is the event handler for all events relating to closing regions on the
- * HMaster. The following event types map to this handler:
- *   - RS_REGION_CLOSING
- *   - RS_REGION_CLOSED
- */
-public class MasterCloseRegionHandler extends HBaseEventHandler
-{
-  private static final Log LOG = LogFactory.getLog(MasterCloseRegionHandler.class);
-  
-  private String regionName;
-  protected byte[] serializedData;
-  RegionTransitionEventData hbEventData;
-  ServerManager serverManager;
-  
-  public MasterCloseRegionHandler(HBaseEventType eventType, 
-                                  ServerManager serverManager, 
-                                  String serverName, 
-                                  String regionName, 
-                                  byte[] serializedData) {
-    super(false, serverName, eventType);
-    this.regionName = regionName;
-    this.serializedData = serializedData;
-    this.serverManager = serverManager;
-  }
-
-  /**
-   * Handle the various events relating to closing regions. We can get the 
-   * following events here:
-   *   - RS_REGION_CLOSING : No-op
-   *   - RS_REGION_CLOSED  : The region is closed. If we are not in a shutdown 
-   *                         state, find the RS to open this region. This could 
-   *                         be a part of a region move, or just that the RS has 
-   *                         died. Should result in a M_REQUEST_OPENREGION event 
-   *                         getting created.
-   */
-  @Override
-  public void process()
-  {
-    LOG.debug("Event = " + getHBEvent() + ", region = " + regionName);
-    // handle RS_REGION_CLOSED events
-    handleRegionClosedEvent();
-  }
-  
-  private void handleRegionClosedEvent() {
-    try {
-      if(hbEventData == null) {
-        hbEventData = new RegionTransitionEventData();
-        Writables.getWritable(serializedData, hbEventData);
-      }
-    } catch (IOException e) {
-      LOG.error("Could not deserialize additional args for Close region", e);
-    }
-    // process the region close - this will cause the reopening of the 
-    // region as a part of the heartbeat of some RS
-    serverManager.processRegionClose(hbEventData.getHmsg().getRegionInfo());
-    LOG.info("Processed close of region " + hbEventData.getHmsg().getRegionInfo().getRegionNameAsString());
-  }
-  
-  public String getRegionName() {
-    return regionName;
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java
deleted file mode 100644
index ee6b92efe28..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.handler;
-
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler;
-import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.ServerManager;
-import org.apache.hadoop.hbase.util.Writables;
-
-/**
- * This is the event handler for all events relating to opening regions on the
- * HMaster. This could be one of the following:
- *   - notification that a region server is "OPENING" a region
- *   - notification that a region server has "OPENED" a region
- * The following event types map to this handler:
- *   - RS_REGION_OPENING
- *   - RS_REGION_OPENED
- */
-public class MasterOpenRegionHandler extends HBaseEventHandler {
-  private static final Log LOG = LogFactory.getLog(MasterOpenRegionHandler.class);
-  // other args passed in a byte array form
-  protected byte[] serializedData;
-  private String regionName;
-  private RegionTransitionEventData hbEventData;
-  ServerManager serverManager;
-
-  public MasterOpenRegionHandler(HBaseEventType eventType, 
-                                 ServerManager serverManager, 
-                                 String serverName, 
-                                 String regionName, 
-                                 byte[] serData) {
-    super(false, serverName, eventType);
-    this.regionName = regionName;
-    this.serializedData = serData;
-    this.serverManager = serverManager;
-  }
-
-  /**
-   * Handle the various events relating to opening regions. We can get the 
-   * following events here:
-   *   - RS_REGION_OPENING : Keep track to see how long the region open takes. 
-   *                         If the RS is taking too long, then revert the 
-   *                         region back to closed state so that it can be 
-   *                         re-assigned.
-   *   - RS_REGION_OPENED  : The region is opened. Add an entry into META for  
-   *                         the RS having opened this region. Then delete this 
-   *                         entry in ZK.
-   */
-  @Override
-  public void process()
-  {
-    LOG.debug("Event = " + getHBEvent() + ", region = " + regionName);
-    if(this.getHBEvent() == HBaseEventType.RS2ZK_REGION_OPENING) {
-      handleRegionOpeningEvent();
-    }
-    else if(this.getHBEvent() == HBaseEventType.RS2ZK_REGION_OPENED) {
-      handleRegionOpenedEvent();
-    }
-  }
-  
-  private void handleRegionOpeningEvent() {
-    // TODO: not implemented. 
-    LOG.debug("NO-OP call to handling region opening event");
-    // Keep track to see how long the region open takes. If the RS is taking too 
-    // long, then revert the region back to closed state so that it can be 
-    // re-assigned.
-  }
-
-  private void handleRegionOpenedEvent() {
-    try {
-      if(hbEventData == null) {
-        hbEventData = new RegionTransitionEventData();
-        Writables.getWritable(serializedData, hbEventData);
-      }
-    } catch (IOException e) {
-      LOG.error("Could not deserialize additional args for Open region", e);
-    }
-    LOG.debug("RS " + hbEventData.getRsName() + " has opened region " + regionName);
-    HServerInfo serverInfo = serverManager.getServerInfo(hbEventData.getRsName());
-    ArrayList<HMsg> returnMsgs = new ArrayList<HMsg>();
-    serverManager.processRegionOpen(serverInfo, hbEventData.getHmsg().getRegionInfo(), returnMsgs);
-    if(returnMsgs.size() > 0) {
-      LOG.error("Open region tried to send message: " + returnMsgs.get(0).getType() + 
-                " about " + returnMsgs.get(0).getRegionInfo().getRegionNameAsString());
-    }
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/AddColumn.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
similarity index 50%
rename from src/main/java/org/apache/hadoop/hbase/master/AddColumn.java
rename to src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
index 686521b0245..2dd5378dbac 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/AddColumn.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
@@ -17,34 +17,36 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
+package org.apache.hadoop.hbase.master.handler;
 
 import java.io.IOException;
+import java.util.List;
 
-/** Instantiated to add a column family to a table */
-class AddColumn extends ColumnOperation {
-  private final HColumnDescriptor newColumn;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.master.MasterServices;
 
-  AddColumn(final HMaster master, final byte [] tableName,
-    final HColumnDescriptor newColumn)
-  throws IOException {
-    super(master, tableName);
-    this.newColumn = newColumn;
+public class ModifyTableHandler extends TableEventHandler {
+  private final HTableDescriptor htd;
+
+  public ModifyTableHandler(final byte [] tableName,
+      final HTableDescriptor htd, final Server server,
+      final MasterServices masterServices) throws IOException {
+    super(EventType.C2M_MODIFY_TABLE, tableName, server, masterServices);
+    this.htd = htd;
   }
 
   @Override
-  protected void postProcessMeta(MetaRegion m, HRegionInterface server)
+  protected void handleTableOperation(List<HRegionInfo> hris)
   throws IOException {
-    for (HRegionInfo i: unservedRegions) {
-      // All we need to do to add a column is add it to the table descriptor.
-      // When the region is brought on-line, it will find the column missing
-      // and create it.
-      i.getTableDesc().addFamily(newColumn);
-      updateRegionInfo(server, m.getRegionName(), i);
+    for (HRegionInfo hri : hris) {
+      // Update region info in META
+      hri.setTableDesc(this.htd);
+      MetaEditor.updateRegionInfo(this.server.getCatalogTracker(), hri);
+      // Update region info in FS
+      this.masterServices.getMasterFileSystem().updateRegionInfo(hri);
     }
   }
 }
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java
new file mode 100644
index 00000000000..2cd2e809143
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java
@@ -0,0 +1,100 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Handles OPENED region event on Master.
+ */
+public class OpenedRegionHandler extends EventHandler implements TotesHRegionInfo {
+  private static final Log LOG = LogFactory.getLog(OpenedRegionHandler.class);
+
+  private final AssignmentManager assignmentManager;
+  private final RegionTransitionData data;
+  private final HRegionInfo regionInfo;
+  private final HServerInfo serverInfo;
+  private final OpenedPriority priority;
+
+  private enum OpenedPriority {
+    ROOT (1),
+    META (2),
+    USER (3);
+
+    private final int value;
+    OpenedPriority(int value) {
+      this.value = value;
+    }
+    public int getValue() {
+      return value;
+    }
+  };
+
+  public OpenedRegionHandler(Server server,
+      AssignmentManager assignmentManager, RegionTransitionData data,
+      HRegionInfo regionInfo, HServerInfo serverInfo) {
+    super(server, EventType.RS2ZK_REGION_OPENED);
+    this.assignmentManager = assignmentManager;
+    this.data = data;
+    this.regionInfo = regionInfo;
+    this.serverInfo = serverInfo;
+    if(regionInfo.isRootRegion()) {
+      priority = OpenedPriority.ROOT;
+    } else if(regionInfo.isMetaRegion()) {
+      priority = OpenedPriority.META;
+    } else {
+      priority = OpenedPriority.USER;
+    }
+  }
+
+  @Override
+  public int getPriority() {
+    return priority.getValue();
+  }
+
+  @Override
+  public HRegionInfo getHRegionInfo() {
+    return this.regionInfo;
+  }
+
+  @Override
+  public void process() {
+    LOG.debug("Handling OPENED event; deleting unassigned node");
+    // TODO: should we check if this table was disabled and get it closed?
+    // Remove region from in-memory transition and unassigned node from ZK
+    try {
+      ZKAssign.deleteOpenedNode(server.getZooKeeper(),
+          regionInfo.getEncodedName());
+    } catch (KeeperException e) {
+      server.abort("Error deleting OPENED node in ZK", e);
+    }
+    assignmentManager.regionOnline(regionInfo, serverInfo);
+    LOG.debug("Opened region " + regionInfo.getRegionNameAsString());
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
new file mode 100644
index 00000000000..14081035c73
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -0,0 +1,160 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.DeadServer;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.zookeeper.KeeperException;
+
+
+public class ServerShutdownHandler extends EventHandler {
+  private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
+  private final HServerInfo hsi;
+  private final Server server;
+  private final MasterServices services;
+  private final DeadServer deadServers;
+
+  public ServerShutdownHandler(final Server server, final MasterServices services,
+      final DeadServer deadServers, final HServerInfo hsi) {
+    super(server, EventType.M_SERVER_SHUTDOWN);
+    this.hsi = hsi;
+    this.server = server;
+    this.services = services;
+    this.deadServers = deadServers;
+    // Add to dead servers.
+    this.deadServers.add(hsi.getServerName());
+  }
+
+  @Override
+  public void process() throws IOException {
+    Pair<Boolean, Boolean> carryingCatalog = null;
+    try {
+      carryingCatalog =
+        this.server.getCatalogTracker().processServerShutdown(this.hsi);
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new IOException("Interrupted", e);
+    } catch (KeeperException e) {
+      this.server.abort("In server shutdown processing", e);
+      throw new IOException("Aborting", e);
+    }
+    final String serverName = this.hsi.getServerName();
+
+    LOG.info("Splitting logs for " + serverName);
+    this.services.getMasterFileSystem().splitLog(serverName);
+
+    // Clean out anything in regions in transition.  Being conservative and
+    // doing after log splitting.  Could do some states before -- OPENING?
+    // OFFLINE? -- and then others after like CLOSING that depend on log
+    // splitting.
+    this.services.getAssignmentManager().processServerShutdown(this.hsi);
+
+    // Assign root and meta if we were carrying them.
+    if (carryingCatalog.getFirst()) { // -ROOT-
+      try {
+        this.services.getAssignmentManager().assignRoot();
+      } catch (KeeperException e) {
+        this.server.abort("In server shutdown processing, assigning root", e);
+        throw new IOException("Aborting", e);
+      }
+    }
+    if (carryingCatalog.getSecond()) { // .META.
+      this.services.getAssignmentManager().assignMeta();
+    }
+
+    // Wait on meta to come online; we need it to progress.
+    try {
+      this.server.getCatalogTracker().waitForMeta();
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new IOException("Interrupted", e);
+    }
+
+    NavigableMap<HRegionInfo, Result> hris =
+      MetaReader.getServerRegions(this.server.getCatalogTracker(), this.hsi);
+    LOG.info("Reassigning the " + hris.size() + " region(s) that " + serverName +
+      " was carrying.");
+
+    // We should encounter -ROOT- and .META. first in the Set given how its
+    // a sorted set.
+    for (Map.Entry<HRegionInfo, Result> e: hris.entrySet()) {
+      // If table is not disabled but the region is offlined,
+      HRegionInfo hri = e.getKey();
+      boolean disabled = this.services.getAssignmentManager().
+        isTableDisabled(hri.getTableDesc().getNameAsString());
+      if (disabled) continue;
+      if (hri.isOffline() && hri.isSplit()) {
+        fixupDaughters(hris, e.getValue());
+        continue;
+      }
+      this.services.getAssignmentManager().assign(hri);
+    }
+    this.deadServers.remove(serverName);
+    LOG.info("Finished processing of shutdown of " + serverName);
+  }
+
+  /**
+   * Check that daughter regions are up in .META. and if not, add them.
+   * @param hris All regions for this server in meta.
+   * @param result The contents of the parent row in .META.
+   * @throws IOException
+   */
+  void fixupDaughters(final NavigableMap<HRegionInfo, Result> hris,
+      final Result result) throws IOException {
+    fixupDaughter(hris, result, HConstants.SPLITA_QUALIFIER);
+    fixupDaughter(hris, result, HConstants.SPLITB_QUALIFIER);
+  }
+
+  /**
+   * Check individual daughter is up in .META.; fixup if its not.
+   * @param hris All regions for this server in meta.
+   * @param result The contents of the parent row in .META.
+   * @param qualifier Which daughter to check for.
+   * @throws IOException
+   */
+  void fixupDaughter(final NavigableMap<HRegionInfo, Result> hris,
+      final Result result, final byte [] qualifier)
+  throws IOException {
+    byte [] bytes = result.getValue(HConstants.CATALOG_FAMILY, qualifier);
+    if (bytes == null || bytes.length <= 0) return;
+    HRegionInfo hri = Writables.getHRegionInfo(bytes);
+    if (!hris.containsKey(hri)) {
+      LOG.info("Fixup; missing daughter " + hri.getEncodedNameAsBytes());
+      MetaEditor.addDaughter(this.server.getCatalogTracker(), hri, null);
+      this.services.getAssignmentManager().assign(hri);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
new file mode 100644
index 00000000000..148d766fe57
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
@@ -0,0 +1,66 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.InvalidFamilyOperationException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Handles adding a new family to an existing table.
+ */
+public class TableAddFamilyHandler extends TableEventHandler {
+
+  private final HColumnDescriptor familyDesc;
+
+  public TableAddFamilyHandler(byte[] tableName, HColumnDescriptor familyDesc,
+      Server server, final MasterServices masterServices) throws IOException {
+    super(EventType.C2M_ADD_FAMILY, tableName, server, masterServices);
+    this.familyDesc = familyDesc;
+  }
+
+  @Override
+  protected void handleTableOperation(List<HRegionInfo> hris)
+  throws IOException {
+    HTableDescriptor htd = hris.get(0).getTableDesc();
+    byte [] familyName = familyDesc.getName();
+    if(htd.hasFamily(familyName)) {
+      throw new InvalidFamilyOperationException(
+          "Family '" + Bytes.toString(familyName) + "' already exists so " +
+          "cannot be added");
+    }
+    for(HRegionInfo hri : hris) {
+      // Update the HTD
+      hri.getTableDesc().addFamily(familyDesc);
+      // Update region in META
+      MetaEditor.updateRegionInfo(this.server.getCatalogTracker(), hri);
+      // Update region info in FS
+      this.masterServices.getMasterFileSystem().updateRegionInfo(hri);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
new file mode 100644
index 00000000000..1e4ee7fd709
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.InvalidFamilyOperationException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Handles adding a new family to an existing table.
+ */
+public class TableDeleteFamilyHandler extends TableEventHandler {
+
+  private final byte [] familyName;
+
+  public TableDeleteFamilyHandler(byte[] tableName, byte [] familyName,
+      Server server, final MasterServices masterServices) throws IOException {
+    super(EventType.C2M_ADD_FAMILY, tableName, server, masterServices);
+    this.familyName = familyName;
+  }
+
+  @Override
+  protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
+    HTableDescriptor htd = hris.get(0).getTableDesc();
+    if(!htd.hasFamily(familyName)) {
+      throw new InvalidFamilyOperationException(
+          "Family '" + Bytes.toString(familyName) + "' does not exist so " +
+          "cannot be deleted");
+    }
+    for (HRegionInfo hri : hris) {
+      // Update the HTD
+      hri.getTableDesc().removeFamily(familyName);
+      // Update region in META
+      MetaEditor.updateRegionInfo(this.server.getCatalogTracker(), hri);
+      MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
+      // Update region info in FS
+      mfs.updateRegionInfo(hri);
+      // Delete directory in FS
+      mfs.deleteFamily(hri, familyName);
+      // Update region info in FS
+      this.masterServices.getMasterFileSystem().updateRegionInfo(hri);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
new file mode 100644
index 00000000000..a8e0a16248d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
@@ -0,0 +1,72 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Base class for performing operations against tables.
+ * Checks on whether the process can go forward are done in constructor rather
+ * than later on in {@link #process()}.  The idea is to fail fast rather than
+ * later down in an async invocation of {@link #process()} (which currently has
+ * no means of reporting back issues once started).
+ */
+public abstract class TableEventHandler extends EventHandler {
+  private static final Log LOG = LogFactory.getLog(TableEventHandler.class);
+  protected final MasterServices masterServices;
+  protected final byte [] tableName;
+
+  public TableEventHandler(EventType eventType, byte [] tableName, Server server,
+      MasterServices masterServices)
+  throws IOException {
+    super(server, eventType);
+    this.masterServices = masterServices;
+    this.tableName = tableName;
+    this.masterServices.checkTableModifiable(tableName);
+  }
+
+  @Override
+  public void process() {
+    try {
+      LOG.info("Handling table operation " + eventType + " on table " +
+          Bytes.toString(tableName));
+      List<HRegionInfo> hris =
+        MetaReader.getTableRegions(this.server.getCatalogTracker(),
+          tableName);
+      handleTableOperation(hris);
+    } catch (IOException e) {
+      LOG.error("Error trying to delete the table " + Bytes.toString(tableName),
+          e);
+    }
+  }
+
+  protected abstract void handleTableOperation(List<HRegionInfo> regions)
+  throws IOException;
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
new file mode 100644
index 00000000000..b420caedff6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
@@ -0,0 +1,65 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.InvalidFamilyOperationException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Handles adding a new family to an existing table.
+ */
+public class TableModifyFamilyHandler extends TableEventHandler {
+
+  private final HColumnDescriptor familyDesc;
+
+  public TableModifyFamilyHandler(byte[] tableName,
+      HColumnDescriptor familyDesc, Server server,
+      final MasterServices masterServices) throws IOException {
+    super(EventType.C2M_MODIFY_FAMILY, tableName, server, masterServices);
+    this.familyDesc = familyDesc;
+  }
+
+  @Override
+  protected void handleTableOperation(List<HRegionInfo> regions) throws IOException {
+    HTableDescriptor htd = regions.get(0).getTableDesc();
+    byte [] familyName = familyDesc.getName();
+    if(!htd.hasFamily(familyName)) {
+      throw new InvalidFamilyOperationException("Family '" +
+        Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
+    }
+    for(HRegionInfo hri : regions) {
+      // Update the HTD
+      hri.getTableDesc().addFamily(familyDesc);
+      // Update region in META
+      MetaEditor.updateRegionInfo(this.server.getCatalogTracker(), hri);
+      // Update region info in FS
+      this.masterServices.getMasterFileSystem().updateRegionInfo(hri);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java
new file mode 100644
index 00000000000..d08f64982fc
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java
@@ -0,0 +1,36 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.beans.EventHandler;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+
+/**
+ * Implementors tote an HRegionInfo instance.
+ * This is a marker interface that can be put on {@link EventHandler}s that
+ * have an {@link HRegionInfo}.
+ */
+public interface TotesHRegionInfo {
+  /**
+   * @return HRegionInfo instance.
+   */
+  public HRegionInfo getHRegionInfo();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index a78eea21322..c27ba7a8d1a 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -35,9 +35,8 @@ import org.apache.hadoop.util.StringUtils;
 /**
  * Compact region on request and then run split if appropriate
  */
-class CompactSplitThread extends Thread {
+public class CompactSplitThread extends Thread implements CompactionRequestor {
   static final Log LOG = LogFactory.getLog(CompactSplitThread.class);
-
   private final long frequency;
   private final ReentrantLock lock = new ReentrantLock();
 
@@ -60,7 +59,7 @@ class CompactSplitThread extends Thread {
   public CompactSplitThread(HRegionServer server) {
     super();
     this.server = server;
-    this.conf = server.conf;
+    this.conf = server.getConfiguration();
     this.regionSplitLimit = conf.getInt("hbase.regionserver.regionSplitLimit",
         Integer.MAX_VALUE);
     this.frequency =
@@ -70,11 +69,11 @@ class CompactSplitThread extends Thread {
 
   @Override
   public void run() {
-    while (!this.server.isStopRequested()) {
+    while (!this.server.isStopped()) {
       HRegion r = null;
       try {
         r = compactionQueue.poll(this.frequency, TimeUnit.MILLISECONDS);
-        if (r != null && !this.server.isStopRequested()) {
+        if (r != null && !this.server.isStopped()) {
           synchronized (regionsInQueue) {
             regionsInQueue.remove(r);
           }
@@ -83,7 +82,7 @@ class CompactSplitThread extends Thread {
             // Don't interrupt us while we are working
             byte [] midKey = r.compactStores();
             if (shouldSplitRegion() && midKey != null &&
-                !this.server.isStopRequested()) {
+                !this.server.isStopped()) {
               split(r, midKey);
             }
           } finally {
@@ -113,13 +112,9 @@ class CompactSplitThread extends Thread {
     LOG.info(getName() + " exiting");
   }
 
-  /**
-   * @param r HRegion store belongs to
-   * @param why Why compaction requested -- used in debug messages
-   */
-  public synchronized void compactionRequested(final HRegion r,
+  public synchronized void requestCompaction(final HRegion r,
       final String why) {
-    compactionRequested(r, false, why);
+    requestCompaction(r, false, why);
   }
 
   /**
@@ -127,9 +122,9 @@ class CompactSplitThread extends Thread {
    * @param force Whether next compaction should be major
    * @param why Why compaction requested -- used in debug messages
    */
-  public synchronized void compactionRequested(final HRegion r,
+  public synchronized void requestCompaction(final HRegion r,
       final boolean force, final String why) {
-    if (this.server.stopRequested.get()) {
+    if (this.server.isStopped()) {
       return;
     }
     r.setForceMajorCompaction(force);
@@ -154,7 +149,7 @@ class CompactSplitThread extends Thread {
     // the prepare call -- we are not ready to split just now.  Just return.
     if (!st.prepare()) return;
     try {
-      st.execute(this.server);
+      st.execute(this.server, this.server);
     } catch (IOException ioe) {
       try {
         LOG.info("Running rollback of failed split of " +
@@ -177,8 +172,9 @@ class CompactSplitThread extends Thread {
     this.server.reportSplit(parent.getRegionInfo(), st.getFirstDaughter(),
       st.getSecondDaughter());
     LOG.info("Region split, META updated, and report to master. Parent=" +
-      parent.getRegionInfo() + ", new regions: " +
-      st.getFirstDaughter() + ", " + st.getSecondDaughter() + ". Split took " +
+      parent.getRegionInfo().getRegionNameAsString() + ", new regions: " +
+      st.getFirstDaughter().getRegionNameAsString() + ", " +
+      st.getSecondDaughter().getRegionNameAsString() + ". Split took " +
       StringUtils.formatTimeDiff(System.currentTimeMillis(), startTime));
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogRollListener.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
similarity index 71%
rename from src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogRollListener.java
rename to src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
index 76514c15b40..b7e868da76c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogRollListener.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2007 The Apache Software Foundation
+ * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -17,13 +17,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package org.apache.hadoop.hbase.regionserver;
 
-package org.apache.hadoop.hbase.regionserver.wal;
-
-/**
- * Mechanism by which the HLog requests a log roll
- */
-public interface LogRollListener {
-  /** Request that the log be rolled */
-  public void logRollRequested();
-}
+public interface CompactionRequestor {
+  /**
+   * @param r Region to compact
+   * @param why Why compaction was requested -- used in debug messages
+   */
+  public void requestCompaction(final HRegion r, final String why);
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java b/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
index 38ac209c55b..b843c91f7c4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
@@ -21,10 +21,7 @@
 package org.apache.hadoop.hbase.regionserver;
 
 /**
- * Implementors of this interface want to be notified when an HRegion
- * determines that a cache flush is needed. A FlushRequester (or null)
- * must be passed to the HRegion constructor so it knows who to call when it
- * has a filled memstore.
+ * Request a flush.
  */
 public interface FlushRequester {
   /**
@@ -32,5 +29,5 @@ public interface FlushRequester {
    *
    * @param region the HRegion requesting the cache flush
    */
-  void request(HRegion region);
+  void requestFlush(HRegion region);
 }
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index a692125ba19..6b8eac8c76d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -211,7 +211,7 @@ public class HRegion implements HeapSize { // , Writable{
 
   final long memstoreFlushSize;
   private volatile long lastFlushTime;
-  final FlushRequester flushListener;
+  final FlushRequester flushRequester;
   private final long blockingMemStoreSize;
   final long threadWakeFrequency;
   // Used to guard splits and closes
@@ -238,7 +238,7 @@ public class HRegion implements HeapSize { // , Writable{
     this.tableDir = null;
     this.blockingMemStoreSize = 0L;
     this.conf = null;
-    this.flushListener = null;
+    this.flushRequester = null;
     this.fs = null;
     this.memstoreFlushSize = 0L;
     this.log = null;
@@ -266,22 +266,19 @@ public class HRegion implements HeapSize { // , Writable{
    * @param conf is global configuration settings.
    * @param regionInfo - HRegionInfo that describes the region
    * is new), then read them from the supplied path.
-   * @param flushListener an object that implements CacheFlushListener or null
-   * making progress to master -- otherwise master might think region deploy
-   * failed.  Can be null.
+   * @param flushRequester an object that implements {@link FlushRequester} or null
    *
    * @see HRegion#newHRegion(Path, HLog, FileSystem, Configuration, org.apache.hadoop.hbase.HRegionInfo, FlushRequester)
-
    */
   public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
-      HRegionInfo regionInfo, FlushRequester flushListener) {
+      HRegionInfo regionInfo, FlushRequester flushRequester) {
     this.tableDir = tableDir;
     this.comparator = regionInfo.getComparator();
     this.log = log;
     this.fs = fs;
     this.conf = conf;
     this.regionInfo = regionInfo;
-    this.flushListener = flushListener;
+    this.flushRequester = flushRequester;
     this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY,
         10 * 1000);
     String encodedNameStr = this.regionInfo.getEncodedName();
@@ -378,7 +375,7 @@ public class HRegion implements HeapSize { // , Writable{
   /**
    * @return True if this region has references.
    */
-  boolean hasReferences() {
+  public boolean hasReferences() {
     for (Store store : this.stores.values()) {
       for (StoreFile sf : store.getStorefiles()) {
         // Found a reference, return.
@@ -943,7 +940,7 @@ public class HRegion implements HeapSize { // , Writable{
     //     and that all updates to the log for this regionName that have lower
     //     log-sequence-ids can be safely ignored.
     if (wal != null) {
-      wal.completeCacheFlush(getRegionName(),
+      wal.completeCacheFlush(this.regionInfo.getEncodedNameAsBytes(),
         regionInfo.getTableDesc().getName(), completeSequenceId,
         this.getRegionInfo().isMetaRegion());
     }
@@ -1517,28 +1514,6 @@ public class HRegion implements HeapSize { // , Writable{
     }
   }
 
-//  /*
-//   * Utility method to verify values length.
-//   * @param batchUpdate The update to verify
-//   * @throws IOException Thrown if a value is too long
-//   */
-//  private void validateValuesLength(Put put)
-//  throws IOException {
-//    Map<byte[], List<KeyValue>> families = put.getFamilyMap();
-//    for(Map.Entry<byte[], List<KeyValue>> entry : families.entrySet()) {
-//      HColumnDescriptor hcd =
-//        this.regionInfo.getTableDesc().getFamily(entry.getKey());
-//      int maxLen = hcd.getMaxValueLength();
-//      for(KeyValue kv : entry.getValue()) {
-//        if(kv.getValueLength() > maxLen) {
-//          throw new ValueOverMaxLengthException("Value in column "
-//            + Bytes.toString(kv.getColumn()) + " is too long. "
-//            + kv.getValueLength() + " > " + maxLen);
-//        }
-//      }
-//    }
-//  }
-
   /*
    * Check if resources to support an update.
    *
@@ -1700,7 +1675,7 @@ public class HRegion implements HeapSize { // , Writable{
   }
 
   private void requestFlush() {
-    if (this.flushListener == null) {
+    if (this.flushRequester == null) {
       return;
     }
     synchronized (writestate) {
@@ -1710,7 +1685,7 @@ public class HRegion implements HeapSize { // , Writable{
       writestate.flushRequested = true;
     }
     // Make request outside of synchronize block; HBASE-818.
-    this.flushListener.request(this);
+    this.flushRequester.requestFlush(this);
     if (LOG.isDebugEnabled()) {
       LOG.debug("Flush requested on " + this);
     }
@@ -1856,7 +1831,7 @@ public class HRegion implements HeapSize { // , Writable{
         // Check this edit is for me. Also, guard against writing the special
         // METACOLUMN info such as HBASE::CACHEFLUSH entries
         if (kv.matchingFamily(HLog.METAFAMILY) ||
-            !Bytes.equals(key.getRegionName(), this.regionInfo.getRegionName())) {
+            !Bytes.equals(key.getEncodedRegionName(), this.regionInfo.getEncodedNameAsBytes())) {
           skippedEdits++;
           continue;
         }
@@ -2394,17 +2369,17 @@ public class HRegion implements HeapSize { // , Writable{
     fs.mkdirs(regionDir);
     HRegion region = HRegion.newHRegion(tableDir,
       new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
-          new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf, null),
+          new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf),
       fs, conf, info, null);
     region.initialize();
     return region;
   }
 
   /**
-   * Convenience method to open a HRegion outside of an HRegionServer context.
+   * Open a Region.
    * @param info Info for region to be opened.
    * @param rootDir Root directory for HBase instance
-   * @param log HLog for region to use. This method will call
+   * @param wal HLog for region to use. This method will call
    * HLog#setSequenceNumber(long) passing the result of the call to
    * HRegion#getMinSequenceId() to ensure the log id is properly kept
    * up.  HRegionStore does this every time it opens a new region.
@@ -2413,8 +2388,29 @@ public class HRegion implements HeapSize { // , Writable{
    *
    * @throws IOException
    */
-  public static HRegion openHRegion(final HRegionInfo info, final Path rootDir,
-    final HLog log, final Configuration conf)
+  public static HRegion openHRegion(final HRegionInfo info, final HLog wal,
+      final Configuration conf)
+  throws IOException {
+    return openHRegion(info, wal, conf, null, null);
+  }
+
+  /**
+   * Open a Region.
+   * @param info Info for region to be opened.
+   * @param wal HLog for region to use. This method will call
+   * HLog#setSequenceNumber(long) passing the result of the call to
+   * HRegion#getMinSequenceId() to ensure the log id is properly kept
+   * up.  HRegionStore does this every time it opens a new region.
+   * @param conf
+   * @param flusher An interface we can request flushes against.
+   * @param reporter An interface we can report progress against.
+   * @return new HRegion
+   *
+   * @throws IOException
+   */
+  public static HRegion openHRegion(final HRegionInfo info, final HLog wal,
+    final Configuration conf, final FlushRequester flusher,
+    final Progressable reporter)
   throws IOException {
     if (LOG.isDebugEnabled()) {
       LOG.debug("Opening region: " + info);
@@ -2422,13 +2418,27 @@ public class HRegion implements HeapSize { // , Writable{
     if (info == null) {
       throw new NullPointerException("Passed region info is null");
     }
-    HRegion r = HRegion.newHRegion(
-        HTableDescriptor.getTableDir(rootDir, info.getTableDesc().getName()),
-        log, FileSystem.get(conf), conf, info, null);
-    long seqid = r.initialize();
-    // If seqid  > current wal seqid, the wal seqid is updated.
-    if (log != null) log.setSequenceNumber(seqid);
-    return r;
+    Path dir = HTableDescriptor.getTableDir(FSUtils.getRootDir(conf),
+      info.getTableDesc().getName());
+    HRegion r = HRegion.newHRegion(dir, wal, FileSystem.get(conf), conf, info,
+      flusher);
+    return r.openHRegion(reporter);
+  }
+
+  /**
+   * Open HRegion.
+   * Calls initialize and sets sequenceid.
+   * @param reporter
+   * @return Returns <code>this</code>
+   * @throws IOException
+   */
+  HRegion openHRegion(final Progressable reporter)
+  throws IOException {
+    long seqid = initialize(reporter);
+    if (this.log != null) {
+      this.log.setSequenceNumber(seqid);
+    }
+    return this;
   }
 
   /**
@@ -3134,7 +3144,7 @@ public class HRegion implements HeapSize { // , Writable{
         + EnvironmentEdgeManager.currentTimeMillis());
     final Path oldLogDir = new Path(c.get("hbase.tmp.dir"),
         HConstants.HREGION_OLDLOGDIR_NAME);
-    final HLog log = new HLog(fs, logdir, oldLogDir, c, null);
+    final HLog log = new HLog(fs, logdir, oldLogDir, c);
     try {
       processTable(fs, tableDir, log, c, majorCompact);
      } finally {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index ac8f9046b94..a65369dcdb5 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -36,13 +36,12 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.NavigableSet;
 import java.util.Random;
 import java.util.Set;
 import java.util.SortedMap;
-import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
-import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
@@ -68,32 +67,31 @@ import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.HServerLoad;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.LeaseListener;
-import org.apache.hadoop.hbase.Leases;
 import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.MasterAddressTracker;
 import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.UnknownRowLockException;
 import org.apache.hadoop.hbase.UnknownScannerException;
 import org.apache.hadoop.hbase.YouAreDeadException;
 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
-import org.apache.hadoop.hbase.HMsg.Type;
-import org.apache.hadoop.hbase.Leases.LeaseStillHeldException;
-import org.apache.hadoop.hbase.client.Action;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.catalog.RootLocationEditor;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.MultiAction;
 import org.apache.hadoop.hbase.client.MultiPut;
 import org.apache.hadoop.hbase.client.MultiPutResponse;
-import org.apache.hadoop.hbase.client.MultiResponse;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Row;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.ServerConnection;
 import org.apache.hadoop.hbase.client.ServerConnectionManager;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
 import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.HBaseRPCErrorHandler;
@@ -101,8 +99,16 @@ import org.apache.hadoop.hbase.ipc.HBaseRPCProtocolVersion;
 import org.apache.hadoop.hbase.ipc.HBaseServer;
 import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException;
+import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler;
+import org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler;
+import org.apache.hadoop.hbase.regionserver.handler.CloseRootHandler;
+import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler;
+import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler;
+import org.apache.hadoop.hbase.regionserver.handler.OpenRootHandler;
 import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.WALObserver;
 import org.apache.hadoop.hbase.replication.regionserver.Replication;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -110,38 +116,29 @@ import org.apache.hadoop.hbase.util.InfoServer;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Sleeper;
 import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.net.DNS;
-import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.EventType;
-import org.apache.zookeeper.Watcher.Event.KeeperState;
+import org.apache.zookeeper.KeeperException;
 
 /**
- * HRegionServer makes a set of HRegions available to clients.  It checks in with
+ * HRegionServer makes a set of HRegions available to clients. It checks in with
  * the HMaster. There are many HRegionServers in a single HBase deployment.
  */
-public class HRegionServer implements HRegionInterface,
-    HBaseRPCErrorHandler, Runnable, Watcher, Stoppable, OnlineRegions {
+public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
+    Runnable, RegionServerServices, Server {
   public static final Log LOG = LogFactory.getLog(HRegionServer.class);
-  private static final HMsg REPORT_EXITING = new HMsg(Type.MSG_REPORT_EXITING);
-  private static final HMsg REPORT_QUIESCED = new HMsg(Type.MSG_REPORT_QUIESCED);
-  private static final HMsg [] EMPTY_HMSG_ARRAY = new HMsg [] {};
 
   // Set when a report to the master comes back with a message asking us to
-  // shutdown.  Also set by call to stop when debugging or running unit tests
-  // of HRegionServer in isolation. We use AtomicBoolean rather than
-  // plain boolean so we can pass a reference to Chore threads.  Otherwise,
-  // Chore threads need to know about the hosting class.
-  protected final AtomicBoolean stopRequested = new AtomicBoolean(false);
+  // shutdown. Also set by call to stop when debugging or running unit tests
+  // of HRegionServer in isolation.
+  protected volatile boolean stopped = false;
 
-  protected final AtomicBoolean quiesced = new AtomicBoolean(false);
-
-  // Go down hard.  Used if file system becomes unavailable and also in
+  // Go down hard. Used if file system becomes unavailable and also in
   // debugging and unit tests.
   protected volatile boolean abortRequested;
 
@@ -159,15 +156,15 @@ public class HRegionServer implements HRegionInterface,
   private Path rootDir;
   private final Random rand = new Random();
 
-  // Key is Bytes.hashCode of region name byte array and the value is HRegion
-  // in both of the maps below.  Use Bytes.mapKey(byte []) generating key for
-  // below maps.
-  protected final Map<Integer, HRegion> onlineRegions =
-    new ConcurrentHashMap<Integer, HRegion>();
+  /**
+   * Map of regions currently being served by this region server. Key is the
+   * encoded region name.
+   */
+  protected final Map<String, HRegion> onlineRegions =
+    new ConcurrentHashMap<String, HRegion>();
 
   protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-  private final LinkedBlockingQueue<HMsg> outboundMsgs =
-    new LinkedBlockingQueue<HMsg>();
+  private final LinkedBlockingQueue<HMsg> outboundMsgs = new LinkedBlockingQueue<HMsg>();
 
   final int numRetries;
   protected final int threadWakeFrequency;
@@ -180,7 +177,7 @@ public class HRegionServer implements HRegionInterface,
   // Remote HMaster
   private HMasterRegionInterface hbaseMaster;
 
-  // Server to handle client requests.  Default access so can be accessed by
+  // Server to handle client requests. Default access so can be accessed by
   // unit tests.
   HBaseServer server;
 
@@ -190,7 +187,7 @@ public class HRegionServer implements HRegionInterface,
   // Request counter
   private volatile AtomicInteger requestCount = new AtomicInteger();
 
-  // Info server.  Default access so can be used by unit tests.  REGIONSERVER
+  // Info server. Default access so can be used by unit tests. REGIONSERVER
   // is name of the webapp and the attribute name used stuffing this instance
   // into web context.
   InfoServer infoServer;
@@ -199,11 +196,11 @@ public class HRegionServer implements HRegionInterface,
   public static final String REGIONSERVER = "regionserver";
 
   /*
-   * Space is reserved in HRS constructor and then released when aborting
-   * to recover from an OOME. See HBASE-706.  TODO: Make this percentage of the
-   * heap or a minimum.
+   * Space is reserved in HRS constructor and then released when aborting to
+   * recover from an OOME. See HBASE-706. TODO: Make this percentage of the heap
+   * or a minimum.
    */
-  private final LinkedList<byte[]> reservedSpace = new LinkedList<byte []>();
+  private final LinkedList<byte[]> reservedSpace = new LinkedList<byte[]>();
 
   private RegionServerMetrics metrics;
 
@@ -213,11 +210,12 @@ public class HRegionServer implements HRegionInterface,
   // Cache flushing
   MemStoreFlusher cacheFlusher;
 
-  /* Check for major compactions.
+  /*
+   * Check for major compactions.
    */
   Chore majorCompactionChecker;
 
-  // HLog and HLog roller.  log is protected rather than private to avoid
+  // HLog and HLog roller. log is protected rather than private to avoid
   // eclipse warning when accessed by inner classes
   protected volatile HLog hlog;
   LogRoller hlogRoller;
@@ -225,123 +223,152 @@ public class HRegionServer implements HRegionInterface,
   // flag set after we're done setting up server threads (used for testing)
   protected volatile boolean isOnline;
 
-  final Map<String, InternalScanner> scanners =
-    new ConcurrentHashMap<String, InternalScanner>();
+  final Map<String, InternalScanner> scanners = new ConcurrentHashMap<String, InternalScanner>();
 
-  private ZooKeeperWrapper zooKeeperWrapper;
+  // zookeeper connection and watcher
+  private ZooKeeperWatcher zooKeeper;
+
+  // master address manager and watcher
+  private MasterAddressTracker masterAddressManager;
+
+  // catalog tracker
+  private CatalogTracker catalogTracker;
+
+  // Cluster Status Tracker
+  private ClusterStatusTracker clusterStatusTracker;
 
   // A sleeper that sleeps for msgInterval.
   private final Sleeper sleeper;
 
   private final long rpcTimeout;
 
-  // Address passed in to constructor.  This is not always the address we run
-  // with.  For example, if passed port is 0, then we are to pick a port.  The
+  // Address passed in to constructor. This is not always the address we run
+  // with. For example, if passed port is 0, then we are to pick a port. The
   // actual address we run with is in the #serverInfo data member.
   private final HServerAddress address;
 
   // The main region server thread.
+  @SuppressWarnings("unused")
   private Thread regionServerThread;
 
   private final String machineName;
 
-  // Replication-related attributes
+  // Instance of the hbase executor service.
+  private ExecutorService service;
+
+  // Replication services. If no replication, this handler will be null.
   private Replication replicationHandler;
-  // End of replication
 
   /**
    * Starts a HRegionServer at the default location
+   *
    * @param conf
    * @throws IOException
+   * @throws InterruptedException 
    */
-  public HRegionServer(Configuration conf) throws IOException {
-    machineName = DNS.getDefaultHost(
-        conf.get("hbase.regionserver.dns.interface","default"),
-        conf.get("hbase.regionserver.dns.nameserver","default"));
-    String addressStr = machineName + ":" +
-      conf.get(HConstants.REGIONSERVER_PORT,
-          Integer.toString(HConstants.DEFAULT_REGIONSERVER_PORT));
-    // This is not necessarily the address we will run with.  The address we
-    // use will be in #serverInfo data member.  For example, we may have been
+  public HRegionServer(Configuration conf) throws IOException, InterruptedException {
+    machineName = DNS.getDefaultHost(conf.get(
+        "hbase.regionserver.dns.interface", "default"), conf.get(
+        "hbase.regionserver.dns.nameserver", "default"));
+    String addressStr = machineName
+        + ":"
+        + conf.get(HConstants.REGIONSERVER_PORT, Integer
+            .toString(HConstants.DEFAULT_REGIONSERVER_PORT));
+    // This is not necessarily the address we will run with. The address we
+    // use will be in #serverInfo data member. For example, we may have been
     // passed a port of 0 which means we should pick some ephemeral port to bind
     // to.
-    address = new HServerAddress(addressStr);
-    LOG.info("My address is " + address);
+    this.address = new HServerAddress(addressStr);
 
-    this.abortRequested = false;
     this.fsOk = true;
     this.conf = conf;
     this.connection = ServerConnectionManager.getConnection(conf);
-
     this.isOnline = false;
 
     // Config'ed params
-    this.numRetries =  conf.getInt("hbase.client.retries.number", 2);
+    this.numRetries = conf.getInt("hbase.client.retries.number", 2);
     this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
         10 * 1000);
     this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 1 * 1000);
 
-    sleeper = new Sleeper(this.msgInterval, this.stopRequested);
+    sleeper = new Sleeper(this.msgInterval, this);
 
     this.maxScannerResultSize = conf.getLong(
-            HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
-            HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
+        HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
+        HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
 
-    // Task thread to process requests from Master
-    this.worker = new Worker();
+    this.numRegionsToReport = conf.getInt(
+        "hbase.regionserver.numregionstoreport", 10);
 
-    this.numRegionsToReport =
-      conf.getInt("hbase.regionserver.numregionstoreport", 10);
+    this.rpcTimeout = conf.getLong(
+        HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
+        HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
 
-    this.rpcTimeout =
-      conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
-          HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
-
-    reinitialize();
+    initialize();
   }
 
   /**
    * Creates all of the state that needs to be reconstructed in case we are
-   * doing a restart. This is shared between the constructor and restart().
-   * Both call it.
+   * doing a restart. This is shared between the constructor and restart(). Both
+   * call it.
+   *
    * @throws IOException
+   * @throws InterruptedException 
    */
-  private void reinitialize() throws IOException {
+  private void initialize() throws IOException, InterruptedException {
     this.abortRequested = false;
-    this.stopRequested.set(false);
+    this.stopped = false;
 
     // Server to handle client requests
-    this.server = HBaseRPC.getServer(this, address.getBindAddress(),
-      address.getPort(), conf.getInt("hbase.regionserver.handler.count", 10),
-      false, conf);
+    this.server = HBaseRPC.getServer(this, address.getBindAddress(), address
+        .getPort(), conf.getInt("hbase.regionserver.handler.count", 10), false,
+        conf);
     this.server.setErrorHandler(this);
     // Address is giving a default IP for the moment. Will be changed after
     // calling the master.
-    this.serverInfo = new HServerInfo(new HServerAddress(
-      new InetSocketAddress(address.getBindAddress(),
-      this.server.getListenerAddress().getPort())), System.currentTimeMillis(),
-      this.conf.getInt("hbase.regionserver.info.port", 60030), machineName);
+    this.serverInfo = new HServerInfo(new HServerAddress(new InetSocketAddress(
+        address.getBindAddress(), this.server.getListenerAddress().getPort())),
+        System.currentTimeMillis(), this.conf.getInt(
+            "hbase.regionserver.info.port", 60030), machineName);
     if (this.serverInfo.getServerAddress() == null) {
-      throw new NullPointerException("Server address cannot be null; " +
-        "hbase-958 debugging");
+      throw new NullPointerException("Server address cannot be null; "
+          + "hbase-958 debugging");
     }
-    reinitializeThreads();
-    reinitializeZooKeeper();
-    int nbBlocks = conf.getInt("hbase.regionserver.nbreservationblocks", 4);
-    for(int i = 0; i < nbBlocks; i++)  {
+    initializeZooKeeper();
+    initializeThreads();
+    int nbBlocks = 0; // TODO: FIX WAS OOME'ing in TESTS ->  conf.getInt("hbase.regionserver.nbreservationblocks", 4);
+    for (int i = 0; i < nbBlocks; i++) {
       reservedSpace.add(new byte[HConstants.DEFAULT_SIZE_RESERVATION_BLOCK]);
     }
   }
 
-  private void reinitializeZooKeeper() throws IOException {
-    zooKeeperWrapper =
-        ZooKeeperWrapper.createInstance(conf, serverInfo.getServerName());
-    zooKeeperWrapper.registerListener(this);
-    watchMasterAddress();
+  private void initializeZooKeeper() throws IOException, InterruptedException {
+    // open connection to zookeeper and set primary watcher
+    zooKeeper = new ZooKeeperWatcher(conf, REGIONSERVER + "-"
+        + serverInfo.getServerName(), this);
+
+    // create the master address manager, register with zk, and start it
+    masterAddressManager = new MasterAddressTracker(zooKeeper, this);
+    masterAddressManager.start();
+
+    // create the catalog tracker and start it
+    this.catalogTracker = new CatalogTracker(this.zooKeeper, this.connection,
+      this, this.conf.getInt("hbase.regionserver.catalog.timeout", -1));
+    catalogTracker.start();
+
+    this.clusterStatusTracker = new ClusterStatusTracker(this.zooKeeper, this);
+    this.clusterStatusTracker.start();
+    this.clusterStatusTracker.blockUntilAvailable();
   }
 
-  private void reinitializeThreads() {
-    this.workerThread = new Thread(worker);
+  /**
+   * @return True if cluster shutdown in progress
+   */
+  private boolean isClusterUp() {
+    return this.clusterStatusTracker.isClusterUp();
+  }
+
+  private void initializeThreads() throws IOException {
 
     // Cache flushing thread.
     this.cacheFlusher = new MemStoreFlusher(conf, this);
@@ -349,202 +376,57 @@ public class HRegionServer implements HRegionInterface,
     // Compaction thread
     this.compactSplitThread = new CompactSplitThread(this);
 
-    // Log rolling thread
-    this.hlogRoller = new LogRoller(this);
-
     // Background thread to check for major compactions; needed if region
-    // has not gotten updates in a while.  Make it run at a lesser frequency.
-    int multiplier = this.conf.getInt(HConstants.THREAD_WAKE_FREQUENCY +
-        ".multiplier", 1000);
+    // has not gotten updates in a while. Make it run at a lesser frequency.
+    int multiplier = this.conf.getInt(HConstants.THREAD_WAKE_FREQUENCY
+        + ".multiplier", 1000);
     this.majorCompactionChecker = new MajorCompactionChecker(this,
-      this.threadWakeFrequency * multiplier,  this.stopRequested);
+        this.threadWakeFrequency * multiplier, this);
 
-    this.leases = new Leases(
-        (int) conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
-            HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD),
+    this.leases = new Leases((int) conf.getLong(
+        HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
+        HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD),
         this.threadWakeFrequency);
   }
 
   /**
-   * We register ourselves as a watcher on the master address ZNode. This is
-   * called by ZooKeeper when we get an event on that ZNode. When this method
-   * is called it means either our master has died, or a new one has come up.
-   * Either way we need to update our knowledge of the master.
-   * @param event WatchedEvent from ZooKeeper.
-   */
-  public void process(WatchedEvent event) {
-    EventType type = event.getType();
-    KeeperState state = event.getState();
-    LOG.info("Got ZooKeeper event, state: " + state + ", type: " +
-      type + ", path: " + event.getPath());
-
-    // Ignore events if we're shutting down.
-    if (this.stopRequested.get()) {
-      LOG.debug("Ignoring ZooKeeper event while shutting down");
-      return;
-    }
-
-    if (state == KeeperState.Expired) {
-      LOG.error("ZooKeeper session expired");
-      boolean restart =
-        this.conf.getBoolean("hbase.regionserver.restart.on.zk.expire", false);
-      if (restart) {
-        restart();
-      } else {
-        abort("ZooKeeper session expired");
-      }
-    } else if (type == EventType.NodeDeleted) {
-      watchMasterAddress();
-    } else if (type == EventType.NodeCreated) {
-      getMaster();
-
-      // ZooKeeper watches are one time only, so we need to re-register our watch.
-      watchMasterAddress();
-    }
-  }
-
-  private void watchMasterAddress() {
-    while (!stopRequested.get() && !zooKeeperWrapper.watchMasterAddress(this)) {
-      LOG.warn("Unable to set watcher on ZooKeeper master address. Retrying.");
-      sleeper.sleep();
-    }
-  }
-
-  private void restart() {
-    abort("Restarting region server");
-    Threads.shutdown(regionServerThread);
-    boolean done = false;
-    while (!done) {
-      try {
-        reinitialize();
-        done = true;
-      } catch (IOException e) {
-        LOG.debug("Error trying to reinitialize ZooKeeper", e);
-      }
-    }
-    Thread t = new Thread(this);
-    String name = regionServerThread.getName();
-    t.setName(name);
-    t.start();
-  }
-
-  /** @return ZooKeeperWrapper used by RegionServer. */
-  public ZooKeeperWrapper getZooKeeperWrapper() {
-    return zooKeeperWrapper;
-  }
-
-  /**
-   * The HRegionServer sticks in this loop until closed. It repeatedly checks
-   * in with the HMaster, sending heartbeats & reports, and receiving HRegion
+   * The HRegionServer sticks in this loop until closed. It repeatedly checks in
+   * with the HMaster, sending heartbeats & reports, and receiving HRegion
    * load/unload instructions.
    */
   public void run() {
-    regionServerThread = Thread.currentThread();
-    boolean quiesceRequested = false;
+    this.regionServerThread = Thread.currentThread();
+    boolean calledCloseUserRegions = false;
     try {
-      MapWritable w = null;
-      while (!stopRequested.get()) {
-        w = reportForDuty();
-        if (w != null) {
-          init(w);
-          break;
-        }
-        sleeper.sleep();
-        LOG.warn("No response from master on reportForDuty. Sleeping and " +
-          "then trying again.");
+      while (!this.stopped) {
+        if (tryReportForDuty()) break;
       }
-      List<HMsg> outboundMessages = new ArrayList<HMsg>();
       long lastMsg = 0;
-      // Now ask master what it wants us to do and tell it what we have done
-      for (int tries = 0; !stopRequested.get() && isHealthy();) {
-        // Try to get the root region location from the master.
-        if (!haveRootRegion.get()) {
-          HServerAddress rootServer = zooKeeperWrapper.readRootRegionLocation();
-          if (rootServer != null) {
-            // By setting the root region location, we bypass the wait imposed on
-            // HTable for all regions being assigned.
-            this.connection.setRootRegionLocation(
-                new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, rootServer));
-            haveRootRegion.set(true);
+      List<HMsg> outboundMessages = new ArrayList<HMsg>();
+      // The main run loop.
+      for (int tries = 0; !this.stopped && isHealthy();) {
+        if (!isClusterUp()) {
+          if (this.onlineRegions.isEmpty()) {
+            stop("Exiting; cluster shutdown set and not carrying any regions");
+          } else if (!calledCloseUserRegions) {
+            closeUserRegions(this.abortRequested);
+            calledCloseUserRegions = true;
           }
         }
+        // Try to get the root region location from zookeeper.
+        checkRootRegionLocation();
         long now = System.currentTimeMillis();
         // Drop into the send loop if msgInterval has elapsed or if something
-        // to send.  If we fail talking to the master, then we'll sleep below
+        // to send. If we fail talking to the master, then we'll sleep below
         // on poll of the outboundMsgs blockingqueue.
         if ((now - lastMsg) >= msgInterval || !outboundMessages.isEmpty()) {
           try {
             doMetrics();
-            MemoryUsage memory =
-              ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
-            HServerLoad hsl = new HServerLoad(requestCount.get(),
-              (int)(memory.getUsed()/1024/1024),
-              (int)(memory.getMax()/1024/1024));
-            for (HRegion r: onlineRegions.values()) {
-              hsl.addRegionInfo(createRegionLoad(r));
-            }
-            this.serverInfo.setLoad(hsl);
-            this.requestCount.set(0);
-            addOutboundMsgs(outboundMessages);
-            HMsg msgs[] = this.hbaseMaster.regionServerReport(
-              serverInfo, outboundMessages.toArray(EMPTY_HMSG_ARRAY),
-              getMostLoadedRegions());
+            tryRegionServerReport(outboundMessages);
             lastMsg = System.currentTimeMillis();
-            updateOutboundMsgs(outboundMessages);
-            outboundMessages.clear();
-            if (this.quiesced.get() && onlineRegions.size() == 0) {
-              // We've just told the master we're exiting because we aren't
-              // serving any regions. So set the stop bit and exit.
-              LOG.info("Server quiesced and not serving any regions. " +
-                "Starting shutdown");
-              stopRequested.set(true);
-              this.outboundMsgs.clear();
-              continue;
-            }
-
-            // Queue up the HMaster's instruction stream for processing
-            boolean restart = false;
-            for(int i = 0;
-                !restart && !stopRequested.get() && i < msgs.length;
-                i++) {
-              LOG.info(msgs[i].toString());
-              this.connection.unsetRootRegionLocation();
-              switch(msgs[i].getType()) {
-
-              case MSG_REGIONSERVER_STOP:
-                stopRequested.set(true);
-                break;
-
-              case MSG_REGIONSERVER_QUIESCE:
-                if (!quiesceRequested) {
-                  try {
-                    toDo.put(new ToDoEntry(msgs[i]));
-                  } catch (InterruptedException e) {
-                    throw new RuntimeException("Putting into msgQueue was " +
-                        "interrupted.", e);
-                  }
-                  quiesceRequested = true;
-                }
-                break;
-
-              default:
-                if (fsOk) {
-                  try {
-                    toDo.put(new ToDoEntry(msgs[i]));
-                  } catch (InterruptedException e) {
-                    throw new RuntimeException("Putting into msgQueue was " +
-                        "interrupted.", e);
-                  }
-                }
-              }
-            }
             // Reset tries count if we had a successful transaction.
             tries = 0;
-
-            if (restart || this.stopRequested.get()) {
-              toDo.clear();
-              continue;
-            }
+            if (this.stopped) continue;
           } catch (Exception e) { // FindBugs REC_CATCH_EXCEPTION
             // Two special exceptions could be printed out here,
             // PleaseHoldException and YouAreDeadException
@@ -560,24 +442,18 @@ public class HRegionServer implements HRegionInterface,
               // Check filesystem every so often.
               checkFileSystem();
             }
-            if (this.stopRequested.get()) {
-              LOG.info("Stop requested, clearing toDo despite exception");
-              toDo.clear();
+            if (this.stopped) {
               continue;
             }
             LOG.warn("Attempt=" + tries, e);
             // No point retrying immediately; this is probably connection to
-            // master issue.  Doing below will cause us to sleep.
+            // master issue. Doing below will cause us to sleep.
             lastMsg = System.currentTimeMillis();
           }
         }
         now = System.currentTimeMillis();
-        HMsg msg = this.outboundMsgs.poll((msgInterval - (now - lastMsg)),
-          TimeUnit.MILLISECONDS);
-        // If we got something, add it to list of things to send.
+        HMsg msg = this.outboundMsgs.poll((msgInterval - (now - lastMsg)), TimeUnit.MILLISECONDS);
         if (msg != null) outboundMessages.add(msg);
-        // Do some housekeeping before going back around
-        housekeeping();
       } // for
     } catch (Throwable t) {
       if (!checkOOME(t)) {
@@ -585,7 +461,6 @@ public class HRegionServer implements HRegionInterface,
       }
     }
     this.leases.closeAfterLeasesExpire();
-    this.worker.stop();
     this.server.stop();
     if (this.infoServer != null) {
       LOG.info("Stopping infoServer");
@@ -596,11 +471,13 @@ public class HRegionServer implements HRegionInterface,
       }
     }
     // Send cache a shutdown.
-    LruBlockCache c = (LruBlockCache)StoreFile.getBlockCache(this.conf);
-    if (c != null) c.shutdown();
+    LruBlockCache c = (LruBlockCache) StoreFile.getBlockCache(this.conf);
+    if (c != null) {
+      c.shutdown();
+    }
 
     // Send interrupts to wake up threads if sleeping so they notice shutdown.
-    // TODO: Should we check they are alive?  If OOME could have exited already
+    // TODO: Should we check they are alive? If OOME could have exited already
     cacheFlusher.interruptIfNecessary();
     compactSplitThread.interruptIfNecessary();
     hlogRoller.interruptIfNecessary();
@@ -610,48 +487,17 @@ public class HRegionServer implements HRegionInterface,
       // Just skip out w/o closing regions.
     } else if (abortRequested) {
       if (this.fsOk) {
-        // Only try to clean up if the file system is available
-        try {
-          if (this.hlog != null) {
-            this.hlog.close();
-            LOG.info("On abort, closed hlog");
-          }
-        } catch (Throwable e) {
-          LOG.error("Unable to close log in abort",
-            RemoteExceptionHandler.checkThrowable(e));
-        }
-        closeAllRegions(); // Don't leave any open file handles
+        closeAllRegions(abortRequested); // Don't leave any open file handles
+        closeWAL(false);
       }
       LOG.info("aborting server at: " + this.serverInfo.getServerName());
     } else {
-      ArrayList<HRegion> closedRegions = closeAllRegions();
-      try {
-        if (this.hlog != null) {
-          hlog.closeAndDelete();
-        }
-      } catch (Throwable e) {
-        LOG.error("Close and delete failed",
-          RemoteExceptionHandler.checkThrowable(e));
-      }
-      try {
-        HMsg[] exitMsg = new HMsg[closedRegions.size() + 1];
-        exitMsg[0] = REPORT_EXITING;
-        // Tell the master what regions we are/were serving
-        int i = 1;
-        for (HRegion region: closedRegions) {
-          exitMsg[i++] = new HMsg(HMsg.Type.MSG_REPORT_CLOSE,
-              region.getRegionInfo());
-        }
-
-        LOG.info("telling master that region server is shutting down at: " +
-            serverInfo.getServerName());
-        hbaseMaster.regionServerReport(serverInfo, exitMsg, (HRegionInfo[])null);
-      } catch (Throwable e) {
-        LOG.warn("Failed to send exiting message to master: ",
-          RemoteExceptionHandler.checkThrowable(e));
-      }
+      closeAllRegions(abortRequested);
+      closeWAL(true);
+      closeAllScanners();
       LOG.info("stopping server at: " + this.serverInfo.getServerName());
     }
+    waitOnAllRegionsToClose();
 
     // Make sure the proxy is down.
     if (this.hbaseMaster != null) {
@@ -659,15 +505,113 @@ public class HRegionServer implements HRegionInterface,
       this.hbaseMaster = null;
     }
 
+    this.zooKeeper.close();
+
     if (!killed) {
-      this.zooKeeperWrapper.close();
       join();
     }
     LOG.info(Thread.currentThread().getName() + " exiting");
   }
 
+  /**
+   * Wait on regions close.
+   */
+  private void waitOnAllRegionsToClose() {
+    // Wait till all regions are closed before going out.
+    int lastCount = -1;
+    while (!this.onlineRegions.isEmpty()) {
+      int count = this.onlineRegions.size();
+      // Only print a message if the count of regions has changed.
+      if (count != lastCount) {
+        lastCount = count;
+        LOG.info("Waiting on " + count + " regions to close");
+        // Only print out regions still closing if a small number else will
+        // swamp the log.
+        if (count < 10) {
+          LOG.debug(this.onlineRegions);
+        }
+      }
+      Threads.sleep(1000);
+    }
+  }
+
+  List<HMsg> tryRegionServerReport(final List<HMsg> outboundMessages)
+  throws IOException {
+    this.serverInfo.setLoad(buildServerLoad());
+    this.requestCount.set(0);
+    addOutboundMsgs(outboundMessages);
+    HMsg [] msgs = this.hbaseMaster.regionServerReport(this.serverInfo,
+      outboundMessages.toArray(HMsg.EMPTY_HMSG_ARRAY),
+      getMostLoadedRegions());
+    updateOutboundMsgs(outboundMessages);
+    outboundMessages.clear();
+
+    for (int i = 0; !this.stopped && msgs != null && i < msgs.length; i++) {
+      LOG.info(msgs[i].toString());
+      // Intercept stop regionserver messages
+      if (msgs[i].getType().equals(HMsg.Type.STOP_REGIONSERVER)) {
+        stop("Received " + msgs[i]);
+        continue;
+      }
+      this.connection.unsetRootRegionLocation();
+      LOG.warn("NOT PROCESSING " + msgs[i] + " -- WHY IS MASTER SENDING IT TO US?");
+    }
+    return outboundMessages;
+  }
+
+  private HServerLoad buildServerLoad() {
+    MemoryUsage memory = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+    HServerLoad hsl = new HServerLoad(requestCount.get(),
+      (int)(memory.getUsed() / 1024 / 1024),
+      (int) (memory.getMax() / 1024 / 1024));
+    for (HRegion r : this.onlineRegions.values()) {
+      hsl.addRegionInfo(createRegionLoad(r));
+    }
+    return hsl;
+  }
+
+  private void checkRootRegionLocation() throws InterruptedException {
+    if (this.haveRootRegion.get()) return;
+    HServerAddress rootServer = catalogTracker.getRootLocation();
+    if (rootServer != null) {
+      // By setting the root region location, we bypass the wait imposed on
+      // HTable for all regions being assigned.
+      HRegionLocation hrl =
+        new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, rootServer);
+      this.connection.setRootRegionLocation(hrl);
+      this.haveRootRegion.set(true);
+    }
+  }
+
+  private void closeWAL(final boolean delete) {
+    try {
+      if (this.hlog != null) {
+        if (delete) {
+          hlog.closeAndDelete();
+        } else {
+          hlog.close();
+        }
+      }
+    } catch (Throwable e) {
+      LOG.error("Close and delete failed", RemoteExceptionHandler.checkThrowable(e));
+    }
+  }
+
+  private void closeAllScanners() {
+    // Close any outstanding scanners. Means they'll get an UnknownScanner
+    // exception next time they come in.
+    for (Map.Entry<String, InternalScanner> e : this.scanners.entrySet()) {
+      try {
+        e.getValue().close();
+      } catch (IOException ioe) {
+        LOG.warn("Closing scanner " + e.getKey(), ioe);
+      }
+    }
+  }
+
   /*
    * Add to the passed <code>msgs</code> messages to pass to the master.
+   *
    * @param msgs Current outboundMsgs array; we'll add messages to this List.
    */
   private void addOutboundMsgs(final List<HMsg> msgs) {
@@ -675,8 +619,8 @@ public class HRegionServer implements HRegionInterface,
       this.outboundMsgs.drainTo(msgs);
       return;
     }
-    OUTER: for (HMsg m: this.outboundMsgs) {
-      for (HMsg mm: msgs) {
+    OUTER: for (HMsg m : this.outboundMsgs) {
+      for (HMsg mm : msgs) {
         // Be careful don't add duplicates.
         if (mm.equals(m)) {
           continue OUTER;
@@ -688,12 +632,15 @@ public class HRegionServer implements HRegionInterface,
 
   /*
    * Remove from this.outboundMsgs those messsages we sent the master.
+   *
    * @param msgs Messages we sent the master.
    */
   private void updateOutboundMsgs(final List<HMsg> msgs) {
-    if (msgs.isEmpty()) return;
-    for (HMsg m: this.outboundMsgs) {
-      for (HMsg mm: msgs) {
+    if (msgs.isEmpty()) {
+      return;
+    }
+    for (HMsg m : this.outboundMsgs) {
+      for (HMsg mm : msgs) {
         if (mm.equals(m)) {
           this.outboundMsgs.remove(m);
           break;
@@ -704,11 +651,12 @@ public class HRegionServer implements HRegionInterface,
 
   /*
    * Run init. Sets up hlog and starts up all server threads.
+   *
    * @param c Extra configuration.
    */
-  protected void init(final MapWritable c) throws IOException {
+  protected void handleReportForDutyResponse(final MapWritable c) throws IOException {
     try {
-      for (Map.Entry<Writable, Writable> e: c.entrySet()) {
+      for (Map.Entry<Writable, Writable> e : c.entrySet()) {
         String key = e.getKey().toString();
         String value = e.getValue().toString();
         if (LOG.isDebugEnabled()) {
@@ -719,17 +667,19 @@ public class HRegionServer implements HRegionInterface,
       // Master may have sent us a new address with the other configs.
       // Update our address in this case. See HBASE-719
       String hra = conf.get("hbase.regionserver.address");
-      // TODO: The below used to be this.address != null.  Was broken by what
+      // TODO: The below used to be this.address != null. Was broken by what
       // looks like a mistake in:
       //
-      // HBASE-1215 migration; metautils scan of meta region was broken; wouldn't see first row
+      // HBASE-1215 migration; metautils scan of meta region was broken;
+      // wouldn't see first row
       // ------------------------------------------------------------------------
-      // r796326 | stack | 2009-07-21 07:40:34 -0700 (Tue, 21 Jul 2009) | 38 lines
+      // r796326 | stack | 2009-07-21 07:40:34 -0700 (Tue, 21 Jul 2009) | 38
+      // lines
       if (hra != null) {
-        HServerAddress hsa = new HServerAddress (hra,
-          this.serverInfo.getServerAddress().getPort());
-        LOG.info("Master passed us address to use. Was=" +
-          this.serverInfo.getServerAddress() + ", Now=" + hra);
+        HServerAddress hsa = new HServerAddress(hra, this.serverInfo
+            .getServerAddress().getPort());
+        LOG.info("Master passed us address to use. Was="
+            + this.serverInfo.getServerAddress() + ", Now=" + hsa.toString());
         this.serverInfo.setServerAddress(hsa);
       }
       
@@ -741,7 +691,7 @@ public class HRegionServer implements HRegionInterface,
       }
       
       // Master sent us hbase.rootdir to use. Should be fully qualified
-      // path with file system specification included.  Set 'fs.defaultFS'
+      // path with file system specification included. Set 'fs.defaultFS'
       // to match the filesystem on hbase.rootdir else underlying hadoop hdfs
       // accessors will be going against wrong filesystem (unless all is set
       // to defaults).
@@ -749,22 +699,24 @@ public class HRegionServer implements HRegionInterface,
       // Get fs instance used by this RS
       this.fs = FileSystem.get(this.conf);
       this.rootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
-      this.hlog = setupHLog();
+      this.hlog = setupWALAndReplication();
       // Init in here rather than in constructor after thread name has been set
       this.metrics = new RegionServerMetrics();
       startServiceThreads();
       isOnline = true;
     } catch (Throwable e) {
       this.isOnline = false;
-      this.stopRequested.set(true);
+      stop("Failed initialization");
       throw convertThrowableToIOE(cleanup(e, "Failed init"),
-        "Region server startup failed");
+          "Region server startup failed");
     }
   }
 
   /*
    * @param r Region to get RegionLoad for.
+   *
    * @return RegionLoad instance.
+   *
    * @throws IOException
    */
   private HServerLoad.RegionLoad createRegionLoad(final HRegion r) {
@@ -772,35 +724,35 @@ public class HRegionServer implements HRegionInterface,
     int stores = 0;
     int storefiles = 0;
     int storefileSizeMB = 0;
-    int memstoreSizeMB = (int)(r.memstoreSize.get()/1024/1024);
+    int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024);
     int storefileIndexSizeMB = 0;
     synchronized (r.stores) {
       stores += r.stores.size();
-      for (Store store: r.stores.values()) {
+      for (Store store : r.stores.values()) {
         storefiles += store.getStorefilesCount();
-        storefileSizeMB +=
-          (int)(store.getStorefilesSize()/1024/1024);
-        storefileIndexSizeMB +=
-          (int)(store.getStorefilesIndexSize()/1024/1024);
+        storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
+        storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024);
       }
     }
     return new HServerLoad.RegionLoad(name, stores, storefiles,
-      storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB);
+        storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB);
   }
 
   /**
-   * @param regionName
+   * @param encodedRegionName
    * @return An instance of RegionLoad.
    * @throws IOException
    */
-  public HServerLoad.RegionLoad createRegionLoad(final byte [] regionName) {
-    return createRegionLoad(this.onlineRegions.get(Bytes.mapKey(regionName)));
+  public HServerLoad.RegionLoad createRegionLoad(final String encodedRegionName) {
+    return createRegionLoad(this.onlineRegions.get(encodedRegionName));
   }
 
   /*
-   * Cleanup after Throwable caught invoking method.  Converts <code>t</code>
-   * to IOE if it isn't already.
+   * Cleanup after Throwable caught invoking method. Converts <code>t</code> to
+   * IOE if it isn't already.
+   *
    * @param t Throwable
+   *
    * @return Throwable converted to an IOE; methods can only let out IOEs.
    */
   private Throwable cleanup(final Throwable t) {
@@ -808,10 +760,13 @@ public class HRegionServer implements HRegionInterface,
   }
 
   /*
-   * Cleanup after Throwable caught invoking method.  Converts <code>t</code>
-   * to IOE if it isn't already.
+   * Cleanup after Throwable caught invoking method. Converts <code>t</code> to
+   * IOE if it isn't already.
+   *
    * @param t Throwable
-   * @param msg Message to log in error.  Can be null.
+   *
+   * @param msg Message to log in error. Can be null.
+   *
    * @return Throwable converted to an IOE; methods can only let out IOEs.
    */
   private Throwable cleanup(final Throwable t, final String msg) {
@@ -833,6 +788,7 @@ public class HRegionServer implements HRegionInterface,
 
   /*
    * @param t
+   *
    * @return Make <code>t</code> an IOE if it isn't already.
    */
   private IOException convertThrowableToIOE(final Throwable t) {
@@ -841,36 +797,38 @@ public class HRegionServer implements HRegionInterface,
 
   /*
    * @param t
+   *
    * @param msg Message to put in new IOE if passed <code>t</code> is not an IOE
+   *
    * @return Make <code>t</code> an IOE if it isn't already.
    */
-  private IOException convertThrowableToIOE(final Throwable t,
-      final String msg) {
-    return (t instanceof IOException? (IOException)t:
-      msg == null || msg.length() == 0?
-        new IOException(t): new IOException(msg, t));
+  private IOException convertThrowableToIOE(final Throwable t, final String msg) {
+    return (t instanceof IOException ? (IOException) t : msg == null
+        || msg.length() == 0 ? new IOException(t) : new IOException(msg, t));
   }
+
   /*
    * Check if an OOME and if so, call abort.
+   *
    * @param e
+   *
    * @return True if we OOME'd and are aborting.
    */
   public boolean checkOOME(final Throwable e) {
     boolean stop = false;
-    if (e instanceof OutOfMemoryError ||
-      (e.getCause() != null && e.getCause() instanceof OutOfMemoryError) ||
-      (e.getMessage() != null &&
-        e.getMessage().contains("java.lang.OutOfMemoryError"))) {
+    if (e instanceof OutOfMemoryError
+        || (e.getCause() != null && e.getCause() instanceof OutOfMemoryError)
+        || (e.getMessage() != null && e.getMessage().contains(
+            "java.lang.OutOfMemoryError"))) {
       abort("OutOfMemoryError, aborting", e);
       stop = true;
     }
     return stop;
   }
 
-
   /**
-   * Checks to see if the file system is still accessible.
-   * If not, sets abortRequested and stopRequested
+   * Checks to see if the file system is still accessible. If not, sets
+   * abortRequested and stopRequested
    *
    * @return false if file system is not available
    */
@@ -893,23 +851,21 @@ public class HRegionServer implements HRegionInterface,
   private static class MajorCompactionChecker extends Chore {
     private final HRegionServer instance;
 
-    MajorCompactionChecker(final HRegionServer h,
-        final int sleepTime, final AtomicBoolean stopper) {
-      super("MajorCompactionChecker", sleepTime, stopper);
+    MajorCompactionChecker(final HRegionServer h, final int sleepTime,
+        final Stoppable stopper) {
+      super("MajorCompactionChecker", sleepTime, h);
       this.instance = h;
       LOG.info("Runs every " + sleepTime + "ms");
     }
 
     @Override
     protected void chore() {
-      Set<Integer> keys = this.instance.onlineRegions.keySet();
-      for (Integer i: keys) {
-        HRegion r = this.instance.onlineRegions.get(i);
+      for (HRegion r : this.instance.onlineRegions.values()) {
         try {
           if (r != null && r.isMajorCompaction()) {
-            // Queue a compaction.  Will recognize if major is needed.
-            this.instance.compactSplitThread.
-              compactionRequested(r, getName() + " requests major compaction");
+            // Queue a compaction. Will recognize if major is needed.
+            this.instance.compactSplitThread.requestCompaction(r, getName()
+                + " requests major compaction");
           }
         } catch (IOException e) {
           LOG.warn("Failed major compaction check on " + r, e);
@@ -919,40 +875,75 @@ public class HRegionServer implements HRegionInterface,
   }
 
   /**
-   * Report the status of the server. A server is online once all the startup
-   * is completed (setting up filesystem, starting service threads, etc.). This
+   * Report the status of the server. A server is online once all the startup is
+   * completed (setting up filesystem, starting service threads, etc.). This
    * method is designed mostly to be useful in tests.
+   *
    * @return true if online, false if not.
    */
   public boolean isOnline() {
     return isOnline;
   }
 
-  private HLog setupHLog() throws IOException {
+  /**
+   * Setup WAL log and replication if enabled.
+   * Replication setup is done in here because it wants to be hooked up to WAL.
+   * @return A WAL instance.
+   * @throws IOException
+   */
+  private HLog setupWALAndReplication() throws IOException {
     final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
     Path logdir = new Path(rootDir, HLog.getHLogDirectoryName(this.serverInfo));
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Log dir " + logdir);
+      LOG.debug("logdir=" + logdir);
     }
-    if (fs.exists(logdir)) {
-      throw new RegionServerRunningException("region server already " +
-        "running at " + this.serverInfo.getServerName() +
-        " because logdir " + logdir.toString() + " exists");
+    if (this.fs.exists(logdir)) {
+      throw new RegionServerRunningException("Region server already "
+          + "running at " + this.serverInfo.getServerName()
+          + " because logdir " + logdir.toString() + " exists");
     }
-    this.replicationHandler = new Replication(this.conf,this.serverInfo,
-        this.fs, logdir, oldLogDir, stopRequested);
-    HLog log = instantiateHLog(logdir, oldLogDir);
-    this.replicationHandler.addLogEntryVisitor(log);
-    return log;
+
+    // Instantiate replication manager if replication enabled.  Pass it the
+    // log directories.
+    try {
+      this.replicationHandler = Replication.isReplication(this.conf)?
+        new Replication(this, this.fs, logdir, oldLogDir): null;
+    } catch (KeeperException e) {
+      throw new IOException("Failed replication handler create", e);
+    }
+    return instantiateHLog(logdir, oldLogDir);
   }
 
-  // instantiate
+  /**
+   * Called by {@link #setupWALAndReplication()} creating WAL instance.
+   * @param logdir
+   * @param oldLogDir
+   * @return WAL instance.
+   * @throws IOException
+   */
   protected HLog instantiateHLog(Path logdir, Path oldLogDir) throws IOException {
-    return new HLog(this.fs, logdir, oldLogDir, this.conf, this.hlogRoller,
-      this.replicationHandler.getReplicationManager(),
-        this.serverInfo.getServerAddress().toString());
+    return new HLog(this.fs, logdir, oldLogDir, this.conf,
+      getWALActionListeners(), this.serverInfo.getServerAddress().toString());
   }
 
+  /**
+   * Called by {@link #instantiateHLog(Path, Path)} setting up WAL instance.
+   * Add any {@link WALObserver}s you want inserted before WAL startup.
+   * @return List of WALActionsListener that will be passed in to
+   * {@link HLog} on construction.
+   */
+  protected List<WALObserver> getWALActionListeners() {
+    List<WALObserver> listeners = new ArrayList<WALObserver>();
+    // Log roller.
+    this.hlogRoller = new LogRoller(this, this);
+    listeners.add(this.hlogRoller);
+    if (this.replicationHandler != null) {
+      listeners = new ArrayList<WALObserver>();
+      // Replication handler is an implementation of WALActionsListener.
+      listeners.add(this.replicationHandler);
+    }
+    return listeners;
+  }
 
   protected LogRoller getLogRoller() {
     return hlogRoller;
@@ -973,19 +964,19 @@ public class HRegionServer implements HRegionInterface,
     this.metrics.regions.set(this.onlineRegions.size());
     this.metrics.incrementRequests(this.requestCount.get());
     // Is this too expensive every three seconds getting a lock on onlineRegions
-    // and then per store carried?  Can I make metrics be sloppier and avoid
+    // and then per store carried? Can I make metrics be sloppier and avoid
     // the synchronizations?
     int stores = 0;
     int storefiles = 0;
     long memstoreSize = 0;
     long storefileIndexSize = 0;
     synchronized (this.onlineRegions) {
-      for (Map.Entry<Integer, HRegion> e: this.onlineRegions.entrySet()) {
+      for (Map.Entry<String, HRegion> e : this.onlineRegions.entrySet()) {
         HRegion r = e.getValue();
         memstoreSize += r.memstoreSize.get();
         synchronized (r.stores) {
           stores += r.stores.size();
-          for(Map.Entry<byte [], Store> ee: r.stores.entrySet()) {
+          for (Map.Entry<byte[], Store> ee : r.stores.entrySet()) {
             Store store = ee.getValue();
             storefiles += store.getStorefilesCount();
             storefileIndexSize += store.getStorefilesIndexSize();
@@ -995,12 +986,13 @@ public class HRegionServer implements HRegionInterface,
     }
     this.metrics.stores.set(stores);
     this.metrics.storefiles.set(storefiles);
-    this.metrics.memstoreSizeMB.set((int)(memstoreSize/(1024*1024)));
-    this.metrics.storefileIndexSizeMB.set((int)(storefileIndexSize/(1024*1024)));
-    this.metrics.compactionQueueSize.set(compactSplitThread.
-      getCompactionQueueSize());
+    this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024)));
+    this.metrics.storefileIndexSizeMB
+        .set((int) (storefileIndexSize / (1024 * 1024)));
+    this.metrics.compactionQueueSize.set(compactSplitThread
+        .getCompactionQueueSize());
 
-    LruBlockCache lruBlockCache = (LruBlockCache)StoreFile.getBlockCache(conf);
+    LruBlockCache lruBlockCache = (LruBlockCache) StoreFile.getBlockCache(conf);
     if (lruBlockCache != null) {
       this.metrics.blockCacheCount.set(lruBlockCache.size());
       this.metrics.blockCacheFree.set(lruBlockCache.getFreeSize());
@@ -1021,14 +1013,14 @@ public class HRegionServer implements HRegionInterface,
   /*
    * Start maintanence Threads, Server, Worker and lease checker threads.
    * Install an UncaughtExceptionHandler that calls abort of RegionServer if we
-   * get an unhandled exception.  We cannot set the handler on all threads.
-   * Server's internal Listener thread is off limits.  For Server, if an OOME,
-   * it waits a while then retries.  Meantime, a flush or a compaction that
-   * tries to run should trigger same critical condition and the shutdown will
-   * run.  On its way out, this server will shut down Server.  Leases are sort
-   * of inbetween. It has an internal thread that while it inherits from
-   * Chore, it keeps its own internal stop mechanism so needs to be stopped
-   * by this hosting server.  Worker logs the exception and exits.
+   * get an unhandled exception. We cannot set the handler on all threads.
+   * Server's internal Listener thread is off limits. For Server, if an OOME, it
+   * waits a while then retries. Meantime, a flush or a compaction that tries to
+   * run should trigger same critical condition and the shutdown will run. On
+   * its way out, this server will shut down Server. Leases are sort of
+   * inbetween. It has an internal thread that while it inherits from Chore, it
+   * keeps its own internal stop mechanism so needs to be stopped by this
+   * hosting server. Worker logs the exception and exits.
    */
   private void startServiceThreads() throws IOException {
     String n = Thread.currentThread().getName();
@@ -1037,17 +1029,31 @@ public class HRegionServer implements HRegionInterface,
         abort("Uncaught exception in service thread " + t.getName(), e);
       }
     };
-    Threads.setDaemonThreadRunning(this.hlogRoller, n + ".logRoller",
-        handler);
+
+    // Start executor services
+    this.service = new ExecutorService(getServerName());
+    this.service.startExecutorService(ExecutorType.RS_OPEN_REGION,
+      conf.getInt("hbase.regionserver.executor.openregion.threads", 5));
+    this.service.startExecutorService(ExecutorType.RS_OPEN_ROOT,
+      conf.getInt("hbase.regionserver.executor.openroot.threads", 1));
+    this.service.startExecutorService(ExecutorType.RS_OPEN_META,
+      conf.getInt("hbase.regionserver.executor.openmeta.threads", 1));
+    this.service.startExecutorService(ExecutorType.RS_CLOSE_REGION,
+      conf.getInt("hbase.regionserver.executor.closeregion.threads", 5));
+    this.service.startExecutorService(ExecutorType.RS_CLOSE_ROOT,
+      conf.getInt("hbase.regionserver.executor.closeroot.threads", 1));
+    this.service.startExecutorService(ExecutorType.RS_CLOSE_META,
+      conf.getInt("hbase.regionserver.executor.closemeta.threads", 1));
+
+    Threads.setDaemonThreadRunning(this.hlogRoller, n + ".logRoller", handler);
     Threads.setDaemonThreadRunning(this.cacheFlusher, n + ".cacheFlusher",
-      handler);
+        handler);
     Threads.setDaemonThreadRunning(this.compactSplitThread, n + ".compactor",
         handler);
-    Threads.setDaemonThreadRunning(this.workerThread, n + ".worker", handler);
-    Threads.setDaemonThreadRunning(this.majorCompactionChecker,
-        n + ".majorCompactionChecker", handler);
+    Threads.setDaemonThreadRunning(this.majorCompactionChecker, n
+        + ".majorCompactionChecker", handler);
 
-    // Leases is not a Thread. Internally it runs a daemon thread.  If it gets
+    // Leases is not a Thread. Internally it runs a daemon thread. If it gets
     // an unhandled exception, it will just exit.
     this.leases.setName(n + ".leaseChecker");
     this.leases.start();
@@ -1055,7 +1061,8 @@ public class HRegionServer implements HRegionInterface,
     int port = this.conf.getInt("hbase.regionserver.info.port", 60030);
     // -1 is for disabling info server
     if (port >= 0) {
-      String addr = this.conf.get("hbase.regionserver.info.bindAddress", "0.0.0.0");
+      String addr = this.conf.get("hbase.regionserver.info.bindAddress",
+          "0.0.0.0");
       // check if auto port bind enabled
       boolean auto = this.conf.getBoolean("hbase.regionserver.info.port.auto",
           false);
@@ -1066,7 +1073,7 @@ public class HRegionServer implements HRegionInterface,
           this.infoServer.start();
           break;
         } catch (BindException e) {
-          if (!auto){
+          if (!auto) {
             // auto bind disabled throw BindException
             throw e;
           }
@@ -1081,13 +1088,15 @@ public class HRegionServer implements HRegionInterface,
       }
     }
 
-    this.replicationHandler.startReplicationServices();
+    if (this.replicationHandler != null) {
+      this.replicationHandler.startReplicationServices();
+    }
 
     // Start Server.  This service is like leases in that it internally runs
     // a thread.
     this.server.start();
-    LOG.info("HRegionServer started at: " +
-      this.serverInfo.getServerAddress().toString());
+    LOG.info("HRegionServer started at: "
+        + this.serverInfo.getServerAddress().toString());
   }
 
   /*
@@ -1099,68 +1108,73 @@ public class HRegionServer implements HRegionInterface,
       return false;
     }
     // Verify that all threads are alive
-    if (!(leases.isAlive() && compactSplitThread.isAlive() &&
-        cacheFlusher.isAlive() && hlogRoller.isAlive() &&
-        workerThread.isAlive() && this.majorCompactionChecker.isAlive())) {
-      // One or more threads are no longer alive - shut down
-      stop();
+    if (!(leases.isAlive() && compactSplitThread.isAlive()
+        && cacheFlusher.isAlive() && hlogRoller.isAlive()
+        && this.majorCompactionChecker.isAlive())) {
+      stop("One or more threads are no longer alive -- stop");
       return false;
     }
     return true;
   }
 
-  /*
-   * Run some housekeeping tasks.
-   */
-  private void housekeeping() {
-    // If the todo list has > 0 messages, iterate looking for open region
-    // messages. Send the master a message that we're working on its
-    // processing so it doesn't assign the region elsewhere.
-    if (this.toDo.isEmpty()) {
-      return;
-    }
-    // This iterator isn't safe if elements are gone and HRS.Worker could
-    // remove them (it already checks for null there). Goes from oldest.
-    for (ToDoEntry e: this.toDo) {
-      if(e == null) {
-        LOG.warn("toDo gave a null entry during iteration");
-        break;
-      }
-      HMsg msg = e.msg;
-      if (msg != null) {
-        if (msg.isType(HMsg.Type.MSG_REGION_OPEN)) {
-          addProcessingMessage(msg.getRegionInfo());
-        }
-      } else {
-        LOG.warn("Message is empty: " + e);
-      }
-    }
-  }
-
   /** @return the HLog */
-  public HLog getLog() {
+  public HLog getWAL() {
     return this.hlog;
   }
 
-  /**
-   * Sets a flag that will cause all the HRegionServer threads to shut down
-   * in an orderly fashion.  Used by unit tests.
-   */
-  public void stop() {
-    this.stopRequested.set(true);
-    synchronized(this) {
+  @Override
+  public CatalogTracker getCatalogTracker() {
+    return this.catalogTracker;
+  }
+
+  @Override
+  public void stop(final String msg) {
+    this.stopped = true;
+    LOG.info("STOPPED: " + msg);
+    synchronized (this) {
       // Wakes run() if it is sleeping
       notifyAll(); // FindBugs NN_NAKED_NOTIFY
     }
   }
 
+  @Override
+  public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct,
+      final boolean daughter)
+  throws KeeperException, IOException {
+    // Do checks to see if we need to compact (references or too many files)
+    if (r.hasReferences() || r.hasTooManyStoreFiles()) {
+      getCompactionRequester().requestCompaction(r,
+        r.hasReferences()? "Region has references on open" :
+          "Region has too many store files");
+    }
+    // Add to online regions
+    addToOnlineRegions(r);
+    // Update ZK, ROOT or META
+    if (r.getRegionInfo().isRootRegion()) {
+      RootLocationEditor.setRootLocation(getZooKeeper(),
+        getServerInfo().getServerAddress());
+    } else if (r.getRegionInfo().isMetaRegion()) {
+      // TODO: doh, this has weird naming between RootEditor/MetaEditor
+      MetaEditor.updateMetaLocation(ct, r.getRegionInfo(), getServerInfo());
+    } else {
+      if (daughter) {
+        // If daughter of a split, update whole row, not just location.
+        MetaEditor.addDaughter(ct, r.getRegionInfo(), getServerInfo());
+      } else {
+        MetaEditor.updateRegionLocation(ct, r.getRegionInfo(), getServerInfo());
+      }
+    }
+  }
+
   /**
-   * Cause the server to exit without closing the regions it is serving, the
-   * log it is using and without notifying the master.
-   * Used unit testing and on catastrophic events such as HDFS is yanked out
-   * from under hbase or we OOME.
-   * @param reason the reason we are aborting
-   * @param cause the exception that caused the abort, or null
+   * Cause the server to exit without closing the regions it is serving, the log
+   * it is using and without notifying the master. Used unit testing and on
+   * catastrophic events such as HDFS is yanked out from under hbase or we OOME.
+   *
+   * @param reason
+   *          the reason we are aborting
+   * @param cause
+   *          the exception that caused the abort, or null
    */
   public void abort(String reason, Throwable cause) {
     if (cause != null) {
@@ -1173,9 +1187,9 @@ public class HRegionServer implements HRegionInterface,
     if (this.metrics != null) {
       LOG.info("Dump of metrics: " + this.metrics);
     }
-    stop();
+    stop(reason);
   }
-  
+
   /**
    * @see HRegionServer#abort(String, Throwable)
    */
@@ -1184,9 +1198,9 @@ public class HRegionServer implements HRegionInterface,
   }
 
   /*
-   * Simulate a kill -9 of this server.
-   * Exits w/o closing regions or cleaninup logs but it does close socket in
-   * case want to bring up server on old hostname+port immediately.
+   * Simulate a kill -9 of this server. Exits w/o closing regions or cleaninup
+   * logs but it does close socket in case want to bring up server on old
+   * hostname+port immediately.
    */
   protected void kill() {
     this.killed = true;
@@ -1194,42 +1208,47 @@ public class HRegionServer implements HRegionInterface,
   }
 
   /**
-   * Wait on all threads to finish.
-   * Presumption is that all closes and stops have already been called.
+   * Wait on all threads to finish. Presumption is that all closes and stops
+   * have already been called.
    */
   protected void join() {
     Threads.shutdown(this.majorCompactionChecker);
-    Threads.shutdown(this.workerThread);
     Threads.shutdown(this.cacheFlusher);
     Threads.shutdown(this.compactSplitThread);
     Threads.shutdown(this.hlogRoller);
-    this.replicationHandler.join();
+    this.service.shutdown();
+    if (this.replicationHandler != null) {
+      this.replicationHandler.join();
+    }
   }
 
+  /**
+   * Get the current master from ZooKeeper and open the RPC connection to it.
+   *
+   * Method will block until a master is available. You can break from this
+   * block by requesting the server stop.
+   *
+   * @return
+   */
   private boolean getMaster() {
     HServerAddress masterAddress = null;
-    while (masterAddress == null) {
-      if (stopRequested.get()) {
+    while ((masterAddress = masterAddressManager.getMasterAddress()) == null) {
+      if (stopped) {
         return false;
       }
-      try {
-        masterAddress = zooKeeperWrapper.readMasterAddressOrThrow();
-      } catch (IOException e) {
-        LOG.warn("Unable to read master address from ZooKeeper. Retrying." +
-                 " Error was:", e);
-        sleeper.sleep();
-      }
+      LOG.debug("No master found, will retry");
+      sleeper.sleep();
     }
-
     LOG.info("Telling master at " + masterAddress + " that we are up");
     HMasterRegionInterface master = null;
-    while (!stopRequested.get() && master == null) {
+    while (!stopped && master == null) {
       try {
-        // Do initial RPC setup.  The final argument indicates that the RPC
+        // Do initial RPC setup. The final argument indicates that the RPC
         // should retry indefinitely.
-        master = (HMasterRegionInterface)HBaseRPC.waitForProxy(
-          HMasterRegionInterface.class, HBaseRPCProtocolVersion.versionID,
-          masterAddress.getInetSocketAddress(), this.conf, -1, this.rpcTimeout);
+        master = (HMasterRegionInterface) HBaseRPC.waitForProxy(
+            HMasterRegionInterface.class, HBaseRPCProtocolVersion.versionID,
+            masterAddress.getInetSocketAddress(), this.conf, -1,
+            this.rpcTimeout);
       } catch (IOException e) {
         LOG.warn("Unable to connect to master. Retrying. Error was:", e);
         sleeper.sleep();
@@ -1239,34 +1258,46 @@ public class HRegionServer implements HRegionInterface,
     return true;
   }
 
+  /**
+   * @return True if successfully invoked {@link #reportForDuty()}
+   * @throws IOException
+   */
+  private boolean tryReportForDuty() throws IOException {
+    MapWritable w = reportForDuty();
+    if (w != null) {
+      handleReportForDutyResponse(w);
+      return true;
+    }
+    sleeper.sleep();
+    LOG.warn("No response on reportForDuty. Sleeping and then retrying.");
+    return false;
+  }
+
   /*
-   * Let the master know we're here
-   * Run initialization using parameters passed us by the master.
+   * Let the master know we're here Run initialization using parameters passed
+   * us by the master.
    */
   private MapWritable reportForDuty() {
-    while (!stopRequested.get() && !getMaster()) {
+    while (!stopped && !getMaster()) {
       sleeper.sleep();
       LOG.warn("Unable to get master for initialization");
     }
 
     MapWritable result = null;
     long lastMsg = 0;
-    while(!stopRequested.get()) {
+    while (!stopped) {
       try {
         this.requestCount.set(0);
-        MemoryUsage memory =
-          ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
-        HServerLoad hsl = new HServerLoad(0, (int)memory.getUsed()/1024/1024,
-          (int)memory.getMax()/1024/1024);
-        this.serverInfo.setLoad(hsl);
-        if (LOG.isDebugEnabled())
-          LOG.debug("sending initial server load: " + hsl);
         lastMsg = System.currentTimeMillis();
-        zooKeeperWrapper.writeRSLocation(this.serverInfo);
+        ZKUtil.setAddressAndWatch(zooKeeper, ZKUtil.joinZNode(
+            zooKeeper.rsZNode, ZKUtil.getNodeName(serverInfo)), address);
+        this.serverInfo.setLoad(buildServerLoad());
         result = this.hbaseMaster.regionServerStartup(this.serverInfo);
         break;
       } catch (IOException e) {
         LOG.warn("error telling master we are up", e);
+      } catch (KeeperException e) {
+        LOG.warn("error putting up ephemeral node in zookeeper", e);
       }
       sleeper.sleep(lastMsg);
     }
@@ -1285,370 +1316,70 @@ public class HRegionServer implements HRegionInterface,
    */
   void reportSplit(HRegionInfo oldRegion, HRegionInfo newRegionA,
       HRegionInfo newRegionB) {
-    this.outboundMsgs.add(new HMsg(HMsg.Type.MSG_REPORT_SPLIT_INCLUDES_DAUGHTERS,
-      oldRegion, newRegionA, newRegionB,
-      Bytes.toBytes("Daughters; " +
-          newRegionA.getRegionNameAsString() + ", " +
-          newRegionB.getRegionNameAsString())));
-  }
-
-  //////////////////////////////////////////////////////////////////////////////
-  // HMaster-given operations
-  //////////////////////////////////////////////////////////////////////////////
-
-  /*
-   * Data structure to hold a HMsg and retries count.
-   */
-  private static final class ToDoEntry {
-    protected final AtomicInteger tries = new AtomicInteger(0);
-    protected final HMsg msg;
-
-    ToDoEntry(final HMsg msg) {
-      this.msg = msg;
-    }
-  }
-
-  final BlockingQueue<ToDoEntry> toDo = new LinkedBlockingQueue<ToDoEntry>();
-  private Worker worker;
-  private Thread workerThread;
-
-  /** Thread that performs long running requests from the master */
-  class Worker implements Runnable {
-    void stop() {
-      synchronized(toDo) {
-        toDo.notifyAll();
-      }
-    }
-
-    public void run() {
-      try {
-        while(!stopRequested.get()) {
-          ToDoEntry e = null;
-          try {
-            e = toDo.poll(threadWakeFrequency, TimeUnit.MILLISECONDS);
-            if(e == null || stopRequested.get()) {
-              continue;
-            }
-            LOG.info("Worker: " + e.msg);
-            HRegion region = null;
-            HRegionInfo info = e.msg.getRegionInfo();
-            switch(e.msg.getType()) {
-
-            case MSG_REGIONSERVER_QUIESCE:
-              closeUserRegions();
-              break;
-
-            case MSG_REGION_OPEN:
-              // Open a region
-              if (!haveRootRegion.get() && !info.isRootRegion()) {
-                // root region is not online yet. requeue this task
-                LOG.info("putting region open request back into queue because" +
-                    " root region is not yet available");
-                try {
-                  toDo.put(e);
-                } catch (InterruptedException ex) {
-                  LOG.warn("insertion into toDo queue was interrupted", ex);
-                  break;
-                }
-              }
-              openRegion(info);
-              break;
-
-            case MSG_REGION_CLOSE:
-              // Close a region
-              closeRegion(e.msg.getRegionInfo(), true);
-              break;
-
-            case MSG_REGION_CLOSE_WITHOUT_REPORT:
-              // Close a region, don't reply
-              closeRegion(e.msg.getRegionInfo(), false);
-              break;
-
-            case MSG_REGION_SPLIT:
-              region = getRegion(info.getRegionName());
-              region.flushcache();
-              region.shouldSplit(true);
-              // force a compaction; split will be side-effect.
-              compactSplitThread.compactionRequested(region,
-                e.msg.getType().name());
-              break;
-
-            case MSG_REGION_MAJOR_COMPACT:
-            case MSG_REGION_COMPACT:
-              // Compact a region
-              region = getRegion(info.getRegionName());
-              compactSplitThread.compactionRequested(region,
-                e.msg.isType(Type.MSG_REGION_MAJOR_COMPACT),
-                e.msg.getType().name());
-              break;
-
-            case MSG_REGION_FLUSH:
-              region = getRegion(info.getRegionName());
-              region.flushcache();
-              break;
-
-            case TESTING_MSG_BLOCK_RS:
-              while (!stopRequested.get()) {
-                Threads.sleep(1000);
-                LOG.info("Regionserver blocked by " +
-                  HMsg.Type.TESTING_MSG_BLOCK_RS + "; " + stopRequested.get());
-              }
-              break;
-
-            default:
-              throw new AssertionError(
-                  "Impossible state during msg processing.  Instruction: "
-                  + e.msg.toString());
-            }
-          } catch (InterruptedException ex) {
-            LOG.warn("Processing Worker queue", ex);
-          } catch (Exception ex) {
-            if (ex instanceof IOException) {
-              ex = RemoteExceptionHandler.checkIOException((IOException) ex);
-            }
-            if(e != null && e.tries.get() < numRetries) {
-              LOG.warn(ex);
-              e.tries.incrementAndGet();
-              try {
-                toDo.put(e);
-              } catch (InterruptedException ie) {
-                throw new RuntimeException("Putting into msgQueue was " +
-                    "interrupted.", ex);
-              }
-            } else {
-              LOG.error("unable to process message" +
-                  (e != null ? (": " + e.msg.toString()) : ""), ex);
-              if (!checkFileSystem()) {
-                break;
-              }
-            }
-          }
-        }
-      } catch(Throwable t) {
-        if (!checkOOME(t)) {
-          LOG.fatal("Unhandled exception", t);
-        }
-      } finally {
-        LOG.info("worker thread exiting");
-      }
-    }
-  }
-
-  void openRegion(final HRegionInfo regionInfo) {
-    Integer mapKey = Bytes.mapKey(regionInfo.getRegionName());
-    HRegion region = this.onlineRegions.get(mapKey);
-    RSZookeeperUpdater zkUpdater = 
-      new RSZookeeperUpdater(conf, serverInfo.getServerName(),
-          regionInfo.getEncodedName());
-    if (region == null) {
-      try {
-        zkUpdater.startRegionOpenEvent(null, true);
-        region = instantiateRegion(regionInfo, this.hlog);
-        // Startup a compaction early if one is needed, if region has references
-        // or if a store has too many store files
-        if (region.hasReferences() || region.hasTooManyStoreFiles()) {
-          this.compactSplitThread.compactionRequested(region,
-            region.hasReferences() ? "Region has references on open" :
-                                     "Region has too many store files");
-        }
-      } catch (Throwable e) {
-        Throwable t = cleanup(e,
-          "Error opening " + regionInfo.getRegionNameAsString());
-        // TODO: add an extra field in HRegionInfo to indicate that there is
-        // an error. We can't do that now because that would be an incompatible
-        // change that would require a migration
-        try {
-          HMsg hmsg = new HMsg(HMsg.Type.MSG_REPORT_CLOSE, 
-                               regionInfo, 
-                               StringUtils.stringifyException(t).getBytes());
-          zkUpdater.abortOpenRegion(hmsg);
-        } catch (IOException e1) {
-          // TODO: Can we recover? Should be throw RTE?
-          LOG.error("Failed to abort open region " + regionInfo.getRegionNameAsString(), e1);
-        }
-        return;
-      }
-      addToOnlineRegions(region);
-    }
-    try {
-      HMsg hmsg = new HMsg(HMsg.Type.MSG_REPORT_OPEN, regionInfo);
-      zkUpdater.finishRegionOpenEvent(hmsg);
-    } catch (IOException e) {
-      LOG.error("Failed to mark region " + regionInfo.getRegionNameAsString() + " as opened", e);
-    }
-  }
-
-  /*
-   * @param regionInfo RegionInfo for the Region we're to instantiate and
-   * initialize.
-   * @param wal Set into here the regions' seqid.
-   * @return
-   * @throws IOException
-   */
-  protected HRegion instantiateRegion(final HRegionInfo regionInfo, final HLog wal)
-  throws IOException {
-    Path dir =
-      HTableDescriptor.getTableDir(rootDir, regionInfo.getTableDesc().getName());
-    HRegion r = HRegion.newHRegion(dir, this.hlog, this.fs, conf, regionInfo,
-      this.cacheFlusher);
-    long seqid = r.initialize(new Progressable() {
-      public void progress() {
-        addProcessingMessage(regionInfo);
-      }
-    });
-    // If seqid  > current wal seqid, the wal seqid is updated.
-    if (wal != null) wal.setSequenceNumber(seqid);
-    return r;
+    this.outboundMsgs.add(new HMsg(
+        HMsg.Type.REGION_SPLIT, oldRegion, newRegionA,
+        newRegionB, Bytes.toBytes("Daughters; "
+            + newRegionA.getRegionNameAsString() + ", "
+            + newRegionB.getRegionNameAsString())));
   }
 
   /**
-   * Add a MSG_REPORT_PROCESS_OPEN to the outbound queue.
-   * This method is called while region is in the queue of regions to process
-   * and then while the region is being opened, it is called from the Worker
-   * thread that is running the region open.
-   * @param hri Region to add the message for
+   * Closes all regions.  Called on our way out.
+   * Assumes that its not possible for new regions to be added to onlineRegions
+   * while this method runs.
    */
-  public void addProcessingMessage(final HRegionInfo hri) {
-    getOutboundMsgs().add(new HMsg(HMsg.Type.MSG_REPORT_PROCESS_OPEN, hri));
-  }
-
-  protected void closeRegion(final HRegionInfo hri, final boolean reportWhenCompleted)
-  throws IOException {
-    RSZookeeperUpdater zkUpdater = null;
-    if(reportWhenCompleted) {
-      zkUpdater = new RSZookeeperUpdater(conf,
-          serverInfo.getServerName(), hri.getEncodedName());
-      zkUpdater.startRegionCloseEvent(null, false);
-    }
-    HRegion region = this.removeFromOnlineRegions(hri);
-    if (region != null) {
-      region.close();
-      if(reportWhenCompleted) {
-        if(zkUpdater != null) {
-          HMsg hmsg = new HMsg(HMsg.Type.MSG_REPORT_CLOSE, hri, null);
-          zkUpdater.finishRegionCloseEvent(hmsg);
-        }
-      }
-    }
-  }
-
-  /** Called either when the master tells us to restart or from stop() */
-  ArrayList<HRegion> closeAllRegions() {
-    ArrayList<HRegion> regionsToClose = new ArrayList<HRegion>();
+  protected void closeAllRegions(final boolean abort) {
+    closeUserRegions(abort);
+    // Only root and meta should remain.  Are we carrying root or meta?
+    HRegion meta = null;
+    HRegion root = null;
     this.lock.writeLock().lock();
     try {
-      regionsToClose.addAll(onlineRegions.values());
-      onlineRegions.clear();
+      for (Map.Entry<String, HRegion> e: onlineRegions.entrySet()) {
+        HRegionInfo hri = e.getValue().getRegionInfo();
+        if (hri.isRootRegion()) {
+          root = e.getValue();
+        } else if (hri.isMetaRegion()) {
+          meta = e.getValue();
+        }
+        if (meta != null && root != null) break;
+      }
     } finally {
       this.lock.writeLock().unlock();
     }
-    // Close any outstanding scanners.  Means they'll get an UnknownScanner
-    // exception next time they come in.
-    for (Map.Entry<String, InternalScanner> e: this.scanners.entrySet()) {
-      try {
-        e.getValue().close();
-      } catch (IOException ioe) {
-        LOG.warn("Closing scanner " + e.getKey(), ioe);
-      }
-    }
-    for (HRegion region: regionsToClose) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("closing region " + Bytes.toString(region.getRegionName()));
-      }
-      try {
-        region.close(abortRequested);
-      } catch (Throwable e) {
-        cleanup(e, "Error closing " + Bytes.toString(region.getRegionName()));
-      }
-    }
-    return regionsToClose;
+    if (meta != null) closeRegion(meta.getRegionInfo(), abort, false);
+    if (root != null) closeRegion(root.getRegionInfo(), abort, false);
   }
 
-  /*
-   * Thread to run close of a region.
+  /**
+   * Schedule closes on all user regions.
+   * @param abort Whether we're running an abort.
    */
-  private static class RegionCloserThread extends Thread {
-    private final HRegion r;
-
-    protected RegionCloserThread(final HRegion r) {
-      super(Thread.currentThread().getName() + ".regionCloser." + r.toString());
-      this.r = r;
-    }
-
-    @Override
-    public void run() {
-      try {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Closing region " + r.toString());
-        }
-        r.close();
-      } catch (Throwable e) {
-        LOG.error("Error closing region " + r.toString(),
-          RemoteExceptionHandler.checkThrowable(e));
-      }
-    }
-  }
-
-  /** Called as the first stage of cluster shutdown. */
-  void closeUserRegions() {
-    ArrayList<HRegion> regionsToClose = new ArrayList<HRegion>();
+  void closeUserRegions(final boolean abort) {
     this.lock.writeLock().lock();
     try {
-      synchronized (onlineRegions) {
-        for (Iterator<Map.Entry<Integer, HRegion>> i =
-            onlineRegions.entrySet().iterator(); i.hasNext();) {
-          Map.Entry<Integer, HRegion> e = i.next();
+      synchronized (this.onlineRegions) {
+        for (Map.Entry<String, HRegion> e: this.onlineRegions.entrySet()) {
           HRegion r = e.getValue();
           if (!r.getRegionInfo().isMetaRegion()) {
-            regionsToClose.add(r);
-            i.remove();
+            // Don't update zk with this close transition; pass false.
+            closeRegion(r.getRegionInfo(), abort, false);
           }
         }
       }
     } finally {
       this.lock.writeLock().unlock();
     }
-    // Run region closes in parallel.
-    Set<Thread> threads = new HashSet<Thread>();
-    try {
-      for (final HRegion r : regionsToClose) {
-        RegionCloserThread t = new RegionCloserThread(r);
-        t.start();
-        threads.add(t);
-      }
-    } finally {
-      for (Thread t : threads) {
-        while (t.isAlive()) {
-          try {
-            t.join();
-          } catch (InterruptedException e) {
-            e.printStackTrace();
-          }
-        }
-      }
-    }
-    this.quiesced.set(true);
-    if (onlineRegions.size() == 0) {
-      outboundMsgs.add(REPORT_EXITING);
-    } else {
-      outboundMsgs.add(REPORT_QUIESCED);
-    }
   }
 
-  //
-  // HRegionInterface
-  //
-
-  public HRegionInfo getRegionInfo(final byte [] regionName)
-  throws NotServingRegionException {
+  public HRegionInfo getRegionInfo(final byte[] regionName)
+      throws NotServingRegionException {
     requestCount.incrementAndGet();
     return getRegion(regionName).getRegionInfo();
   }
 
-
-  public Result getClosestRowBefore(final byte [] regionName,
-    final byte [] row, final byte [] family)
-  throws IOException {
+  public Result getClosestRowBefore(final byte[] regionName, final byte[] row,
+      final byte[] family) throws IOException {
     checkOpen();
     requestCount.incrementAndGet();
     try {
@@ -1664,33 +1395,34 @@ public class HRegionServer implements HRegionInterface,
   }
 
   /** {@inheritDoc} */
-  public Result get(byte [] regionName, Get get) throws IOException {
+  public Result get(byte[] regionName, Get get) throws IOException {
     checkOpen();
     requestCount.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
       return region.get(get, getLockFromId(get.getLockId()));
-    } catch(Throwable t) {
+    } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
     }
   }
 
-  public boolean exists(byte [] regionName, Get get) throws IOException {
+  public boolean exists(byte[] regionName, Get get) throws IOException {
     checkOpen();
     requestCount.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
       Result r = region.get(get, getLockFromId(get.getLockId()));
       return r != null && !r.isEmpty();
-    } catch(Throwable t) {
+    } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
     }
   }
 
-  public void put(final byte [] regionName, final Put put)
-  throws IOException {
-    if (put.getRow() == null)
+  public void put(final byte[] regionName, final Put put) throws IOException {
+    if (put.getRow() == null) {
       throw new IllegalArgumentException("update has null row");
+    }
+
     checkOpen();
     this.requestCount.incrementAndGet();
     HRegion region = getRegion(regionName);
@@ -1706,7 +1438,7 @@ public class HRegionServer implements HRegionInterface,
   }
 
   public int put(final byte[] regionName, final List<Put> puts)
-  throws IOException {
+      throws IOException {
     checkOpen();
     HRegion region = null;
     try {
@@ -1714,21 +1446,22 @@ public class HRegionServer implements HRegionInterface,
       if (!region.getRegionInfo().isMetaTable()) {
         this.cacheFlusher.reclaimMemStoreMemory();
       }
-      
+
       @SuppressWarnings("unchecked")
       Pair<Put, Integer>[] putsWithLocks = new Pair[puts.size()];
-      
+
       int i = 0;
       for (Put p : puts) {
         Integer lock = getLockFromId(p.getLockId());
         putsWithLocks[i++] = new Pair<Put, Integer>(p, lock);
       }
-      
+
       this.requestCount.addAndGet(puts.size());
       OperationStatusCode[] codes = region.put(putsWithLocks);
       for (i = 0; i < codes.length; i++) {
-        if (codes[i] != OperationStatusCode.SUCCESS)
+        if (codes[i] != OperationStatusCode.SUCCESS) {
           return i;
+        }
       }
       return -1;
     } catch (Throwable t) {
@@ -1736,8 +1469,8 @@ public class HRegionServer implements HRegionInterface,
     }
   }
 
-  private boolean checkAndMutate(final byte[] regionName, final byte [] row,
-      final byte [] family, final byte [] qualifier, final byte [] value,
+  private boolean checkAndMutate(final byte[] regionName, final byte[] row,
+      final byte[] family, final byte[] qualifier, final byte[] value,
       final Writable w, Integer lock) throws IOException {
     checkOpen();
     this.requestCount.incrementAndGet();
@@ -1746,28 +1479,28 @@ public class HRegionServer implements HRegionInterface,
       if (!region.getRegionInfo().isMetaTable()) {
         this.cacheFlusher.reclaimMemStoreMemory();
       }
-      return region.checkAndMutate(row, family, qualifier, value, w, lock,
-          true);
+      return region
+          .checkAndMutate(row, family, qualifier, value, w, lock, true);
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
     }
   }
 
-
   /**
    *
    * @param regionName
    * @param row
    * @param family
    * @param qualifier
-   * @param value the expected value
+   * @param value
+   *          the expected value
    * @param put
    * @throws IOException
    * @return true if the new put was execute, false otherwise
    */
-  public boolean checkAndPut(final byte[] regionName, final byte [] row,
-      final byte [] family, final byte [] qualifier, final byte [] value,
-      final Put put) throws IOException{
+  public boolean checkAndPut(final byte[] regionName, final byte[] row,
+      final byte[] family, final byte[] qualifier, final byte[] value,
+      final Put put) throws IOException {
     return checkAndMutate(regionName, row, family, qualifier, value, put,
         getLockFromId(put.getLockId()));
   }
@@ -1778,14 +1511,15 @@ public class HRegionServer implements HRegionInterface,
    * @param row
    * @param family
    * @param qualifier
-   * @param value the expected value
+   * @param value
+   *          the expected value
    * @param delete
    * @throws IOException
    * @return true if the new put was execute, false otherwise
    */
-  public boolean checkAndDelete(final byte[] regionName, final byte [] row,
-      final byte [] family, final byte [] qualifier, final byte [] value,
-      final Delete delete) throws IOException{
+  public boolean checkAndDelete(final byte[] regionName, final byte[] row,
+      final byte[] family, final byte[] qualifier, final byte[] value,
+      final Delete delete) throws IOException {
     return checkAndMutate(regionName, row, family, qualifier, value, delete,
         getLockFromId(delete.getLockId()));
   }
@@ -1794,8 +1528,7 @@ public class HRegionServer implements HRegionInterface,
   // remote scanner interface
   //
 
-  public long openScanner(byte [] regionName, Scan scan)
-  throws IOException {
+  public long openScanner(byte[] regionName, Scan scan) throws IOException {
     checkOpen();
     NullPointerException npe = null;
     if (regionName == null) {
@@ -1820,20 +1553,19 @@ public class HRegionServer implements HRegionInterface,
     scannerId = rand.nextLong();
     String scannerName = String.valueOf(scannerId);
     scanners.put(scannerName, s);
-    this.leases.
-      createLease(scannerName, new ScannerListener(scannerName));
+    this.leases.createLease(scannerName, new ScannerListener(scannerName));
     return scannerId;
   }
 
   public Result next(final long scannerId) throws IOException {
-    Result [] res = next(scannerId, 1);
-    if(res == null || res.length == 0) {
+    Result[] res = next(scannerId, 1);
+    if (res == null || res.length == 0) {
       return null;
     }
     return res[0];
   }
 
-  public Result [] next(final long scannerId, int nbRows) throws IOException {
+  public Result[] next(final long scannerId, int nbRows) throws IOException {
     try {
       String scannerName = String.valueOf(scannerId);
       InternalScanner s = this.scanners.get(scannerName);
@@ -1852,7 +1584,8 @@ public class HRegionServer implements HRegionInterface,
       List<Result> results = new ArrayList<Result>(nbRows);
       long currentScanResultSize = 0;
       List<KeyValue> values = new ArrayList<KeyValue>();
-      for (int i = 0; i < nbRows && currentScanResultSize < maxScannerResultSize; i++) {
+      for (int i = 0; i < nbRows
+          && currentScanResultSize < maxScannerResultSize; i++) {
         requestCount.incrementAndGet();
         // Collect values to be returned here
         boolean moreRows = s.next(values);
@@ -1868,13 +1601,13 @@ public class HRegionServer implements HRegionInterface,
         values.clear();
       }
       // Below is an ugly hack where we cast the InternalScanner to be a
-      // HRegion.RegionScanner.  The alternative is to change InternalScanner
+      // HRegion.RegionScanner. The alternative is to change InternalScanner
       // interface but its used everywhere whereas we just need a bit of info
       // from HRegion.RegionScanner, IF its filter if any is done with the scan
-      // and wants to tell the client to stop the scan.  This is done by passing
+      // and wants to tell the client to stop the scan. This is done by passing
       // a null result.
-      return ((HRegion.RegionScanner)s).isFilterDone() && results.isEmpty()?
-        null: results.toArray(new Result[0]);
+      return ((HRegion.RegionScanner) s).isFilterDone() && results.isEmpty() ? null
+          : results.toArray(new Result[0]);
     } catch (Throwable t) {
       if (t instanceof NotServingRegionException) {
         String scannerName = String.valueOf(scannerId);
@@ -1900,8 +1633,8 @@ public class HRegionServer implements HRegionInterface,
   }
 
   /**
-   * Instantiated as a scanner lease.
-   * If the lease times out, the scanner is closed
+   * Instantiated as a scanner lease. If the lease times out, the scanner is
+   * closed
    */
   private class ScannerListener implements LeaseListener {
     private final String scannerName;
@@ -1926,8 +1659,8 @@ public class HRegionServer implements HRegionInterface,
   //
   // Methods that do the actual work for the remote API
   //
-  public void delete(final byte [] regionName, final Delete delete)
-  throws IOException {
+  public void delete(final byte[] regionName, final Delete delete)
+      throws IOException {
     checkOpen();
     try {
       boolean writeToWAL = true;
@@ -1944,7 +1677,7 @@ public class HRegionServer implements HRegionInterface,
   }
 
   public int delete(final byte[] regionName, final List<Delete> deletes)
-  throws IOException {
+      throws IOException {
     // Count of Deletes processed.
     int i = 0;
     checkOpen();
@@ -1957,7 +1690,7 @@ public class HRegionServer implements HRegionInterface,
       }
       int size = deletes.size();
       Integer[] locks = new Integer[size];
-      for (Delete delete: deletes) {
+      for (Delete delete : deletes) {
         this.requestCount.incrementAndGet();
         locks[i] = getLockFromId(delete.getLockId());
         region.delete(delete, locks[i], writeToWAL);
@@ -1974,16 +1707,15 @@ public class HRegionServer implements HRegionInterface,
     return -1;
   }
 
-  public long lockRow(byte [] regionName, byte [] row)
-  throws IOException {
+  public long lockRow(byte[] regionName, byte[] row) throws IOException {
     checkOpen();
     NullPointerException npe = null;
-    if(regionName == null) {
+    if (regionName == null) {
       npe = new NullPointerException("regionName is null");
-    } else if(row == null) {
+    } else if (row == null) {
       npe = new NullPointerException("row to lock is null");
     }
-    if(npe != null) {
+    if (npe != null) {
       IOException io = new IOException("Invalid arguments to lockRow");
       io.initCause(npe);
       throw io;
@@ -1992,34 +1724,36 @@ public class HRegionServer implements HRegionInterface,
     try {
       HRegion region = getRegion(regionName);
       Integer r = region.obtainRowLock(row);
-      long lockId = addRowLock(r,region);
+      long lockId = addRowLock(r, region);
       LOG.debug("Row lock " + lockId + " explicitly acquired by client");
       return lockId;
     } catch (Throwable t) {
-      throw convertThrowableToIOE(cleanup(t,
-        "Error obtaining row lock (fsOk: " + this.fsOk + ")"));
+      throw convertThrowableToIOE(cleanup(t, "Error obtaining row lock (fsOk: "
+          + this.fsOk + ")"));
     }
   }
 
-  protected long addRowLock(Integer r, HRegion region) throws LeaseStillHeldException {
+  protected long addRowLock(Integer r, HRegion region)
+      throws LeaseStillHeldException {
     long lockId = -1L;
     lockId = rand.nextLong();
     String lockName = String.valueOf(lockId);
     rowlocks.put(lockName, r);
-    this.leases.
-      createLease(lockName, new RowLockListener(lockName, region));
+    this.leases.createLease(lockName, new RowLockListener(lockName, region));
     return lockId;
   }
 
   /**
-   * Method to get the Integer lock identifier used internally
-   * from the long lock identifier used by the client.
-   * @param lockId long row lock identifier from client
+   * Method to get the Integer lock identifier used internally from the long
+   * lock identifier used by the client.
+   *
+   * @param lockId
+   *          long row lock identifier from client
    * @return intId Integer row lock used internally in HRegion
-   * @throws IOException Thrown if this is not a valid client lock id.
+   * @throws IOException
+   *           Thrown if this is not a valid client lock id.
    */
-  Integer getLockFromId(long lockId)
-  throws IOException {
+  Integer getLockFromId(long lockId) throws IOException {
     if (lockId == -1L) {
       return null;
     }
@@ -2032,16 +1766,15 @@ public class HRegionServer implements HRegionInterface,
     return rl;
   }
 
-  public void unlockRow(byte [] regionName, long lockId)
-  throws IOException {
+  public void unlockRow(byte[] regionName, long lockId) throws IOException {
     checkOpen();
     NullPointerException npe = null;
-    if(regionName == null) {
+    if (regionName == null) {
       npe = new NullPointerException("regionName is null");
-    } else if(lockId == -1L) {
+    } else if (lockId == -1L) {
       npe = new NullPointerException("lockId is null");
     }
-    if(npe != null) {
+    if (npe != null) {
       IOException io = new IOException("Invalid arguments to unlockRow");
       io.initCause(npe);
       throw io;
@@ -2051,31 +1784,30 @@ public class HRegionServer implements HRegionInterface,
       HRegion region = getRegion(regionName);
       String lockName = String.valueOf(lockId);
       Integer r = rowlocks.remove(lockName);
-      if(r == null) {
+      if (r == null) {
         throw new UnknownRowLockException(lockName);
       }
       region.releaseRowLock(r);
       this.leases.cancelLease(lockName);
-      LOG.debug("Row lock " + lockId + " has been explicitly released by client");
+      LOG.debug("Row lock " + lockId
+          + " has been explicitly released by client");
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
     }
   }
 
   @Override
-  public void bulkLoadHFile(
-      String hfilePath, byte[] regionName, byte[] familyName)
-  throws IOException {
+  public void bulkLoadHFile(String hfilePath, byte[] regionName,
+      byte[] familyName) throws IOException {
     HRegion region = getRegion(regionName);
     region.bulkLoadHFile(hfilePath, familyName);
   }
 
-  Map<String, Integer> rowlocks =
-    new ConcurrentHashMap<String, Integer>();
+  Map<String, Integer> rowlocks = new ConcurrentHashMap<String, Integer>();
 
   /**
-   * Instantiated as a row lock lease.
-   * If the lease times out, the row lock is released
+   * Instantiated as a row lock lease. If the lease times out, the row lock is
+   * released
    */
   private class RowLockListener implements LeaseListener {
     private final String lockName;
@@ -2089,12 +1821,94 @@ public class HRegionServer implements HRegionInterface,
     public void leaseExpired() {
       LOG.info("Row Lock " + this.lockName + " lease expired");
       Integer r = rowlocks.remove(this.lockName);
-      if(r != null) {
+      if (r != null) {
         region.releaseRowLock(r);
       }
     }
   }
 
+  // Region open/close direct RPCs
+
+  @Override
+  public void openRegion(HRegionInfo region) {
+    LOG.info("Received request to open region: " +
+      region.getRegionNameAsString());
+    if(region.isRootRegion()) {
+      this.service.submit(new OpenRootHandler(this, this, region));
+    } else if(region.isMetaRegion()) {
+      this.service.submit(new OpenMetaHandler(this, this, region));
+    } else {
+      this.service.submit(new OpenRegionHandler(this, this, region));
+    }
+  }
+
+  @Override
+  public boolean closeRegion(HRegionInfo region)
+  throws NotServingRegionException {
+    LOG.info("Received close region: " + region.getRegionNameAsString());
+    // TODO: Need to check if this is being served here but currently undergoing
+    // a split (so master needs to retry close after split is complete)
+    if (!onlineRegions.containsKey(region.getEncodedName())) {
+      LOG.warn("Received close for region we are not serving");
+      throw new NotServingRegionException("Received close for "
+          + region.getRegionNameAsString() + " but we are not serving it");
+    }
+    return closeRegion(region, false, true);
+  }
+
+  /**
+   * @param region Region to close
+   * @param abort True if we are aborting
+   * @param zk True if we are to update zk about the region close; if the close
+   * was orchestrated by master, then update zk.  If the close is being run by
+   * the regionserver because its going down, don't update zk.
+   * @return
+   */
+  protected boolean closeRegion(HRegionInfo region, final boolean abort,
+      final boolean zk) {
+    CloseRegionHandler crh = null;
+    if (region.isRootRegion()) {
+      crh = new CloseRootHandler(this, this, region, abort, zk);
+    } else if (region.isMetaRegion()) {
+      crh = new CloseMetaHandler(this, this, region, abort, zk);
+    } else {
+      crh = new CloseRegionHandler(this, this, region, abort, zk);
+    }
+    this.service.submit(crh);
+    return true;
+  }
+
+  // Manual remote region administration RPCs
+
+  @Override
+  public void flushRegion(HRegionInfo regionInfo)
+      throws NotServingRegionException, IOException {
+    HRegion region = getRegion(regionInfo.getRegionName());
+    region.flushcache();
+  }
+
+  @Override
+  public void splitRegion(HRegionInfo regionInfo)
+      throws NotServingRegionException, IOException {
+    HRegion region = getRegion(regionInfo.getRegionName());
+    region.flushcache();
+    region.shouldSplit(true);
+    // force a compaction, split will be side-effect
+    // TODO: flush/compact/split refactor will make it trivial to do this
+    // sync/async (and won't require us to do a compaction to split!)
+    compactSplitThread.requestCompaction(region, "User-triggered split");
+  }
+
+  @Override
+  public void compactRegion(HRegionInfo regionInfo, boolean major)
+      throws NotServingRegionException, IOException {
+    HRegion region = getRegion(regionInfo.getRegionName());
+    region.flushcache();
+    region.shouldSplit(true);
+    compactSplitThread.requestCompaction(region, major, "User-triggered "
+        + (major ? "major " : "") + "compaction");
+  }
+
   /** @return the info server */
   public InfoServer getInfoServer() {
     return infoServer;
@@ -2103,8 +1917,8 @@ public class HRegionServer implements HRegionInterface,
   /**
    * @return true if a stop has been requested.
    */
-  public boolean isStopRequested() {
-    return this.stopRequested.get();
+  public boolean isStopped() {
+    return this.stopped;
   }
 
   /**
@@ -2120,53 +1934,52 @@ public class HRegionServer implements HRegionInterface,
     return lock.writeLock();
   }
 
-  /**
-   * @return Immutable list of this servers regions.
-   */
-  public Collection<HRegion> getOnlineRegions() {
-    return Collections.unmodifiableCollection(onlineRegions.values());
-  }
-
-  public HRegion [] getOnlineRegionsAsArray() {
-    return getOnlineRegions().toArray(new HRegion[0]);
-  }
-
-  /**
-   * @return The HRegionInfos from online regions sorted
-   */
-  public SortedSet<HRegionInfo> getSortedOnlineRegionInfos() {
-    SortedSet<HRegionInfo> result = new TreeSet<HRegionInfo>();
+  @Override
+  public NavigableSet<HRegionInfo> getOnlineRegions() {
+    NavigableSet<HRegionInfo> sortedset = new TreeSet<HRegionInfo>();
     synchronized(this.onlineRegions) {
-      for (HRegion r: this.onlineRegions.values()) {
-        result.add(r.getRegionInfo());
+      for (Map.Entry<String,HRegion> e: this.onlineRegions.entrySet()) {
+        sortedset.add(e.getValue().getRegionInfo());
       }
     }
-    return result;
+    return sortedset;
   }
 
-  public void addToOnlineRegions(final HRegion r) {
-    this.lock.writeLock().lock();
+  /**
+   * For tests and web ui.
+   * This method will only work if HRegionServer is in the same JVM as client;
+   * HRegion cannot be serialized to cross an rpc.
+   * @see #getOnlineRegions()
+   */
+  public Collection<HRegion> getOnlineRegionsLocalContext() {
+    return Collections.unmodifiableCollection(this.onlineRegions.values());
+  }
+
+  @Override
+  public void addToOnlineRegions(HRegion region) {
+    lock.writeLock().lock();
     try {
-      this.onlineRegions.put(Bytes.mapKey(r.getRegionInfo().getRegionName()), r);
+      onlineRegions.put(region.getRegionInfo().getEncodedName(), region);
     } finally {
-      this.lock.writeLock().unlock();
+      lock.writeLock().unlock();
     }
   }
 
-  public HRegion removeFromOnlineRegions(HRegionInfo hri) {
+  @Override
+  public boolean removeFromOnlineRegions(final String encodedName) {
     this.lock.writeLock().lock();
     HRegion toReturn = null;
     try {
-      toReturn = onlineRegions.remove(Bytes.mapKey(hri.getRegionName()));
+      toReturn = onlineRegions.remove(encodedName);
     } finally {
       this.lock.writeLock().unlock();
     }
-    return toReturn;
+    return toReturn != null;
   }
 
   /**
    * @return A new Map of online regions sorted by region size with the first
-   * entry being the biggest.
+   *         entry being the biggest.
    */
   public SortedMap<Long, HRegion> getCopyOfOnlineRegionsSortedBySize() {
     // we'll sort the regions in reverse
@@ -2185,13 +1998,18 @@ public class HRegionServer implements HRegionInterface,
     return sortedRegions;
   }
 
+  @Override
+  public HRegion getFromOnlineRegions(final String encodedRegionName) {
+    return onlineRegions.get(encodedRegionName);
+  }
+
   /**
    * @param regionName
-   * @return HRegion for the passed <code>regionName</code> or null if named
-   * region is not member of the online regions.
+   * @return HRegion for the passed binary <code>regionName</code> or null if
+   *         named region is not member of the online regions.
    */
-  public HRegion getOnlineRegion(final byte [] regionName) {
-    return onlineRegions.get(Bytes.mapKey(regionName));
+  public HRegion getOnlineRegion(final byte[] regionName) {
+    return getFromOnlineRegions(HRegionInfo.encodeRegionName(regionName));
   }
 
   /** @return the request count */
@@ -2206,18 +2024,21 @@ public class HRegionServer implements HRegionInterface,
 
   /**
    * Protected utility method for safely obtaining an HRegion handle.
-   * @param regionName Name of online {@link HRegion} to return
+   *
+   * @param regionName
+   *          Name of online {@link HRegion} to return
    * @return {@link HRegion} for <code>regionName</code>
    * @throws NotServingRegionException
    */
-  protected HRegion getRegion(final byte [] regionName)
-  throws NotServingRegionException {
+  protected HRegion getRegion(final byte[] regionName)
+      throws NotServingRegionException {
     HRegion region = null;
     this.lock.readLock().lock();
     try {
-      region = onlineRegions.get(Integer.valueOf(Bytes.hashCode(regionName)));
+      region = getOnlineRegion(regionName);
       if (region == null) {
-        throw new NotServingRegionException(regionName);
+        throw new NotServingRegionException("Region is not online: " +
+          Bytes.toStringBinary(regionName));
       }
       return region;
     } finally {
@@ -2226,10 +2047,10 @@ public class HRegionServer implements HRegionInterface,
   }
 
   /**
-   * Get the top N most loaded regions this server is serving so we can
-   * tell the master which regions it can reallocate if we're overloaded.
-   * TODO: actually calculate which regions are most loaded. (Right now, we're
-   * just grabbing the first N regions being served regardless of load.)
+   * Get the top N most loaded regions this server is serving so we can tell the
+   * master which regions it can reallocate if we're overloaded. TODO: actually
+   * calculate which regions are most loaded. (Right now, we're just grabbing
+   * the first N regions being served regardless of load.)
    */
   protected HRegionInfo[] getMostLoadedRegions() {
     ArrayList<HRegionInfo> regions = new ArrayList<HRegionInfo>();
@@ -2254,9 +2075,9 @@ public class HRegionServer implements HRegionInterface,
    * @throws IOException
    */
   protected void checkOpen() throws IOException {
-    if (this.stopRequested.get() || this.abortRequested) {
-      throw new IOException("Server not running" +
-        (this.abortRequested? ", aborting": ""));
+    if (this.stopped || this.abortRequested) {
+      throw new IOException("Server not running"
+          + (this.abortRequested ? ", aborting" : ""));
     }
     if (!fsOk) {
       throw new IOException("File system not available");
@@ -2264,12 +2085,12 @@ public class HRegionServer implements HRegionInterface,
   }
 
   /**
-   * @return Returns list of non-closed regions hosted on this server.  If no
-   * regions to check, returns an empty list.
+   * @return Returns list of non-closed regions hosted on this server. If no
+   *         regions to check, returns an empty list.
    */
   protected Set<HRegion> getRegionsToCheck() {
     HashSet<HRegion> regionsToCheck = new HashSet<HRegion>();
-    //TODO: is this locking necessary?
+    // TODO: is this locking necessary?
     lock.readLock().lock();
     try {
       regionsToCheck.addAll(this.onlineRegions.values());
@@ -2286,9 +2107,8 @@ public class HRegionServer implements HRegionInterface,
     return regionsToCheck;
   }
 
-  public long getProtocolVersion(final String protocol,
-      final long clientVersion)
-  throws IOException {
+  public long getProtocolVersion(final String protocol, final long clientVersion)
+      throws IOException {
     if (protocol.equals(HRegionInterface.class.getName())) {
       return HBaseRPCProtocolVersion.versionID;
     }
@@ -2304,6 +2124,7 @@ public class HRegionServer implements HRegionInterface,
 
   /**
    * Return the total size of all memstores in every region.
+   *
    * @return memstore size in bytes
    */
   public long getGlobalMemStoreSize() {
@@ -2340,17 +2161,19 @@ public class HRegionServer implements HRegionInterface,
   /**
    * @return Info on port this server has bound to, etc.
    */
-  public HServerInfo getServerInfo() { return this.serverInfo; }
+  public HServerInfo getServerInfo() {
+    return this.serverInfo;
+  }
 
   /** {@inheritDoc} */
-  public long incrementColumnValue(byte [] regionName, byte [] row,
-      byte [] family, byte [] qualifier, long amount, boolean writeToWAL)
-  throws IOException {
+  public long incrementColumnValue(byte[] regionName, byte[] row,
+      byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
+      throws IOException {
     checkOpen();
 
     if (regionName == null) {
-      throw new IOException("Invalid arguments to incrementColumnValue " +
-      "regionName is null");
+      throw new IOException("Invalid arguments to incrementColumnValue "
+          + "regionName is null");
     }
     requestCount.incrementAndGet();
     try {
@@ -2369,7 +2192,7 @@ public class HRegionServer implements HRegionInterface,
   public HRegionInfo[] getRegionsAssignment() throws IOException {
     HRegionInfo[] regions = new HRegionInfo[onlineRegions.size()];
     Iterator<HRegion> ite = onlineRegions.values().iterator();
-    for(int i = 0; ite.hasNext(); i++) {
+    for (int i = 0; ite.hasNext(); i++) {
       regions[i] = ite.next().getRegionInfo();
     }
     return regions;
@@ -2379,64 +2202,19 @@ public class HRegionServer implements HRegionInterface,
   public HServerInfo getHServerInfo() throws IOException {
     return serverInfo;
   }
-  
-  @Override
-  public MultiResponse multi(MultiAction multi) throws IOException {
-    MultiResponse response = new MultiResponse();
-    for (Map.Entry<byte[], List<Action>> e : multi.actions.entrySet()) {
-      byte[] regionName = e.getKey();
-      List<Action> actionsForRegion = e.getValue();
-      // sort based on the row id - this helps in the case where we reach the
-      // end of a region, so that we don't have to try the rest of the 
-      // actions in the list.
-      Collections.sort(actionsForRegion);
-      Row action = null;
-      try {
-        for (Action a : actionsForRegion) {
-          action = a.getAction();
-          if (action instanceof Delete) {
-            delete(regionName, (Delete) action);
-            response.add(regionName, new Pair<Integer, Result>(
-                a.getOriginalIndex(), new Result()));
-          } else if (action instanceof Get) {
-            response.add(regionName, new Pair<Integer, Result>(
-                a.getOriginalIndex(), get(regionName, (Get) action)));
-          } else if (action instanceof Put) {
-            put(regionName, (Put) action);
-            response.add(regionName, new Pair<Integer, Result>(
-                a.getOriginalIndex(), new Result()));
-          } else {
-            LOG.debug("Error: invalid Action, row must be a Get, Delete or Put.");
-            throw new IllegalArgumentException("Invalid Action, row must be a Get, Delete or Put.");
-          }
-        }
-      } catch (IOException ioe) {
-          if (multi.size() == 1) {
-            throw ioe;
-          } else {
-            LOG.error("Exception found while attempting " + action.toString()
-                + " " + StringUtils.stringifyException(ioe));
-            response.add(regionName,null);
-            // stop processing on this region, continue to the next.
-          }
-        }
-      }
-      
-      return response;
-    }
 
-  /**
-   * @deprecated Use HRegionServer.multi( MultiAction action) instead
-   */
   @Override
   public MultiPutResponse multiPut(MultiPut puts) throws IOException {
     MultiPutResponse resp = new MultiPutResponse();
+
     // do each region as it's own.
-    for( Map.Entry<byte[], List<Put>> e: puts.puts.entrySet()) {
+    for (Map.Entry<byte[], List<Put>> e : puts.puts.entrySet()) {
       int result = put(e.getKey(), e.getValue());
       resp.addResult(e.getKey(), result);
+
       e.getValue().clear(); // clear some RAM
     }
+
     return resp;
   }
 
@@ -2446,12 +2224,28 @@ public class HRegionServer implements HRegionInterface,
 
   /**
    * Interval at which threads should run
+   *
    * @return the interval
    */
   public int getThreadWakeFrequency() {
     return threadWakeFrequency;
   }
-  
+
+  @Override
+  public ZooKeeperWatcher getZooKeeper() {
+    return zooKeeper;
+  }
+
+  @Override
+  public String getServerName() {
+    return serverInfo.getServerName();
+  }
+
+  @Override
+  public CompactionRequestor getCompactionRequester() {
+    return this.compactSplitThread;
+  }
+
   //
   // Main program and support routines
   //
@@ -2462,9 +2256,9 @@ public class HRegionServer implements HRegionInterface,
    * @throws IOException
    */
   public static Thread startRegionServer(final HRegionServer hrs)
-  throws IOException {
-    return startRegionServer(hrs,
-      "regionserver" + hrs.getServerInfo().getServerAddress().getPort());
+      throws IOException {
+    return startRegionServer(hrs, "regionserver"
+        + hrs.getServerInfo().getServerAddress().getPort());
   }
 
   /**
@@ -2474,15 +2268,14 @@ public class HRegionServer implements HRegionInterface,
    * @throws IOException
    */
   public static Thread startRegionServer(final HRegionServer hrs,
-      final String name)
-  throws IOException {
+      final String name) throws IOException {
     Thread t = new Thread(hrs);
     t.setName(name);
     t.start();
     // Install shutdown hook that will catch signals and run an orderly shutdown
     // of the hrs.
-    ShutdownHook.install(hrs.getConfiguration(),
-      FileSystem.get(hrs.getConfiguration()), hrs, t);
+    ShutdownHook.install(hrs.getConfiguration(), FileSystem.get(hrs
+        .getConfiguration()), hrs, t);
     return t;
   }
 
@@ -2500,33 +2293,39 @@ public class HRegionServer implements HRegionInterface,
 
   /**
    * Utility for constructing an instance of the passed HRegionServer class.
+   *
    * @param regionServerClass
    * @param conf2
    * @return HRegionServer instance.
    */
-  public static HRegionServer constructRegionServer(Class<? extends HRegionServer> regionServerClass,
-      final Configuration conf2)  {
+  public static HRegionServer constructRegionServer(
+      Class<? extends HRegionServer> regionServerClass,
+      final Configuration conf2) {
     try {
-      Constructor<? extends HRegionServer> c =
-        regionServerClass.getConstructor(Configuration.class);
+      Constructor<? extends HRegionServer> c = regionServerClass
+          .getConstructor(Configuration.class);
       return c.newInstance(conf2);
     } catch (Exception e) {
-      throw new RuntimeException("Failed construction of " +
-        "Master: " + regionServerClass.toString(), e);
+      throw new RuntimeException("Failed construction of " + "Master: "
+          + regionServerClass.toString(), e);
     }
   }
 
   @Override
-  public void replicateLogEntries(HLog.Entry[] entries) throws IOException {
+  public void replicateLogEntries(final HLog.Entry[] entries)
+  throws IOException {
+    if (this.replicationHandler == null) return;
     this.replicationHandler.replicateLogEntries(entries);
   }
 
   /**
    * Do class main.
+   *
    * @param args
-   * @param regionServerClass HRegionServer to instantiate.
+   * @param regionServerClass
+   *          HRegionServer to instantiate.
    */
-  protected static void doMain(final String [] args,
+  protected static void doMain(final String[] args,
       final Class<? extends HRegionServer> regionServerClass) {
     Configuration conf = HBaseConfiguration.create();
 
@@ -2549,11 +2348,11 @@ public class HRegionServer implements HRegionInterface,
 
       if (cmd.getArgList().contains("start")) {
         try {
-          // If 'local', don't start a region server here.  Defer to
-          // LocalHBaseCluster.  It manages 'local' clusters.
+          // If 'local', don't start a region server here. Defer to
+          // LocalHBaseCluster. It manages 'local' clusters.
           if (LocalHBaseCluster.isLocal(conf)) {
-            LOG.warn("Not starting a distinct region server because " +
-              HConstants.CLUSTER_DISTRIBUTED + " is false");
+            LOG.warn("Not starting a distinct region server because "
+                + HConstants.CLUSTER_DISTRIBUTED + " is false");
           } else {
             RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
             if (runtime != null) {
@@ -2584,12 +2383,11 @@ public class HRegionServer implements HRegionInterface,
   /**
    * @param args
    */
-  public static void main(String [] args) {
+  public static void main(String[] args) {
     Configuration conf = HBaseConfiguration.create();
     @SuppressWarnings("unchecked")
-    Class<? extends HRegionServer> regionServerClass =
-      (Class<? extends HRegionServer>) conf.getClass(HConstants.REGION_SERVER_IMPL,
-        HRegionServer.class);
+    Class<? extends HRegionServer> regionServerClass = (Class<? extends HRegionServer>) conf
+        .getClass(HConstants.REGION_SERVER_IMPL, HRegionServer.class);
     doMain(args, regionServerClass);
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/LeaseException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java
similarity index 92%
rename from src/main/java/org/apache/hadoop/hbase/LeaseException.java
rename to src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java
index c48cc7fee6c..cafbb28496d 100644
--- a/src/main/java/org/apache/hadoop/hbase/LeaseException.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java
@@ -17,7 +17,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase;
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.DoNotRetryIOException;
 
 /**
  * Reports a problem with a lease
diff --git a/src/main/java/org/apache/hadoop/hbase/LeaseListener.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseListener.java
similarity index 96%
rename from src/main/java/org/apache/hadoop/hbase/LeaseListener.java
rename to src/main/java/org/apache/hadoop/hbase/regionserver/LeaseListener.java
index 54b34528818..a8437364a9b 100644
--- a/src/main/java/org/apache/hadoop/hbase/LeaseListener.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseListener.java
@@ -17,7 +17,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase;
+package org.apache.hadoop.hbase.regionserver;
 
 
 /**
diff --git a/src/main/java/org/apache/hadoop/hbase/Leases.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Leases.java
similarity index 99%
rename from src/main/java/org/apache/hadoop/hbase/Leases.java
rename to src/main/java/org/apache/hadoop/hbase/regionserver/Leases.java
index e502cc47a00..15f7453d6b0 100644
--- a/src/main/java/org/apache/hadoop/hbase/Leases.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Leases.java
@@ -17,7 +17,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase;
+package org.apache.hadoop.hbase.regionserver;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index 7b8fa567035..a8fba1501dc 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -21,9 +21,15 @@ package org.apache.hadoop.hbase.regionserver;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
-import org.apache.hadoop.hbase.regionserver.wal.LogRollListener;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.regionserver.wal.WALObserver;
 import org.apache.hadoop.hbase.util.Bytes;
 
 import java.io.IOException;
@@ -37,26 +43,31 @@ import java.util.concurrent.locks.ReentrantLock;
  * can be interrupted when there is something to do, rather than the Chore
  * sleep time which is invariant.
  */
-class LogRoller extends Thread implements LogRollListener {
+class LogRoller extends Thread implements WALObserver {
   static final Log LOG = LogFactory.getLog(LogRoller.class);
   private final ReentrantLock rollLock = new ReentrantLock();
   private final AtomicBoolean rollLog = new AtomicBoolean(false);
-  private final HRegionServer server;
+  private final Server server;
+  private final RegionServerServices services;
   private volatile long lastrolltime = System.currentTimeMillis();
   // Period to roll log.
   private final long rollperiod;
+  private final int threadWakeFrequency;
 
   /** @param server */
-  public LogRoller(final HRegionServer server) {
+  public LogRoller(final Server server, final RegionServerServices services) {
     super();
     this.server = server;
-    this.rollperiod =
-      this.server.conf.getLong("hbase.regionserver.logroll.period", 3600000);
+    this.services = services;
+    this.rollperiod = this.server.getConfiguration().
+      getLong("hbase.regionserver.logroll.period", 3600000);
+    this.threadWakeFrequency = this.server.getConfiguration().
+      getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
   }
 
   @Override
   public void run() {
-    while (!server.isStopRequested()) {
+    while (!server.isStopped()) {
       long now = System.currentTimeMillis();
       boolean periodic = false;
       if (!rollLog.get()) {
@@ -64,7 +75,7 @@ class LogRoller extends Thread implements LogRollListener {
         if (!periodic) {
           synchronized (rollLog) {
             try {
-              rollLog.wait(server.threadWakeFrequency);
+              rollLog.wait(this.threadWakeFrequency);
             } catch (InterruptedException e) {
               // Fall through
             }
@@ -79,27 +90,21 @@ class LogRoller extends Thread implements LogRollListener {
       rollLock.lock(); // FindBugs UL_UNRELEASED_LOCK_EXCEPTION_PATH
       try {
         this.lastrolltime = now;
-        byte [][] regionsToFlush = server.getLog().rollWriter();
+        // This is array of actual region names.
+        byte [][] regionsToFlush = this.services.getWAL().rollWriter();
         if (regionsToFlush != null) {
           for (byte [] r: regionsToFlush) scheduleFlush(r);
         }
       } catch (FailedLogCloseException e) {
-        LOG.fatal("Forcing server shutdown", e);
-        server.checkFileSystem();
         server.abort("Failed log close in log roller", e);
       } catch (java.net.ConnectException e) {
-        LOG.fatal("Forcing server shutdown", e);
-        server.checkFileSystem();
-        server.abort("Failed connect in log roller", e);
+        server.abort("Failed log close in log roller", e);
       } catch (IOException ex) {
-        LOG.fatal("Log rolling failed with ioe: ",
-          RemoteExceptionHandler.checkIOException(ex));
-        server.checkFileSystem();
         // Abort if we get here.  We probably won't recover an IOE. HBASE-1132
-        server.abort("IOE in log roller", ex);
+        server.abort("IOE in log roller",
+          RemoteExceptionHandler.checkIOException(ex));
       } catch (Exception ex) {
         LOG.error("Log rolling failed", ex);
-        server.checkFileSystem();
         server.abort("Log rolling failed", ex);
       } finally {
         rollLog.set(false);
@@ -109,14 +114,17 @@ class LogRoller extends Thread implements LogRollListener {
     LOG.info("LogRoller exiting.");
   }
 
+  /**
+   * @param region Encoded name of region to flush.
+   */
   private void scheduleFlush(final byte [] region) {
     boolean scheduled = false;
-    HRegion r = this.server.getOnlineRegion(region);
+    HRegion r = this.services.getFromOnlineRegions(Bytes.toString(region));
     FlushRequester requester = null;
     if (r != null) {
-      requester = this.server.getFlushRequester();
+      requester = this.services.getFlushRequester();
       if (requester != null) {
-        requester.request(r);
+        requester.requestFlush(r);
         scheduled = true;
       }
     }
@@ -145,4 +153,15 @@ class LogRoller extends Thread implements LogRollListener {
       rollLock.unlock();
     }
   }
+
+  @Override
+  public void logRolled(Path newFile) {
+    // Not interested
+  }
+
+  @Override
+  public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey,
+      WALEdit logEdit) {
+    // Not interested.
+  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index b45ed0259a7..b2357e9e734 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -136,7 +136,7 @@ class MemStoreFlusher extends Thread implements FlushRequester {
 
   @Override
   public void run() {
-    while (!this.server.isStopRequested()) {
+    while (!this.server.isStopped()) {
       FlushQueueEntry fqe = null;
       try {
         fqe = flushQueue.poll(threadWakeFrequency, TimeUnit.MILLISECONDS);
@@ -164,7 +164,7 @@ class MemStoreFlusher extends Thread implements FlushRequester {
     LOG.info(getName() + " exiting");
   }
 
-  public void request(HRegion r) {
+  public void requestFlush(HRegion r) {
     synchronized (regionsInQueue) {
       if (!regionsInQueue.containsKey(r)) {
         // This entry has no delay so it will be added at the top of the flush
@@ -212,7 +212,7 @@ class MemStoreFlusher extends Thread implements FlushRequester {
           LOG.warn("Region " + region.getRegionNameAsString() + " has too many " +
             "store files; delaying flush up to " + this.blockingWaitTime + "ms");
         }
-        this.server.compactSplitThread.compactionRequested(region, getName());
+        this.server.compactSplitThread.requestCompaction(region, getName());
         // Put back on the queue.  Have it come back out of the queue
         // after a delay of this.blockingWaitTime / 100 ms.
         this.flushQueue.add(fqe.requeue(this.blockingWaitTime / 100));
@@ -247,7 +247,7 @@ class MemStoreFlusher extends Thread implements FlushRequester {
     }
     try {
       if (region.flushcache()) {
-        server.compactSplitThread.compactionRequested(region, getName());
+        server.compactSplitThread.requestCompaction(region, getName());
       }
     } catch (DroppedSnapshotException ex) {
       // Cache flush can fail in a few places. If it fails in a critical
@@ -325,7 +325,7 @@ class MemStoreFlusher extends Thread implements FlushRequester {
       regionsToCompact.add(biggestMemStoreRegion);
     }
     for (HRegion region : regionsToCompact) {
-      server.compactSplitThread.compactionRequested(region, getName());
+      server.compactSplitThread.requestCompaction(region, getName());
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java b/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
index 62f6eab238c..3c90ed153c4 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
@@ -19,23 +19,32 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
-import org.apache.hadoop.hbase.HRegionInfo;
-
 /**
- * Add and remove online regions.
+ * Interface to Map of online regions.  In the  Map, the key is the region's
+ * encoded name and the value is an {@link HRegion} instance.
  */
 interface OnlineRegions {
   /**
    * Add to online regions.
    * @param r
    */
-  void addToOnlineRegions(final HRegion r);
+  public void addToOnlineRegions(final HRegion r);
 
   /**
    * This method removes HRegion corresponding to hri from the Map of onlineRegions.
    *
-   * @param hri the HRegionInfo corresponding to the HRegion to-be-removed.
-   * @return the removed HRegion, or null if the HRegion was not in onlineRegions.
+   * @param encodedRegionName
+   * @return True if we removed a region from online list.
    */
-  HRegion removeFromOnlineRegions(HRegionInfo hri);
-}
+  public boolean removeFromOnlineRegions(String encodedRegionName);
+
+  /**
+   * Return {@link HRegion} instance.
+   * Only works if caller is in same context, in same JVM. HRegion is not
+   * serializable.
+   * @param encodedRegionName
+   * @return HRegion for the passed encoded <code>encodedRegionName</code> or
+   * null if named region is not member of the online regions.
+   */
+  public HRegion getFromOnlineRegions(String encodedRegionName);
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RSZookeeperUpdater.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RSZookeeperUpdater.java
deleted file mode 100644
index 310b063dd04..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RSZookeeperUpdater.java
+++ /dev/null
@@ -1,164 +0,0 @@
-package org.apache.hadoop.hbase.regionserver;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.data.Stat;
-
-/**
- * This is a helper class for region servers to update various states in 
- * Zookeeper. The various updates are abstracted out here. 
- * 
- * The "startRegionXXX" methods are to be called first, followed by the 
- * "finishRegionXXX" methods. Supports updating zookeeper periodically as a 
- * part of the "startRegionXXX". Currently handles the following state updates:
- *   - Close region
- *   - Open region
- */
-// TODO: make this thread local, in which case it is re-usable per thread
-public class RSZookeeperUpdater {
-  private static final Log LOG = LogFactory.getLog(RSZookeeperUpdater.class);
-  private final String regionServerName;
-  private String regionName = null;
-  private String regionZNode = null;
-  private ZooKeeperWrapper zkWrapper = null;
-  private int zkVersion = 0;
-  HBaseEventType lastUpdatedState;
-
-  public RSZookeeperUpdater(Configuration conf,
-                            String regionServerName, String regionName) {
-    this(conf, regionServerName, regionName, 0);
-  }
-  
-  public RSZookeeperUpdater(Configuration conf, String regionServerName,
-                            String regionName, int zkVersion) {
-    this.zkWrapper = ZooKeeperWrapper.getInstance(conf, regionServerName);
-    this.regionServerName = regionServerName;
-    this.regionName = regionName;
-    // get the region ZNode we have to create
-    this.regionZNode = zkWrapper.getZNode(zkWrapper.getRegionInTransitionZNode(), regionName);
-    this.zkVersion = zkVersion;
-  }
-  
-  /**
-   * This method updates the various states in ZK to inform the master that the 
-   * region server has started closing the region.
-   * @param updatePeriodically - if true, periodically updates the state in ZK
-   */
-  public void startRegionCloseEvent(HMsg hmsg, boolean updatePeriodically) throws IOException {
-    // if this ZNode already exists, something is wrong
-    if(zkWrapper.exists(regionZNode, true)) {
-      String msg = "ZNode " + regionZNode + " already exists in ZooKeeper, will NOT close region.";
-      LOG.error(msg);
-      throw new IOException(msg);
-    }
-    
-    // create the region node in the unassigned directory first
-    zkWrapper.createZNodeIfNotExists(regionZNode, null, CreateMode.PERSISTENT, true);
-
-    // update the data for "regionName" ZNode in unassigned to CLOSING
-    updateZKWithEventData(HBaseEventType.RS2ZK_REGION_CLOSING, hmsg);
-    
-    // TODO: implement the updatePeriodically logic here
-  }
-
-  /**
-   * This method updates the states in ZK to signal that the region has been 
-   * closed. This will stop the periodic updater thread if one was started.
-   * @throws IOException
-   */
-  public void finishRegionCloseEvent(HMsg hmsg) throws IOException {    
-    // TODO: stop the updatePeriodically here
-
-    // update the data for "regionName" ZNode in unassigned to CLOSED
-    updateZKWithEventData(HBaseEventType.RS2ZK_REGION_CLOSED, hmsg);
-  }
-  
-  /**
-   * This method updates the various states in ZK to inform the master that the 
-   * region server has started opening the region.
-   * @param updatePeriodically - if true, periodically updates the state in ZK
-   */
-  public void startRegionOpenEvent(HMsg hmsg, boolean updatePeriodically) throws IOException {
-    Stat stat = new Stat();
-    byte[] data = zkWrapper.readZNode(regionZNode, stat);
-    // if there is no ZNode for this region, something is wrong
-    if(data == null) {
-      String msg = "ZNode " + regionZNode + " does not exist in ZooKeeper, will NOT open region.";
-      LOG.error(msg);
-      throw new IOException(msg);
-    }
-    // if the ZNode is not in the closed state, something is wrong
-    HBaseEventType rsEvent = HBaseEventType.fromByte(data[0]);
-    if(rsEvent != HBaseEventType.RS2ZK_REGION_CLOSED && rsEvent != HBaseEventType.M2ZK_REGION_OFFLINE) {
-      String msg = "ZNode " + regionZNode + " is not in CLOSED/OFFLINE state (state = " + rsEvent + "), will NOT open region.";
-      LOG.error(msg);
-      throw new IOException(msg);
-    }
-
-    // get the version to update from ZK
-    zkVersion = stat.getVersion();
-
-    // update the data for "regionName" ZNode in unassigned to CLOSING
-    updateZKWithEventData(HBaseEventType.RS2ZK_REGION_OPENING, hmsg);
-    
-    // TODO: implement the updatePeriodically logic here
-  }
-  
-  /**
-   * This method updates the states in ZK to signal that the region has been 
-   * opened. This will stop the periodic updater thread if one was started.
-   * @throws IOException
-   */
-  public void finishRegionOpenEvent(HMsg hmsg) throws IOException {
-    // TODO: stop the updatePeriodically here
-
-    // update the data for "regionName" ZNode in unassigned to CLOSED
-    updateZKWithEventData(HBaseEventType.RS2ZK_REGION_OPENED, hmsg);
-  }
-  
-  public boolean isClosingRegion() {
-    return (lastUpdatedState == HBaseEventType.RS2ZK_REGION_CLOSING);
-  }
-
-  public boolean isOpeningRegion() {
-    return (lastUpdatedState == HBaseEventType.RS2ZK_REGION_OPENING);
-  }
-
-  public void abortOpenRegion(HMsg hmsg) throws IOException {
-    LOG.error("Aborting open of region " + regionName);
-
-    // TODO: stop the updatePeriodically for start open region here
-
-    // update the data for "regionName" ZNode in unassigned to CLOSED
-    updateZKWithEventData(HBaseEventType.RS2ZK_REGION_CLOSED, hmsg);
-  }
-
-  private void updateZKWithEventData(HBaseEventType hbEventType, HMsg hmsg) throws IOException {
-    // update the data for "regionName" ZNode in unassigned to "hbEventType"
-    byte[] data = null;
-    try {
-      data = Writables.getBytes(new RegionTransitionEventData(hbEventType, regionServerName, hmsg));
-    } catch (IOException e) {
-      LOG.error("Error creating event data for " + hbEventType, e);
-    }
-    LOG.debug("Updating ZNode " + regionZNode + 
-              " with [" + hbEventType + "]" +
-              " expected version = " + zkVersion);
-    lastUpdatedState = hbEventType;
-    zkWrapper.writeZNode(regionZNode, data, zkVersion, true);
-    zkVersion++;
-  }
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
new file mode 100644
index 00000000000..4ba63d82fb5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -0,0 +1,63 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Services provided by {@link HRegionServer}
+ */
+public interface RegionServerServices extends OnlineRegions {
+  public HLog getWAL();
+
+  /**
+   * @return Implementation of {@link CompactionRequestor} or null.
+   */
+  public CompactionRequestor getCompactionRequester();
+  
+  /**
+   * @return Implementation of {@link FlushRequester} or null.
+   */
+  public FlushRequester getFlushRequester();
+
+  /**
+   * Return data structure that has Server address and startcode.
+   * @return The HServerInfo for this RegionServer.
+   */
+  public HServerInfo getServerInfo();
+
+  /**
+   * Tasks to perform after region open to complete deploy of region on
+   * regionserver
+   * @param r Region to open.
+   * @param ct Instance of {@link CatalogTracker}
+   * @param daughter True if this is daughter of a split
+   * @throws KeeperException
+   * @throws IOException
+   */
+  public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct,
+      final boolean daughter)
+  throws KeeperException, IOException;
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
index 719c59d6156..b25e575112c 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
@@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.util.Threads;
 
 /**
@@ -99,7 +100,7 @@ class ShutdownHook {
       LOG.info("Shutdown hook starting; " + RUN_SHUTDOWN_HOOK + "=" + b +
         "; fsShutdownHook=" + this.fsShutdownHook);
       if (b) {
-        this.stop.stop();
+        this.stop.stop("Shutdown hook");
         Threads.shutdown(this.threadToJoin);
         if (this.fsShutdownHook != null) {
           LOG.info("Starting fs shutdown hook thread.");
@@ -198,8 +199,14 @@ class ShutdownHook {
   // Stoppable with nothing to stop.  Used below in main testing.
   static class DoNothingStoppable implements Stoppable {
     @Override
-    public void stop() {
-      // Nothing to do.
+    public boolean isStopped() {
+      // TODO Auto-generated method stub
+      return false;
+    }
+
+    @Override
+    public void stop(String why) {
+      // TODO Auto-generated method stub
     }
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index 3507c0d5b55..87c03bcb9cb 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -30,16 +30,16 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
 import org.apache.hadoop.hbase.io.Reference.Range;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.PairOfSameType;
-import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.util.Progressable;
+import org.apache.zookeeper.KeeperException;
 
 /**
  * Executes region split as a "transaction".  Call {@link #prepare()} to setup
@@ -112,6 +112,8 @@ class SplitTransaction {
 
   /**
    * Constructor
+   * @param services So we can online new servces.  If null, we'll skip onlining
+   * (Useful testing).
    * @param c Configuration to use running split
    * @param r Region to split
    * @param splitrow Row to split around
@@ -176,35 +178,23 @@ class SplitTransaction {
 
   /**
    * Run the transaction.
-   * @param or Object that can online/offline parent region.
+   * @param server Hosting server instance.
+   * @param services Used to online/offline regions.
    * @throws IOException If thrown, transaction failed. Call {@link #rollback(OnlineRegions)}
    * @return Regions created
    * @see #rollback(OnlineRegions)
    */
-  public PairOfSameType<HRegion> execute(final OnlineRegions or) throws IOException {
-    return execute(or, or != null);
-  }
-
-  /**
-   * Run the transaction.
-   * @param or Object that can online/offline parent region.  Can be null (Tests
-   * will pass null).
-   * @param If <code>true</code>, update meta (set to false when testing).
-   * @throws IOException If thrown, transaction failed. Call {@link #rollback(OnlineRegions)}
-   * @return Regions created
-   * @see #rollback(OnlineRegions)
-   */
-  PairOfSameType<HRegion> execute(final OnlineRegions or, final boolean updateMeta)
+  PairOfSameType<HRegion> execute(final Server server,
+      final RegionServerServices services)
   throws IOException {
     LOG.info("Starting split of region " + this.parent);
     if (!this.parent.lock.writeLock().isHeldByCurrentThread()) {
       throw new SplitAndCloseWriteLockNotHeld();
     }
 
-    // We'll need one of these later but get it now because if we fail there
-    // is nothing to undo.
-    HTable t = null;
-    if (updateMeta) t = getTable(this.parent.getConf());
+    // If true, no cluster to write meta edits into.
+    boolean testing =
+      server.getConfiguration().getBoolean("hbase.testing.nocluster", false);
 
     createSplitDir(this.parent.getFilesystem(), this.splitdir);
     this.journal.add(JournalEntry.CREATE_SPLIT_DIR);
@@ -212,7 +202,9 @@ class SplitTransaction {
     List<StoreFile> hstoreFilesToSplit = this.parent.close(false);
     this.journal.add(JournalEntry.CLOSED_PARENT_REGION);
 
-    if (or != null) or.removeFromOnlineRegions(this.parent.getRegionInfo());
+    if (!testing) {
+      services.removeFromOnlineRegions(this.parent.getRegionInfo().getEncodedName());
+    }
     this.journal.add(JournalEntry.OFFLINED_PARENT);
 
     splitStoreFiles(this.splitdir, hstoreFilesToSplit);
@@ -225,52 +217,39 @@ class SplitTransaction {
     // stuff in fs that needs cleanup -- a storefile or two.  Thats why we
     // add entry to journal BEFORE rather than AFTER the change.
     this.journal.add(JournalEntry.STARTED_REGION_A_CREATION);
-    HRegion a = createDaughterRegion(this.hri_a);
+    HRegion a = createDaughterRegion(this.hri_a, this.parent.flushRequester);
 
     // Ditto
     this.journal.add(JournalEntry.STARTED_REGION_B_CREATION);
-    HRegion b = createDaughterRegion(this.hri_b);
+    HRegion b = createDaughterRegion(this.hri_b, this.parent.flushRequester);
 
-    Put editParentPut = createOfflineParentPut();
-    if (t != null) t.put(editParentPut);
-
-    // The is the point of no return.  We are committed to the split now.  Up to
-    // a failure editing parent in meta or a crash of the hosting regionserver,
-    // we could rollback (or, if crash, we could cleanup on redeploy) but now
-    // meta has been changed, we can only go forward.  If the below last steps
-    // do not complete, repair has to be done by another agent.  For example,
-    // basescanner, at least up till master rewrite, would add daughter rows if
-    // missing from meta.  It could do this because the parent edit includes the
-    // daughter specs.  In Bigtable paper, they have another mechanism where
-    // some feedback to the master somehow flags it that split is incomplete and
-    // needs fixup.  Whatever the mechanism, its a TODO that we have some fixup.
-    
-    // I looked at writing the put of the parent edit above out to the WAL log
-    // before changing meta with the notion that should we fail, then on replay
-    // the offlining of the parent and addition of daughters up into meta could
-    // be reinserted.  The edits would have to be 'special' and given how our
-    // splits work, splitting by region, I think the replay would have to happen
-    // inside in the split code -- as soon as it saw one of these special edits,
-    // rather than write the edit out a file for the .META. region to replay or
-    // somehow, write it out to this regions edits file for it to handle on
-    // redeploy -- this'd be whacky, we'd be telling meta about a split during
-    // the deploy of the parent -- instead we'd have to play the edit inside
-    // in the split code somehow; this would involve a stop-the-splitting till
-    // meta had been edited which might hold up splitting a good while.
-
-    // Finish up the meta edits.  If these fail, another agent needs to do fixup
-    HRegionInfo hri = this.hri_a;
-    try {
-      if (t != null) t.put(createDaughterPut(hri));
-      hri = this.hri_b;
-      if (t != null) t.put(createDaughterPut(hri));
-    } catch (IOException e) {
-      // Don't let this out or we'll run rollback.
-      LOG.warn("Failed adding daughter " + hri.toString());
+    // Edit parent in meta
+    if (!testing) {
+      MetaEditor.offlineParentInMeta(server.getCatalogTracker(),
+        this.parent.getRegionInfo(), a.getRegionInfo(), b.getRegionInfo());
+    }
+
+    // The is the point of no return.  We are committed to the split now.  We
+    // have still the daughter regions to open but meta has been changed.
+    // If we fail from here on out, we can not rollback so, we'll just abort.
+    // The meta has been changed though so there will need to be a fixup run
+    // during processing of the crashed server by master (TODO: Verify this in place).
+
+    // TODO: Could we be smarter about the sequence in which we do these steps?
+
+    if (!testing) {
+      // Open daughters in parallel.
+      DaughterOpener aOpener = new DaughterOpener(server, services, a);
+      DaughterOpener bOpener = new DaughterOpener(server, services, b);
+      aOpener.start();
+      bOpener.start();
+      try {
+        aOpener.join();
+        bOpener.join();
+      } catch (InterruptedException e) {
+        server.abort("Exception running daughter opens", e);
+      }
     }
-    // This should not fail because the HTable instance we are using is not
-    // running a buffer -- its immediately flushing its puts.
-    if (t != null) t.close();
 
     // Unlock if successful split.
     this.parent.lock.writeLock().unlock();
@@ -281,6 +260,70 @@ class SplitTransaction {
     return new PairOfSameType<HRegion>(a, b);
   }
 
+  class DaughterOpener extends Thread {
+    private final RegionServerServices services;
+    private final Server server;
+    private final HRegion r;
+
+    DaughterOpener(final Server s, final RegionServerServices services,
+        final HRegion r) {
+      super(s.getServerName() + "-daughterOpener=" + r.getRegionInfo().getEncodedName());
+      setDaemon(true);
+      this.services = services;
+      this.server = s;
+      this.r = r;
+    }
+
+    @Override
+    public void run() {
+      try {
+        openDaughterRegion(this.server, this.services, r);
+      } catch (Throwable t) {
+        this.server.abort("Failed open of daughter " +
+          this.r.getRegionInfo().getRegionNameAsString(), t);
+      }
+    }
+  }
+
+  /**
+   * Open daughter regions, add them to online list and update meta.
+   * @param server
+   * @param services
+   * @param daughter
+   * @throws IOException
+   * @throws KeeperException
+   */
+  void openDaughterRegion(final Server server,
+      final RegionServerServices services, final HRegion daughter)
+  throws IOException, KeeperException {
+    HRegionInfo hri = daughter.getRegionInfo();
+    LoggingProgressable reporter =
+      new LoggingProgressable(hri, server.getConfiguration());
+    HRegion r = daughter.openHRegion(reporter);
+    services.postOpenDeployTasks(r, server.getCatalogTracker(), true);
+  }
+
+  static class LoggingProgressable implements Progressable {
+    private final HRegionInfo hri;
+    private long lastLog = -1;
+    private final long interval;
+
+    LoggingProgressable(final HRegionInfo hri, final Configuration c) {
+      this.hri = hri;
+      this.interval = c.getLong("hbase.regionserver.split.daughter.open.log.interval",
+        10000);
+    }
+
+    @Override
+    public void progress() {
+      long now = System.currentTimeMillis();
+      if (now - lastLog > this.interval) {
+        LOG.info("Opening " + this.hri.getRegionNameAsString());
+        this.lastLog = now;
+      }
+    }
+  }
+
   private static Path getSplitDir(final HRegion r) {
     return new Path(r.getRegionDir(), SPLITDIR);
   }
@@ -348,12 +391,14 @@ class SplitTransaction {
   }
 
   /**
-   * @param hri
+   * @param hri Spec. for daughter region to open.
+   * @param flusher Flusher this region should use.
    * @return Created daughter HRegion.
    * @throws IOException
    * @see #cleanupDaughterRegion(FileSystem, Path, HRegionInfo)
    */
-  HRegion createDaughterRegion(final HRegionInfo hri)
+  HRegion createDaughterRegion(final HRegionInfo hri,
+      final FlushRequester flusher)
   throws IOException {
     // Package private so unit tests have access.
     FileSystem fs = this.parent.getFilesystem();
@@ -361,7 +406,7 @@ class SplitTransaction {
       this.splitdir, hri);
     HRegion r = HRegion.newHRegion(this.parent.getTableDir(),
       this.parent.getLog(), fs, this.parent.getConf(),
-      hri, null);
+      hri, flusher);
     HRegion.moveInitialFilesIntoPlace(fs, regionDir, r.getRegionDir());
     return r;
   }
@@ -389,56 +434,6 @@ class SplitTransaction {
     return new Path(splitdir, hri.getEncodedName());
   }
 
-  /*
-   * @param r Parent region we want to edit.
-   * @return An HTable instance against the meta table that holds passed
-   * <code>r</code>; it has autoFlush enabled so we immediately send puts (No
-   * buffering enabled).
-   * @throws IOException
-   */
-  private HTable getTable(final Configuration conf) throws IOException {
-    // When a region is split, the META table needs to updated if we're
-    // splitting a 'normal' region, and the ROOT table needs to be
-    // updated if we are splitting a META region.
-    HTable t = null;
-    if (this.parent.getRegionInfo().isMetaTable()) {
-      t = new HTable(conf, HConstants.ROOT_TABLE_NAME);
-    } else {
-      t = new HTable(conf, HConstants.META_TABLE_NAME);
-    }
-    // Flush puts as we send them -- no buffering.
-    t.setAutoFlush(true);
-    return t;
-  }
-
-
-  private Put createOfflineParentPut() throws IOException  {
-    HRegionInfo editedParentRegionInfo =
-      new HRegionInfo(this.parent.getRegionInfo());
-    editedParentRegionInfo.setOffline(true);
-    editedParentRegionInfo.setSplit(true);
-    Put put = new Put(editedParentRegionInfo.getRegionName());
-    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
-      Writables.getBytes(editedParentRegionInfo));
-    put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
-        HConstants.EMPTY_BYTE_ARRAY);
-    put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
-        HConstants.EMPTY_BYTE_ARRAY);
-    put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
-      Writables.getBytes(this.hri_a));
-    put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,
-      Writables.getBytes(this.hri_b));
-    return put;
-  }
-
-  private Put createDaughterPut(final HRegionInfo daughter)
-  throws IOException {
-    Put p = new Put(daughter.getRegionName());
-    p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
-      Writables.getBytes(daughter));
-    return p;
-  }
-
   /**
    * @param or Object that can online/offline parent region.  Can be passed null
    * by unit tests.
@@ -537,4 +532,4 @@ class SplitTransaction {
     cleanupSplitDir(r.getFilesystem(), splitdir);
     LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Stoppable.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Stoppable.java
deleted file mode 100644
index 1fd71460091..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/Stoppable.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-/**
- * Implementations are stoppable.
- */
-interface Stoppable {
-  // Starting small, just doing a stoppable/stop for now and keeping it package
-  // protected for now.  Needed so don't have to pass RegionServer instance
-  // everywhere.  Doing Lifecycle seemed a stretch since none of our servers
-  // do natural start/stop, etc. RegionServer is hosted in a Thread (can't do
-  // 'stop' on a Thread and 'start' has special meaning for Threads) and then
-  // Master is implemented differently again (it is a Thread itself). We
-  // should move to redoing Master and RegionServer servers to use Spring or
-  // some such container but for now, I just need stop -- St.Ack.
-  /**
-   * Stop service.
-   */
-  public void stop();
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 107d641c6c0..fb4752cfe36 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -291,9 +291,6 @@ public class StoreFile {
    * @return This files maximum edit sequence id.
    */
   public long getMaxSequenceId() {
-    if (this.sequenceid == -1) {
-      throw new IllegalAccessError("Has not been initialized");
-    }
     return this.sequenceid;
   }
 
@@ -370,11 +367,9 @@ public class StoreFile {
    * @see #closeReader()
    */
   private Reader open() throws IOException {
-
     if (this.reader != null) {
       throw new IllegalAccessError("Already open");
     }
-
     if (isReference()) {
       this.reader = new HalfStoreFileReader(this.fs, this.referencePath,
           getBlockCache(), this.reference);
@@ -382,7 +377,6 @@ public class StoreFile {
       this.reader = new Reader(this.fs, this.path, getBlockCache(),
           this.inMemory);
     }
-
     // Load up indices and fileinfo.
     metadataMap = Collections.unmodifiableMap(this.reader.loadFileInfo());
     // Read in our metadata.
@@ -409,6 +403,10 @@ public class StoreFile {
       } else {
         this.majorCompaction.set(mc);
       }
+    } else {
+      // Presume it is not major compacted if it doesn't explicity say so
+      // HFileOutputFormat explicitly sets the major compacted key.
+      this.majorCompaction = new AtomicBoolean(false);
     }
 
     if (this.bloomType != BloomType.NONE) {
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java
index 1c136396178..8706e65f17d 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java
@@ -59,5 +59,4 @@ interface StoreFlusher {
    * @throws IOException
    */
   boolean commit() throws IOException;
-
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java
new file mode 100644
index 00000000000..3963b387cf6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java
@@ -0,0 +1,43 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.handler;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+
+/**
+ * Handles closing of the root region on a region server.
+ */
+public class CloseMetaHandler extends CloseRegionHandler {
+  // Called when master tells us shutdown a region via close rpc
+  public CloseMetaHandler(final Server server,
+      final RegionServerServices rsServices, final HRegionInfo regionInfo) {
+    this(server, rsServices, regionInfo, false, true);
+  }
+
+  // Called when regionserver determines its to go down; not master orchestrated
+  public CloseMetaHandler(final Server server,
+      final RegionServerServices rsServices,
+      final HRegionInfo regionInfo,
+      final boolean abort, final boolean zk) {
+    super(server, rsServices, regionInfo, abort, zk, EventType.M2RS_CLOSE_META);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
new file mode 100644
index 00000000000..47a5ed4a45b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
@@ -0,0 +1,187 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.handler;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Handles closing of a region on a region server.
+ */
+public class CloseRegionHandler extends EventHandler {
+  // NOTE on priorities shutting down.  There are none for close. There are some
+  // for open.  I think that is right.  On shutdown, we want the meta to close
+  // before root and both to close after the user regions have closed.  What
+  // about the case where master tells us to shutdown a catalog region and we
+  // have a running queue of user regions to close?
+  private static final Log LOG = LogFactory.getLog(CloseRegionHandler.class);
+
+  private final int FAILED = -1;
+
+  private final RegionServerServices rsServices;
+
+  private final HRegionInfo regionInfo;
+
+  // If true, the hosting server is aborting.  Region close process is different
+  // when we are aborting.
+  private final boolean abort;
+
+  // Update zk on closing transitions. Usually true.  Its false if cluster
+  // is going down.  In this case, its the rs that initiates the region
+  // close -- not the master process so state up in zk will unlikely be
+  // CLOSING.
+  private final boolean zk;
+
+  // This is executed after receiving an CLOSE RPC from the master.
+  public CloseRegionHandler(final Server server,
+      final RegionServerServices rsServices, HRegionInfo regionInfo) {
+    this(server, rsServices, regionInfo, false, true);
+  }
+
+  /**
+   * This method used internally by the RegionServer to close out regions.
+   * @param server
+   * @param rsServices
+   * @param regionInfo
+   * @param abort If the regionserver is aborting.
+   * @param zk If the close should be noted out in zookeeper.
+   */
+  public CloseRegionHandler(final Server server,
+      final RegionServerServices rsServices,
+      final HRegionInfo regionInfo, final boolean abort, final boolean zk) {
+    this(server, rsServices,  regionInfo, abort, zk, EventType.M2RS_CLOSE_REGION);
+  }
+
+  protected CloseRegionHandler(final Server server,
+      final RegionServerServices rsServices, HRegionInfo regionInfo,
+      boolean abort, final boolean zk, EventType eventType) {
+    super(server, eventType);
+    this.server = server;
+    this.rsServices = rsServices;
+    this.regionInfo = regionInfo;
+    this.abort = abort;
+    this.zk = zk;
+  }
+
+  public HRegionInfo getRegionInfo() {
+    return regionInfo;
+  }
+
+  @Override
+  public void process() {
+    String name = regionInfo.getRegionNameAsString();
+    LOG.debug("Processing close of " + name);
+    String encodedRegionName = regionInfo.getEncodedName();
+    // Check that this region is being served here
+    HRegion region = this.rsServices.getFromOnlineRegions(encodedRegionName);
+    if (region == null) {
+      LOG.warn("Received CLOSE for region " + name + " but currently not serving");
+      return;
+    }
+
+    int expectedVersion = FAILED;
+    if (this.zk) {
+      expectedVersion = setClosingState();
+      if (expectedVersion == FAILED) return;
+    }
+
+    // Close the region
+    try {
+      // TODO: If we need to keep updating CLOSING stamp to prevent against
+      //       a timeout if this is long-running, need to spin up a thread?
+      this.rsServices.removeFromOnlineRegions(regionInfo.getEncodedName());
+      region.close(abort);
+    } catch (IOException e) {
+      LOG.error("IOException closing region for " + regionInfo);
+      if (this.zk) deleteClosingState();
+    }
+
+    if (this.zk) setClosedState(expectedVersion, region);
+
+    // Done!  Successful region open
+    LOG.debug("Closed region " + region.getRegionNameAsString());
+  }
+
+  /**
+   * Transition ZK node to CLOSED
+   * @param expectedVersion
+   */
+  private void setClosedState(final int expectedVersion, final HRegion region) {
+    try {
+      if (ZKAssign.transitionNodeClosed(server.getZooKeeper(), regionInfo,
+          server.getServerName(), expectedVersion) == FAILED) {
+        LOG.warn("Completed the CLOSE of a region but when transitioning from " +
+            " CLOSING to CLOSED got a version mismatch, someone else clashed " +
+            "so now unassigning");
+        region.close();
+        return;
+      }
+    } catch (NullPointerException e) {
+      // I've seen NPE when table was deleted while close was running in unit tests.
+      LOG.warn("NPE during close -- catching and continuing...", e);
+    } catch (KeeperException e) {
+      LOG.error("Failed transitioning node from CLOSING to CLOSED", e);
+      return;
+    } catch (IOException e) {
+      LOG.error("Failed to close region after failing to transition", e);
+      return;
+    }
+  }
+
+  /**
+   * @return True if succeeded, false otherwise.
+   */
+  private void deleteClosingState() {
+    try {
+      ZKAssign.deleteClosingNode(server.getZooKeeper(),
+          this.regionInfo.getEncodedName()); 
+    } catch (KeeperException e1) {
+      LOG.error("Error deleting CLOSING node");
+    }
+  }
+
+  /**
+   * Create ZK node in CLOSING state.
+   * @return The expectedVersion.  If -1, we failed setting CLOSING.
+   */
+  private int setClosingState() {
+    int expectedVersion = FAILED;
+    try {
+      if ((expectedVersion = ZKAssign.createNodeClosing(
+          server.getZooKeeper(), regionInfo, server.getServerName())) == FAILED) {
+        LOG.warn("Error creating node in CLOSING state, aborting close of "
+            + regionInfo.getRegionNameAsString());
+      }
+    } catch (KeeperException e) {
+      LOG.warn("Error creating node in CLOSING state, aborting close of "
+          + regionInfo.getRegionNameAsString());
+    }
+    return expectedVersion;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRootHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRootHandler.java
new file mode 100644
index 00000000000..e96f70c6d67
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRootHandler.java
@@ -0,0 +1,43 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.handler;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+
+/**
+ * Handles closing of the root region on a region server.
+ */
+public class CloseRootHandler extends CloseRegionHandler {
+  // This is executed after receiving an CLOSE RPC from the master for root.
+  public CloseRootHandler(final Server server,
+      final RegionServerServices rsServices, HRegionInfo regionInfo) {
+    this(server, rsServices, regionInfo, false, true);
+  }
+
+  // This is called directly by the regionserver when its determined its
+  // shutting down.
+  public CloseRootHandler(final Server server,
+      final RegionServerServices rsServices, HRegionInfo regionInfo,
+      final boolean abort, final boolean zk) {
+    super(server, rsServices, regionInfo, abort, zk, EventType.M2RS_CLOSE_ROOT);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/executor/NamedThreadFactory.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenMetaHandler.java
similarity index 58%
rename from src/main/java/org/apache/hadoop/hbase/executor/NamedThreadFactory.java
rename to src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenMetaHandler.java
index 87ea97d1a82..dfd2157b964 100644
--- a/src/main/java/org/apache/hadoop/hbase/executor/NamedThreadFactory.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenMetaHandler.java
@@ -17,27 +17,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase.executor;
+package org.apache.hadoop.hbase.regionserver.handler;
 
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 
 /**
- * Returns a named thread with a specified prefix.
- *
- * TODO: Use guava (com.google.common.util.concurrent.NamingThreadFactory)
+ * Handles opening of a meta region on a region server.
+ * <p>
+ * This is executed after receiving an OPEN RPC from the master for meta.
  */
-public class NamedThreadFactory implements ThreadFactory
-{
-  private String threadPrefix;
-  private AtomicInteger threadId = new AtomicInteger(0);
-
-  public NamedThreadFactory(String threadPrefix) {
-    this.threadPrefix = threadPrefix;
-  }
-
-  @Override
-  public Thread newThread(Runnable r) {
-    return new Thread(r, threadPrefix + "-" + threadId.incrementAndGet());
+public class OpenMetaHandler extends OpenRegionHandler {
+  public OpenMetaHandler(final Server server,
+      final RegionServerServices rsServices, HRegionInfo regionInfo) {
+    super(server,rsServices,  regionInfo, EventType.M2RS_OPEN_META);
   }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
new file mode 100644
index 00000000000..7b48025c30f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -0,0 +1,194 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.handler;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.util.Progressable;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
+
+/**
+ * Handles opening of a region on a region server.
+ * <p>
+ * This is executed after receiving an OPEN RPC from the master or client.
+ */
+public class OpenRegionHandler extends EventHandler {
+  private static final Log LOG = LogFactory.getLog(OpenRegionHandler.class);
+
+  private final RegionServerServices rsServices;
+
+  private final HRegionInfo regionInfo;
+
+  public OpenRegionHandler(final Server server,
+      final RegionServerServices rsServices, HRegionInfo regionInfo) {
+    this(server, rsServices, regionInfo, EventType.M2RS_OPEN_REGION);
+  }
+
+  protected OpenRegionHandler(final Server server,
+      final RegionServerServices rsServices, final HRegionInfo regionInfo,
+      EventType eventType) {
+    super(server, eventType);
+    this.rsServices = rsServices;
+    this.regionInfo = regionInfo;
+  }
+
+  public HRegionInfo getRegionInfo() {
+    return regionInfo;
+  }
+
+  @Override
+  public void process() throws IOException {
+    LOG.debug("Processing open of " + regionInfo.getRegionNameAsString());
+    final String encodedName = regionInfo.getEncodedName();
+
+    // TODO: Previously we would check for root region availability (but only that it
+    // was initially available, does not check if it later went away)
+    // Do we need to wait on both root and meta to be available to open a region
+    // now since we edit meta?
+
+    // Check that this region is not already online
+    HRegion region = this.rsServices.getFromOnlineRegions(encodedName);
+    if (region != null) {
+      LOG.warn("Attempting open of " + regionInfo.getRegionNameAsString() +
+        " but it's already online on this server");
+      return;
+    }
+
+    int openingVersion = transitionZookeeperOfflineToOpening(encodedName);
+    if (openingVersion == -1) return;
+
+    // Open the region
+    final AtomicInteger openingInteger = new AtomicInteger(openingVersion);
+    try {
+      // Instantiate the region.  This also periodically updates OPENING.
+      region = HRegion.openHRegion(regionInfo, this.rsServices.getWAL(),
+          server.getConfiguration(), this.rsServices.getFlushRequester(),
+          new Progressable() {
+            public void progress() {
+              try {
+                int vsn = ZKAssign.retransitionNodeOpening(
+                    server.getZooKeeper(), regionInfo, server.getServerName(),
+                    openingInteger.get());
+                if (vsn == -1) {
+                  throw KeeperException.create(Code.BADVERSION);
+                }
+                openingInteger.set(vsn);
+              } catch (KeeperException e) {
+                server.abort("ZK exception refreshing OPENING node", e);
+              }
+            }
+      });
+    } catch (IOException e) {
+      LOG.error("IOException instantiating region for " + regionInfo +
+        "; resetting state of transition node from OPENING to OFFLINE");
+      try {
+        // TODO: We should rely on the master timing out OPENING instead of this
+        // TODO: What if this was a split open?  The RS made the OFFLINE
+        // znode, not the master.
+        ZKAssign.forceNodeOffline(server.getZooKeeper(), regionInfo,
+            server.getServerName());
+      } catch (KeeperException e1) {
+        LOG.error("Error forcing node back to OFFLINE from OPENING");
+        return;
+      }
+      return;
+    }
+
+    // Re-transition node to OPENING again to verify no one has stomped on us
+    openingVersion = openingInteger.get();
+    try {
+      if((openingVersion = ZKAssign.retransitionNodeOpening(
+          server.getZooKeeper(), regionInfo, server.getServerName(),
+          openingVersion)) == -1) {
+        LOG.warn("Completed the OPEN of a region but when transitioning from " +
+            " OPENING to OPENING got a version mismatch, someone else clashed " +
+            "so now unassigning");
+        region.close();
+        return;
+      }
+    } catch (KeeperException e) {
+      LOG.error("Failed transitioning node from OPENING to OPENED", e);
+      return;
+    } catch (IOException e) {
+      LOG.error("Failed to close region after failing to transition", e);
+      return;
+    }
+
+    // Update ZK, ROOT or META
+    try {
+      this.rsServices.postOpenDeployTasks(region,
+        this.server.getCatalogTracker(), false);
+    } catch (IOException e) {
+      // TODO: rollback the open?
+      LOG.error("Error updating region location in catalog table", e);
+    } catch (KeeperException e) {
+      // TODO: rollback the open?
+      LOG.error("ZK Error updating region location in catalog table", e);
+    }
+
+    // Finally, Transition ZK node to OPENED
+    try {
+      if(ZKAssign.transitionNodeOpened(server.getZooKeeper(), regionInfo,
+          server.getServerName(), openingVersion) == -1) {
+        LOG.warn("Completed the OPEN of a region but when transitioning from " +
+            " OPENING to OPENED got a version mismatch, someone else clashed " +
+            "so now unassigning");
+        region.close();
+        return;
+      }
+    } catch (KeeperException e) {
+      LOG.error("Failed transitioning node from OPENING to OPENED", e);
+      return;
+    } catch (IOException e) {
+      LOG.error("Failed to close region after failing to transition", e);
+      return;
+    }
+
+    // Done!  Successful region open
+    LOG.debug("Opened " + region.getRegionNameAsString());
+  }
+
+  int transitionZookeeperOfflineToOpening(final String encodedName) {
+    // Transition ZK node from OFFLINE to OPENING
+    // TODO: should also handle transition from CLOSED?
+    int openingVersion = -1;
+    try {
+      if ((openingVersion = ZKAssign.transitionNodeOpening(server.getZooKeeper(),
+          regionInfo, server.getServerName())) == -1) {
+        LOG.warn("Error transitioning node from OFFLINE to OPENING, " +
+            "aborting open");
+      }
+    } catch (KeeperException e) {
+      LOG.error("Error transitioning node from OFFLINE to OPENING for region " +
+        encodedName, e);
+    }
+    return openingVersion;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRootHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRootHandler.java
new file mode 100644
index 00000000000..94a1ed95e11
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRootHandler.java
@@ -0,0 +1,36 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.handler;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+
+/**
+ * Handles opening of the root region on a region server.
+ * <p>
+ * This is executed after receiving an OPEN RPC from the master for root.
+ */
+public class OpenRootHandler extends OpenRegionHandler {
+  public OpenRootHandler(final Server server,
+      final RegionServerServices rsServices, HRegionInfo regionInfo) {
+    super(server, rsServices, regionInfo, EventType.M2RS_OPEN_ROOT);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 20a535c086e..49845c505b6 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -47,6 +47,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -78,8 +79,9 @@ import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.StringUtils;
 
-import com.google.common.util.concurrent.NamingThreadFactory;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 /**
  * HLog stores all the edits to the HStore.  Its the hbase write-ahead-log
@@ -136,15 +138,15 @@ public class HLog implements Syncable {
   private final FileSystem fs;
   private final Path dir;
   private final Configuration conf;
-  private final LogRollListener listener;
+  // Listeners that are called on WAL events.
+  private List<WALObserver> listeners =
+    new CopyOnWriteArrayList<WALObserver>();
   private final long optionalFlushInterval;
   private final long blocksize;
   private final int flushlogentries;
   private final String prefix;
   private final AtomicInteger unflushedEntries = new AtomicInteger(0);
   private final Path oldLogDir;
-  private final List<LogActionsListener> actionListeners =
-      Collections.synchronizedList(new ArrayList<LogActionsListener>());
 
 
   private static Class<? extends Writer> logWriterClass;
@@ -187,7 +189,8 @@ public class HLog implements Syncable {
     Collections.synchronizedSortedMap(new TreeMap<Long, Path>());
 
   /*
-   * Map of regions to first sequence/edit id in their memstore.
+   * Map of regions to most recent sequence/edit id in their memstore.
+   * Key is encoded region name.
    */
   private final ConcurrentSkipListMap<byte [], Long> lastSeqWritten =
     new ConcurrentSkipListMap<byte [], Long>(Bytes.BYTES_COMPARATOR);
@@ -228,9 +231,6 @@ public class HLog implements Syncable {
    */
   private final LogSyncer logSyncerThread;
 
-  private final List<LogEntryVisitor> logEntryVisitors =
-      new CopyOnWriteArrayList<LogEntryVisitor>();
-
   /**
    * Pattern used to validate a HLog file name
    */
@@ -278,19 +278,18 @@ public class HLog implements Syncable {
   }
 
   /**
-   * HLog creating with a null actions listener.
+   * Constructor.
    *
    * @param fs filesystem handle
    * @param dir path to where hlogs are stored
    * @param oldLogDir path to where hlogs are archived
    * @param conf configuration to use
-   * @param listener listerner used to request log rolls
    * @throws IOException
    */
   public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
-              final Configuration conf, final LogRollListener listener)
+              final Configuration conf)
   throws IOException {
-    this(fs, dir, oldLogDir, conf, listener, null, null);
+    this(fs, dir, oldLogDir, conf, null, null);
   }
 
   /**
@@ -304,22 +303,27 @@ public class HLog implements Syncable {
    * @param dir path to where hlogs are stored
    * @param oldLogDir path to where hlogs are archived
    * @param conf configuration to use
-   * @param listener listerner used to request log rolls
-   * @param actionListener optional listener for hlog actions like archiving
+   * @param listeners Listeners on WAL events. Listeners passed here will
+   * be registered before we do anything else; e.g. the
+   * Constructor {@link #rollWriter().
    * @param prefix should always be hostname and port in distributed env and
    *        it will be URL encoded before being used.
    *        If prefix is null, "hlog" will be used
    * @throws IOException
    */
   public HLog(final FileSystem fs, final Path dir, final Path oldLogDir,
-              final Configuration conf, final LogRollListener listener,
-              final LogActionsListener actionListener, final String prefix)
+    final Configuration conf, final List<WALObserver> listeners,
+    final String prefix)
   throws IOException {
     super();
     this.fs = fs;
     this.dir = dir;
     this.conf = conf;
-    this.listener = listener;
+    if (listeners != null) {
+      for (WALObserver i: listeners) {
+        registerWALActionsListener(i);
+      }
+    }
     this.flushlogentries =
       conf.getInt("hbase.regionserver.flushlogentries", 1);
     this.blocksize = conf.getLong("hbase.regionserver.hlog.blocksize",
@@ -339,14 +343,12 @@ public class HLog implements Syncable {
     }
     this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 32);
     this.enabled = conf.getBoolean("hbase.regionserver.hlog.enabled", true);
-    LOG.info("HLog configuration: blocksize=" + this.blocksize +
-      ", rollsize=" + this.logrollsize +
+    LOG.info("HLog configuration: blocksize=" +
+      StringUtils.byteDesc(this.blocksize) +
+      ", rollsize=" + StringUtils.byteDesc(this.logrollsize) +
       ", enabled=" + this.enabled +
       ", flushlogentries=" + this.flushlogentries +
       ", optionallogflushinternal=" + this.optionalFlushInterval + "ms");
-    if (actionListener != null) {
-      addLogActionsListerner(actionListener);
-    }
     // If prefix is null||empty then just name it hlog
     this.prefix = prefix == null || prefix.isEmpty() ?
         "hlog" : URLEncoder.encode(prefix, "UTF8");
@@ -355,22 +357,26 @@ public class HLog implements Syncable {
 
     // handle the reflection necessary to call getNumCurrentReplicas()
     this.getNumCurrentReplicas = null;
-    if(this.hdfs_out != null) {
+    Exception exception = null;
+    if (this.hdfs_out != null) {
       try {
         this.getNumCurrentReplicas = this.hdfs_out.getClass().
           getMethod("getNumCurrentReplicas", new Class<?> []{});
         this.getNumCurrentReplicas.setAccessible(true);
       } catch (NoSuchMethodException e) {
         // Thrown if getNumCurrentReplicas() function isn't available
+        exception = e;
       } catch (SecurityException e) {
         // Thrown if we can't get access to getNumCurrentReplicas()
+        exception = e;
         this.getNumCurrentReplicas = null; // could happen on setAccessible()
       }
     }
-    if(this.getNumCurrentReplicas != null) {
+    if (this.getNumCurrentReplicas != null) {
       LOG.info("Using getNumCurrentReplicas--HDFS-826");
     } else {
-      LOG.info("getNumCurrentReplicas--HDFS-826 not available" );
+      LOG.info("getNumCurrentReplicas--HDFS-826 not available; hdfs_out=" +
+        this.hdfs_out + ", exception=" + exception.getMessage());
     }
 
     logSyncerThread = new LogSyncer(this.optionalFlushInterval);
@@ -378,6 +384,14 @@ public class HLog implements Syncable {
         Thread.currentThread().getName() + ".logSyncer");
   }
 
+  public void registerWALActionsListener (final WALObserver listener) {
+    this.listeners.add(listener);
+  }
+
+  public boolean unregisterWALActionsListener(final WALObserver listener) {
+    return this.listeners.remove(listener);
+  }
+
   /**
    * @return Current state of the monotonically increasing file id.
    */
@@ -429,7 +443,8 @@ public class HLog implements Syncable {
    * for the lock on this and consequently never release the cacheFlushLock
    *
    * @return If lots of logs, flush the returned regions so next time through
-   * we can clean logs. Returns null if nothing to flush.
+   * we can clean logs. Returns null if nothing to flush.  Names are actual
+   * region names as returned by {@link HRegionInfo#getEncodedName()}
    * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException
    * @throws IOException
    */
@@ -475,14 +490,14 @@ public class HLog implements Syncable {
         this.numEntries.set(0);
       }
       // Tell our listeners that a new log was created
-      if (!this.actionListeners.isEmpty()) {
-        for (LogActionsListener list : this.actionListeners) {
-          list.logRolled(newPath);
+      if (!this.listeners.isEmpty()) {
+        for (WALObserver i : this.listeners) {
+          i.logRolled(newPath);
         }
       }
       // Can we delete any of the old log files?
       if (this.outputfiles.size() > 0) {
-        if (this.lastSeqWritten.size() <= 0) {
+        if (this.lastSeqWritten.isEmpty()) {
           LOG.debug("Last sequenceid written is empty. Deleting all old hlogs");
           // If so, then no new writes have come in since all regions were
           // flushed (and removed from the lastSeqWritten map). Means can
@@ -557,7 +572,8 @@ public class HLog implements Syncable {
   /*
    * Clean up old commit logs.
    * @return If lots of logs, flush the returned region so next time through
-   * we can clean logs. Returns null if nothing to flush.
+   * we can clean logs. Returns null if nothing to flush.  Returns array of
+   * encoded region names to flush.
    * @throws IOException
    */
   private byte [][] cleanOldLogs() throws IOException {
@@ -584,10 +600,12 @@ public class HLog implements Syncable {
     }
 
     // If too many log files, figure which regions we need to flush.
+    // Array is an array of encoded region names.
     byte [][] regions = null;
     int logCount = this.outputfiles.size() - logsToRemove;
     if (logCount > this.maxLogs && this.outputfiles != null &&
         this.outputfiles.size() > 0) {
+      // This is an array of encoded region names.
       regions = findMemstoresWithEditsOlderThan(this.outputfiles.firstKey(),
         this.lastSeqWritten);
       StringBuilder sb = new StringBuilder();
@@ -631,6 +649,10 @@ public class HLog implements Syncable {
     return Collections.min(this.lastSeqWritten.values());
   }
 
+  /**
+   * @param oldestOutstandingSeqNum
+   * @return (Encoded) name of oldest outstanding region.
+   */
   private byte [] getOldestRegion(final Long oldestOutstandingSeqNum) {
     byte [] oldestRegion = null;
     for (Map.Entry<byte [], Long> e: this.lastSeqWritten.entrySet()) {
@@ -758,7 +780,7 @@ public class HLog implements Syncable {
     final long now,
     final boolean isMetaRegion)
   throws IOException {
-    byte [] regionName = regionInfo.getRegionName();
+    byte [] regionName = regionInfo.getEncodedNameAsBytes();
     byte [] tableName = regionInfo.getTableDesc().getName();
     this.append(regionInfo, makeKey(regionName, tableName, -1, now), logEdit);
   }
@@ -787,7 +809,6 @@ public class HLog implements Syncable {
     if (this.closed) {
       throw new IOException("Cannot append; log is closed");
     }
-    byte [] regionName = regionInfo.getRegionName();
     synchronized (updateLock) {
       long seqNum = obtainSeqNum();
       logKey.setLogSeqNum(seqNum);
@@ -796,7 +817,8 @@ public class HLog implements Syncable {
       // memstore). When the cache is flushed, the entry for the
       // region being flushed is removed if the sequence number of the flush
       // is greater than or equal to the value in lastSeqWritten.
-      this.lastSeqWritten.putIfAbsent(regionName, Long.valueOf(seqNum));
+      this.lastSeqWritten.putIfAbsent(regionInfo.getEncodedNameAsBytes(),
+        Long.valueOf(seqNum));
       doWrite(regionInfo, logKey, logEdit);
       this.unflushedEntries.incrementAndGet();
       this.numEntries.incrementAndGet();
@@ -807,8 +829,8 @@ public class HLog implements Syncable {
   }
 
   /**
-   * Append a set of edits to the log. Log edits are keyed by regionName,
-   * rowname, and log-sequence-id.
+   * Append a set of edits to the log. Log edits are keyed by (encoded)
+   * regionName, rowname, and log-sequence-id.
    *
    * Later, if we sort by these keys, we obtain all the relevant edits for a
    * given key-range of the HRegion (TODO). Any edits that do not have a
@@ -833,8 +855,6 @@ public class HLog implements Syncable {
     final long now)
   throws IOException {
     if (edits.isEmpty()) return;
-    
-    byte[] regionName = info.getRegionName();
     if (this.closed) {
       throw new IOException("Cannot append; log is closed");
     }
@@ -845,8 +865,11 @@ public class HLog implements Syncable {
       // memstore). . When the cache is flushed, the entry for the
       // region being flushed is removed if the sequence number of the flush
       // is greater than or equal to the value in lastSeqWritten.
-      this.lastSeqWritten.putIfAbsent(regionName, seqNum);
-      HLogKey logKey = makeKey(regionName, tableName, seqNum, now);
+      // Use encoded name.  Its shorter, guaranteed unique and a subset of
+      // actual  name.
+      byte [] hriKey = info.getEncodedNameAsBytes();
+      this.lastSeqWritten.putIfAbsent(hriKey, seqNum);
+      HLogKey logKey = makeKey(hriKey, tableName, seqNum, now);
       doWrite(info, logKey, edits);
       this.numEntries.incrementAndGet();
 
@@ -910,7 +933,7 @@ public class HLog implements Syncable {
         LOG.error("Error while syncing, requesting close of hlog ", e);
         requestLogRoll();
       } catch (InterruptedException e) {
-        LOG.debug(getName() + "interrupted while waiting for sync requests");
+        LOG.debug(getName() + " interrupted while waiting for sync requests");
       } finally {
         syncerShuttingDown = true;
         syncDone.signalAll();
@@ -1043,8 +1066,10 @@ public class HLog implements Syncable {
   }
 
   private void requestLogRoll() {
-    if (this.listener != null) {
-      this.listener.logRollRequested();
+    if (!this.listeners.isEmpty()) {
+      for (WALObserver i: this.listeners) {
+        i.logRollRequested();
+      }
     }
   }
 
@@ -1053,9 +1078,9 @@ public class HLog implements Syncable {
     if (!this.enabled) {
       return;
     }
-    if (!this.logEntryVisitors.isEmpty()) {
-      for (LogEntryVisitor visitor : this.logEntryVisitors) {
-        visitor.visitLogEntryBeforeWrite(info, logKey, logEdit);
+    if (!this.listeners.isEmpty()) {
+      for (WALObserver i: this.listeners) {
+        i.visitLogEntryBeforeWrite(info, logKey, logEdit);
       }
     }
     try {
@@ -1115,14 +1140,13 @@ public class HLog implements Syncable {
    *
    * Protected by cacheFlushLock
    *
-   * @param regionName
+   * @param encodedRegionName
    * @param tableName
    * @param logSeqId
    * @throws IOException
    */
-  public void completeCacheFlush(final byte [] regionName, final byte [] tableName,
-    final long logSeqId,
-    final boolean isMetaRegion)
+  public void completeCacheFlush(final byte [] encodedRegionName,
+      final byte [] tableName, final long logSeqId, final boolean isMetaRegion)
   throws IOException {
     try {
       if (this.closed) {
@@ -1131,15 +1155,15 @@ public class HLog implements Syncable {
       synchronized (updateLock) {
         long now = System.currentTimeMillis();
         WALEdit edit = completeCacheFlushLogEdit();
-        HLogKey key = makeKey(regionName, tableName, logSeqId,
+        HLogKey key = makeKey(encodedRegionName, tableName, logSeqId,
             System.currentTimeMillis());
         this.writer.append(new Entry(key, edit));
         writeTime += System.currentTimeMillis() - now;
         writeOps++;
         this.numEntries.incrementAndGet();
-        Long seq = this.lastSeqWritten.get(regionName);
+        Long seq = this.lastSeqWritten.get(encodedRegionName);
         if (seq != null && logSeqId >= seq.longValue()) {
-          this.lastSeqWritten.remove(regionName);
+          this.lastSeqWritten.remove(encodedRegionName);
         }
       }
       // sync txn to file system
@@ -1475,9 +1499,10 @@ public class HLog implements Syncable {
       conf.getInt("hbase.regionserver.hlog.splitlog.writer.threads", 3);
     boolean skipErrors = conf.getBoolean("hbase.skip.errors", false);
     HashMap<byte[], Future> writeFutureResult = new HashMap<byte[], Future>();
-    NamingThreadFactory f  = new NamingThreadFactory(
-            "SplitWriter-%1$d", Executors.defaultThreadFactory());
-    ThreadPoolExecutor threadPool = (ThreadPoolExecutor)Executors.newFixedThreadPool(logWriterThreads, f);
+    ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
+    builder.setNameFormat("SplitWriter-%1$d");
+    ThreadFactory factory = builder.build();
+    ThreadPoolExecutor threadPool = (ThreadPoolExecutor)Executors.newFixedThreadPool(logWriterThreads, factory);
     for (final byte [] region : splitLogsMap.keySet()) {
       Callable splitter = createNewSplitter(rootDir, logWriters, splitLogsMap, region, fs, conf);
       writeFutureResult.put(region, threadPool.submit(splitter));
@@ -1558,7 +1583,7 @@ public class HLog implements Syncable {
     try {
       Entry entry;
       while ((entry = in.next()) != null) {
-        byte[] region = entry.getKey().getRegionName();
+        byte[] region = entry.getKey().getEncodedRegionName();
         LinkedList<Entry> queue = splitLogsMap.get(region);
         if (queue == null) {
           queue = new LinkedList<Entry>();
@@ -1682,7 +1707,7 @@ public class HLog implements Syncable {
     Path tableDir = HTableDescriptor.getTableDir(rootDir,
       logEntry.getKey().getTablename());
     Path regiondir = HRegion.getRegionDir(tableDir,
-      HRegionInfo.encodeRegionName(logEntry.getKey().getRegionName()));
+      Bytes.toString(logEntry.getKey().getEncodedRegionName()));
     Path dir = getRegionDirRecoveredEditsDir(regiondir);
     if (!fs.exists(dir)) {
       if (!fs.mkdirs(dir)) LOG.warn("mkdir failed on " + dir);
@@ -1759,32 +1784,6 @@ public class HLog implements Syncable {
     return new Path(regiondir, RECOVERED_EDITS_DIR);
   }
 
-  /**
-   *
-   * @param visitor
-   */
-  public void addLogEntryVisitor(LogEntryVisitor visitor) {
-    this.logEntryVisitors.add(visitor);
-  }
-
-  /**
-   * 
-   * @param visitor
-   */
-  public void removeLogEntryVisitor(LogEntryVisitor visitor) {
-    this.logEntryVisitors.remove(visitor);
-  }
-
-
-  public void addLogActionsListerner(LogActionsListener list) {
-    LOG.info("Adding a listener");
-    this.actionListeners.add(list);
-  }
-
-  public boolean removeLogActionsListener(LogActionsListener list) {
-    return this.actionListeners.remove(list);
-  }
-
   private static void usage() {
     System.err.println("Usage: java org.apache.hbase.HLog" +
         " {--dump <logfile>... | --split <logdir>...}");
@@ -1846,5 +1845,4 @@ public class HLog implements Syncable {
   public static final long FIXED_OVERHEAD = ClassSize.align(
       ClassSize.OBJECT + (5 * ClassSize.REFERENCE) +
       ClassSize.ATOMIC_INTEGER + Bytes.SIZEOF_INT + (3 * Bytes.SIZEOF_LONG));
-
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
index 3036be30b25..ed7849f4354 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
@@ -39,7 +39,8 @@ import org.apache.hadoop.io.WritableComparable;
  * associated row.
  */
 public class HLogKey implements WritableComparable<HLogKey> {
-  private byte [] regionName;
+  //  The encoded region name.
+  private byte [] encodedRegionName;
   private byte [] tablename;
   private long logSeqNum;
   // Time at which this edit was written.
@@ -57,27 +58,24 @@ public class HLogKey implements WritableComparable<HLogKey> {
    * We maintain the tablename mainly for debugging purposes.
    * A regionName is always a sub-table object.
    *
-   * @param regionName  - name of region
+   * @param encodedRegionName Encoded name of the region as returned by
+   * {@link HRegionInfo#getEncodedNameAsBytes()}.
    * @param tablename   - name of table
    * @param logSeqNum   - log sequence number
    * @param now Time at which this edit was written.
    */
-  public HLogKey(final byte [] regionName, final byte [] tablename,
+  public HLogKey(final byte [] encodedRegionName, final byte [] tablename,
       long logSeqNum, final long now) {
-    this.regionName = regionName;
+    this.encodedRegionName = encodedRegionName;
     this.tablename = tablename;
     this.logSeqNum = logSeqNum;
     this.writeTime = now;
     this.clusterId = HConstants.DEFAULT_CLUSTER_ID;
   }
 
-  //////////////////////////////////////////////////////////////////////////////
-  // A bunch of accessors
-  //////////////////////////////////////////////////////////////////////////////
-
-  /** @return region name */
-  public byte [] getRegionName() {
-    return regionName;
+  /** @return encoded region name */
+  public byte [] getEncodedRegionName() {
+    return encodedRegionName;
   }
 
   /** @return table name */
@@ -119,7 +117,7 @@ public class HLogKey implements WritableComparable<HLogKey> {
 
   @Override
   public String toString() {
-    return Bytes.toString(tablename) + "/" + Bytes.toString(regionName) + "/" +
+    return Bytes.toString(tablename) + "/" + Bytes.toString(encodedRegionName) + "/" +
       logSeqNum;
   }
 
@@ -136,7 +134,7 @@ public class HLogKey implements WritableComparable<HLogKey> {
 
   @Override
   public int hashCode() {
-    int result = Bytes.hashCode(this.regionName);
+    int result = Bytes.hashCode(this.encodedRegionName);
     result ^= this.logSeqNum;
     result ^= this.writeTime;
     result ^= this.clusterId;
@@ -144,7 +142,7 @@ public class HLogKey implements WritableComparable<HLogKey> {
   }
 
   public int compareTo(HLogKey o) {
-    int result = Bytes.compareTo(this.regionName, o.regionName);
+    int result = Bytes.compareTo(this.encodedRegionName, o.encodedRegionName);
     if (result == 0) {
       if (this.logSeqNum < o.logSeqNum) {
         result = -1;
@@ -163,7 +161,7 @@ public class HLogKey implements WritableComparable<HLogKey> {
   }
 
   public void write(DataOutput out) throws IOException {
-    Bytes.writeByteArray(out, this.regionName);
+    Bytes.writeByteArray(out, this.encodedRegionName);
     Bytes.writeByteArray(out, this.tablename);
     out.writeLong(this.logSeqNum);
     out.writeLong(this.writeTime);
@@ -171,7 +169,7 @@ public class HLogKey implements WritableComparable<HLogKey> {
   }
 
   public void readFields(DataInput in) throws IOException {
-    this.regionName = Bytes.readByteArray(in);
+    this.encodedRegionName = Bytes.readByteArray(in);
     this.tablename = Bytes.readByteArray(in);
     this.logSeqNum = in.readLong();
     this.writeTime = in.readLong();
@@ -181,5 +179,4 @@ public class HLogKey implements WritableComparable<HLogKey> {
       // Means it's an old key, just continue
     }
   }
-
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogEntryVisitor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogEntryVisitor.java
deleted file mode 100644
index 3518fba2a81..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogEntryVisitor.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.apache.hadoop.hbase.regionserver.wal;
-
-import org.apache.hadoop.hbase.HRegionInfo;
-
-public interface LogEntryVisitor {
-
-  /**
-   *
-   * @param info
-   * @param logKey
-   * @param logEdit
-   */
-  public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey,
-                                       WALEdit logEdit);
-}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogActionsListener.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALObserver.java
similarity index 66%
rename from src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogActionsListener.java
rename to src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALObserver.java
index 110d35d6115..cc360c41f49 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogActionsListener.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALObserver.java
@@ -20,17 +20,30 @@
 package org.apache.hadoop.hbase.regionserver.wal;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
 
 /**
- * Interface that defines all actions that can be listened to coming
- * from the HLog. The calls are done in sync with what happens over in the
- * HLog so make sure your implementation is fast.
+ * Get notification of {@link HLog}/WAL log events. The invocations are inline
+ * so make sure your implementation is fast else you'll slow hbase.
  */
-public interface LogActionsListener {
-
+public interface WALObserver {
   /**
-   * Notify the listener that a new file is available
+   * The WAL was rolled.
    * @param newFile the path to the new hlog
    */
   public void logRolled(Path newFile);
+
+  /**
+   * A request was made that the WAL be rolled.
+   */
+  public void logRollRequested();
+
+  /**
+  * Called before each write.
+  * @param info
+  * @param logKey
+  * @param logEdit
+  */
+ public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey,
+   WALEdit logEdit);
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeperWrapper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
similarity index 56%
rename from src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeperWrapper.java
rename to src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
index 1007aeb4699..6e21afd8f37 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeperWrapper.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
@@ -19,29 +19,32 @@
  */
 package org.apache.hadoop.hbase.replication;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.SortedMap;
-import java.util.TreeMap;
 import java.util.SortedSet;
+import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+
 /**
  * This class serves as a helper for all things related to zookeeper
  * in replication.
@@ -70,16 +73,15 @@ import java.util.concurrent.atomic.AtomicBoolean;
  *    ...
  * </pre>
  */
-public class ReplicationZookeeperWrapper {
-
+public class ReplicationZookeeper {
   private static final Log LOG =
-      LogFactory.getLog(ReplicationZookeeperWrapper.class);
+    LogFactory.getLog(ReplicationZookeeper.class);
   // Name of znode we use to lock when failover
   private final static String RS_LOCK_ZNODE = "lock";
   // Our handle on zookeeper
-  private final ZooKeeperWrapper zookeeperWrapper;
+  private final ZooKeeperWatcher zookeeper;
   // Map of addresses of peer clusters with their ZKW
-  private final Map<String, ZooKeeperWrapper> peerClusters;
+  private final Map<String, ReplicationZookeeper> peerClusters;
   // Path to the root replication znode
   private final String replicationZNode;
   // Path to the peer clusters znode
@@ -97,22 +99,22 @@ public class ReplicationZookeeperWrapper {
   private final AtomicBoolean replicating;
   // Byte (stored as string here) that identifies this cluster
   private final String clusterId;
+  // Abortable
+  private final Abortable abortable;
 
   /**
    * Constructor used by region servers, connects to the peer cluster right away.
    *
-   * @param zookeeperWrapper zkw to wrap
-   * @param conf             conf to use
+   * @param zookeeper
    * @param replicating    atomic boolean to start/stop replication
-   * @param rsName      the name of this region server, null if
-   *                         using RZH only to use the helping methods
    * @throws IOException
+   * @throws KeeperException 
    */
-  public ReplicationZookeeperWrapper(
-      ZooKeeperWrapper zookeeperWrapper, Configuration conf,
-      final AtomicBoolean replicating, String rsName) throws IOException {
-    this.zookeeperWrapper = zookeeperWrapper;
-    this.conf = conf;
+  public ReplicationZookeeper(final Server server, final AtomicBoolean replicating)
+  throws IOException, KeeperException {
+    this.abortable = server;
+    this.zookeeper = server.getZooKeeper();
+    this.conf = server.getConfiguration();
     String replicationZNodeName =
         conf.get("zookeeper.znode.replication", "replication");
     String peersZNodeName =
@@ -129,37 +131,39 @@ public class ReplicationZookeeperWrapper {
           this.conf.get("hbase.zookeeper.property.clientPort") + ":" +
           this.conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
 
-    this.peerClusters = new HashMap<String, ZooKeeperWrapper>();
-    this.replicationZNode = zookeeperWrapper.getZNode(
-        zookeeperWrapper.getParentZNode(), replicationZNodeName);
-    this.peersZNode =
-        zookeeperWrapper.getZNode(replicationZNode, peersZNodeName);
-    this.rsZNode =
-        zookeeperWrapper.getZNode(replicationZNode, rsZNodeName);
+    this.peerClusters = new HashMap<String, ReplicationZookeeper>();
+    this.replicationZNode =
+      ZKUtil.joinZNode(this.zookeeper.baseZNode, replicationZNodeName);
+    this.peersZNode = ZKUtil.joinZNode(replicationZNode, peersZNodeName);
+    this.rsZNode = ZKUtil.joinZNode(replicationZNode, rsZNodeName);
 
     this.replicating = replicating;
     setReplicating();
-    String idResult = Bytes.toString(
-        this.zookeeperWrapper.getData(this.replicationZNode,
-        clusterIdZNodeName));
-    this.clusterId =
-        idResult == null ?
-            Byte.toString(HConstants.DEFAULT_CLUSTER_ID) : idResult;
-    String address = Bytes.toString(
-        this.zookeeperWrapper.getData(this.replicationZNode,
-          repMasterZNodeName));
+    String znode = ZKUtil.joinZNode(this.replicationZNode, clusterIdZNodeName);
+    byte [] data = ZKUtil.getData(this.zookeeper, znode);
+    String idResult = Bytes.toString(data);
+    this.clusterId = idResult == null?
+      Byte.toString(HConstants.DEFAULT_CLUSTER_ID): idResult;
+
+    znode = ZKUtil.joinZNode(this.replicationZNode, repMasterZNodeName);
+    data = ZKUtil.getData(this.zookeeper, znode);
+    String address = Bytes.toString(data);
     this.replicationMaster = thisCluster.equals(address);
-    LOG.info("This cluster (" + thisCluster + ") is a "
-          + (this.replicationMaster ? "master" : "slave") + " for replication" +
-          ", compared with (" + address + ")");
-    if (rsName != null) {
-      this.rsServerNameZnode =
-          this.zookeeperWrapper.getZNode(rsZNode, rsName);
-      List<String> znodes = this.zookeeperWrapper.listZnodes(this.peersZNode,
-          new ReplicationStatusWatcher());
+    LOG.info("This cluster (" + thisCluster + ") is a " +
+      (this.replicationMaster ? "master" : "slave") + " for replication" +
+        ", compared with (" + address + ")");
+
+    if (server.getServerName() != null) {
+      this.rsServerNameZnode = ZKUtil.joinZNode(rsZNode, server.getServerName());
+      // Set a tracker on replicationStateNodeNode
+      ReplicationStatusTracker tracker =
+        new ReplicationStatusTracker(this.zookeeper, getRepStateNode(), server);
+      tracker.start();
+
+      List<String> znodes = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
       if (znodes != null) {
-        for (String znode : znodes) {
-          connectToPeer(znode);
+        for (String z : znodes) {
+          connectToPeer(z);
         }
       }
     } else {
@@ -178,34 +182,72 @@ public class ReplicationZookeeperWrapper {
     if (this.peerClusters.size() == 0) {
       return new ArrayList<HServerAddress>(0);
     }
-    ZooKeeperWrapper zkw = this.peerClusters.get(peerClusterId);
+    ReplicationZookeeper zkw = this.peerClusters.get(peerClusterId);
     return zkw == null?
-        new ArrayList<HServerAddress>(0) : zkw.scanRSDirectory();
+      new ArrayList<HServerAddress>(0):
+      zkw.scanAddressDirectory(this.zookeeper.rsZNode);
+  }
+
+  /**
+   * Scan a directory of address data.
+   * @param znode The parent node
+   * @return The directory contents as HServerAddresses
+   */
+  public List<HServerAddress> scanAddressDirectory(String znode) {
+    List<HServerAddress> list = new ArrayList<HServerAddress>();
+    List<String> nodes = null;
+    try {
+      nodes = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
+    } catch (KeeperException e) {
+      this.abortable.abort("Scanning " + znode, e);
+    }
+    if (nodes == null) {
+      return list;
+    }
+    for (String node : nodes) {
+      String path = ZKUtil.joinZNode(znode, node);
+      list.add(readAddress(path));
+    }
+    return list;
+  }
+
+  private HServerAddress readAddress(String znode) {
+    byte [] data = null;
+    try {
+      data = ZKUtil.getData(this.zookeeper, znode);
+    } catch (KeeperException e) {
+      this.abortable.abort("Getting address", e);
+    }
+    return new HServerAddress(Bytes.toString(data));
   }
 
   /**
    * This method connects this cluster to another one and registers it
    * in this region server's replication znode
    * @param peerId id of the peer cluster
+   * @throws KeeperException 
    */
-  private void connectToPeer(String peerId) throws IOException {
-    String[] ensemble =
-        Bytes.toString(this.zookeeperWrapper.getData(this.peersZNode, peerId)).
-            split(":");
+  private void connectToPeer(String peerId) throws IOException, KeeperException {
+    String znode = ZKUtil.joinZNode(this.peersZNode, peerId);
+    byte [] data = ZKUtil.getData(this.zookeeper, znode);
+    String [] ensemble = Bytes.toString(data).split(":");
     if (ensemble.length != 3) {
       throw new IllegalArgumentException("Wrong format of cluster address: " +
-          this.zookeeperWrapper.getData(this.peersZNode, peerId));
+        Bytes.toStringBinary(data));
     }
     Configuration otherConf = new Configuration(this.conf);
     otherConf.set(HConstants.ZOOKEEPER_QUORUM, ensemble[0]);
     otherConf.set("hbase.zookeeper.property.clientPort", ensemble[1]);
     otherConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, ensemble[2]);
+    // REENABLE -- FIX!!!!
+    /*
     ZooKeeperWrapper zkw = ZooKeeperWrapper.createInstance(otherConf,
         "connection to cluster: " + peerId);
     zkw.registerListener(new ReplicationStatusWatcher());
     this.peerClusters.put(peerId, zkw);
     this.zookeeperWrapper.ensureExists(this.zookeeperWrapper.getZNode(
         this.rsServerNameZnode, peerId));
+        */
     LOG.info("Added new peer cluster " + StringUtils.arrayToString(ensemble));
   }
 
@@ -213,16 +255,24 @@ public class ReplicationZookeeperWrapper {
    * This reads the state znode for replication and sets the atomic boolean
    */
   private void setReplicating() {
-    String value = Bytes.toString(this.zookeeperWrapper.getDataAndWatch(
-        this.replicationZNode, this.replicationStateNodeName,
-        new ReplicationStatusWatcher()));
-    if (value != null) {
-      this.replicating.set(value.equals("true"));
-      LOG.info("Replication is now " + (this.replicating.get() ?
+    try {
+      byte [] data = ZKUtil.getDataAndWatch(this.zookeeper, getRepStateNode());
+      String value = Bytes.toString(data);
+      if (value == null) LOG.info(getRepStateNode() + " data is null");
+      else {
+        this.replicating.set(Boolean.parseBoolean(value));
+        LOG.info("Replication is now " + (this.replicating.get()?
           "started" : "stopped"));
+      }
+    } catch (KeeperException e) {
+      this.abortable.abort("Failed getting data on from " + getRepStateNode(), e);
     }
   }
 
+  private String getRepStateNode() {
+    return ZKUtil.joinZNode(this.replicationZNode, this.replicationStateNodeName);
+  }
+
   /**
    * Add a new log to the list of hlogs in zookeeper
    * @param filename name of the hlog's znode
@@ -230,13 +280,11 @@ public class ReplicationZookeeperWrapper {
    */
   public void addLogToList(String filename, String clusterId) {
     try {
-      this.zookeeperWrapper.writeZNode(
-          this.zookeeperWrapper.getZNode(
-              this.rsServerNameZnode, clusterId), filename, "");
-    } catch (InterruptedException e) {
-      LOG.error(e);
+      String znode = ZKUtil.joinZNode(this.rsServerNameZnode, clusterId);
+      znode = ZKUtil.joinZNode(znode, filename);
+      ZKUtil.createAndWatch(this.zookeeper, znode, Bytes.toBytes(""));
     } catch (KeeperException e) {
-      LOG.error(e);
+      this.abortable.abort("Failed add log to list", e);
     }
   }
 
@@ -247,13 +295,11 @@ public class ReplicationZookeeperWrapper {
    */
   public void removeLogFromList(String filename, String clusterId) {
     try {
-      this.zookeeperWrapper.deleteZNode(
-          this.zookeeperWrapper.getZNode(this.rsServerNameZnode,
-              this.zookeeperWrapper.getZNode(clusterId, filename)));
-    } catch (InterruptedException e) {
-      LOG.error(e);
+      String znode = ZKUtil.joinZNode(rsServerNameZnode, clusterId);
+      znode = ZKUtil.joinZNode(znode, filename);
+      ZKUtil.deleteChildrenRecursively(this.zookeeper, znode);
     } catch (KeeperException e) {
-      LOG.error(e);
+      this.abortable.abort("Failed remove from list", e);
     }
   }
 
@@ -265,16 +311,15 @@ public class ReplicationZookeeperWrapper {
    * @throws IOException
    */
   public void writeReplicationStatus(String filename, String clusterId,
-                                     long position) {
+      long position) {
     try {
-      String clusterZNode = this.zookeeperWrapper.getZNode(
-          this.rsServerNameZnode, clusterId);
-      this.zookeeperWrapper.writeZNode(clusterZNode, filename,
-          Long.toString(position));
-    } catch (InterruptedException e) {
-      LOG.error(e);
+      String znode = ZKUtil.joinZNode(this.rsServerNameZnode, clusterId);
+      znode = ZKUtil.joinZNode(znode, filename);
+      // Why serialize String of Long and note Long as bytes?
+      ZKUtil.createAndWatch(this.zookeeper, znode,
+        Bytes.toBytes(Long.toString(position)));
     } catch (KeeperException e) {
-      LOG.error(e);
+      this.abortable.abort("Writing replication status", e);
     }
   }
 
@@ -285,8 +330,15 @@ public class ReplicationZookeeperWrapper {
    * @return a list of server nanes
    */
   public List<String> getRegisteredRegionServers(Watcher watch) {
-    return this.zookeeperWrapper.listZnodes(
-        this.zookeeperWrapper.getRsZNode(), watch);
+    List<String> result = null;
+    try {
+      // TODO: This is rsZNode from zk which is like getListOfReplicators
+      // but maybe these are from different zk instances?
+      result = ZKUtil.listChildrenNoWatch(this.zookeeper, rsZNode);
+    } catch (KeeperException e) {
+      this.abortable.abort("Get list of registered region servers", e);
+    }
+    return result;
   }
 
   /**
@@ -295,8 +347,14 @@ public class ReplicationZookeeperWrapper {
    * @param watch the watche to set
    * @return a list of server names
    */
-  public List<String> getListOfReplicators(Watcher watch) {
-    return this.zookeeperWrapper.listZnodes(rsZNode, watch);
+  public List<String> getListOfReplicators() {
+    List<String> result = null;
+    try {
+      result = ZKUtil.listChildrenNoWatch(this.zookeeper, rsZNode);
+    } catch (KeeperException e) {
+      this.abortable.abort("Get list of replicators", e);
+    }
+    return result;
   }
 
   /**
@@ -305,9 +363,15 @@ public class ReplicationZookeeperWrapper {
    * @param watch the watch to set
    * @return a list of peer cluster
    */
-  public List<String> getListPeersForRS(String rs, Watcher watch) {
-    return this.zookeeperWrapper.listZnodes(
-        zookeeperWrapper.getZNode(rsZNode, rs), watch);
+  public List<String> getListPeersForRS(String rs) {
+    String znode = ZKUtil.joinZNode(rsZNode, rs);
+    List<String> result = null;
+    try {
+      result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
+    } catch (KeeperException e) {
+      this.abortable.abort("Get list of peers for rs", e);
+    }
+    return result;
   }
 
   /**
@@ -317,9 +381,16 @@ public class ReplicationZookeeperWrapper {
    * @param watch the watch to set
    * @return a list of hlogs
    */
-  public List<String> getListHLogsForPeerForRS(String rs, String id, Watcher watch) {
-    return this.zookeeperWrapper.listZnodes(
-        zookeeperWrapper.getZNode(zookeeperWrapper.getZNode(rsZNode, rs), id), watch);
+  public List<String> getListHLogsForPeerForRS(String rs, String id) {
+    String znode = ZKUtil.joinZNode(rsZNode, rs);
+    znode = ZKUtil.joinZNode(znode, id);
+    List<String> result = null;
+    try {
+      result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
+    } catch (KeeperException e) {
+      this.abortable.abort("Get list of hlogs for peer", e);
+    }
+    return result;
   }
 
   /**
@@ -329,17 +400,11 @@ public class ReplicationZookeeperWrapper {
    */
   public boolean lockOtherRS(String znode) {
     try {
-      this.zookeeperWrapper.writeZNode(
-          this.zookeeperWrapper.getZNode(this.rsZNode, znode),
-          RS_LOCK_ZNODE, rsServerNameZnode, true);
-
-    } catch (InterruptedException e) {
-      LOG.error(e);
-      return false;
+      String parent = ZKUtil.joinZNode(this.rsZNode, znode);
+      String p = ZKUtil.joinZNode(parent, RS_LOCK_ZNODE);
+      ZKUtil.createAndWatch(this.zookeeper, p, Bytes.toBytes(rsServerNameZnode));
     } catch (KeeperException e) {
-      LOG.debug("Won't lock " + znode + " because " + e.getMessage());
-      // TODO see if the other still exists!!
-      return false;
+      this.abortable.abort("Failed lock other rs", e);
     }
     return true;
   }
@@ -357,8 +422,9 @@ public class ReplicationZookeeperWrapper {
     SortedMap<String,SortedSet<String>> queues =
         new TreeMap<String,SortedSet<String>>();
     try {
-      String nodePath = this.zookeeperWrapper.getZNode(rsZNode, znode);
-      List<String> clusters = this.zookeeperWrapper.listZnodes(nodePath, null);
+      String nodePath = ZKUtil.joinZNode(rsZNode, znode);
+      List<String> clusters =
+        ZKUtil.listChildrenNoWatch(this.zookeeper, nodePath);
       // We have a lock znode in there, it will count as one.
       if (clusters == null || clusters.size() <= 1) {
         return queues;
@@ -370,11 +436,11 @@ public class ReplicationZookeeperWrapper {
         // do that for queues that were recovered 10 times giving a znode like
         // number-startcode-number-otherstartcode-number-anotherstartcode-etc
         String newCluster = cluster+"-"+znode;
-        String newClusterZnode =
-            this.zookeeperWrapper.getZNode(rsServerNameZnode, newCluster);
-        this.zookeeperWrapper.ensureExists(newClusterZnode);
-        String clusterPath = this.zookeeperWrapper.getZNode(nodePath, cluster);
-        List<String> hlogs = this.zookeeperWrapper.listZnodes(clusterPath, null);
+        String newClusterZnode = ZKUtil.joinZNode(rsServerNameZnode, newCluster);
+        ZKUtil.createNodeIfNotExistsAndWatch(this.zookeeper, newClusterZnode,
+          HConstants.EMPTY_BYTE_ARRAY);
+        String clusterPath = ZKUtil.joinZNode(nodePath, cluster);
+        List<String> hlogs = ZKUtil.listChildrenNoWatch(this.zookeeper, clusterPath);
         // That region server didn't have anything to replicate for this cluster
         if (hlogs == null || hlogs.size() == 0) {
           continue;
@@ -382,19 +448,16 @@ public class ReplicationZookeeperWrapper {
         SortedSet<String> logQueue = new TreeSet<String>();
         queues.put(newCluster, logQueue);
         for (String hlog : hlogs) {
-          String position = Bytes.toString(
-              this.zookeeperWrapper.getData(clusterPath, hlog));
-          LOG.debug("Creating " + hlog + " with data " + position);
-          this.zookeeperWrapper.writeZNode(newClusterZnode, hlog, position);
+          String z = ZKUtil.joinZNode(clusterPath, hlog);
+          byte [] position = ZKUtil.getData(this.zookeeper, z);
+          LOG.debug("Creating " + hlog + " with data " + Bytes.toString(position));
+          String child = ZKUtil.joinZNode(newClusterZnode, hlog);
+          ZKUtil.createAndWatch(this.zookeeper, child, position);
           logQueue.add(hlog);
         }
       }
-    } catch (InterruptedException e) {
-      LOG.warn(e);
-      return null;
     } catch (KeeperException e) {
-      LOG.warn(e);
-      return null;
+      this.abortable.abort("Copy queues from rs", e);
     }
     return queues;
   }
@@ -405,12 +468,10 @@ public class ReplicationZookeeperWrapper {
    */
   public void deleteSource(String peerZnode) {
     try {
-      this.zookeeperWrapper.deleteZNode(
-          this.zookeeperWrapper.getZNode(rsServerNameZnode, peerZnode), true);
-    } catch (InterruptedException e) {
-      LOG.error(e);
+      ZKUtil.deleteChildrenRecursively(this.zookeeper,
+          ZKUtil.joinZNode(rsServerNameZnode, peerZnode));
     } catch (KeeperException e) {
-      LOG.error(e);
+      this.abortable.abort("Failed delete of " + peerZnode, e);
     }
   }
 
@@ -420,12 +481,10 @@ public class ReplicationZookeeperWrapper {
    */
   public void deleteRsQueues(String znode) {
     try {
-      this.zookeeperWrapper.deleteZNode(
-          this.zookeeperWrapper.getZNode(rsZNode, znode), true);
-    } catch (InterruptedException e) {
-      LOG.error(e);
+      ZKUtil.deleteChildrenRecursively(this.zookeeper,
+          ZKUtil.joinZNode(rsZNode, znode));
     } catch (KeeperException e) {
-      LOG.error(e);
+      this.abortable.abort("Failed delete of " + znode, e);
     }
   }
 
@@ -441,12 +500,13 @@ public class ReplicationZookeeperWrapper {
    * @param peerId znode of the peer cluster
    * @param hlog name of the hlog
    * @return the position in that hlog
+   * @throws KeeperException 
    */
-  public long getHLogRepPosition(String peerId, String hlog) {
-    String clusterZnode =
-        this.zookeeperWrapper.getZNode(rsServerNameZnode, peerId);
-    String data = Bytes.toString(
-        this.zookeeperWrapper.getData(clusterZnode, hlog));
+  public long getHLogRepPosition(String peerId, String hlog)
+  throws KeeperException {
+    String clusterZnode = ZKUtil.joinZNode(rsServerNameZnode, peerId);
+    String znode = ZKUtil.joinZNode(clusterZnode, hlog);
+    String data = Bytes.toString(ZKUtil.getData(this.zookeeper, znode));
     return data == null || data.length() == 0 ? 0 : Long.parseLong(data);
   }
 
@@ -472,22 +532,23 @@ public class ReplicationZookeeperWrapper {
    * Get a map of all peer clusters
    * @return map of peer cluster, zk address to ZKW
    */
-  public Map<String, ZooKeeperWrapper> getPeerClusters() {
+  public Map<String, ReplicationZookeeper> getPeerClusters() {
     return this.peerClusters;
   }
 
   /**
-   * Watcher for the status of the replication
+   * Tracker for status of the replication
    */
-  public class ReplicationStatusWatcher implements Watcher {
+  public class ReplicationStatusTracker extends ZooKeeperNodeTracker {
+    public ReplicationStatusTracker(ZooKeeperWatcher watcher, String node,
+        Abortable abortable) {
+      super(watcher, node, abortable);
+    }
+
     @Override
-    public void process(WatchedEvent watchedEvent) {
-      Event.EventType type = watchedEvent.getType();
-      LOG.info("Got event " + type + " with path " + watchedEvent.getPath());
-      if (type.equals(Event.EventType.NodeDataChanged)) {
-        setReplicating();
-      }
+    public synchronized void nodeDataChanged(String path) {
+      super.nodeDataChanged(path);
+      setReplicating();
     }
   }
-
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index 4d4b00a4da4..b4c881497e1 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -25,8 +25,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.LogCleanerDelegate;
-import org.apache.hadoop.hbase.replication.ReplicationZookeeperWrapper;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
+// REENALBE import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
 
@@ -45,7 +45,7 @@ public class ReplicationLogCleaner implements LogCleanerDelegate, Watcher {
   private static final Log LOG =
     LogFactory.getLog(ReplicationLogCleaner.class);
   private Configuration conf;
-  private ReplicationZookeeperWrapper zkHelper;
+  private ReplicationZookeeper zkHelper;
   private Set<String> hlogs = new HashSet<String>();
 
   /**
@@ -78,30 +78,31 @@ public class ReplicationLogCleaner implements LogCleanerDelegate, Watcher {
   private boolean refreshHLogsAndSearch(String searchedLog) {
     this.hlogs.clear();
     final boolean lookForLog = searchedLog != null;
-    List<String> rss = zkHelper.getListOfReplicators(this);
-    if (rss == null) {
-      LOG.debug("Didn't find any region server that replicates, deleting: " +
-          searchedLog);
-      return false;
-    }
-    for (String rs: rss) {
-      List<String> listOfPeers = zkHelper.getListPeersForRS(rs, this);
-      // if rs just died, this will be null
-      if (listOfPeers == null) {
-        continue;
-      }
-      for (String id : listOfPeers) {
-        List<String> peersHlogs = zkHelper.getListHLogsForPeerForRS(rs, id, this);
-        if (peersHlogs != null) {
-          this.hlogs.addAll(peersHlogs);
-        }
-        // early exit if we found the log
-        if(lookForLog && this.hlogs.contains(searchedLog)) {
-          LOG.debug("Found log in ZK, keeping: " + searchedLog);
-          return true;
-        }
-      }
-    }
+// REENALBE
+//    List<String> rss = zkHelper.getListOfReplicators(this);
+//    if (rss == null) {
+//      LOG.debug("Didn't find any region server that replicates, deleting: " +
+//          searchedLog);
+//      return false;
+//    }
+//    for (String rs: rss) {
+//      List<String> listOfPeers = zkHelper.getListPeersForRS(rs, this);
+//      // if rs just died, this will be null
+//      if (listOfPeers == null) {
+//        continue;
+//      }
+//      for (String id : listOfPeers) {
+//        List<String> peersHlogs = zkHelper.getListHLogsForPeerForRS(rs, id, this);
+//        if (peersHlogs != null) {
+//          this.hlogs.addAll(peersHlogs);
+//        }
+//        // early exit if we found the log
+//        if(lookForLog && this.hlogs.contains(searchedLog)) {
+//          LOG.debug("Found log in ZK, keeping: " + searchedLog);
+//          return true;
+//        }
+//      }
+//    }
     LOG.debug("Didn't find this log in ZK, deleting: " + searchedLog);
     return false;
   }
@@ -109,14 +110,15 @@ public class ReplicationLogCleaner implements LogCleanerDelegate, Watcher {
   @Override
   public void setConf(Configuration conf) {
     this.conf = conf;
-    try {
-      this.zkHelper = new ReplicationZookeeperWrapper(
-          ZooKeeperWrapper.createInstance(this.conf,
-              HMaster.class.getName()),
-          this.conf, new AtomicBoolean(true), null);
-    } catch (IOException e) {
-      LOG.error(e);
-    }
+//    try {
+      // REENABLE
+//      this.zkHelper = new ReplicationZookeeperWrapper(
+//          ZooKeeperWrapper.createInstance(this.conf,
+//              HMaster.class.getName()),
+//          this.conf, new AtomicBoolean(true), null);
+//    } catch (IOException e) {
+//      LOG.error(e);
+//    }
     refreshHLogsAndSearch(null);
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index 52cb8e8ad20..d3595b5b766 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -19,71 +19,75 @@
  */
 package org.apache.hadoop.hbase.replication.regionserver;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
-import org.apache.hadoop.hbase.regionserver.wal.LogEntryVisitor;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.replication.ReplicationZookeeperWrapper;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-
 import java.io.IOException;
 import java.util.NavigableMap;
 import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-/**
- * Replication serves as an umbrella over the setup of replication and
- * is used by HRS.
- */
-public class Replication implements LogEntryVisitor {
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.regionserver.wal.WALObserver;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.zookeeper.KeeperException;
 
+/**
+ * Gateway to Replication.  Used by {@link HRegionServer}.
+ */
+public class Replication implements WALObserver {
   private final boolean replication;
   private final ReplicationSourceManager replicationManager;
   private boolean replicationMaster;
   private final AtomicBoolean replicating = new AtomicBoolean(true);
-  private final ReplicationZookeeperWrapper zkHelper;
+  private final ReplicationZookeeper zkHelper;
   private final Configuration conf;
-  private final AtomicBoolean  stopRequested;
   private ReplicationSink replicationSink;
+  // Hosting server
+  private final Server server;
 
   /**
    * Instantiate the replication management (if rep is enabled).
-   * @param conf conf to use
-   * @param hsi the info if this region server
+   * @param server Hosting server
    * @param fs handle to the filesystem
+   * @param logDir
    * @param oldLogDir directory where logs are archived
-   * @param stopRequested boolean that tells us if we are shutting down
    * @throws IOException
+   * @throws KeeperException 
    */
-  public Replication(Configuration conf, HServerInfo hsi,
-                     FileSystem fs, Path logDir, Path oldLogDir,
-                     AtomicBoolean stopRequested) throws IOException {
-    this.conf = conf;
-    this.stopRequested = stopRequested;
-    this.replication =
-        conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false);
+  public Replication(final Server server, final FileSystem fs,
+      final Path logDir, final Path oldLogDir)
+  throws IOException, KeeperException {
+    this.server = server;
+    this.conf = this.server.getConfiguration();
+    this.replication = isReplication(this.conf);
     if (replication) {
-      this.zkHelper = new ReplicationZookeeperWrapper(
-        ZooKeeperWrapper.getInstance(conf, hsi.getServerName()), conf,
-        this.replicating, hsi.getServerName());
+      this.zkHelper = new ReplicationZookeeper(server, this.replicating);
       this.replicationMaster = zkHelper.isReplicationMaster();
       this.replicationManager = this.replicationMaster ?
-        new ReplicationSourceManager(zkHelper, conf, stopRequested,
+        new ReplicationSourceManager(zkHelper, conf, this.server,
           fs, this.replicating, logDir, oldLogDir) : null;
     } else {
-      replicationManager = null;
-      zkHelper = null;
+      this.replicationManager = null;
+      this.zkHelper = null;
     }
   }
 
+  /**
+   * @param c Configuration to look at
+   * @return True if replication is enabled.
+   */
+  public static boolean isReplication(final Configuration c) {
+    return c.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false);
+  }
+
   /**
    * Join with the replication threads
    */
@@ -92,7 +96,7 @@ public class Replication implements LogEntryVisitor {
       if (this.replicationMaster) {
         this.replicationManager.join();
       }
-      this.zkHelper.deleteOwnRSZNode();
+        this.zkHelper.deleteOwnRSZNode();
     }
   }
 
@@ -117,8 +121,7 @@ public class Replication implements LogEntryVisitor {
       if (this.replicationMaster) {
         this.replicationManager.init();
       } else {
-        this.replicationSink =
-            new ReplicationSink(this.conf, this.stopRequested);
+        this.replicationSink = new ReplicationSink(this.conf, this.server);
       }
     }
   }
@@ -128,12 +131,12 @@ public class Replication implements LogEntryVisitor {
    * @return the manager if replication is enabled, else returns false
    */
   public ReplicationSourceManager getReplicationManager() {
-    return replicationManager;
+    return this.replicationManager;
   }
 
   @Override
   public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey,
-                                       WALEdit logEdit) {
+      WALEdit logEdit) {
     NavigableMap<byte[], Integer> scopes =
         new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
     byte[] family;
@@ -150,13 +153,13 @@ public class Replication implements LogEntryVisitor {
     }
   }
 
-  /**
-   * Add this class as a log entry visitor for HLog if replication is enabled
-   * @param hlog log that was add ourselves on
-   */
-  public void addLogEntryVisitor(HLog hlog) {
-    if (replication) {
-      hlog.addLogEntryVisitor(this);
-    }
+  @Override
+  public void logRolled(Path p) {
+    getReplicationManager().logRolled(p);
   }
-}
+
+  @Override
+  public void logRollRequested() {
+    // Not interested
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index 3bed8bb13d6..cd9dd5886cb 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.Stoppable;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -60,8 +61,8 @@ public class ReplicationSink {
   private final Configuration conf;
   // Pool used to replicated
   private final HTablePool pool;
-  // boolean coming from HRS to know when the process stops
-  private final AtomicBoolean stop;
+  // Chain to pull on when we want all to stop.
+  private final Stoppable stopper;
   private final ReplicationSinkMetrics metrics;
 
   /**
@@ -71,12 +72,12 @@ public class ReplicationSink {
    * @param stopper             boolean to tell this thread to stop
    * @throws IOException thrown when HDFS goes bad or bad file name
    */
-  public ReplicationSink(Configuration conf, AtomicBoolean stopper)
+  public ReplicationSink(Configuration conf, Stoppable stopper)
       throws IOException {
     this.conf = conf;
     this.pool = new HTablePool(this.conf,
         conf.getInt("replication.sink.htablepool.capacity", 10));
-    this.stop = stopper;
+    this.stopper = stopper;
     this.metrics = new ReplicationSinkMetrics();
   }
 
@@ -146,14 +147,14 @@ public class ReplicationSink {
       } else {
         // Should we log rejected edits in a file for replay?
         LOG.error("Unable to accept edit because", ex);
-        this.stop.set(true);
+        this.stopper.stop("Unable to accept edit because " + ex.getMessage());
         throw ex;
       }
     } catch (RuntimeException re) {
       if (re.getCause() instanceof TableNotFoundException) {
         LOG.warn("Losing edits because: ", re);
       } else {
-        this.stop.set(true);
+        this.stopper.stop("Replication stopped us because " + re.getMessage());
         throw re;
       }
     }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 32508de9279..8f35d1b2487 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -19,25 +19,6 @@
  */
 package org.apache.hadoop.hbase.replication.regionserver;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.replication.ReplicationZookeeperWrapper;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Threads;
-
 import java.io.EOFException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -54,6 +35,26 @@ import java.util.concurrent.PriorityBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
+
 /**
  * Class that handles the source of a replication stream.
  * Currently does not handle more than 1 slave
@@ -76,7 +77,7 @@ public class ReplicationSource extends Thread
   private HLog.Entry[] entriesArray;
   private HConnection conn;
   // Helper class for zookeeper
-  private ReplicationZookeeperWrapper zkHelper;
+  private ReplicationZookeeper zkHelper;
   private Configuration conf;
   // ratio of region servers to chose from a slave cluster
   private float ratio;
@@ -88,7 +89,7 @@ public class ReplicationSource extends Thread
   // The manager of all sources to which we ping back our progress
   private ReplicationSourceManager manager;
   // Should we stop everything?
-  private AtomicBoolean stop;
+  private Stoppable stopper;
   // List of chosen sinks (region servers)
   private List<HServerAddress> currentPeers;
   // How long should we sleep for each retry
@@ -139,11 +140,11 @@ public class ReplicationSource extends Thread
   public void init(final Configuration conf,
                    final FileSystem fs,
                    final ReplicationSourceManager manager,
-                   final AtomicBoolean stopper,
+                   final Stoppable stopper,
                    final AtomicBoolean replicating,
                    final String peerClusterZnode)
       throws IOException {
-    this.stop = stopper;
+    this.stopper = stopper;
     this.conf = conf;
     this.replicationQueueSizeCapacity =
         this.conf.getLong("replication.source.size.capacity", 1024*1024*64);
@@ -224,18 +225,18 @@ public class ReplicationSource extends Thread
   public void run() {
     connectToPeers();
     // We were stopped while looping to connect to sinks, just abort
-    if (this.stop.get()) {
+    if (this.stopper.isStopped()) {
       return;
     }
     // If this is recovered, the queue is already full and the first log
     // normally has a position (unless the RS failed between 2 logs)
     if (this.queueRecovered) {
-      this.position = this.zkHelper.getHLogRepPosition(
-          this.peerClusterZnode, this.queue.peek().getName());
+//      this.position = this.zkHelper.getHLogRepPosition(
+//          this.peerClusterZnode, this.queue.peek().getName());
     }
     int sleepMultiplier = 1;
     // Loop until we close down
-    while (!stop.get() && this.running) {
+    while (!stopper.isStopped() && this.running) {
       // Get a new path
       if (!getNextPath()) {
         if (sleepForRetries("No log to process", sleepMultiplier)) {
@@ -311,7 +312,7 @@ public class ReplicationSource extends Thread
       // If we didn't get anything to replicate, or if we hit a IOE,
       // wait a bit and retry.
       // But if we need to stop, don't bother sleeping
-      if (!stop.get() && (gotIOE || currentNbEntries == 0)) {
+      if (!stopper.isStopped() && (gotIOE || currentNbEntries == 0)) {
         if (sleepForRetries("Nothing to replicate", sleepMultiplier)) {
           sleepMultiplier++;
         }
@@ -373,7 +374,7 @@ public class ReplicationSource extends Thread
 
   private void connectToPeers() {
     // Connect to peer cluster first, unless we have to stop
-    while (!this.stop.get() && this.currentPeers.size() == 0) {
+    while (!this.stopper.isStopped() && this.currentPeers.size() == 0) {
       try {
         chooseSinks();
         Thread.sleep(this.sleepForRetries);
@@ -517,7 +518,7 @@ public class ReplicationSource extends Thread
    */
   protected void shipEdits() {
     int sleepMultiplier = 1;
-    while (!stop.get()) {
+    while (!this.stopper.isStopped()) {
       try {
         HRegionInterface rrs = getRS();
         LOG.debug("Replicating " + currentNbEntries);
@@ -549,7 +550,7 @@ public class ReplicationSource extends Thread
                 chooseSinks();
               }
             }
-          } while (!stop.get() && down);
+          } while (!this.stopper.isStopped() && down);
         } catch (InterruptedException e) {
           LOG.debug("Interrupted while trying to contact the peer cluster");
         }
@@ -688,5 +689,4 @@ public class ReplicationSource extends Thread
       return Long.parseLong(parts[parts.length-1]);
     }
   }
-
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
index f6e72dde1e9..b0ca50a9d73 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
@@ -19,13 +19,13 @@
  */
 package org.apache.hadoop.hbase.replication.regionserver;
 
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.hbase.Stoppable;
 
 /**
  * Interface that defines a replication source
@@ -45,7 +45,7 @@ public interface ReplicationSourceInterface {
   public void init(final Configuration conf,
                    final FileSystem fs,
                    final ReplicationSourceManager manager,
-                   final AtomicBoolean stopper,
+                   final Stoppable stopper,
                    final AtomicBoolean replicating,
                    final String peerClusterId) throws IOException;
 
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 8046b7389ed..95cd32873d6 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -20,16 +20,6 @@
 
 package org.apache.hadoop.hbase.replication.regionserver;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.regionserver.wal.LogActionsListener;
-import org.apache.hadoop.hbase.replication.ReplicationZookeeperWrapper;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -39,6 +29,16 @@ import java.util.SortedSet;
 import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+
 /**
  * This class is responsible to manage all the replication
  * sources. There are two classes of sources:
@@ -50,8 +50,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
  * tries to grab a lock in order to transfer all the queues in a local
  * old source.
  */
-public class ReplicationSourceManager implements LogActionsListener {
-
+public class ReplicationSourceManager {
   private static final Log LOG =
       LogFactory.getLog(ReplicationSourceManager.class);
   // List of all the sources that read this RS's logs
@@ -61,9 +60,9 @@ public class ReplicationSourceManager implements LogActionsListener {
   // Indicates if we are currently replicating
   private final AtomicBoolean replicating;
   // Helper for zookeeper
-  private final ReplicationZookeeperWrapper zkHelper;
-  // Indicates if the region server is closing
-  private final AtomicBoolean stopper;
+  private final ReplicationZookeeper zkHelper;
+  // All about stopping
+  private final Stoppable stopper;
   // All logs we are currently trackign
   private final SortedSet<String> hlogs;
   private final Configuration conf;
@@ -88,9 +87,9 @@ public class ReplicationSourceManager implements LogActionsListener {
    * @param logDir the directory that contains all hlog directories of live RSs
    * @param oldLogDir the directory where old logs are archived
    */
-  public ReplicationSourceManager(final ReplicationZookeeperWrapper zkHelper,
+  public ReplicationSourceManager(final ReplicationZookeeper zkHelper,
                                   final Configuration conf,
-                                  final AtomicBoolean stopper,
+                                  final Stoppable stopper,
                                   final FileSystem fs,
                                   final AtomicBoolean replicating,
                                   final Path logDir,
@@ -146,7 +145,7 @@ public class ReplicationSourceManager implements LogActionsListener {
       ReplicationSourceInterface src = addSource(id);
       src.startup();
     }
-    List<String> currentReplicators = this.zkHelper.getListOfReplicators(null);
+    List<String> currentReplicators = this.zkHelper.getListOfReplicators();
     synchronized (otherRegionServers) {
       LOG.info("Current list of replicators: " + currentReplicators
           + " other RSs: " + otherRegionServers);
@@ -198,7 +197,7 @@ public class ReplicationSourceManager implements LogActionsListener {
    * @return a sorted set of hlog names
    */
   protected SortedSet<String> getHLogs() {
-    return new TreeSet(this.hlogs);
+    return new TreeSet<String>(this.hlogs);
   }
 
   /**
@@ -209,8 +208,7 @@ public class ReplicationSourceManager implements LogActionsListener {
     return this.sources;
   }
 
-  @Override
-  public void logRolled(Path newLog) {
+  void logRolled(Path newLog) {
     if (this.sources.size() > 0) {
       this.zkHelper.addLogToList(newLog.getName(),
           this.sources.get(0).getPeerClusterZnode());
@@ -229,7 +227,7 @@ public class ReplicationSourceManager implements LogActionsListener {
    * Get the ZK help of this manager
    * @return the helper
    */
-  public ReplicationZookeeperWrapper getRepZkWrapper() {
+  public ReplicationZookeeper getRepZkWrapper() {
     return zkHelper;
   }
 
@@ -248,11 +246,12 @@ public class ReplicationSourceManager implements LogActionsListener {
       final Configuration conf,
       final FileSystem fs,
       final ReplicationSourceManager manager,
-      final AtomicBoolean stopper,
+      final Stoppable stopper,
       final AtomicBoolean replicating,
       final String peerClusterId) throws IOException {
     ReplicationSourceInterface src;
     try {
+      @SuppressWarnings("rawtypes")
       Class c = Class.forName(conf.get("replication.replicationsource.implementation",
           ReplicationSource.class.getCanonicalName()));
       src = (ReplicationSourceInterface) c.newInstance();
@@ -276,7 +275,7 @@ public class ReplicationSourceManager implements LogActionsListener {
    */
   public void transferQueues(String rsZnode) {
     // We try to lock that rs' queue directory
-    if (this.stopper.get()) {
+    if (this.stopper.isStopped()) {
       LOG.info("Not transferring queue since we are shutting down");
       return;
     }
@@ -372,5 +371,4 @@ public class ReplicationSourceManager implements LogActionsListener {
   public FileSystem getFs() {
     return this.fs;
   }
-
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
index 441917bd385..6702bf0ae3f 100644
--- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
@@ -18,6 +18,17 @@
 
 package org.apache.hadoop.hbase.thrift;
 
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.HelpFormatter;
@@ -35,7 +46,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -73,17 +83,6 @@ import org.apache.thrift.transport.TServerSocket;
 import org.apache.thrift.transport.TServerTransport;
 import org.apache.thrift.transport.TTransportFactory;
 
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
 /**
  * ThriftServer - this class starts up a Thrift server which implements the
  * Hbase API specified in the Hbase.thrift IDL file.
@@ -108,7 +107,6 @@ public class ThriftServer {
       protected Map<String, HTable> initialValue() {
         return new TreeMap<String, HTable>();
       }
-
     };
 
     /**
@@ -183,11 +181,16 @@ public class ThriftServer {
 
     /**
      * Constructs an HBaseHandler object.
-     *
-     * @throws MasterNotRunningException
+     * @throws IOException 
      */
-    HBaseHandler() throws MasterNotRunningException {
-      conf = HBaseConfiguration.create();
+    HBaseHandler()
+    throws IOException {
+      this(HBaseConfiguration.create());
+    }
+
+    HBaseHandler(final Configuration c)
+    throws IOException {
+      this.conf = c;
       admin = new HBaseAdmin(conf);
       scannerMap = new HashMap<Integer, ResultScanner>();
     }
@@ -210,7 +213,7 @@ public class ThriftServer {
 
     public boolean isTableEnabled(final byte[] tableName) throws IOError {
       try {
-        return HTable.isTableEnabled(tableName);
+        return HTable.isTableEnabled(this.conf, tableName);
       } catch (IOException e) {
         throw new IOError(e.getMessage());
       }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 5cf3481e021..e26dbfbdd52 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -394,7 +394,7 @@ public class FSUtils {
   public static Map<String, Integer> getTableFragmentation(
     final HMaster master)
   throws IOException {
-    Path path = master.getRootDir();
+    Path path = getRootDir(master.getConfiguration());
     // since HMaster.getFileSystem() is package private
     FileSystem fs = path.getFileSystem(master.getConfiguration());
     return getTableFragmentation(fs, path);
diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseConfTool.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseConfTool.java
similarity index 77%
rename from src/main/java/org/apache/hadoop/hbase/HBaseConfTool.java
rename to src/main/java/org/apache/hadoop/hbase/util/HBaseConfTool.java
index 5b9ad3f9b8f..225f92c7883 100644
--- a/src/main/java/org/apache/hadoop/hbase/HBaseConfTool.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseConfTool.java
@@ -18,17 +18,24 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hbase;
+package org.apache.hadoop.hbase.util;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 
+/**
+ * Tool that prints out a configuration.
+ * Pass the configuration key on the command-line.
+ */
 public class HBaseConfTool {
-
   public static void main(String args[]) {
-    if (args.length < 1)
+    if (args.length < 1) {
+      System.err.println("Usage: HBaseConfTool <CONFIGURATION_KEY>");
+      System.exit(1);
       return;
+    }
 
     Configuration conf = HBaseConfiguration.create();
     System.out.println(conf.get(args[0]));
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/HMerge.java b/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
similarity index 89%
rename from src/main/java/org/apache/hadoop/hbase/HMerge.java
rename to src/main/java/org/apache/hadoop/hbase/util/HMerge.java
index 12b4b7b5ad8..c9190b14206 100644
--- a/src/main/java/org/apache/hadoop/hbase/HMerge.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
@@ -17,14 +17,21 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase;
+package org.apache.hadoop.hbase.util;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.TableNotDisabledException;
 import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
@@ -35,8 +42,6 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Writables;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -75,8 +80,32 @@ class HMerge {
   public static void merge(Configuration conf, FileSystem fs,
     final byte [] tableName)
   throws IOException {
-    HConnection connection = HConnectionManager.getConnection(conf);
-    boolean masterIsRunning = connection.isMasterRunning();
+    merge(conf, fs, tableName, true);
+  }
+
+  /**
+   * Scans the table and merges two adjacent regions if they are small. This
+   * only happens when a lot of rows are deleted.
+   *
+   * When merging the META region, the HBase instance must be offline.
+   * When merging a normal table, the HBase instance must be online, but the
+   * table must be disabled.
+   *
+   * @param conf        - configuration object for HBase
+   * @param fs          - FileSystem where regions reside
+   * @param tableName   - Table to be compacted
+   * @param testMasterRunning True if we are to verify master is down before
+   * running merge
+   * @throws IOException
+   */
+  public static void merge(Configuration conf, FileSystem fs,
+    final byte [] tableName, final boolean testMasterRunning)
+  throws IOException {
+    boolean masterIsRunning = false;
+    if (testMasterRunning) {
+      HConnection connection = HConnectionManager.getConnection(conf);
+      masterIsRunning = connection.isMasterRunning();
+    }
     HConnectionManager.deleteConnectionInfo(conf, false);
     if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
       if (masterIsRunning) {
@@ -89,6 +118,10 @@ class HMerge {
         throw new IllegalStateException(
             "HBase instance must be running to merge a normal table");
       }
+      HBaseAdmin admin = new HBaseAdmin(conf);
+      if (!admin.isTableDisabled(tableName)) {
+        throw new TableNotDisabledException(tableName);
+      }
       new OnlineMerger(conf, fs, tableName).process();
     }
   }
@@ -116,8 +149,7 @@ class HMerge {
       Path logdir = new Path(tabledir, "merge_" + System.currentTimeMillis() +
           HConstants.HREGION_LOGDIR_NAME);
       Path oldLogDir = new Path(tabledir, HConstants.HREGION_OLDLOGDIR_NAME);
-      this.hlog =
-        new HLog(fs, logdir, oldLogDir, conf, null);
+      this.hlog = new HLog(fs, logdir, oldLogDir, conf);
     }
 
     void process() throws IOException {
@@ -227,7 +259,6 @@ class HMerge {
         if (!Bytes.equals(region.getTableDesc().getName(), this.tableName)) {
           return null;
         }
-        checkOfflined(region);
         return region;
       } catch (IOException e) {
         e = RemoteExceptionHandler.checkIOException(e);
@@ -237,14 +268,6 @@ class HMerge {
       }
     }
 
-    protected void checkOfflined(final HRegionInfo hri)
-    throws TableNotDisabledException {
-      if (!hri.isOffline()) {
-        throw new TableNotDisabledException("Region " +
-          hri.getRegionNameAsString() + " is not disabled");
-      }
-    }
-
     /*
      * Check current row has a HRegionInfo.  Skip to next row if HRI is empty.
      * @return A Map of the row content else null if we are off the end.
diff --git a/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
index 280b91da4a9..4a9f1c39e4b 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
@@ -41,7 +42,7 @@ public class JVMClusterUtil {
     private final HRegionServer regionServer;
 
     public RegionServerThread(final HRegionServer r, final int index) {
-      super(r, "RegionServer:" + index);
+      super(r, "RegionServer:" + index + ";" + r.getServerName());
       this.regionServer = r;
     }
 
@@ -60,7 +61,7 @@ public class JVMClusterUtil {
       // cases, we'll jump out of the run without setting online flag.  Check
       // stopRequested so we don't wait here a flag that will never be flipped.
       while (!this.regionServer.isOnline() &&
-          !this.regionServer.isStopRequested()) {
+          !this.regionServer.isStopped()) {
         try {
           Thread.sleep(1000);
         } catch (InterruptedException e) {
@@ -80,17 +81,22 @@ public class JVMClusterUtil {
    * @return Region server added.
    */
   public static JVMClusterUtil.RegionServerThread createRegionServerThread(final Configuration c,
-    final Class<? extends HRegionServer> hrsc, final int index)
+      final Class<? extends HRegionServer> hrsc, final int index)
   throws IOException {
-      HRegionServer server;
-      try {
-        server = hrsc.getConstructor(Configuration.class).newInstance(c);
-      } catch (Exception e) {
-        IOException ioe = new IOException();
-        ioe.initCause(e);
-        throw ioe;
-      }
-      return new JVMClusterUtil.RegionServerThread(server, index);
+    HRegionServer server;
+    try {
+      server = hrsc.getConstructor(Configuration.class).newInstance(c);
+    } catch (InvocationTargetException ite) {
+      Throwable target = ite.getTargetException();
+      throw new RuntimeException("Failed construction of RegionServer: " +
+        hrsc.toString() + ((target.getCause() != null)?
+          target.getCause().getMessage(): ""), target);
+    } catch (Exception e) {
+      IOException ioe = new IOException();
+      ioe.initCause(e);
+      throw ioe;
+    }
+    return new JVMClusterUtil.RegionServerThread(server, index);
   }
 
   /**
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Merge.java b/src/main/java/org/apache/hadoop/hbase/util/Merge.java
index c78110eafee..2b03ed920e3 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Merge.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Merge.java
@@ -277,9 +277,9 @@ public class Merge extends Configured implements Tool {
     }
     HRegion merged = null;
     HLog log = utils.getLog();
-    HRegion r1 = HRegion.openHRegion(info1, this.rootdir, log, getConf());
+    HRegion r1 = HRegion.openHRegion(info1, log, getConf());
     try {
-      HRegion r2 = HRegion.openHRegion(info2, this.rootdir, log, getConf());
+      HRegion r2 = HRegion.openHRegion(info2, log, getConf());
       try {
         merged = HRegion.merge(r1, r2);
       } finally {
diff --git a/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java b/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java
index 4481b12ed4f..9e212111603 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java
@@ -103,7 +103,7 @@ public class MetaUtils {
           HConstants.HREGION_LOGDIR_NAME + "_" + System.currentTimeMillis());
       Path oldLogDir = new Path(this.fs.getHomeDirectory(),
           HConstants.HREGION_OLDLOGDIR_NAME);
-      this.log = new HLog(this.fs, logdir, oldLogDir, this.conf, null);
+      this.log = new HLog(this.fs, logdir, oldLogDir, this.conf);
     }
     return this.log;
   }
@@ -266,15 +266,14 @@ public class MetaUtils {
     if (this.rootRegion != null) {
       return this.rootRegion;
     }
-    this.rootRegion = HRegion.openHRegion(HRegionInfo.ROOT_REGIONINFO,
-      this.rootdir, getLog(), this.conf);
+    this.rootRegion = HRegion.openHRegion(HRegionInfo.ROOT_REGIONINFO, getLog(),
+      this.conf);
     this.rootRegion.compactStores();
     return this.rootRegion;
   }
 
   private HRegion openMetaRegion(HRegionInfo metaInfo) throws IOException {
-    HRegion meta =
-      HRegion.openHRegion(metaInfo, this.rootdir, getLog(), this.conf);
+    HRegion meta = HRegion.openHRegion(metaInfo, getLog(), this.conf);
     meta.compactStores();
     return meta;
   }
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java b/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java
index e5b4a5f9aa9..011dcbee686 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java
@@ -21,8 +21,7 @@ package org.apache.hadoop.hbase.util;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
-import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.hbase.Stoppable;
 
 /**
  * Sleeper for current thread.
@@ -33,7 +32,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public class Sleeper {
   private final Log LOG = LogFactory.getLog(this.getClass().getName());
   private final int period;
-  private final AtomicBoolean stop;
+  private final Stoppable stopper;
   private static final long MINIMAL_DELTA_FOR_LOGGING = 10000;
 
   private final Object sleepLock = new Object();
@@ -41,11 +40,12 @@ public class Sleeper {
 
   /**
    * @param sleep sleep time in milliseconds
-   * @param stop flag for when we stop
+   * @param stopper When {@link Stoppable#isStopped()} is true, this thread will
+   * cleanup and exit cleanly.
    */
-  public Sleeper(final int sleep, final AtomicBoolean stop) {
+  public Sleeper(final int sleep, final Stoppable stopper) {
     this.period = sleep;
-    this.stop = stop;
+    this.stopper = stopper;
   }
 
   /**
@@ -72,7 +72,7 @@ public class Sleeper {
    * will be docked current time minus passed <code>startTime<code>.
    */
   public void sleep(final long startTime) {
-    if (this.stop.get()) {
+    if (this.stopper.isStopped()) {
       return;
     }
     long now = System.currentTimeMillis();
@@ -101,7 +101,7 @@ public class Sleeper {
       } catch(InterruptedException iex) {
         // We we interrupted because we're meant to stop?  If not, just
         // continue ignoring the interruption
-        if (this.stop.get()) {
+        if (this.stopper.isStopped()) {
           return;
         }
       }
@@ -111,4 +111,4 @@ public class Sleeper {
     }
     triggerWake = false;
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java
new file mode 100644
index 00000000000..5974681bf8e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java
@@ -0,0 +1,86 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Tracker on cluster settings up in zookeeper.
+ * This is not related to {@link ClusterStatus}.  That class is a data structure
+ * that holds snapshot of current view on cluster.  This class is about tracking
+ * cluster attributes up in zookeeper.
+ *
+ */
+public class ClusterStatusTracker extends ZooKeeperNodeTracker {
+  private static final Log LOG = LogFactory.getLog(ClusterStatusTracker.class);
+
+  /**
+   * Creates a cluster status tracker.
+   *
+   * <p>After construction, use {@link #start} to kick off tracking.
+   *
+   * @param watcher
+   * @param abortable
+   */
+  public ClusterStatusTracker(ZooKeeperWatcher watcher, Abortable abortable) {
+    super(watcher, watcher.clusterStateZNode, abortable);
+  }
+
+  /**
+   * Checks if cluster is up.
+   * @return true if root region location is available, false if not
+   */
+  public boolean isClusterUp() {
+    return super.getData() != null;
+  }
+
+  /**
+   * Sets the cluster as up.
+   * @throws KeeperException unexpected zk exception
+   */
+  public void setClusterUp()
+  throws KeeperException {
+    byte [] upData = Bytes.toBytes(new java.util.Date().toString());
+    try {
+      ZKUtil.createAndWatch(watcher, watcher.clusterStateZNode, upData);
+    } catch(KeeperException.NodeExistsException nee) {
+      ZKUtil.setData(watcher, watcher.clusterStateZNode, upData);
+    }
+  }
+
+  /**
+   * Sets the cluster as down by deleting the znode.
+   * @throws KeeperException unexpected zk exception
+   */
+  public void setClusterDown()
+  throws KeeperException {
+    try {
+      ZKUtil.deleteNode(watcher, watcher.clusterStateZNode);
+    } catch(KeeperException.NoNodeException nne) {
+      LOG.warn("Attempted to set cluster as down but already down, cluster " +
+          "state node (" + watcher.clusterStateZNode + ") not found");
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
index 71f1ab583b8..b5f763f7b13 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
@@ -19,21 +19,8 @@
  */
 package org.apache.hadoop.hbase.zookeeper;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.net.DNS;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.zookeeper.server.ServerConfig;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
-import org.apache.zookeeper.server.quorum.QuorumPeerMain;
-
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.PrintWriter;
 import java.net.InetAddress;
 import java.net.NetworkInterface;
@@ -41,29 +28,27 @@ import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
-import java.util.Map.Entry;
 import java.util.Properties;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.net.DNS;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.zookeeper.server.ServerConfig;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.apache.zookeeper.server.quorum.QuorumPeerMain;
 
 /**
  * HBase's version of ZooKeeper's QuorumPeer. When HBase is set to manage
  * ZooKeeper, this class is used to start up QuorumPeer instances. By doing
  * things in here rather than directly calling to ZooKeeper, we have more
- * control over the process. Currently, this class allows us to parse the
+ * control over the process. This class uses {@link ZKConfig} to parse the
  * zoo.cfg and inject variables from HBase's site.xml configuration in.
  */
 public class HQuorumPeer {
-  private static final Log LOG = LogFactory.getLog(HQuorumPeer.class);
-
-  private static final String VARIABLE_START = "${";
-  private static final int VARIABLE_START_LENGTH = VARIABLE_START.length();
-  private static final String VARIABLE_END = "}";
-  private static final int VARIABLE_END_LENGTH = VARIABLE_END.length();
-
-  private static final String ZK_CFG_PROPERTY = "hbase.zookeeper.property.";
-  private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
-  private static final String ZK_CLIENT_PORT_KEY = ZK_CFG_PROPERTY
-      + "clientPort";
-
+  
   /**
    * Parse ZooKeeper configuration from HBase XML config and run a QuorumPeer.
    * @param args String[] of command line arguments. Not used.
@@ -71,7 +56,7 @@ public class HQuorumPeer {
   public static void main(String[] args) {
     Configuration conf = HBaseConfiguration.create();
     try {
-      Properties zkProperties = makeZKProps(conf);
+      Properties zkProperties = ZKConfig.makeZKProps(conf);
       writeMyID(zkProperties);
       QuorumPeerConfig zkConfig = new QuorumPeerConfig();
       zkConfig.parseProperties(zkProperties);
@@ -158,195 +143,4 @@ public class HQuorumPeer {
     w.println(myId);
     w.close();
   }
-
-  /**
-   * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
-   * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
-   * the corresponding config options from the HBase XML configs and generate
-   * the appropriate ZooKeeper properties.
-   * @param conf Configuration to read from.
-   * @return Properties holding mappings representing ZooKeeper zoo.cfg file.
-   */
-  public static Properties makeZKProps(Configuration conf) {
-    // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read
-    // it and grab its configuration properties.
-    ClassLoader cl = HQuorumPeer.class.getClassLoader();
-    final InputStream inputStream =
-      cl.getResourceAsStream(HConstants.ZOOKEEPER_CONFIG_NAME);
-    if (inputStream != null) {
-      try {
-        return parseZooCfg(conf, inputStream);
-      } catch (IOException e) {
-        LOG.warn("Cannot read " + HConstants.ZOOKEEPER_CONFIG_NAME +
-                 ", loading from XML files", e);
-      }
-    }
-
-    // Otherwise, use the configuration options from HBase's XML files.
-    Properties zkProperties = new Properties();
-
-    // Directly map all of the hbase.zookeeper.property.KEY properties.
-    for (Entry<String, String> entry : conf) {
-      String key = entry.getKey();
-      if (key.startsWith(ZK_CFG_PROPERTY)) {
-        String zkKey = key.substring(ZK_CFG_PROPERTY_SIZE);
-        String value = entry.getValue();
-        // If the value has variables substitutions, need to do a get.
-        if (value.contains(VARIABLE_START)) {
-          value = conf.get(key);
-        }
-        zkProperties.put(zkKey, value);
-      }
-    }
-
-    // If clientPort is not set, assign the default
-    if (zkProperties.getProperty(ZK_CLIENT_PORT_KEY) == null) {
-      zkProperties.put(ZK_CLIENT_PORT_KEY,
-                       HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
-    }
-
-    // Create the server.X properties.
-    int peerPort = conf.getInt("hbase.zookeeper.peerport", 2888);
-    int leaderPort = conf.getInt("hbase.zookeeper.leaderport", 3888);
-
-    final String[] serverHosts = conf.getStrings(HConstants.ZOOKEEPER_QUORUM,
-                                                 "localhost");
-    for (int i = 0; i < serverHosts.length; ++i) {
-      String serverHost = serverHosts[i];
-      String address = serverHost + ":" + peerPort + ":" + leaderPort;
-      String key = "server." + i;
-      zkProperties.put(key, address);
-    }
-
-    return zkProperties;
-  }
-  
-  /**
-   * Return the ZK Quorum servers string given zk properties returned by 
-   * makeZKProps
-   * @param properties
-   * @return
-   */
-  public static String getZKQuorumServersString(Properties properties) {
-    String clientPort = null;
-    List<String> servers = new ArrayList<String>();
-
-    // The clientPort option may come after the server.X hosts, so we need to
-    // grab everything and then create the final host:port comma separated list.
-    boolean anyValid = false;
-    for (Entry<Object,Object> property : properties.entrySet()) {
-      String key = property.getKey().toString().trim();
-      String value = property.getValue().toString().trim();
-      if (key.equals("clientPort")) {
-        clientPort = value;
-      }
-      else if (key.startsWith("server.")) {
-        String host = value.substring(0, value.indexOf(':'));
-        servers.add(host);
-        try {
-          //noinspection ResultOfMethodCallIgnored
-          InetAddress.getByName(host);
-          anyValid = true;
-        } catch (UnknownHostException e) {
-          LOG.warn(StringUtils.stringifyException(e));
-        }
-      }
-    }
-
-    if (!anyValid) {
-      LOG.error("no valid quorum servers found in " + HConstants.ZOOKEEPER_CONFIG_NAME);
-      return null;
-    }
-
-    if (clientPort == null) {
-      LOG.error("no clientPort found in " + HConstants.ZOOKEEPER_CONFIG_NAME);
-      return null;
-    }
-
-    if (servers.isEmpty()) {
-      LOG.fatal("No server.X lines found in conf/zoo.cfg. HBase must have a " +
-                "ZooKeeper cluster configured for its operation.");
-      return null;
-    }
-
-    StringBuilder hostPortBuilder = new StringBuilder();
-    for (int i = 0; i < servers.size(); ++i) {
-      String host = servers.get(i);
-      if (i > 0) {
-        hostPortBuilder.append(',');
-      }
-      hostPortBuilder.append(host);
-      hostPortBuilder.append(':');
-      hostPortBuilder.append(clientPort);
-    }
-
-    return hostPortBuilder.toString();
-  }
-
-  /**
-   * Parse ZooKeeper's zoo.cfg, injecting HBase Configuration variables in.
-   * This method is used for testing so we can pass our own InputStream.
-   * @param conf HBaseConfiguration to use for injecting variables.
-   * @param inputStream InputStream to read from.
-   * @return Properties parsed from config stream with variables substituted.
-   * @throws IOException if anything goes wrong parsing config
-   */
-  public static Properties parseZooCfg(Configuration conf,
-      InputStream inputStream) throws IOException {
-    Properties properties = new Properties();
-    try {
-      properties.load(inputStream);
-    } catch (IOException e) {
-      final String msg = "fail to read properties from "
-        + HConstants.ZOOKEEPER_CONFIG_NAME;
-      LOG.fatal(msg);
-      throw new IOException(msg, e);
-    }
-    for (Entry<Object, Object> entry : properties.entrySet()) {
-      String value = entry.getValue().toString().trim();
-      String key = entry.getKey().toString().trim();
-      StringBuilder newValue = new StringBuilder();
-      int varStart = value.indexOf(VARIABLE_START);
-      int varEnd = 0;
-      while (varStart != -1) {
-        varEnd = value.indexOf(VARIABLE_END, varStart);
-        if (varEnd == -1) {
-          String msg = "variable at " + varStart + " has no end marker";
-          LOG.fatal(msg);
-          throw new IOException(msg);
-        }
-        String variable = value.substring(varStart + VARIABLE_START_LENGTH, varEnd);
-
-        String substituteValue = System.getProperty(variable);
-        if (substituteValue == null) {
-          substituteValue = conf.get(variable);
-        }
-        if (substituteValue == null) {
-          String msg = "variable " + variable + " not set in system property "
-                     + "or hbase configs";
-          LOG.fatal(msg);
-          throw new IOException(msg);
-        }
-
-        newValue.append(substituteValue);
-
-        varEnd += VARIABLE_END_LENGTH;
-        varStart = value.indexOf(VARIABLE_START, varEnd);
-      }
-      // Special case for 'hbase.cluster.distributed' property being 'true'
-      if (key.startsWith("server.")) {
-        if (conf.get(HConstants.CLUSTER_DISTRIBUTED).equals(HConstants.CLUSTER_IS_DISTRIBUTED)
-            && value.startsWith("localhost")) {
-          String msg = "The server in zoo.cfg cannot be set to localhost " +
-              "in a fully-distributed setup because it won't be reachable. " +
-              "See \"Getting Started\" for more information.";
-          LOG.fatal(msg);
-          throw new IOException(msg);
-        }
-      }
-      newValue.append(value.substring(varEnd));
-      properties.setProperty(key, newValue.toString());
-    }
-    return properties;
-  }
 }
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java
new file mode 100644
index 00000000000..87905383132
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java
@@ -0,0 +1,73 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+
+/**
+ * Tracks the unassigned zookeeper node used by the META table.
+ * <p>
+ * A callback is made into a {@link CatalogTracker} when META completes a new
+ * assignment.
+ * <p>
+ * If META is already assigned when instantiating this class, you will not
+ * receive any notification for that assignment.  You will receive a
+ * notification after META has been successfully assigned to a new location.
+ */
+public class MetaNodeTracker extends ZooKeeperListener {
+  private static final Log LOG = LogFactory.getLog(MetaNodeTracker.class);
+
+  private final String node;
+
+  /** Catalog tracker to notify when META has a new assignment completed. */
+  private final CatalogTracker catalogTracker;
+
+  /**
+   * Creates a meta node tracker.
+   * @param watcher
+   * @param abortable
+   */
+  public MetaNodeTracker(ZooKeeperWatcher watcher,
+      CatalogTracker catalogTracker) {
+    super(watcher);
+    this.catalogTracker = catalogTracker;
+    node = ZKUtil.joinZNode(watcher.assignmentZNode,
+            HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
+  }
+
+  @Override
+  public void nodeDeleted(String path) {
+    if(path.equals(node)) {
+      LOG.info("Detected completed assignment of META, notifying catalog " +
+          "tracker");
+      try {
+        catalogTracker.waitForMetaServerConnectionDefault();
+      } catch (IOException e) {
+        LOG.warn("Tried to reset META server location after seeing the " +
+            "completion of a new META assignment but got an IOE", e);
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
similarity index 96%
rename from src/main/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java
rename to src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
index ff9ce2b9e00..05e32f78644 100644
--- a/src/main/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
@@ -17,7 +17,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase;
+package org.apache.hadoop.hbase.zookeeper;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -62,6 +62,10 @@ public class MiniZooKeeperCluster {
     this.clientPort = clientPort;
   }
 
+  public int getClientPort() {
+    return clientPort;
+  }
+
   public void setTickTime(int tickTime) {
     this.tickTime = tickTime;
   }
@@ -104,7 +108,7 @@ public class MiniZooKeeperCluster {
         standaloneServerFactory =
           new NIOServerCnxn.Factory(new InetSocketAddress(clientPort));
       } catch (BindException e) {
-        LOG.info("Faild binding ZK Server to client port: " + clientPort);
+        LOG.info("Failed binding ZK Server to client port: " + clientPort);
         //this port is already in use. try to use another
         clientPort++;
         continue;
@@ -118,7 +122,7 @@ public class MiniZooKeeperCluster {
     }
 
     started = true;
-
+    LOG.info("Started MiniZK Server on client port: " + clientPort);
     return clientPort;
   }
 
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
new file mode 100644
index 00000000000..0437484100e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
@@ -0,0 +1,101 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Tracks the online region servers via ZK.
+ *
+ * <p>Handling of new RSs checking in is done via RPC.  This class
+ * is only responsible for watching for expired nodes.  It handles
+ * listening for changes in the RS node list and watching each node.
+ *
+ * <p>If an RS node gets deleted, this automatically handles calling of
+ * {@link ServerManager#expireServer(org.apache.hadoop.hbase.HServerInfo)}.
+ */
+public class RegionServerTracker extends ZooKeeperListener {
+  private static final Log LOG = LogFactory.getLog(RegionServerTracker.class);
+
+  private ServerManager serverManager;
+  private Abortable abortable;
+
+  public RegionServerTracker(ZooKeeperWatcher watcher,
+      Abortable abortable, ServerManager serverManager) {
+    super(watcher);
+    this.abortable = abortable;
+    this.serverManager = serverManager;
+  }
+
+  /**
+   * Starts the tracking of online RegionServers.
+   *
+   * <p>All RSs will be tracked after this method is called.
+   *
+   * @throws KeeperException
+   */
+  public void start() throws KeeperException {
+    watcher.registerListener(this);
+    ZKUtil.watchAndGetNewChildren(watcher, watcher.rsZNode);
+  }
+
+  @Override
+  public void nodeDeleted(String path) {
+    if(path.startsWith(watcher.rsZNode)) {
+      String serverName = ZKUtil.getNodeName(path);
+      LOG.info("RegionServer ephemeral node deleted, processing expiration [" +
+          serverName + "]");
+      HServerInfo hsi = serverManager.getServerInfo(serverName);
+      if(hsi == null) {
+        LOG.info("No HServerInfo found for " + serverName);
+        return;
+      }
+      serverManager.expireServer(hsi);
+    }
+  }
+
+  @Override
+  public void nodeChildrenChanged(String path) {
+    if(path.equals(watcher.rsZNode)) {
+      try {
+        ZKUtil.watchAndGetNewChildren(watcher, watcher.rsZNode);
+      } catch (KeeperException e) {
+        abortable.abort("Unexpected zk exception getting RS nodes", e);
+      }
+    }
+  }
+
+  /**
+   * Gets the online servers.
+   * @return list of online servers from zk
+   * @throws KeeperException
+   */
+  public List<HServerAddress> getOnlineServers() throws KeeperException {
+    return ZKUtil.listChildrenAndGetAsAddresses(watcher, watcher.rsZNode);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java
new file mode 100644
index 00000000000..19d7c31b851
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java
@@ -0,0 +1,84 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.catalog.RootLocationEditor;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Tracks the root region server location node in zookeeper.
+ * Root region location is set by {@link RootLocationEditor} usually called
+ * out of {@link RegionServerServices#postOpenDeployTasks(org.apache.hadoop.hbase.regionserver.HRegion, org.apache.hadoop.hbase.catalog.CatalogTracker)}
+ */
+public class RootRegionTracker extends ZooKeeperNodeTracker {
+  /**
+   * Creates a root region location tracker.
+   *
+   * <p>After construction, use {@link #start} to kick off tracking.
+   *
+   * @param watcher
+   * @param abortable
+   */
+  public RootRegionTracker(ZooKeeperWatcher watcher, Abortable abortable) {
+    super(watcher, watcher.rootServerZNode, abortable);
+  }
+
+  /**
+   * Checks if the root region location is available.
+   * @return true if root region location is available, false if not
+   */
+  public boolean isLocationAvailable() {
+    return super.getData() != null;
+  }
+
+  /**
+   * Gets the root region location, if available.  Null if not.  Does not block.
+   * @return server address for server hosting root region, null if none available
+   * @throws InterruptedException 
+   */
+  public HServerAddress getRootRegionLocation() throws InterruptedException {
+    return dataToHServerAddress(super.getData());
+  }
+
+  /**
+   * Gets the root region location, if available, and waits for up to the
+   * specified timeout if not immediately available.
+   * @param timeout maximum time to wait, in millis, use {@link ZooKeeperNodeTracker#NO_TIMEOUT} for
+   * forever
+   * @return server address for server hosting root region, null if timed out
+   * @throws InterruptedException if interrupted while waiting
+   */
+  public HServerAddress waitRootRegionLocation(long timeout)
+  throws InterruptedException {
+    return dataToHServerAddress(super.blockUntilAvailable(timeout));
+  }
+
+  /*
+   * @param data
+   * @return Returns null if <code>data</code> is null else converts passed data
+   * to an HServerAddress instance.
+   */
+  private static HServerAddress dataToHServerAddress(final byte [] data) {
+    return data == null ? null: new HServerAddress(Bytes.toString(data));
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
new file mode 100644
index 00000000000..c88a38fac4c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
@@ -0,0 +1,659 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.KeeperException.NodeExistsException;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ * Utility class for doing region assignment in ZooKeeper.  This class extends
+ * stuff done in {@link ZKUtil} to cover specific assignment operations.
+ * <p>
+ * Contains only static methods and constants.
+ * <p>
+ * Used by both the Master and RegionServer.
+ * <p>
+ * All valid transitions outlined below:
+ * <p>
+ * <b>MASTER</b>
+ * <ol>
+ *   <li>
+ *     Master creates an unassigned node as OFFLINE.
+ *     - Cluster startup and table enabling.
+ *   </li>
+ *   <li>
+ *     Master forces an existing unassigned node to OFFLINE.
+ *     - RegionServer failure.
+ *     - Allows transitions from all states to OFFLINE.
+ *   </li>
+ *   <li>
+ *     Master deletes an unassigned node that was in a OPENED state.
+ *     - Normal region transitions.  Besides cluster startup, no other deletions
+ *     of unassigned nodes is allowed.
+ *   </li>
+ *   <li>
+ *     Master deletes all unassigned nodes regardless of state.
+ *     - Cluster startup before any assignment happens.
+ *   </li>
+ * </ol>
+ * <p>
+ * <b>REGIONSERVER</b>
+ * <ol>
+ *   <li>
+ *     RegionServer creates an unassigned node as CLOSING.
+ *     - All region closes will do this in response to a CLOSE RPC from Master.
+ *     - A node can never be transitioned to CLOSING, only created.
+ *   </li>
+ *   <li>
+ *     RegionServer transitions an unassigned node from CLOSING to CLOSED.
+ *     - Normal region closes.  CAS operation.
+ *   </li>
+ *   <li>
+ *     RegionServer transitions an unassigned node from OFFLINE to OPENING.
+ *     - All region opens will do this in response to an OPEN RPC from the Master.
+ *     - Normal region opens.  CAS operation.
+ *   </li>
+ *   <li>
+ *     RegionServer transitions an unassigned node from OPENING to OPENED.
+ *     - Normal region opens.  CAS operation.
+ *   </li>
+ * </ol>
+ */
+public class ZKAssign {
+
+  /**
+   * Gets the full path node name for the unassigned node for the specified
+   * region.
+   * @param zkw zk reference
+   * @param regionName region name
+   * @return full path node name
+   */
+  private static String getNodeName(ZooKeeperWatcher zkw, String regionName) {
+    return ZKUtil.joinZNode(zkw.assignmentZNode, regionName);
+  }
+
+  /**
+   * Gets the region name from the full path node name of an unassigned node.
+   * @param path full zk path
+   * @return region name
+   */
+  public static String getRegionName(ZooKeeperWatcher zkw, String path) {
+    return path.substring(zkw.assignmentZNode.length()+1);
+  }
+
+  // Master methods
+
+  /**
+   * Creates a new unassigned node in the OFFLINE state for the specified region.
+   *
+   * <p>Does not transition nodes from other states.  If a node already exists
+   * for this region, a {@link NodeExistsException} will be thrown.
+   *
+   * <p>Sets a watcher on the unassigned region node if the method is successful.
+   *
+   * <p>This method should only be used during cluster startup and the enabling
+   * of a table.
+   *
+   * @param zkw zk reference
+   * @param region region to be created as offline
+   * @param serverName server event originates from
+   * @throws KeeperException if unexpected zookeeper exception
+   * @throws KeeperException.NodeExistsException if node already exists
+   */
+  public static void createNodeOffline(ZooKeeperWatcher zkw, HRegionInfo region,
+      String serverName)
+  throws KeeperException, KeeperException.NodeExistsException {
+    createNodeOffline(zkw, region, serverName, EventType.M2ZK_REGION_OFFLINE);
+  }
+
+  public static void createNodeOffline(ZooKeeperWatcher zkw, HRegionInfo region,
+      String serverName, final EventType event)
+  throws KeeperException, KeeperException.NodeExistsException {
+    zkw.debug("Creating an unassigned node for " + region.getEncodedName() +
+        " in an OFFLINE state");
+    RegionTransitionData data = new RegionTransitionData(event,
+      region.getRegionName(), serverName);
+    synchronized(zkw.getNodes()) {
+      String node = getNodeName(zkw, region.getEncodedName());
+      zkw.getNodes().add(node);
+      ZKUtil.createAndWatch(zkw, node, data.getBytes());
+    }
+  }
+
+  /**
+   * Forces an existing unassigned node to the OFFLINE state for the specified
+   * region.
+   *
+   * <p>Does not create a new node.  If a node does not already exist for this
+   * region, a {@link NoNodeException} will be thrown.
+   *
+   * <p>Sets a watcher on the unassigned region node if the method is
+   * successful.
+   *
+   * <p>This method should only be used during recovery of regionserver failure.
+   *
+   * @param zkw zk reference
+   * @param region region to be forced as offline
+   * @param serverName server event originates from
+   * @throws KeeperException if unexpected zookeeper exception
+   * @throws KeeperException.NoNodeException if node does not exist
+   */
+  public static void forceNodeOffline(ZooKeeperWatcher zkw, HRegionInfo region,
+      String serverName)
+  throws KeeperException, KeeperException.NoNodeException {
+    zkw.debug("Forcing an existing unassigned node for " +
+        region.getEncodedName() + " to an OFFLINE state");
+    RegionTransitionData data = new RegionTransitionData(
+        EventType.M2ZK_REGION_OFFLINE, region.getRegionName(), serverName);
+    synchronized(zkw.getNodes()) {
+      String node = getNodeName(zkw, region.getEncodedName());
+      zkw.getNodes().add(node);
+      ZKUtil.setData(zkw, node, data.getBytes());
+    }
+  }
+
+
+  /**
+   * Creates or force updates an unassigned node to the OFFLINE state for the
+   * specified region.
+   * <p>
+   * Attempts to create the node but if it exists will force it to transition to
+   * and OFFLINE state.
+   *
+   * <p>Sets a watcher on the unassigned region node if the method is
+   * successful.
+   *
+   * <p>This method should be used when assigning a region.
+   *
+   * @param zkw zk reference
+   * @param region region to be created as offline
+   * @param serverName server event originates from
+   * @throws KeeperException if unexpected zookeeper exception
+   * @throws KeeperException.NodeExistsException if node already exists
+   */
+  public static boolean createOrForceNodeOffline(ZooKeeperWatcher zkw,
+      HRegionInfo region, String serverName)
+  throws KeeperException {
+    zkw.debug("Creating or updating an unassigned node for " +
+        region.getEncodedName() + " with an OFFLINE state");
+    RegionTransitionData data = new RegionTransitionData(
+        EventType.M2ZK_REGION_OFFLINE, region.getRegionName(), serverName);
+    synchronized(zkw.getNodes()) {
+      String node = getNodeName(zkw, region.getEncodedName());
+      zkw.getNodes().add(node);
+      int version = ZKUtil.checkExists(zkw, node);
+      if(version == -1) {
+        ZKUtil.createAndWatch(zkw, node, data.getBytes());
+        return true;
+      } else {
+        return ZKUtil.setData(zkw, node, data.getBytes(), version);
+      }
+    }
+  }
+
+  /**
+   * Deletes an existing unassigned node that is in the OPENED state for the
+   * specified region.
+   *
+   * <p>If a node does not already exist for this region, a
+   * {@link NoNodeException} will be thrown.
+   *
+   * <p>No watcher is set whether this succeeds or not.
+   *
+   * <p>Returns false if the node was not in the proper state but did exist.
+   *
+   * <p>This method is used during normal region transitions when a region
+   * finishes successfully opening.  This is the Master acknowledging completion
+   * of the specified regions transition.
+   *
+   * @param zkw zk reference
+   * @param region opened region to be deleted from zk
+   * @throws KeeperException if unexpected zookeeper exception
+   * @throws KeeperException.NoNodeException if node does not exist
+   */
+  public static boolean deleteOpenedNode(ZooKeeperWatcher zkw,
+      String regionName)
+  throws KeeperException, KeeperException.NoNodeException {
+    return deleteNode(zkw, regionName, EventType.RS2ZK_REGION_OPENED);
+  }
+
+  /**
+   * Deletes an existing unassigned node that is in the CLOSED state for the
+   * specified region.
+   *
+   * <p>If a node does not already exist for this region, a
+   * {@link NoNodeException} will be thrown.
+   *
+   * <p>No watcher is set whether this succeeds or not.
+   *
+   * <p>Returns false if the node was not in the proper state but did exist.
+   *
+   * <p>This method is used during table disables when a region finishes
+   * successfully closing.  This is the Master acknowledging completion
+   * of the specified regions transition to being closed.
+   *
+   * @param zkw zk reference
+   * @param region closed region to be deleted from zk
+   * @throws KeeperException if unexpected zookeeper exception
+   * @throws KeeperException.NoNodeException if node does not exist
+   */
+  public static boolean deleteClosedNode(ZooKeeperWatcher zkw,
+      String regionName)
+  throws KeeperException, KeeperException.NoNodeException {
+    return deleteNode(zkw, regionName, EventType.RS2ZK_REGION_CLOSED);
+  }
+
+  /**
+   * Deletes an existing unassigned node that is in the CLOSING state for the
+   * specified region.
+   *
+   * <p>If a node does not already exist for this region, a
+   * {@link NoNodeException} will be thrown.
+   *
+   * <p>No watcher is set whether this succeeds or not.
+   *
+   * <p>Returns false if the node was not in the proper state but did exist.
+   *
+   * <p>This method is used during table disables when a region finishes
+   * successfully closing.  This is the Master acknowledging completion
+   * of the specified regions transition to being closed.
+   *
+   * @param zkw zk reference
+   * @param region closing region to be deleted from zk
+   * @throws KeeperException if unexpected zookeeper exception
+   * @throws KeeperException.NoNodeException if node does not exist
+   */
+  public static boolean deleteClosingNode(ZooKeeperWatcher zkw,
+      String regionName)
+  throws KeeperException, KeeperException.NoNodeException {
+    return deleteNode(zkw, regionName, EventType.RS2ZK_REGION_CLOSING);
+  }
+
+  /**
+   * Deletes an existing unassigned node that is in the specified state for the
+   * specified region.
+   *
+   * <p>If a node does not already exist for this region, a
+   * {@link NoNodeException} will be thrown.
+   *
+   * <p>No watcher is set whether this succeeds or not.
+   *
+   * <p>Returns false if the node was not in the proper state but did exist.
+   *
+   * <p>This method is used during table disables when a region finishes
+   * successfully closing.  This is the Master acknowledging completion
+   * of the specified regions transition to being closed.
+   *
+   * @param zkw zk reference
+   * @param region region to be deleted from zk
+   * @param expectedState state region must be in for delete to complete
+   * @throws KeeperException if unexpected zookeeper exception
+   * @throws KeeperException.NoNodeException if node does not exist
+   */
+  private static boolean deleteNode(ZooKeeperWatcher zkw, String regionName,
+      EventType expectedState)
+  throws KeeperException, KeeperException.NoNodeException {
+    zkw.debug("Deleting an existing unassigned node for " + regionName +
+        " that is in expected state " + expectedState);
+    String node = getNodeName(zkw, regionName);
+    Stat stat = new Stat();
+    byte [] bytes = ZKUtil.getDataNoWatch(zkw, node, stat);
+    if(bytes == null) {
+      throw KeeperException.create(Code.NONODE);
+    }
+    RegionTransitionData data = RegionTransitionData.fromBytes(bytes);
+    if(!data.getEventType().equals(expectedState)) {
+      zkw.warn("Attempting to delete an unassigned node in " + expectedState +
+          " state but node is in " + data.getEventType() + " state");
+      return false;
+    }
+    synchronized(zkw.getNodes()) {
+      // TODO: Does this go here or only if we successfully delete node?
+      zkw.getNodes().remove(node);
+      if(!ZKUtil.deleteNode(zkw, node, stat.getVersion())) {
+        zkw.warn("Attempting to delete an unassigned node in " + expectedState +
+            " state but " +
+            "after verifying it was in OPENED state, we got a version mismatch");
+        return false;
+      }
+      return true;
+    }
+  }
+
+  /**
+   * Deletes all unassigned nodes regardless of their state.
+   *
+   * <p>No watchers are set.
+   *
+   * <p>This method is used by the Master during cluster startup to clear out
+   * any existing state from other cluster runs.
+   *
+   * @param zkw zk reference
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static void deleteAllNodes(ZooKeeperWatcher zkw)
+  throws KeeperException {
+    zkw.debug("Deleting any existing unassigned nodes");
+    ZKUtil.deleteChildrenRecursively(zkw, zkw.assignmentZNode);
+  }
+
+  // RegionServer methods
+
+  /**
+   * Creates a new unassigned node in the CLOSING state for the specified
+   * region.
+   *
+   * <p>Does not transition nodes from any states.  If a node already exists
+   * for this region, a {@link NodeExistsException} will be thrown.
+   *
+   * <p>If creation is successful, returns the version number of the CLOSING
+   * node created.
+   *
+   * <p>Does not set any watches.
+   *
+   * <p>This method should only be used by a RegionServer when initiating a
+   * close of a region after receiving a CLOSE RPC from the Master.
+   *
+   * @param zkw zk reference
+   * @param region region to be created as closing
+   * @param serverName server event originates from
+   * @return version of node after transition, -1 if unsuccessful transition
+   * @throws KeeperException if unexpected zookeeper exception
+   * @throws KeeperException.NodeExistsException if node already exists
+   */
+  public static int createNodeClosing(ZooKeeperWatcher zkw, HRegionInfo region,
+      String serverName)
+  throws KeeperException, KeeperException.NodeExistsException {
+    zkw.debug("Creating an unassigned node for " + region.getEncodedName() +
+    " in a CLOSING state");
+    RegionTransitionData data = new RegionTransitionData(
+        EventType.RS2ZK_REGION_CLOSING, region.getRegionName(), serverName);
+    synchronized(zkw.getNodes()) {
+      String node = getNodeName(zkw, region.getEncodedName());
+      zkw.getNodes().add(node);
+      return ZKUtil.createAndWatch(zkw, node, data.getBytes());
+    }
+  }
+
+  /**
+   * Transitions an existing unassigned node for the specified region which is
+   * currently in the CLOSING state to be in the CLOSED state.
+   *
+   * <p>Does not transition nodes from other states.  If for some reason the
+   * node could not be transitioned, the method returns -1.  If the transition
+   * is successful, the version of the node after transition is returned.
+   *
+   * <p>This method can fail and return false for three different reasons:
+   * <ul><li>Unassigned node for this region does not exist</li>
+   * <li>Unassigned node for this region is not in CLOSING state</li>
+   * <li>After verifying CLOSING state, update fails because of wrong version
+   * (someone else already transitioned the node)</li>
+   * </ul>
+   *
+   * <p>Does not set any watches.
+   *
+   * <p>This method should only be used by a RegionServer when initiating a
+   * close of a region after receiving a CLOSE RPC from the Master.
+   *
+   * @param zkw zk reference
+   * @param region region to be transitioned to closed
+   * @param serverName server event originates from
+   * @return version of node after transition, -1 if unsuccessful transition
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static int transitionNodeClosed(ZooKeeperWatcher zkw,
+      HRegionInfo region, String serverName, int expectedVersion)
+  throws KeeperException {
+    return transitionNode(zkw, region, serverName,
+        EventType.RS2ZK_REGION_CLOSING,
+        EventType.RS2ZK_REGION_CLOSED, expectedVersion);
+  }
+
+  /**
+   * Transitions an existing unassigned node for the specified region which is
+   * currently in the OFFLINE state to be in the OPENING state.
+   *
+   * <p>Does not transition nodes from other states.  If for some reason the
+   * node could not be transitioned, the method returns -1.  If the transition
+   * is successful, the version of the node written as OPENING is returned.
+   *
+   * <p>This method can fail and return -1 for three different reasons:
+   * <ul><li>Unassigned node for this region does not exist</li>
+   * <li>Unassigned node for this region is not in OFFLINE state</li>
+   * <li>After verifying OFFLINE state, update fails because of wrong version
+   * (someone else already transitioned the node)</li>
+   * </ul>
+   *
+   * <p>Does not set any watches.
+   *
+   * <p>This method should only be used by a RegionServer when initiating an
+   * open of a region after receiving an OPEN RPC from the Master.
+   *
+   * @param zkw zk reference
+   * @param region region to be transitioned to opening
+   * @param serverName server event originates from
+   * @return version of node after transition, -1 if unsuccessful transition
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static int transitionNodeOpening(ZooKeeperWatcher zkw,
+      HRegionInfo region, String serverName)
+  throws KeeperException {
+    return transitionNodeOpening(zkw, region, serverName,
+      EventType.M2ZK_REGION_OFFLINE);
+  }
+
+  public static int transitionNodeOpening(ZooKeeperWatcher zkw,
+      HRegionInfo region, String serverName, final EventType beginState)
+  throws KeeperException {
+    return transitionNode(zkw, region, serverName, beginState,
+      EventType.RS2ZK_REGION_OPENING, -1);
+  }
+
+  /**
+   * Retransitions an existing unassigned node for the specified region which is
+   * currently in the OPENING state to be in the OPENING state.
+   *
+   * <p>Does not transition nodes from other states.  If for some reason the
+   * node could not be transitioned, the method returns -1.  If the transition
+   * is successful, the version of the node rewritten as OPENING is returned.
+   *
+   * <p>This method can fail and return -1 for three different reasons:
+   * <ul><li>Unassigned node for this region does not exist</li>
+   * <li>Unassigned node for this region is not in OPENING state</li>
+   * <li>After verifying OPENING state, update fails because of wrong version
+   * (someone else already transitioned the node)</li>
+   * </ul>
+   *
+   * <p>Does not set any watches.
+   *
+   * <p>This method should only be used by a RegionServer when initiating an
+   * open of a region after receiving an OPEN RPC from the Master.
+   *
+   * @param zkw zk reference
+   * @param region region to be transitioned to opening
+   * @param serverName server event originates from
+   * @return version of node after transition, -1 if unsuccessful transition
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static int retransitionNodeOpening(ZooKeeperWatcher zkw,
+      HRegionInfo region, String serverName, int expectedVersion)
+  throws KeeperException {
+    return transitionNode(zkw, region, serverName,
+        EventType.RS2ZK_REGION_OPENING,
+        EventType.RS2ZK_REGION_OPENING, expectedVersion);
+  }
+
+  /**
+   * Transitions an existing unassigned node for the specified region which is
+   * currently in the OPENING state to be in the OPENED state.
+   *
+   * <p>Does not transition nodes from other states.  If for some reason the
+   * node could not be transitioned, the method returns -1.  If the transition
+   * is successful, the version of the node after transition is returned.
+   *
+   * <p>This method can fail and return false for three different reasons:
+   * <ul><li>Unassigned node for this region does not exist</li>
+   * <li>Unassigned node for this region is not in OPENING state</li>
+   * <li>After verifying OPENING state, update fails because of wrong version
+   * (this should never actually happen since an RS only does this transition
+   * following a transition to OPENING.  if two RS are conflicting, one would
+   * fail the original transition to OPENING and not this transition)</li>
+   * </ul>
+   *
+   * <p>Does not set any watches.
+   *
+   * <p>This method should only be used by a RegionServer when completing the
+   * open of a region.
+   *
+   * @param zkw zk reference
+   * @param region region to be transitioned to opened
+   * @param serverName server event originates from
+   * @return version of node after transition, -1 if unsuccessful transition
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static int transitionNodeOpened(ZooKeeperWatcher zkw,
+      HRegionInfo region, String serverName, int expectedVersion)
+  throws KeeperException {
+    return transitionNode(zkw, region, serverName,
+        EventType.RS2ZK_REGION_OPENING,
+        EventType.RS2ZK_REGION_OPENED, expectedVersion);
+  }
+
+  /**
+   * Private method that actually performs unassigned node transitions.
+   *
+   * <p>Attempts to transition the unassigned node for the specified region
+   * from the expected state to the state in the specified transition data.
+   *
+   * <p>Method first reads existing data and verifies it is in the expected
+   * state.  If the node does not exist or the node is not in the expected
+   * state, the method returns -1.  If the transition is successful, the
+   * version number of the node following the transition is returned.
+   *
+   * <p>If the read state is what is expected, it attempts to write the new
+   * state and data into the node.  When doing this, it includes the expected
+   * version (determined when the existing state was verified) to ensure that
+   * only one transition is successful.  If there is a version mismatch, the
+   * method returns -1.
+   *
+   * <p>If the write is successful, no watch is set and the method returns true.
+   *
+   * @param zkw zk reference
+   * @param region region to be transitioned to opened
+   * @param serverName server event originates from
+   * @param beginState state the node must currently be in to do transition
+   * @param endState state to transition node to if all checks pass
+   * @return version of node after transition, -1 if unsuccessful transition
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  private static int transitionNode(ZooKeeperWatcher zkw, HRegionInfo region,
+      String serverName, EventType beginState, EventType endState,
+      int expectedVersion)
+  throws KeeperException {
+    String encoded = region.getEncodedName();
+    if(zkw.isDebugEnabled()) {
+      zkw.debug("Attempting to transition node " +
+        HRegionInfo.prettyPrint(encoded) +
+        " from " + beginState.toString() + " to " + endState.toString());
+    }
+
+    String node = getNodeName(zkw, encoded);
+
+    // Read existing data of the node
+    Stat stat = new Stat();
+    byte [] existingBytes =
+      ZKUtil.getDataNoWatch(zkw, node, stat);
+    RegionTransitionData existingData =
+      RegionTransitionData.fromBytes(existingBytes);
+
+    // Verify it is the expected version
+    if(expectedVersion != -1 && stat.getVersion() != expectedVersion) {
+      zkw.warn("Attempt to transition the unassigned node for " + encoded +
+          " from " + beginState + " to " + endState + " failed, " +
+          "the node existed but was version " + stat.getVersion() +
+          " not the expected version " + expectedVersion);
+        return -1;
+    }
+
+    // Verify it is in expected state
+    if(!existingData.getEventType().equals(beginState)) {
+      zkw.warn("Attempt to transition the unassigned node for " + encoded +
+        " from " + beginState + " to " + endState + " failed, " +
+        "the node existed but was in the state " + existingData.getEventType());
+      return -1;
+    }
+
+    // Write new data, ensuring data has not changed since we last read it
+    try {
+      RegionTransitionData data = new RegionTransitionData(endState,
+          region.getRegionName(), serverName);
+      if(!ZKUtil.setData(zkw, node, data.getBytes(), stat.getVersion())) {
+        zkw.warn("Attempt to transition the unassigned node for " + encoded +
+        " from " + beginState + " to " + endState + " failed, " +
+        "the node existed and was in the expected state but then when " +
+        "setting data we got a version mismatch");
+        return -1;
+      }
+      if(zkw.isDebugEnabled()) {
+        zkw.debug("Successfully transitioned node " + encoded +
+          " from " + beginState + " to " + endState);
+      }
+      return stat.getVersion() + 1;
+    } catch (KeeperException.NoNodeException nne) {
+      zkw.warn("Attempt to transition the unassigned node for " + encoded +
+        " from " + beginState + " to " + endState + " failed, " +
+        "the node existed and was in the expected state but then when " +
+        "setting data it no longer existed");
+      return -1;
+    }
+  }
+
+  /**
+   * Gets the current data in the unassigned node for the specified region name
+   * or fully-qualified path.
+   *
+   * <p>Returns null if the region does not currently have a node.
+   *
+   * <p>Sets a watch on the node if the node exists.
+   *
+   * @param watcher zk reference
+   * @param pathOrRegionName fully-specified path or region name
+   * @return data for the unassigned node
+   * @throws KeeperException
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static RegionTransitionData getData(ZooKeeperWatcher zkw,
+      String pathOrRegionName)
+  throws KeeperException {
+    String node = pathOrRegionName.startsWith("/") ?
+        pathOrRegionName : getNodeName(zkw, pathOrRegionName);
+    byte [] data = ZKUtil.getDataAndWatch(zkw, node);
+    if(data == null) {
+      return null;
+    }
+    return RegionTransitionData.fromBytes(data);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
new file mode 100644
index 00000000000..c60150f9277
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
@@ -0,0 +1,252 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Map.Entry;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Utility methods for reading, parsing, and building zookeeper configuration.
+ */
+public class ZKConfig {
+  private static final Log LOG = LogFactory.getLog(ZKConfig.class);
+
+  private static final String VARIABLE_START = "${";
+  private static final int VARIABLE_START_LENGTH = VARIABLE_START.length();
+  private static final String VARIABLE_END = "}";
+  private static final int VARIABLE_END_LENGTH = VARIABLE_END.length();
+
+  private static final String ZK_CFG_PROPERTY = "hbase.zookeeper.property.";
+  private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
+  private static final String ZK_CLIENT_PORT_KEY = ZK_CFG_PROPERTY
+      + "clientPort";
+
+  /**
+   * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
+   * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
+   * the corresponding config options from the HBase XML configs and generate
+   * the appropriate ZooKeeper properties.
+   * @param conf Configuration to read from.
+   * @return Properties holding mappings representing ZooKeeper zoo.cfg file.
+   */
+  public static Properties makeZKProps(Configuration conf) {
+    // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read
+    // it and grab its configuration properties.
+    ClassLoader cl = HQuorumPeer.class.getClassLoader();
+    final InputStream inputStream =
+      cl.getResourceAsStream(HConstants.ZOOKEEPER_CONFIG_NAME);
+    if (inputStream != null) {
+      try {
+        return parseZooCfg(conf, inputStream);
+      } catch (IOException e) {
+        LOG.warn("Cannot read " + HConstants.ZOOKEEPER_CONFIG_NAME +
+                 ", loading from XML files", e);
+      }
+    }
+
+    // Otherwise, use the configuration options from HBase's XML files.
+    Properties zkProperties = new Properties();
+
+    // Directly map all of the hbase.zookeeper.property.KEY properties.
+    for (Entry<String, String> entry : conf) {
+      String key = entry.getKey();
+      if (key.startsWith(ZK_CFG_PROPERTY)) {
+        String zkKey = key.substring(ZK_CFG_PROPERTY_SIZE);
+        String value = entry.getValue();
+        // If the value has variables substitutions, need to do a get.
+        if (value.contains(VARIABLE_START)) {
+          value = conf.get(key);
+        }
+        zkProperties.put(zkKey, value);
+      }
+    }
+
+    // If clientPort is not set, assign the default
+    if (zkProperties.getProperty(ZK_CLIENT_PORT_KEY) == null) {
+      zkProperties.put(ZK_CLIENT_PORT_KEY,
+                       HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
+    }
+
+    // Create the server.X properties.
+    int peerPort = conf.getInt("hbase.zookeeper.peerport", 2888);
+    int leaderPort = conf.getInt("hbase.zookeeper.leaderport", 3888);
+
+    final String[] serverHosts = conf.getStrings(HConstants.ZOOKEEPER_QUORUM,
+                                                 "localhost");
+    for (int i = 0; i < serverHosts.length; ++i) {
+      String serverHost = serverHosts[i];
+      String address = serverHost + ":" + peerPort + ":" + leaderPort;
+      String key = "server." + i;
+      zkProperties.put(key, address);
+    }
+
+    return zkProperties;
+  }
+
+  /**
+   * Parse ZooKeeper's zoo.cfg, injecting HBase Configuration variables in.
+   * This method is used for testing so we can pass our own InputStream.
+   * @param conf HBaseConfiguration to use for injecting variables.
+   * @param inputStream InputStream to read from.
+   * @return Properties parsed from config stream with variables substituted.
+   * @throws IOException if anything goes wrong parsing config
+   */
+  public static Properties parseZooCfg(Configuration conf,
+      InputStream inputStream) throws IOException {
+    Properties properties = new Properties();
+    try {
+      properties.load(inputStream);
+    } catch (IOException e) {
+      final String msg = "fail to read properties from "
+        + HConstants.ZOOKEEPER_CONFIG_NAME;
+      LOG.fatal(msg);
+      throw new IOException(msg, e);
+    }
+    for (Entry<Object, Object> entry : properties.entrySet()) {
+      String value = entry.getValue().toString().trim();
+      String key = entry.getKey().toString().trim();
+      StringBuilder newValue = new StringBuilder();
+      int varStart = value.indexOf(VARIABLE_START);
+      int varEnd = 0;
+      while (varStart != -1) {
+        varEnd = value.indexOf(VARIABLE_END, varStart);
+        if (varEnd == -1) {
+          String msg = "variable at " + varStart + " has no end marker";
+          LOG.fatal(msg);
+          throw new IOException(msg);
+        }
+        String variable = value.substring(varStart + VARIABLE_START_LENGTH, varEnd);
+
+        String substituteValue = System.getProperty(variable);
+        if (substituteValue == null) {
+          substituteValue = conf.get(variable);
+        }
+        if (substituteValue == null) {
+          String msg = "variable " + variable + " not set in system property "
+                     + "or hbase configs";
+          LOG.fatal(msg);
+          throw new IOException(msg);
+        }
+
+        newValue.append(substituteValue);
+
+        varEnd += VARIABLE_END_LENGTH;
+        varStart = value.indexOf(VARIABLE_START, varEnd);
+      }
+      // Special case for 'hbase.cluster.distributed' property being 'true'
+      if (key.startsWith("server.")) {
+        if (conf.get(HConstants.CLUSTER_DISTRIBUTED).equals(HConstants.CLUSTER_IS_DISTRIBUTED)
+            && value.startsWith("localhost")) {
+          String msg = "The server in zoo.cfg cannot be set to localhost " +
+              "in a fully-distributed setup because it won't be reachable. " +
+              "See \"Getting Started\" for more information.";
+          LOG.fatal(msg);
+          throw new IOException(msg);
+        }
+      }
+      newValue.append(value.substring(varEnd));
+      properties.setProperty(key, newValue.toString());
+    }
+    return properties;
+  }
+
+  /**
+   * Return the ZK Quorum servers string given zk properties returned by
+   * makeZKProps
+   * @param properties
+   * @return
+   */
+  public static String getZKQuorumServersString(Properties properties) {
+    String clientPort = null;
+    List<String> servers = new ArrayList<String>();
+
+    // The clientPort option may come after the server.X hosts, so we need to
+    // grab everything and then create the final host:port comma separated list.
+    boolean anyValid = false;
+    for (Entry<Object,Object> property : properties.entrySet()) {
+      String key = property.getKey().toString().trim();
+      String value = property.getValue().toString().trim();
+      if (key.equals("clientPort")) {
+        clientPort = value;
+      }
+      else if (key.startsWith("server.")) {
+        String host = value.substring(0, value.indexOf(':'));
+        servers.add(host);
+        try {
+          //noinspection ResultOfMethodCallIgnored
+          InetAddress.getByName(host);
+          anyValid = true;
+        } catch (UnknownHostException e) {
+          LOG.warn(StringUtils.stringifyException(e));
+        }
+      }
+    }
+
+    if (!anyValid) {
+      LOG.error("no valid quorum servers found in " + HConstants.ZOOKEEPER_CONFIG_NAME);
+      return null;
+    }
+
+    if (clientPort == null) {
+      LOG.error("no clientPort found in " + HConstants.ZOOKEEPER_CONFIG_NAME);
+      return null;
+    }
+
+    if (servers.isEmpty()) {
+      LOG.fatal("No server.X lines found in conf/zoo.cfg. HBase must have a " +
+                "ZooKeeper cluster configured for its operation.");
+      return null;
+    }
+
+    StringBuilder hostPortBuilder = new StringBuilder();
+    for (int i = 0; i < servers.size(); ++i) {
+      String host = servers.get(i);
+      if (i > 0) {
+        hostPortBuilder.append(',');
+      }
+      hostPortBuilder.append(host);
+      hostPortBuilder.append(':');
+      hostPortBuilder.append(clientPort);
+    }
+
+    return hostPortBuilder.toString();
+  }
+
+  /**
+   * Return the ZK Quorum servers string given the specified configuration.
+   * @param properties
+   * @return
+   */
+  public static String getZKQuorumServersString(Configuration conf) {
+    return getZKQuorumServersString(makeZKProps(conf));
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
index 94806bdaa06..ae1c34eb22d 100644
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
@@ -20,15 +20,14 @@
 
 package org.apache.hadoop.hbase.zookeeper;
 
+import java.util.Properties;
+import java.util.Map.Entry;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-
-import java.util.Map.Entry;
-import java.util.Properties;
 
 /**
- * Tool for reading ZooKeeper servers from HBase XML configuation and producing
+ * Tool for reading ZooKeeper servers from HBase XML configuration and producing
  * a line-by-line list for use by bash scripts.
  */
 public class ZKServerTool {
@@ -41,7 +40,7 @@ public class ZKServerTool {
     // Note that we do not simply grab the property
     // HConstants.ZOOKEEPER_QUORUM from the HBaseConfiguration because the
     // user may be using a zoo.cfg file.
-    Properties zkProps = HQuorumPeer.makeZKProps(conf);
+    Properties zkProps = ZKConfig.makeZKProps(conf);
     for (Entry<Object, Object> entry : zkProps.entrySet()) {
       String key = entry.getKey().toString().trim();
       String value = entry.getValue().toString().trim();
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableDisable.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableDisable.java
new file mode 100644
index 00000000000..147cd1898b0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableDisable.java
@@ -0,0 +1,70 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.util.List;
+
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Helper class for table disable tracking in zookeeper.
+ * <p>
+ * The node /disabled will contain a child node for every table which should be
+ * disabled, for example, /disabled/table.
+ */
+public class ZKTableDisable {
+
+  /**
+   * Sets the specified table as disabled in zookeeper.  Fails silently if the
+   * table is already disabled in zookeeper.  Sets no watches.
+   * @param zkw
+   * @param tableName
+   * @throws KeeperException unexpected zookeeper exception
+   */
+  public static void disableTable(ZooKeeperWatcher zkw, String tableName)
+  throws KeeperException {
+    ZKUtil.createAndFailSilent(zkw, ZKUtil.joinZNode(zkw.tableZNode,
+        tableName));
+  }
+
+  /**
+   * Unsets the specified table as disabled in zookeeper.  Fails silently if the
+   * table is not currently disabled in zookeeper.  Sets no watches.
+   * @param zkw
+   * @param tableName
+   * @throws KeeperException unexpected zookeeper exception
+   */
+  public static void undisableTable(ZooKeeperWatcher zkw, String tableName)
+  throws KeeperException {
+    ZKUtil.deleteNodeFailSilent(zkw, ZKUtil.joinZNode(zkw.tableZNode,
+        tableName));
+  }
+
+  /**
+   * Gets a list of all the tables set as disabled in zookeeper.
+   * @param zkw
+   * @return list of disabled tables, empty list if none
+   * @throws KeeperException
+   */
+  public static List<String> getDisabledTables(ZooKeeperWatcher zkw)
+  throws KeeperException {
+    return ZKUtil.listChildrenNoWatch(zkw, zkw.tableZNode);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
new file mode 100644
index 00000000000..1595d234b97
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -0,0 +1,999 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ * Internal HBase utility class for ZooKeeper.
+ *
+ * <p>Contains only static methods and constants.
+ *
+ * <p>Methods all throw {@link KeeperException} if there is an unexpected
+ * zookeeper exception, so callers of these methods must handle appropriately.
+ * If ZK is required for the operation, the server will need to be aborted.
+ */
+public class ZKUtil {
+  private static final Log LOG = LogFactory.getLog(ZKUtil.class);
+
+  // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
+  private static final char ZNODE_PATH_SEPARATOR = '/';
+
+  /**
+   * Creates a new connection to ZooKeeper, pulling settings and quorum config
+   * from the specified configuration object using methods from {@link ZKConfig}.
+   *
+   * Sets the connection status monitoring watcher to the specified watcher.
+   *
+   * @param conf configuration to pull quorum and other settings from
+   * @param watcher watcher to monitor connection changes
+   * @return connection to zookeeper
+   * @throws IOException if unable to connect to zk or config problem
+   */
+  public static ZooKeeper connect(Configuration conf, Watcher watcher)
+  throws IOException {
+    Properties properties = ZKConfig.makeZKProps(conf);
+    String quorum = ZKConfig.getZKQuorumServersString(properties);
+    return connect(conf, quorum, watcher);
+  }
+
+  public static ZooKeeper connect(Configuration conf, String quorum,
+      Watcher watcher)
+  throws IOException {
+    if(quorum == null) {
+      throw new IOException("Unable to determine ZooKeeper quorum");
+    }
+    int timeout = conf.getInt("zookeeper.session.timeout", 60 * 1000);
+    LOG.debug("Opening connection to ZooKeeper with quorum (" + quorum + ")");
+    return new ZooKeeper(quorum, timeout, watcher);
+  }
+
+  //
+  // Helper methods
+  //
+
+  /**
+   * Join the prefix znode name with the suffix znode name to generate a proper
+   * full znode name.
+   *
+   * Assumes prefix does not end with slash and suffix does not begin with it.
+   *
+   * @param prefix beginning of znode name
+   * @param suffix ending of znode name
+   * @return result of properly joining prefix with suffix
+   */
+  public static String joinZNode(String prefix, String suffix) {
+    return prefix + ZNODE_PATH_SEPARATOR + suffix;
+  }
+
+  /**
+   * Returns the full path of the immediate parent of the specified node.
+   * @param node path to get parent of
+   * @return parent of path, null if passed the root node or an invalid node
+   */
+  public static String getParent(String node) {
+    int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
+    return idx <= 0 ? null : node.substring(0, idx);
+  }
+
+  /**
+   * Get the unique node-name for the specified regionserver.
+   *
+   * Used when a server puts up an ephemeral node for itself and needs to use
+   * a unique name.
+   *
+   * @param serverInfo server information
+   * @return unique, zookeeper-safe znode path for the server instance
+   */
+  public static String getNodeName(HServerInfo serverInfo) {
+    return serverInfo.getServerName();
+  }
+
+  /**
+   * Get the name of the current node from the specified fully-qualified path.
+   * @param path fully-qualified path
+   * @return name of the current node
+   */
+  public static String getNodeName(String path) {
+    return path.substring(path.lastIndexOf("/")+1);
+  }
+
+  /**
+   * Get the key to the ZK ensemble for this configuration without
+   * adding a name at the end
+   * @param conf Configuration to use to build the key
+   * @return ensemble key without a name
+   */
+  public static String getZooKeeperClusterKey(Configuration conf) {
+    return getZooKeeperClusterKey(conf, null);
+  }
+
+  /**
+   * Get the key to the ZK ensemble for this configuration and append
+   * a name at the end
+   * @param conf Configuration to use to build the key
+   * @param name Name that should be appended at the end if not empty or null
+   * @return ensemble key with a name (if any)
+   */
+  public static String getZooKeeperClusterKey(Configuration conf, String name) {
+    String quorum = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll(
+        "[\\t\\n\\x0B\\f\\r]", ""));
+    StringBuilder builder = new StringBuilder(quorum);
+    builder.append(":");
+    builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
+    if (name != null && !name.isEmpty()) {
+      builder.append(",");
+      builder.append(name);
+    }
+    return builder.toString();
+  }
+
+  //
+  // Existence checks and watches
+  //
+
+  /**
+   * Watch the specified znode for delete/create/change events.  The watcher is
+   * set whether or not the node exists.  If the node already exists, the method
+   * returns true.  If the node does not exist, the method returns false.
+   *
+   * @param zkw zk reference
+   * @param znode path of node to watch
+   * @return true if znode exists, false if does not exist or error
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
+  throws KeeperException {
+    try {
+      Stat s = zkw.getZooKeeper().exists(znode, zkw);
+      zkw.debug("Set watcher on existing znode " + znode);
+      return s != null ? true : false;
+    } catch (KeeperException e) {
+      zkw.warn("Unable to set watcher on znode " + znode, e);
+      zkw.keeperException(e);
+      return false;
+    } catch (InterruptedException e) {
+      zkw.warn("Unable to set watcher on znode " + znode, e);
+      zkw.interruptedException(e);
+      return false;
+    }
+  }
+
+  /**
+   * Check if the specified node exists.  Sets no watches.
+   *
+   * Returns true if node exists, false if not.  Returns an exception if there
+   * is an unexpected zookeeper exception.
+   *
+   * @param zkw zk reference
+   * @param znode path of node to watch
+   * @return version of the node if it exists, -1 if does not exist
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static int checkExists(ZooKeeperWatcher zkw, String znode)
+  throws KeeperException {
+    try {
+      Stat s = zkw.getZooKeeper().exists(znode, null);
+      return s != null ? s.getVersion() : -1;
+    } catch (KeeperException e) {
+      zkw.warn("Unable to set watcher on znode (" + znode + ")", e);
+      zkw.keeperException(e);
+      return -1;
+    } catch (InterruptedException e) {
+      zkw.warn("Unable to set watcher on znode (" + znode + ")", e);
+      zkw.interruptedException(e);
+      return -1;
+    }
+  }
+
+  //
+  // Znode listings
+  //
+
+  /**
+   * Lists the children znodes of the specified znode.  Also sets a watch on
+   * the specified znode which will capture a NodeDeleted event on the specified
+   * znode as well as NodeChildrenChanged if any children of the specified znode
+   * are created or deleted.
+   *
+   * Returns null if the specified node does not exist.  Otherwise returns a
+   * list of children of the specified node.  If the node exists but it has no
+   * children, an empty list will be returned.
+   *
+   * @param zkw zk reference
+   * @param znode path of node to list and watch children of
+   * @returns list of children of the specified node, an empty list if the node
+   *          exists but has no children, and null if the node does not exist
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static List<String> listChildrenAndWatchForNewChildren(
+      ZooKeeperWatcher zkw, String znode)
+  throws KeeperException {
+    try {
+      List<String> children = zkw.getZooKeeper().getChildren(znode, zkw);
+      return children;
+    } catch(KeeperException.NoNodeException ke) {
+      zkw.debug("Unable to list children of znode " + znode + " " +
+          "because node does not exist (not an error)");
+      return null;
+    } catch (KeeperException e) {
+      zkw.warn("Unable to list children of znode " + znode + " ", e);
+      zkw.keeperException(e);
+      return null;
+    } catch (InterruptedException e) {
+      zkw.warn("Unable to list children of znode " + znode + " ", e);
+      zkw.interruptedException(e);
+      return null;
+    }
+  }
+
+  /**
+   * Lists the children of the specified znode, retrieving the data of each
+   * child as a server address.
+   *
+   * Used to list the currently online regionservers and their addresses.
+   *
+   * Sets no watches at all, this method is best effort.
+   *
+   * Returns an empty list if the node has no children.  Returns null if the
+   * parent node itself does not exist.
+   *
+   * @param zkw zookeeper reference
+   * @param znode node to get children of as addresses
+   * @return list of data of children of specified znode, empty if no children,
+   *         null if parent does not exist
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static List<HServerAddress> listChildrenAndGetAsAddresses(
+      ZooKeeperWatcher zkw, String znode)
+  throws KeeperException {
+    List<String> children = listChildrenNoWatch(zkw, znode);
+    if(children == null) {
+      return null;
+    }
+    List<HServerAddress> addresses =
+      new ArrayList<HServerAddress>(children.size());
+    for(String child : children) {
+      addresses.add(getDataAsAddress(zkw, joinZNode(znode, child)));
+    }
+    return addresses;
+  }
+
+  /**
+   * Lists the children of the specified znode without setting any watches.
+   *
+   * Used to list the currently online regionservers and their addresses.
+   *
+   * Sets no watches at all, this method is best effort.
+   *
+   * Returns an empty list if the node has no children.  Returns null if the
+   * parent node itself does not exist.
+   *
+   * @param zkw zookeeper reference
+   * @param znode node to get children of as addresses
+   * @return list of data of children of specified znode, empty if no children,
+   *         null if parent does not exist
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static List<String> listChildrenNoWatch(
+      ZooKeeperWatcher zkw, String znode)
+  throws KeeperException {
+    List<String> children = null;
+    try {
+      // List the children without watching
+      children = zkw.getZooKeeper().getChildren(znode, null);
+    } catch(KeeperException.NoNodeException nne) {
+      return null;
+    } catch(InterruptedException ie) {
+      zkw.interruptedException(ie);
+    }
+    return children;
+  }
+
+  /**
+   * Atomically add watches and read data from all unwatched unassigned nodes.
+   *
+   * <p>This works because master is the only person deleting nodes.
+   */
+  public static List<NodeAndData> watchAndGetNewChildren(ZooKeeperWatcher zkw,
+      String baseNode)
+  throws KeeperException {
+    List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
+    synchronized(zkw.getNodes()) {
+      List<String> nodes =
+        ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
+      for(String node : nodes) {
+        String nodePath = ZKUtil.joinZNode(baseNode, node);
+        if(!zkw.getNodes().contains(nodePath)) {
+          byte [] data = ZKUtil.getDataAndWatch(zkw, nodePath);
+          newNodes.add(new NodeAndData(nodePath, data));
+          zkw.getNodes().add(nodePath);
+        }
+      }
+    }
+    return newNodes;
+  }
+
+  /**
+   * Simple class to hold a node path and node data.
+   */
+  public static class NodeAndData {
+    private String node;
+    private byte [] data;
+    public NodeAndData(String node, byte [] data) {
+      this.node = node;
+      this.data = data;
+    }
+    public String getNode() {
+      return node;
+    }
+    public byte [] getData() {
+      return data;
+    }
+    @Override
+    public String toString() {
+      return node + " (" + RegionTransitionData.fromBytes(data) + ")";
+    }
+  }
+
+  /**
+   * Checks if the specified znode has any children.  Sets no watches.
+   *
+   * Returns true if the node exists and has children.  Returns false if the
+   * node does not exist or if the node does not have any children.
+   *
+   * Used during master initialization to determine if the master is a
+   * failed-over-to master or the first master during initial cluster startup.
+   * If the directory for regionserver ephemeral nodes is empty then this is
+   * a cluster startup, if not then it is not cluster startup.
+   *
+   * @param zkw zk reference
+   * @param znode path of node to check for children of
+   * @return true if node has children, false if not or node does not exist
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
+  throws KeeperException {
+    try {
+      return !zkw.getZooKeeper().getChildren(znode, null).isEmpty();
+    } catch(KeeperException.NoNodeException ke) {
+      zkw.debug("Unable to list children of znode " + znode + " " +
+      "because node does not exist (not an error)");
+      return false;
+    } catch (KeeperException e) {
+      zkw.warn("Unable to list children of znode " + znode, e);
+      zkw.keeperException(e);
+      return false;
+    } catch (InterruptedException e) {
+      zkw.warn("Unable to list children of znode " + znode, e);
+      zkw.interruptedException(e);
+      return false;
+    }
+  }
+
+  /**
+   * Get the number of children of the specified node.
+   *
+   * If the node does not exist or has no children, returns 0.
+   *
+   * Sets no watches at all.
+   *
+   * @param zkw zk reference
+   * @param znode path of node to count children of
+   * @return number of children of specified node, 0 if none or parent does not
+   *         exist
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode)
+  throws KeeperException {
+    try {
+      Stat stat = zkw.getZooKeeper().exists(znode, null);
+      return stat == null ? 0 : stat.getNumChildren();
+    } catch(KeeperException e) {
+      zkw.warn("Unable to get children of node " + znode);
+      zkw.keeperException(e);
+    } catch(InterruptedException e) {
+      zkw.interruptedException(e);
+    }
+    return 0;
+  }
+
+  //
+  // Data retrieval
+  //
+
+  /**
+   * Get znode data. Does not set a watcher.
+   * @return ZNode data
+   */
+  public static byte [] getData(ZooKeeperWatcher zkw, String znode)
+  throws KeeperException {
+    try {
+      byte [] data = zkw.getZooKeeper().getData(znode, null, null);
+      zkw.debug("Retrieved " + data.length + " bytes of data from znode " + znode);
+      return data;
+    } catch (KeeperException.NoNodeException e) {
+      zkw.debug("Unable to get data of znode " + znode + " " +
+          "because node does not exist (not an error)");
+      return null;
+    } catch (KeeperException e) {
+      zkw.warn("Unable to get data of znode " + znode, e);
+      zkw.keeperException(e);
+      return null;
+    } catch (InterruptedException e) {
+      zkw.warn("Unable to get data of znode " + znode, e);
+      zkw.interruptedException(e);
+      return null;
+    }
+  }
+
+  /**
+   * Get the data at the specified znode and set a watch.
+   *
+   * Returns the data and sets a watch if the node exists.  Returns null and no
+   * watch is set if the node does not exist or there is an exception.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @return data of the specified znode, or null
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
+  throws KeeperException {
+    try {
+      byte [] data = zkw.getZooKeeper().getData(znode, zkw, null);
+      zkw.debug("Retrieved " + data.length + " bytes of data from znode " +
+          znode + " and set a watcher");
+      return data;
+    } catch (KeeperException.NoNodeException e) {
+      zkw.debug("Unable to get data of znode " + znode + " " +
+          "because node does not exist (not an error)");
+      return null;
+    } catch (KeeperException e) {
+      zkw.warn("Unable to get data of znode " + znode, e);
+      zkw.keeperException(e);
+      return null;
+    } catch (InterruptedException e) {
+      zkw.warn("Unable to get data of znode " + znode, e);
+      zkw.interruptedException(e);
+      return null;
+    }
+  }
+
+  /**
+   * Get the data at the specified znode without setting a watch.
+   *
+   * Returns the data if the node exists.  Returns null if the node does not
+   * exist.
+   *
+   * Sets the stats of the node in the passed Stat object.  Pass a null stat if
+   * not interested.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @param stat node status to set if node exists
+   * @return data of the specified znode, or null if does not exist
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
+      Stat stat)
+  throws KeeperException {
+    try {
+      byte [] data = zkw.getZooKeeper().getData(znode, zkw, stat);
+      zkw.debug("Retrieved " + data.length + " bytes of data from znode " + znode);
+      return data;
+    } catch (KeeperException.NoNodeException e) {
+      zkw.debug("Unable to get data of znode " + znode + " " +
+          "because node does not exist (not necessarily an error)");
+      return null;
+    } catch (KeeperException e) {
+      zkw.warn("Unable to get data of znode " + znode, e);
+      zkw.keeperException(e);
+      return null;
+    } catch (InterruptedException e) {
+      zkw.warn("Unable to get data of znode " + znode, e);
+      zkw.interruptedException(e);
+      return null;
+    }
+  }
+
+  /**
+   * Get the data at the specified znode, deserialize it as an HServerAddress,
+   * and set a watch.
+   *
+   * Returns the data as a server address and sets a watch if the node exists.
+   * Returns null and no watch is set if the node does not exist or there is an
+   * exception.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @return data of the specified node as a server address, or null
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static HServerAddress getDataAsAddress(ZooKeeperWatcher zkw,
+      String znode)
+  throws KeeperException {
+    byte [] data = getDataAndWatch(zkw, znode);
+    if(data == null) {
+      return null;
+    }
+    String addrString = Bytes.toString(data);
+    zkw.debug("Read server address from znode " + znode + ": " + addrString);
+    return new HServerAddress(addrString);
+  }
+
+  /**
+   * Update the data of an existing node with the expected version to have the
+   * specified data.
+   *
+   * Throws an exception if there is a version mismatch or some other problem.
+   *
+   * Sets no watches under any conditions.
+   *
+   * @param zkw zk reference
+   * @param znode
+   * @param data
+   * @param expectedVersion
+   * @throws KeeperException if unexpected zookeeper exception
+   * @throws KeeperException.BadVersionException if version mismatch
+   */
+  public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
+      byte [] data, int expectedVersion)
+  throws KeeperException {
+    try {
+      zkw.getZooKeeper().setData(znode, data, expectedVersion);
+    } catch(InterruptedException ie) {
+      zkw.interruptedException(ie);
+    }
+  }
+
+  //
+  // Data setting
+  //
+
+  /**
+   * Set the specified znode to be an ephemeral node carrying the specified
+   * server address.  Used by masters for their ephemeral node and regionservers
+   * for their ephemeral node.
+   *
+   * If the node is created successfully, a watcher is also set on the node.
+   *
+   * If the node is not created successfully because it already exists, this
+   * method will also set a watcher on the node.
+   *
+   * If there is another problem, a KeeperException will be thrown.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @param address server address
+   * @return true if address set, false if not, watch set in both cases
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static boolean setAddressAndWatch(ZooKeeperWatcher zkw,
+      String znode, HServerAddress address)
+  throws KeeperException {
+    return createEphemeralNodeAndWatch(zkw, znode,
+        Bytes.toBytes(address.toString()));
+  }
+
+  /**
+   * Sets the data of the existing znode to be the specified data.  Ensures that
+   * the current data has the specified expected version.
+   *
+   * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
+   *
+   * <p>If their is a version mismatch, method returns null.
+   *
+   * <p>No watches are set but setting data will trigger other watchers of this
+   * node.
+   *
+   * <p>If there is another problem, a KeeperException will be thrown.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @param data data to set for node
+   * @param expectedVersion version expected when setting data
+   * @return true if data set, false if version mismatch
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static boolean setData(ZooKeeperWatcher zkw, String znode,
+      byte [] data, int expectedVersion)
+  throws KeeperException, KeeperException.NoNodeException {
+    try {
+      return zkw.getZooKeeper().setData(znode, data, expectedVersion) != null;
+    } catch (InterruptedException e) {
+      zkw.interruptedException(e);
+      return false;
+    }
+  }
+
+  /**
+   * Sets the data of the existing znode to be the specified data.  The node
+   * must exist but no checks are done on the existing data or version.
+   *
+   * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
+   *
+   * <p>No watches are set but setting data will trigger other watchers of this
+   * node.
+   *
+   * <p>If there is another problem, a KeeperException will be thrown.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @param data data to set for node
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static void setData(ZooKeeperWatcher zkw, String znode,
+      byte [] data)
+  throws KeeperException, KeeperException.NoNodeException {
+    setData(zkw, znode, data, -1);
+  }
+
+  //
+  // Node creation
+  //
+
+  /**
+   *
+   * Set the specified znode to be an ephemeral node carrying the specified
+   * data.
+   *
+   * If the node is created successfully, a watcher is also set on the node.
+   *
+   * If the node is not created successfully because it already exists, this
+   * method will also set a watcher on the node.
+   *
+   * If there is another problem, a KeeperException will be thrown.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @param data data of node
+   * @return true if node created, false if not, watch set in both cases
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
+      String znode, byte [] data)
+  throws KeeperException {
+    try {
+      zkw.getZooKeeper().create(znode, data, Ids.OPEN_ACL_UNSAFE,
+          CreateMode.EPHEMERAL);
+    } catch (KeeperException.NodeExistsException nee) {
+      if(!watchAndCheckExists(zkw, znode)) {
+        // It did exist but now it doesn't, try again
+        return createEphemeralNodeAndWatch(zkw, znode, data);
+      }
+      return false;
+    } catch (InterruptedException e) {
+      LOG.info("Interrupted", e);
+    }
+    return true;
+  }
+
+  /**
+   * Creates the specified znode to be a persistent node carrying the specified
+   * data.
+   *
+   * Returns true if the node was successfully created, false if the node
+   * already existed.
+   *
+   * If the node is created successfully, a watcher is also set on the node.
+   *
+   * If the node is not created successfully because it already exists, this
+   * method will also set a watcher on the node but return false.
+   *
+   * If there is another problem, a KeeperException will be thrown.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @param data data of node
+   * @return true if node created, false if not, watch set in both cases
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static boolean createNodeIfNotExistsAndWatch(
+      ZooKeeperWatcher zkw, String znode, byte [] data)
+  throws KeeperException {
+    try {
+      zkw.getZooKeeper().create(znode, data, Ids.OPEN_ACL_UNSAFE,
+          CreateMode.PERSISTENT);
+    } catch (KeeperException.NodeExistsException nee) {
+      try {
+        zkw.getZooKeeper().exists(znode, zkw);
+      } catch (InterruptedException e) {
+        zkw.interruptedException(e);
+        return false;
+      }
+      return false;
+    } catch (InterruptedException e) {
+      zkw.interruptedException(e);
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Creates the specified node with the specified data and watches it.
+   *
+   * <p>Throws an exception if the node already exists.
+   *
+   * <p>The node created is persistent and open access.
+   *
+   * <p>Returns the version number of the created node if successful.
+   *
+   * @param zkw zk reference
+   * @param znode path of node to create
+   * @param data data of node to create
+   * @return version of node created
+   * @throws KeeperException if unexpected zookeeper exception
+   * @throws KeeperException.NodeExistsException if node already exists
+   */
+  public static int createAndWatch(ZooKeeperWatcher zkw,
+      String znode, byte [] data)
+  throws KeeperException, KeeperException.NodeExistsException {
+    try {
+      zkw.getZooKeeper().create(znode, data, Ids.OPEN_ACL_UNSAFE,
+          CreateMode.PERSISTENT);
+      return zkw.getZooKeeper().exists(znode, zkw).getVersion();
+    } catch (InterruptedException e) {
+      zkw.interruptedException(e);
+      return -1;
+    }
+  }
+
+  /**
+   * Creates the specified node, if the node does not exist.  Does not set a
+   * watch and fails silently if the node already exists.
+   *
+   * The node created is persistent and open access.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static void createAndFailSilent(ZooKeeperWatcher zkw,
+      String znode)
+  throws KeeperException {
+    try {
+      zkw.getZooKeeper().create(znode, new byte[0], Ids.OPEN_ACL_UNSAFE,
+          CreateMode.PERSISTENT);
+    } catch(KeeperException.NodeExistsException nee) {
+    } catch(InterruptedException ie) {
+      zkw.interruptedException(ie);
+    }
+  }
+
+  /**
+   * Creates the specified node and all parent nodes required for it to exist.
+   *
+   * No watches are set and no errors are thrown if the node already exists.
+   *
+   * The nodes created are persistent and open access.
+   *
+   * @param zkw zk reference
+   * @param znode path of node
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static void createWithParents(ZooKeeperWatcher zkw,
+      String znode)
+  throws KeeperException {
+    try {
+      if(znode == null) {
+        return;
+      }
+      zkw.getZooKeeper().create(znode, new byte[0], Ids.OPEN_ACL_UNSAFE,
+          CreateMode.PERSISTENT);
+    } catch(KeeperException.NodeExistsException nee) {
+      return;
+    } catch(KeeperException.NoNodeException nne) {
+      createWithParents(zkw, getParent(znode));
+      createWithParents(zkw, znode);
+    } catch(InterruptedException ie) {
+      zkw.interruptedException(ie);
+    }
+  }
+
+  //
+  // Deletes
+  //
+
+  /**
+   * Delete the specified node.  Sets no watches.  Throws all exceptions.
+   */
+  public static void deleteNode(ZooKeeperWatcher zkw, String node)
+  throws KeeperException {
+    deleteNode(zkw, node, -1);
+  }
+
+  /**
+   * Delete the specified node with the specified version.  Sets no watches.
+   * Throws all exceptions.
+   */
+  public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
+      int version)
+  throws KeeperException {
+    try {
+      zkw.getZooKeeper().delete(node, version);
+      return true;
+    } catch(KeeperException.BadVersionException bve) {
+      return false;
+    } catch(InterruptedException ie) {
+      zkw.interruptedException(ie);
+      return false;
+    }
+  }
+
+  /**
+   * Deletes the specified node.  Fails silent if the node does not exist.
+   * @param zkw
+   * @param joinZNode
+   * @throws KeeperException
+   */
+  public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
+  throws KeeperException {
+    try {
+      zkw.getZooKeeper().delete(node, -1);
+    } catch(KeeperException.NoNodeException nne) {
+    } catch(InterruptedException ie) {
+      zkw.interruptedException(ie);
+    }
+  }
+
+  /**
+   * Delete the specified node and all of it's children.
+   *
+   * Sets no watches.  Throws all exceptions besides dealing with deletion of
+   * children.
+   */
+  public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
+  throws KeeperException {
+    try {
+      List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
+      if(!children.isEmpty()) {
+        for(String child : children) {
+          deleteNodeRecursively(zkw, joinZNode(node, child));
+        }
+      }
+      zkw.getZooKeeper().delete(node, -1);
+    } catch(InterruptedException ie) {
+      zkw.interruptedException(ie);
+    }
+  }
+
+  /**
+   * Delete all the children of the specified node but not the node itself.
+   *
+   * Sets no watches.  Throws all exceptions besides dealing with deletion of
+   * children.
+   */
+  public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
+  throws KeeperException {
+    List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
+    if(!children.isEmpty()) {
+      for(String child : children) {
+        deleteNodeRecursively(zkw, joinZNode(node, child));
+      }
+    }
+  }
+
+  //
+  // ZooKeeper cluster information
+  //
+
+  /** @return String dump of everything in ZooKeeper. */
+  public static String dump(ZooKeeperWatcher zkw) {
+    StringBuilder sb = new StringBuilder();
+    try {
+      sb.append("\nHBase tree in ZooKeeper is rooted at ").append(zkw.baseZNode);
+      sb.append("\n  Cluster up? ").append(checkExists(zkw, zkw.clusterStateZNode));
+      sb.append("\n  Master address: ").append(
+          getDataAsAddress(zkw, zkw.masterAddressZNode));
+      sb.append("\n  Region server holding ROOT: ").append(
+          getDataAsAddress(zkw, zkw.rootServerZNode));
+      sb.append("\n  Region servers:");
+      for (HServerAddress address : listChildrenAndGetAsAddresses(zkw,
+          zkw.rsZNode)) {
+        sb.append("\n    - ").append(address);
+      }
+      sb.append("\n  Quorum Server Statistics:");
+      String[] servers = zkw.getQuorum().split(",");
+      for (String server : servers) {
+        sb.append("\n    - ").append(server);
+        try {
+          String[] stat = getServerStats(server);
+          for (String s : stat) {
+            sb.append("\n        ").append(s);
+          }
+        } catch (Exception e) {
+          sb.append("\n        ERROR: ").append(e.getMessage());
+        }
+      }
+    } catch(KeeperException ke) {
+      sb.append("\n  FATAL ZooKeeper Exception!\n");
+      sb.append("\n  " + ke.getMessage());
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Gets the statistics from the given server. Uses a 1 minute timeout.
+   *
+   * @param server  The server to get the statistics from.
+   * @return The array of response strings.
+   * @throws IOException When the socket communication fails.
+   */
+  public static String[] getServerStats(String server)
+  throws IOException {
+    return getServerStats(server, 60 * 1000);
+  }
+
+  /**
+   * Gets the statistics from the given server.
+   *
+   * @param server  The server to get the statistics from.
+   * @param timeout  The socket timeout to use.
+   * @return The array of response strings.
+   * @throws IOException When the socket communication fails.
+   */
+  public static String[] getServerStats(String server, int timeout)
+  throws IOException {
+    String[] sp = server.split(":");
+    Socket socket = new Socket(sp[0],
+      sp.length > 1 ? Integer.parseInt(sp[1]) : 2181);
+    socket.setSoTimeout(timeout);
+    PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
+    BufferedReader in = new BufferedReader(new InputStreamReader(
+      socket.getInputStream()));
+    out.println("stat");
+    out.flush();
+    ArrayList<String> res = new ArrayList<String>();
+    while (true) {
+      String line = in.readLine();
+      if (line != null) {
+        res.add(line);
+      } else {
+        break;
+      }
+    }
+    socket.close();
+    return res.toArray(new String[res.size()]);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java
new file mode 100644
index 00000000000..97e3af641ce
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java
@@ -0,0 +1,78 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+
+/**
+ * Base class for internal listeners of ZooKeeper events.
+ *
+ * The {@link ZooKeeperWatcher} for a process will execute the appropriate
+ * methods of implementations of this class.  In order to receive events from
+ * the watcher, every listener must register itself via {@link ZooKeeperWatcher#registerListener}.
+ *
+ * Subclasses need only override those methods in which they are interested.
+ *
+ * Note that the watcher will be blocked when invoking methods in listeners so
+ * they must not be long-running.
+ */
+public class ZooKeeperListener {
+
+  // Reference to the zk watcher which also contains configuration and constants
+  protected ZooKeeperWatcher watcher;
+
+  /**
+   * Construct a ZooKeeper event listener.
+   */
+  public ZooKeeperListener(ZooKeeperWatcher watcher) {
+    this.watcher = watcher;
+  }
+
+  /**
+   * Called when a new node has been created.
+   * @param path full path of the new node
+   */
+  public void nodeCreated(String path) {
+    // no-op
+  }
+
+  /**
+   * Called when a node has been deleted
+   * @param path full path of the deleted node
+   */
+  public void nodeDeleted(String path) {
+    // no-op
+  }
+
+  /**
+   * Called when an existing node has changed data.
+   * @param path full path of the updated node
+   */
+  public void nodeDataChanged(String path) {
+    // no-op
+  }
+
+  /**
+   * Called when an existing node has a child node added or removed.
+   * @param path full path of the node whose children have changed
+   */
+  public void nodeChildrenChanged(String path) {
+    // no-op
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
new file mode 100644
index 00000000000..e5073695d52
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
@@ -0,0 +1,173 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Tracks the availability and value of a single ZooKeeper node.
+ *
+ * <p>Utilizes the {@link ZooKeeperListener} interface to get the necessary
+ * ZooKeeper events related to the node.
+ *
+ * <p>This is the base class used by trackers in both the Master and
+ * RegionServers.
+ */
+public abstract class ZooKeeperNodeTracker extends ZooKeeperListener {
+  /**
+   * Pass this if you do not want a timeout.
+   */
+  public final static long NO_TIMEOUT = -1;
+
+  /** Path of node being tracked */
+  protected final String node;
+
+  /** Data of the node being tracked */
+  private byte [] data;
+
+  /** Used to abort if a fatal error occurs */
+  protected final Abortable abortable;
+
+  /**
+   * Constructs a new ZK node tracker.
+   *
+   * <p>After construction, use {@link #start} to kick off tracking.
+   *
+   * @param watcher
+   * @param node
+   * @param abortable
+   */
+  public ZooKeeperNodeTracker(ZooKeeperWatcher watcher, String node,
+      Abortable abortable) {
+    super(watcher);
+    this.node = node;
+    this.abortable = abortable;
+    this.data = null;
+  }
+
+  /**
+   * Starts the tracking of the node in ZooKeeper.
+   *
+   * <p>Use {@link blockUntilAvailable} to block until the node is available
+   * or {@link getData} to get the data of the node if it is available.
+   */
+  public synchronized void start() {
+    this.watcher.registerListener(this);
+    try {
+      if(ZKUtil.watchAndCheckExists(watcher, node)) {
+        byte [] data = ZKUtil.getDataAndWatch(watcher, node);
+        if(data != null) {
+          this.data = data;
+        } else {
+          // It existed but now does not, try again to ensure a watch is set
+          start();
+        }
+      }
+    } catch (KeeperException e) {
+      abortable.abort("Unexpected exception during initialization, aborting", e);
+    }
+  }
+
+  /**
+   * Gets the data of the node, blocking until the node is available.
+   *
+   * @return data of the node
+   * @throws InterruptedException if the waiting thread is interrupted
+   */
+  public synchronized byte [] blockUntilAvailable()
+  throws InterruptedException {
+    return blockUntilAvailable(NO_TIMEOUT);
+  }
+
+  /**
+   * Gets the data of the node, blocking until the node is available or the
+   * specified timeout has elapsed.
+   *
+   * @param timeout maximum time to wait for the node data to be available,
+   *                in milliseconds.  Pass {@link #NO_TIMEOUT} for no timeout.
+   * @return data of the node
+   * @throws InterruptedException if the waiting thread is interrupted
+   */
+  public synchronized byte [] blockUntilAvailable(long timeout)
+  throws InterruptedException {
+    if (timeout != NO_TIMEOUT && timeout < 0) throw new IllegalArgumentException();
+    long startTime = System.currentTimeMillis();
+    long remaining = timeout;
+    while ((remaining == NO_TIMEOUT || remaining > 0) && this.data == null) {
+      if (remaining == NO_TIMEOUT) wait();
+      else wait(remaining);
+      remaining = timeout - (System.currentTimeMillis() - startTime);
+    }
+    return data;
+  }
+
+  /**
+   * Gets the data of the node.
+   *
+   * <p>If the node is currently available, the most up-to-date known version of
+   * the data is returned.  If the node is not currently available, null is
+   * returned.
+   *
+   * @return data of the node, null if unavailable
+   */
+  public synchronized byte [] getData() {
+    return data;
+  }
+
+  @Override
+  public synchronized void nodeCreated(String path) {
+    if(path.equals(node)) {
+      try {
+        byte [] data = ZKUtil.getDataAndWatch(watcher, node);
+        if(data != null) {
+          this.data = data;
+          notifyAll();
+        } else {
+          nodeDeleted(path);
+        }
+      } catch(KeeperException e) {
+        abortable.abort("Unexpected exception handling nodeCreated event", e);
+      }
+    }
+  }
+
+  @Override
+  public synchronized void nodeDeleted(String path) {
+    if(path.equals(node)) {
+      try {
+        if(ZKUtil.watchAndCheckExists(watcher, node)) {
+          nodeCreated(path);
+        } else {
+          this.data = null;
+        }
+      } catch(KeeperException e) {
+        abortable.abort("Unexpected exception handling nodeDeleted event", e);
+      }
+    }
+  }
+
+  @Override
+  public synchronized void nodeDataChanged(String path) {
+    if(path.equals(node)) {
+      nodeCreated(path);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
new file mode 100644
index 00000000000..0957f9c1c7b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -0,0 +1,361 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+
+/**
+ * Acts as the single ZooKeeper Watcher.  One instance of this is instantiated
+ * for each Master, RegionServer, and client process.
+ *
+ * <p>This is the only class that implements {@link Watcher}.  Other internal
+ * classes which need to be notified of ZooKeeper events must register with
+ * the local instance of this watcher via {@link #registerListener}.
+ *
+ * <p>This class also holds and manages the connection to ZooKeeper.  Code to deal
+ * with connection related events and exceptions are handled here.
+ */
+public class ZooKeeperWatcher implements Watcher {
+  private static final Log LOG = LogFactory.getLog(ZooKeeperWatcher.class);
+
+  // name of this watcher (for logging only)
+  private String name;
+
+  // zookeeper quorum
+  private String quorum;
+
+  // zookeeper connection
+  private ZooKeeper zooKeeper;
+
+  // abortable in case of zk failure
+  private Abortable abortable;
+
+  // listeners to be notified
+  private final Set<ZooKeeperListener> listeners =
+    new CopyOnWriteArraySet<ZooKeeperListener>();
+
+  // set of unassigned nodes watched
+  private Set<String> unassignedNodes = new HashSet<String>();
+
+  // node names
+
+  // base znode for this cluster
+  public String baseZNode;
+  // znode containing location of server hosting root region
+  public String rootServerZNode;
+  // znode containing ephemeral nodes of the regionservers
+  public String rsZNode;
+  // znode of currently active master
+  public String masterAddressZNode;
+  // znode containing the current cluster state
+  public String clusterStateZNode;
+  // znode used for region transitioning and assignment
+  public String assignmentZNode;
+  // znode used for table disabling/enabling
+  public String tableZNode;
+
+  /**
+   * Instantiate a ZooKeeper connection and watcher.
+   * @param name name of this watcher, for logging/debug purposes only
+   * @throws IOException
+   */
+  public ZooKeeperWatcher(Configuration conf, String name,
+      Abortable abortable)
+  throws IOException {
+    this.name = name;
+    this.quorum = ZKConfig.getZKQuorumServersString(conf);
+    this.zooKeeper = ZKUtil.connect(conf, quorum, this);
+    this.abortable = abortable;
+    info("Connected to ZooKeeper");
+    setNodeNames(conf);
+    try {
+      // Create all the necessary "directories" of znodes
+      // TODO: Move this to an init method somewhere so not everyone calls it?
+      ZKUtil.createAndFailSilent(this, baseZNode);
+      ZKUtil.createAndFailSilent(this, assignmentZNode);
+      ZKUtil.createAndFailSilent(this, rsZNode);
+      ZKUtil.createAndFailSilent(this, tableZNode);
+    } catch (KeeperException e) {
+      error("Unexpected KeeperException creating base node", e);
+      error("Message: " + e.getMessage());
+      throw new IOException(e);
+    }
+  }
+
+  /**
+   * Set the local variable node names using the specified configuration.
+   */
+  private void setNodeNames(Configuration conf) {
+    baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
+        HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
+    rootServerZNode = ZKUtil.joinZNode(baseZNode,
+        conf.get("zookeeper.znode.rootserver", "root-region-server"));
+    rsZNode = ZKUtil.joinZNode(baseZNode,
+        conf.get("zookeeper.znode.rs", "rs"));
+    masterAddressZNode = ZKUtil.joinZNode(baseZNode,
+        conf.get("zookeeper.znode.master", "master"));
+    clusterStateZNode = ZKUtil.joinZNode(baseZNode,
+        conf.get("zookeeper.znode.state", "shutdown"));
+    assignmentZNode = ZKUtil.joinZNode(baseZNode,
+        conf.get("zookeeper.znode.unassigned", "unassigned"));
+    tableZNode = ZKUtil.joinZNode(baseZNode,
+        conf.get("zookeeper.znode.tableEnableDisable", "table"));
+  }
+
+  /**
+   * Register the specified listener to receive ZooKeeper events.
+   * @param listener
+   */
+  public void registerListener(ZooKeeperListener listener) {
+    listeners.add(listener);
+  }
+
+  /**
+   * Get the connection to ZooKeeper.
+   * @return connection reference to zookeeper
+   */
+  public ZooKeeper getZooKeeper() {
+    return zooKeeper;
+  }
+
+  /**
+   * Get the quorum address of this instance.
+   * @returns quorum string of this zookeeper connection instance
+   */
+  public String getQuorum() {
+    return quorum;
+  }
+
+  /**
+   * Method called from ZooKeeper for events and connection status.
+   *
+   * Valid events are passed along to listeners.  Connection status changes
+   * are dealt with locally.
+   */
+  @Override
+  public void process(WatchedEvent event) {
+    LOG.debug("<" + name + "> Received ZooKeeper Event, " +
+        "type=" + event.getType() + ", " +
+        "state=" + event.getState() + ", " +
+        "path=" + event.getPath());
+
+    // While we are still using both ZKWs, need to call parent process()
+//    super.process(event);
+
+    switch(event.getType()) {
+
+      // If event type is NONE, this is a connection status change
+      case None: {
+        connectionEvent(event);
+        break;
+      }
+
+      // Otherwise pass along to the listeners
+
+      case NodeCreated: {
+        for(ZooKeeperListener listener : listeners) {
+          listener.nodeCreated(event.getPath());
+        }
+        break;
+      }
+
+      case NodeDeleted: {
+        for(ZooKeeperListener listener : listeners) {
+          listener.nodeDeleted(event.getPath());
+        }
+        break;
+      }
+
+      case NodeDataChanged: {
+        for(ZooKeeperListener listener : listeners) {
+          listener.nodeDataChanged(event.getPath());
+        }
+        break;
+      }
+
+      case NodeChildrenChanged: {
+        for(ZooKeeperListener listener : listeners) {
+          listener.nodeChildrenChanged(event.getPath());
+        }
+        break;
+      }
+    }
+  }
+
+  // Connection management
+
+  /**
+   * Called when there is a connection-related event via the Watcher callback.
+   *
+   * If Disconnected or Expired, this should shutdown the cluster.
+   *
+   * @param event
+   */
+  private void connectionEvent(WatchedEvent event) {
+    switch(event.getState()) {
+      // SyncConnected is normal, ignore
+      case SyncConnected:
+        break;
+
+      // Abort the server if Disconnected or Expired
+      // TODO: Åny reason to handle these two differently?
+      case Disconnected:
+        info("Received Disconnected from ZooKeeper, ignoring");
+        break;
+      case Expired:
+        error("Received Expired from ZooKeeper, aborting server");
+        if(abortable != null) {
+          abortable.abort("Received Expired from ZooKeeper, aborting server", null);
+        }
+        break;
+    }
+  }
+
+  /**
+   * Get the set of already watched unassigned nodes.
+   * @return
+   */
+  public Set<String> getNodes() {
+    return unassignedNodes;
+  }
+
+  /**
+   * Handles KeeperExceptions in client calls.
+   *
+   * This may be temporary but for now this gives one place to deal with these.
+   *
+   * TODO: Currently this method rethrows the exception to let the caller handle
+   *
+   * @param ke
+   * @throws KeeperException
+   */
+  public void keeperException(KeeperException ke)
+  throws KeeperException {
+    error("Received unexpected KeeperException, re-throwing exception", ke);
+    throw ke;
+  }
+
+  /**
+   * Handles InterruptedExceptions in client calls.
+   *
+   * This may be temporary but for now this gives one place to deal with these.
+   *
+   * TODO: Currently, this method does nothing.
+   *       Is this ever expected to happen?  Do we abort or can we let it run?
+   *       Maybe this should be logged as WARN?  It shouldn't happen?
+   *
+   * @param ie
+   */
+  public void interruptedException(InterruptedException ie) {
+    debug("Received InterruptedException, doing nothing here", ie);
+    // no-op
+  }
+
+  // Logging methods
+
+  /**
+   * Exposed info logging method so our zookeeper output is named.
+   * @param string log line
+   */
+  public void info(String string) {
+    LOG.info("<" + name + "> " + string);
+  }
+
+  /**
+   * Exposed debug logging method so our zookeeper output is named.
+   * @param string log line
+   */
+  public void debug(String string) {
+    LOG.debug("<" + name + "> " + string);
+  }
+
+  /**
+   * Exposed debug logging method so our zookeeper output is named.
+   * @param string log line
+   */
+  public void debug(String string, Throwable t) {
+    LOG.debug("<" + name + "> " + string, t);
+  }
+
+  /**
+   * Exposed warn logging method so our zookeeper output is named.
+   * @param string log line
+   */
+  public void warn(String string) {
+    LOG.warn("<" + name + "> " + string);
+  }
+
+  /**
+   * Exposed warn logging method so our zookeeper output is named.
+   * @param string log line
+   * @param t exception
+   */
+  public void warn(String string, Throwable t) {
+    LOG.warn("<" + name + "> " + string, t);
+  }
+
+  /**
+   * Exposed error logging method so our zookeeper output is named.
+   * @param string log line
+   */
+  public void error(String string) {
+    LOG.error("<" + name + "> " + string);
+  }
+
+  /**
+   * Exposed error logging method so our zookeeper output is named.
+   * @param string log line
+   * @param t exception
+   */
+  public void error(String string, Throwable t) {
+    LOG.error("<" + name + "> " + string, t);
+  }
+
+  public boolean isDebugEnabled() {
+    return LOG.isDebugEnabled();
+  }
+
+  /**
+   * Close the connection to ZooKeeper.
+   * @throws InterruptedException
+   */
+  public void close() {
+    try {
+      if(zooKeeper != null) {
+        zooKeeper.close();
+//        super.close();
+      }
+    } catch (InterruptedException e) {
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
deleted file mode 100644
index 3256ac9d506..00000000000
--- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
+++ /dev/null
@@ -1,1286 +0,0 @@
-/**
- * Copyright 2009 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.zookeeper;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.net.Socket;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper.States;
-import org.apache.zookeeper.data.Stat;
-
-/**
- * Wraps a ZooKeeper instance and adds HBase specific functionality.
- *
- * This class provides methods to:
- * - read/write/delete the root region location in ZooKeeper.
- * - set/check out of safe mode flag.
- *
- * ------------------------------------------
- * The following STATIC ZNodes are created:
- * ------------------------------------------
- * - parentZNode     : All the HBase directories are hosted under this parent
- *                     node, default = "/hbase"
- * - rsZNode         : This is the directory where the RS's create ephemeral
- *                     nodes. The master watches these nodes, and their expiry
- *                     indicates RS death. The default location is "/hbase/rs"
- *
- * ------------------------------------------
- * The following DYNAMIC ZNodes are created:
- * ------------------------------------------
- * - rootRegionZNode     : Specifies the RS hosting root.
- * - masterElectionZNode : ZNode used for election of the primary master when
- *                         there are secondaries. All the masters race to write
- *                         their addresses into this location, the one that
- *                         succeeds is the primary. Others block.
- * - clusterStateZNode   : Determines if the cluster is running. Its default
- *                         location is "/hbase/shutdown". It always has a value
- *                         of "up". If present with the valus, cluster is up
- *                         and running. If deleted, the cluster is shutting
- *                         down.
- * - rgnsInTransitZNode  : All the nodes under this node are names of regions
- *                         in transition. The first byte of the data for each
- *                         of these nodes is the event type. This is used to
- *                         deserialize the rest of the data.
- */
-public class ZooKeeperWrapper implements Watcher {
-  protected static final Log LOG = LogFactory.getLog(ZooKeeperWrapper.class);
-
-  // instances of the watcher
-  private static Map<String,ZooKeeperWrapper> INSTANCES =
-    new HashMap<String,ZooKeeperWrapper>();
-  // lock for ensuring a singleton per instance type
-  private static Lock createLock = new ReentrantLock();
-  // name of this instance
-  private String instanceName;
-
-  // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
-  private static final char ZNODE_PATH_SEPARATOR = '/';
-
-  private String quorumServers = null;
-  private final int sessionTimeout;
-  private ZooKeeper zooKeeper;
-
-  /*
-   * All the HBase directories are hosted under this parent
-   */
-  public final String parentZNode;
-  /*
-   * Specifies the RS hosting root
-   */
-  private final String rootRegionZNode;
-  /*
-   * This is the directory where the RS's create ephemeral nodes. The master
-   * watches these nodes, and their expiry indicates RS death.
-   */
-  private final String rsZNode;
-  /*
-   * ZNode used for election of the primary master when there are secondaries.
-   */
-  private final String masterElectionZNode;
-  /*
-   * State of the cluster - if up and running or shutting down
-   */
-  public final String clusterStateZNode;
-  /*
-   * Regions that are in transition
-   */
-  private final String rgnsInTransitZNode;
-  /*
-   * List of ZNodes in the unassgined region that are already being watched
-   */
-  private Set<String> unassignedZNodesWatched = new HashSet<String>();
-
-  private List<Watcher> listeners = new ArrayList<Watcher>();
-
-  // return the singleton given the name of the instance
-  public static ZooKeeperWrapper getInstance(Configuration conf, String name) {
-    name = getZookeeperClusterKey(conf, name);
-    return INSTANCES.get(name);
-  }
-  // creates only one instance
-  public static ZooKeeperWrapper createInstance(Configuration conf, String name) {
-    if (getInstance(conf, name) != null) {
-      return getInstance(conf, name);
-    }
-    ZooKeeperWrapper.createLock.lock();
-    try {
-      if (getInstance(conf, name) == null) {
-        try {
-          String fullname = getZookeeperClusterKey(conf, name);
-          ZooKeeperWrapper instance = new ZooKeeperWrapper(conf, fullname);
-          INSTANCES.put(fullname, instance);
-        }
-        catch (Exception e) {
-          LOG.error("<" + name + ">" + "Error creating a ZooKeeperWrapper " + e);
-        }
-      }
-    }
-    finally {
-      createLock.unlock();
-    }
-    return getInstance(conf, name);
-  }
-
-  /**
-   * Create a ZooKeeperWrapper. The Zookeeper wrapper listens to all messages
-   * from Zookeeper, and notifies all the listeners about all the messages. Any
-   * component can subscribe to these messages by adding itself as a listener,
-   * and remove itself from being a listener.
-   *
-   * @param conf HBaseConfiguration to read settings from.
-   * @throws IOException If a connection error occurs.
-   */
-  private ZooKeeperWrapper(Configuration conf, String instanceName)
-  throws IOException {
-    this.instanceName = instanceName;
-    Properties properties = HQuorumPeer.makeZKProps(conf);
-    quorumServers = HQuorumPeer.getZKQuorumServersString(properties);
-    if (quorumServers == null) {
-      throw new IOException("Could not read quorum servers from " +
-                            HConstants.ZOOKEEPER_CONFIG_NAME);
-    }
-    sessionTimeout = conf.getInt("zookeeper.session.timeout", 60 * 1000);
-    reconnectToZk();
-
-    parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
-
-    String rootServerZNodeName = conf.get("zookeeper.znode.rootserver", "root-region-server");
-    String rsZNodeName         = conf.get("zookeeper.znode.rs", "rs");
-    String masterAddressZNodeName = conf.get("zookeeper.znode.master", "master");
-    String stateZNodeName      = conf.get("zookeeper.znode.state", "shutdown");
-    String regionsInTransitZNodeName = conf.get("zookeeper.znode.regionInTransition", "UNASSIGNED");
-
-    rootRegionZNode     = getZNode(parentZNode, rootServerZNodeName);
-    rsZNode             = getZNode(parentZNode, rsZNodeName);
-    rgnsInTransitZNode  = getZNode(parentZNode, regionsInTransitZNodeName);
-    masterElectionZNode = getZNode(parentZNode, masterAddressZNodeName);
-    clusterStateZNode   = getZNode(parentZNode, stateZNodeName);
-  }
-
-  public void reconnectToZk() throws IOException {
-    try {
-      LOG.info("Reconnecting to zookeeper");
-      if(zooKeeper != null) {
-        zooKeeper.close();
-        LOG.debug("<" + instanceName + ">" + "Closed existing zookeeper client");
-      }
-      zooKeeper = new ZooKeeper(quorumServers, sessionTimeout, this);
-      LOG.debug("<" + instanceName + ">" + "Connected to zookeeper again");
-    } catch (IOException e) {
-      LOG.error("<" + instanceName + ">" + "Failed to create ZooKeeper object: " + e);
-      throw new IOException(e);
-    } catch (InterruptedException e) {
-      LOG.error("<" + instanceName + ">" + "Error closing ZK connection: " + e);
-      throw new IOException(e);
-    }
-  }
-
-  public synchronized void registerListener(Watcher watcher) {
-    listeners.add(watcher);
-  }
-
-  public synchronized void unregisterListener(Watcher watcher) {
-    listeners.remove(watcher);
-  }
-
-  /**
-   * This is the primary ZK watcher
-   * @see org.apache.zookeeper.Watcher#process(org.apache.zookeeper.WatchedEvent)
-   */
-  @Override
-  public synchronized void process(WatchedEvent event) {
-    for(Watcher w : listeners) {
-      try {
-        w.process(event);
-      } catch (Throwable t) {
-        LOG.error("<"+instanceName+">" + "ZK updates listener threw an exception in process()", t);
-      }
-    }
-  }
-
-  /** @return String dump of everything in ZooKeeper. */
-  @SuppressWarnings({"ConstantConditions"})
-  public String dump() {
-    StringBuilder sb = new StringBuilder();
-    sb.append("\nHBase tree in ZooKeeper is rooted at ").append(parentZNode);
-    sb.append("\n  Cluster up? ").append(exists(clusterStateZNode, true));
-    sb.append("\n  Master address: ").append(readMasterAddress(null));
-    sb.append("\n  Region server holding ROOT: ").append(readRootRegionLocation());
-    sb.append("\n  Region servers:");
-    for (HServerAddress address : scanRSDirectory()) {
-      sb.append("\n    - ").append(address);
-    }
-    sb.append("\n  Quorum Server Statistics:");
-    String[] servers = quorumServers.split(",");
-    for (String server : servers) {
-      sb.append("\n    - ").append(server);
-      try {
-        String[] stat = getServerStats(server);
-        for (String s : stat) {
-          sb.append("\n        ").append(s);
-        }
-      } catch (Exception e) {
-        sb.append("\n        ERROR: ").append(e.getMessage());
-      }
-    }
-    return sb.toString();
-  }
-
-  /**
-   * Gets the statistics from the given server. Uses a 1 minute timeout.
-   *
-   * @param server  The server to get the statistics from.
-   * @return The array of response strings.
-   * @throws IOException When the socket communication fails.
-   */
-  public String[] getServerStats(String server)
-  throws IOException {
-    return getServerStats(server, 60 * 1000);
-  }
-
-  /**
-   * Gets the statistics from the given server.
-   *
-   * @param server  The server to get the statistics from.
-   * @param timeout  The socket timeout to use.
-   * @return The array of response strings.
-   * @throws IOException When the socket communication fails.
-   */
-  public String[] getServerStats(String server, int timeout)
-  throws IOException {
-    String[] sp = server.split(":");
-    Socket socket = new Socket(sp[0],
-      sp.length > 1 ? Integer.parseInt(sp[1]) : 2181);
-    socket.setSoTimeout(timeout);
-    PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
-    BufferedReader in = new BufferedReader(new InputStreamReader(
-      socket.getInputStream()));
-    out.println("stat");
-    out.flush();
-    ArrayList<String> res = new ArrayList<String>();
-    while (true) {
-      String line = in.readLine();
-      if (line != null) res.add(line);
-      else break;
-    }
-    socket.close();
-    return res.toArray(new String[res.size()]);
-  }
-
-  public boolean exists(String znode, boolean watch) {
-    try {
-      return zooKeeper.exists(getZNode(parentZNode, znode), watch?this:null) != null;
-    } catch (KeeperException.SessionExpiredException e) {
-      // if the session has expired try to reconnect to ZK, then perform query
-      try {
-        // TODO: ZK-REFACTOR: We should not reconnect - we should just quit and restart.
-        reconnectToZk();
-        return zooKeeper.exists(getZNode(parentZNode, znode), watch?this:null) != null;
-      } catch (IOException e1) {
-        LOG.error("Error reconnecting to zookeeper", e1);
-        throw new RuntimeException("Error reconnecting to zookeeper", e1);
-      } catch (KeeperException e1) {
-        LOG.error("Error reading after reconnecting to zookeeper", e1);
-        throw new RuntimeException("Error reading after reconnecting to zookeeper", e1);
-      } catch (InterruptedException e1) {
-        LOG.error("Error reading after reconnecting to zookeeper", e1);
-        throw new RuntimeException("Error reading after reconnecting to zookeeper", e1);
-      }
-    } catch (KeeperException e) {
-      return false;
-    } catch (InterruptedException e) {
-      return false;
-    }
-  }
-
-  /** @return ZooKeeper used by this wrapper. */
-  public ZooKeeper getZooKeeper() {
-    return zooKeeper;
-  }
-
-  /**
-   * This is for testing KeeperException.SessionExpiredException.
-   * See HBASE-1232.
-   * @return long session ID of this ZooKeeper session.
-   */
-  public long getSessionID() {
-    return zooKeeper.getSessionId();
-  }
-
-  /**
-   * This is for testing KeeperException.SessionExpiredException.
-   * See HBASE-1232.
-   * @return byte[] password of this ZooKeeper session.
-   */
-  public byte[] getSessionPassword() {
-    return zooKeeper.getSessionPasswd();
-  }
-
-  /** @return host:port list of quorum servers. */
-  public String getQuorumServers() {
-    return quorumServers;
-  }
-
-  /** @return true if currently connected to ZooKeeper, false otherwise. */
-  public boolean isConnected() {
-    return zooKeeper.getState() == States.CONNECTED;
-  }
-
-  /**
-   * Read location of server storing root region.
-   * @return HServerAddress pointing to server serving root region or null if
-   *         there was a problem reading the ZNode.
-   */
-  public HServerAddress readRootRegionLocation() {
-    return readAddress(rootRegionZNode, null);
-  }
-
-  /**
-   * Read address of master server.
-   * @return HServerAddress of master server.
-   * @throws IOException if there's a problem reading the ZNode.
-   */
-  public HServerAddress readMasterAddressOrThrow() throws IOException {
-    return readAddressOrThrow(masterElectionZNode, null);
-  }
-
-  /**
-   * Read master address and set a watch on it.
-   * @param watcher Watcher to set on master address ZNode if not null.
-   * @return HServerAddress of master or null if there was a problem reading the
-   *         ZNode. The watcher is set only if the result is not null.
-   */
-  public HServerAddress readMasterAddress(Watcher watcher) {
-    return readAddress(masterElectionZNode, watcher);
-  }
-
-  /**
-   * Watch the state of the cluster, up or down
-   * @param watcher Watcher to set on cluster state node
-   */
-  public void setClusterStateWatch() {
-    try {
-      zooKeeper.exists(clusterStateZNode, this);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to check on ZNode " + clusterStateZNode, e);
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to check on ZNode " + clusterStateZNode, e);
-    }
-  }
-
-  /**
-   * Set the cluster state, up or down
-   * @param up True to write the node, false to delete it
-   * @return true if it worked, else it's false
-   */
-  public boolean setClusterState(boolean up) {
-    if (!ensureParentExists(clusterStateZNode)) {
-      return false;
-    }
-    try {
-      if(up) {
-        byte[] data = Bytes.toBytes("up");
-        zooKeeper.create(clusterStateZNode, data,
-            Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-        LOG.debug("<" + instanceName + ">" + "State node wrote in ZooKeeper");
-      } else {
-        zooKeeper.delete(clusterStateZNode, -1);
-        LOG.debug("<" + instanceName + ">" + "State node deleted in ZooKeeper");
-      }
-      return true;
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to set state node in ZooKeeper", e);
-    } catch (KeeperException e) {
-      if(e.code() == KeeperException.Code.NODEEXISTS) {
-        LOG.debug("<" + instanceName + ">" + "State node exists.");
-      } else {
-        LOG.warn("<" + instanceName + ">" + "Failed to set state node in ZooKeeper", e);
-      }
-    }
-
-    return false;
-  }
-
-  /**
-   * Set a watcher on the master address ZNode. The watcher will be set unless
-   * an exception occurs with ZooKeeper.
-   * @param watcher Watcher to set on master address ZNode.
-   * @return true if watcher was set, false otherwise.
-   */
-  public boolean watchMasterAddress(Watcher watcher) {
-    try {
-      zooKeeper.exists(masterElectionZNode, watcher);
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to set watcher on ZNode " + masterElectionZNode, e);
-      return false;
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to set watcher on ZNode " + masterElectionZNode, e);
-      return false;
-    }
-    LOG.debug("<" + instanceName + ">" + "Set watcher on master address ZNode " + masterElectionZNode);
-    return true;
-  }
-  
-  /**
-   * @return true if zookeeper has a master address.
-   */
-  public boolean masterAddressExists() {
-    return checkExistenceOf(masterElectionZNode);
-  }
-
-  private HServerAddress readAddress(String znode, Watcher watcher) {
-    try {
-      LOG.debug("<" + instanceName + ">" + "Trying to read " + znode);
-      return readAddressOrThrow(znode, watcher);
-    } catch (IOException e) {
-      LOG.debug("<" + instanceName + ">" + "Failed to read " + e.getMessage());
-      return null;
-    }
-  }
-
-  private HServerAddress readAddressOrThrow(String znode, Watcher watcher) throws IOException {
-    byte[] data;
-    try {
-      data = zooKeeper.getData(znode, watcher, null);
-    } catch (InterruptedException e) {
-      throw new IOException(e);
-    } catch (KeeperException e) {
-      throw new IOException(e);
-    }
-
-    String addressString = Bytes.toString(data);
-    LOG.debug("<" + instanceName + ">" + "Read ZNode " + znode + " got " + addressString);
-    return new HServerAddress(addressString);
-  }
-
-  /**
-   * Make sure this znode exists by creating it if it's missing
-   * @param znode full path to znode
-   * @return true if it works
-   */
-  public boolean ensureExists(final String znode) {
-    try {
-      Stat stat = zooKeeper.exists(znode, false);
-      if (stat != null) {
-        return true;
-      }
-      zooKeeper.create(znode, new byte[0],
-                       Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-      LOG.debug("<" + instanceName + ">" + "Created ZNode " + znode);
-      return true;
-    } catch (KeeperException.NodeExistsException e) {
-      return true;      // ok, move on.
-    } catch (KeeperException.NoNodeException e) {
-      return ensureParentExists(znode) && ensureExists(znode);
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to create " + znode +
-        " -- check quorum servers, currently=" + this.quorumServers, e);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to create " + znode +
-        " -- check quorum servers, currently=" + this.quorumServers, e);
-    }
-    return false;
-  }
-
-  private boolean ensureParentExists(final String znode) {
-    int index = znode.lastIndexOf(ZNODE_PATH_SEPARATOR);
-    if (index <= 0) {   // Parent is root, which always exists.
-      return true;
-    }
-    return ensureExists(znode.substring(0, index));
-  }
-
-  /**
-   * Delete ZNode containing root region location.
-   * @return true if operation succeeded, false otherwise.
-   */
-  public boolean deleteRootRegionLocation()  {
-    if (!ensureParentExists(rootRegionZNode)) {
-      return false;
-    }
-
-    try {
-      deleteZNode(rootRegionZNode);
-      return true;
-    } catch (KeeperException.NoNodeException e) {
-      return true;    // ok, move on.
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to delete " + rootRegionZNode + ": " + e);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to delete " + rootRegionZNode + ": " + e);
-    }
-
-    return false;
-  }
-
-  /**
-   * Unrecursive deletion of specified znode
-   * @param znode
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-  public void deleteZNode(String znode)
-      throws KeeperException, InterruptedException {
-    deleteZNode(znode, false);
-  }
-
-  /**
-   * Optionnally recursive deletion of specified znode
-   * @param znode
-   * @param recursive
-   * @throws KeeperException
-   * @throws InterruptedException
-   */
-  public void deleteZNode(String znode, boolean recursive)
-    throws KeeperException, InterruptedException {
-    if (recursive) {
-      LOG.info("<" + instanceName + ">" + "deleteZNode get children for " + znode);
-      List<String> znodes = this.zooKeeper.getChildren(znode, false);
-      if (znodes != null && znodes.size() > 0) {
-        for (String child : znodes) {
-          String childFullPath = getZNode(znode, child);
-          LOG.info("<" + instanceName + ">" + "deleteZNode recursive call " + childFullPath);
-          this.deleteZNode(childFullPath, true);
-        }
-      }
-    }
-    this.zooKeeper.delete(znode, -1);
-    LOG.debug("<" + instanceName + ">" + "Deleted ZNode " + znode);
-  }
-
-  private boolean createRootRegionLocation(String address) {
-    byte[] data = Bytes.toBytes(address);
-    try {
-      zooKeeper.create(rootRegionZNode, data, Ids.OPEN_ACL_UNSAFE,
-                       CreateMode.PERSISTENT);
-      LOG.debug("<" + instanceName + ">" + "Created ZNode " + rootRegionZNode + " with data " + address);
-      return true;
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to create root region in ZooKeeper: " + e);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to create root region in ZooKeeper: " + e);
-    }
-
-    return false;
-  }
-
-  private boolean updateRootRegionLocation(String address) {
-    byte[] data = Bytes.toBytes(address);
-    try {
-      zooKeeper.setData(rootRegionZNode, data, -1);
-      LOG.debug("<" + instanceName + ">" + "SetData of ZNode " + rootRegionZNode + " with " + address);
-      return true;
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to set root region location in ZooKeeper: " + e);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to set root region location in ZooKeeper: " + e);
-    }
-
-    return false;
-  }
-
-  /**
-   * Write root region location to ZooKeeper. If address is null, delete ZNode.
-   * containing root region location.
-   * @param address HServerAddress to write to ZK.
-   * @return true if operation succeeded, false otherwise.
-   */
-  public boolean writeRootRegionLocation(HServerAddress address) {
-    if (address == null) {
-      return deleteRootRegionLocation();
-    }
-
-    if (!ensureParentExists(rootRegionZNode)) {
-      return false;
-    }
-
-    String addressString = address.toString();
-
-    if (checkExistenceOf(rootRegionZNode)) {
-      return updateRootRegionLocation(addressString);
-    }
-
-    return createRootRegionLocation(addressString);
-  }
-
-  /**
-   * Write address of master to ZooKeeper.
-   * @param address HServerAddress of master.
-   * @return true if operation succeeded, false otherwise.
-   */
-  public boolean writeMasterAddress(final HServerAddress address) {
-    LOG.debug("<" + instanceName + ">" + "Writing master address " + address.toString() + " to znode " + masterElectionZNode);
-    if (!ensureParentExists(masterElectionZNode)) {
-      return false;
-    }
-    LOG.debug("<" + instanceName + ">" + "Znode exists : " + masterElectionZNode);
-
-    String addressStr = address.toString();
-    byte[] data = Bytes.toBytes(addressStr);
-    try {
-      zooKeeper.create(masterElectionZNode, data, Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
-      LOG.debug("<" + instanceName + ">" + "Wrote master address " + address + " to ZooKeeper");
-      return true;
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to write master address " + address + " to ZooKeeper", e);
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to write master address " + address + " to ZooKeeper", e);
-    }
-
-    return false;
-  }
-
-  /**
-   * Write in ZK this RS startCode and address.
-   * Ensures that the full path exists.
-   * @param info The RS info
-   * @return true if the location was written, false if it failed
-   */
-  public boolean writeRSLocation(HServerInfo info) {
-    ensureExists(rsZNode);
-    byte[] data = Bytes.toBytes(info.getServerAddress().toString());
-    String znode = joinPath(rsZNode, info.getServerName());
-    try {
-      zooKeeper.create(znode, data, Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
-      LOG.debug("<" + instanceName + ">" + "Created ZNode " + znode
-          + " with data " + info.getServerAddress().toString());
-      return true;
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to create " + znode + " znode in ZooKeeper: " + e);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to create " + znode + " znode in ZooKeeper: " + e);
-    }
-    return false;
-  }
-
-  /**
-   * Update the RS address and set a watcher on the znode
-   * @param info The RS info
-   * @param watcher The watcher to put on the znode
-   * @return true if the update is done, false if it failed
-   */
-  public boolean updateRSLocationGetWatch(HServerInfo info, Watcher watcher) {
-    byte[] data = Bytes.toBytes(info.getServerAddress().toString());
-    String znode = rsZNode + ZNODE_PATH_SEPARATOR + info.getServerName();
-    try {
-      zooKeeper.setData(znode, data, -1);
-      LOG.debug("<" + instanceName + ">" + "Updated ZNode " + znode
-          + " with data " + info.getServerAddress().toString());
-      zooKeeper.getData(znode, watcher, null);
-      return true;
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to update " + znode + " znode in ZooKeeper: " + e);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to update " + znode + " znode in ZooKeeper: " + e);
-    }
-
-    return false;
-  }
-
-  /**
-   * Scans the regions servers directory
-   * @return A list of server addresses
-   */
-  public List<HServerAddress> scanRSDirectory() {
-    return scanAddressDirectory(rsZNode, null);
-  }
-
-  /**
-   * Scans the regions servers directory and sets a watch on each znode
-   * @param watcher a watch to use for each znode
-   * @return A list of server addresses
-   */
-  public List<HServerAddress> scanRSDirectory(Watcher watcher) {
-    return scanAddressDirectory(rsZNode, watcher);
-  }
-
-  /**
-   * Method used to make sure the region server directory is empty.
-   *
-   */
-  public void clearRSDirectory() {
-    try {
-      List<String> nodes = zooKeeper.getChildren(rsZNode, false);
-      for (String node : nodes) {
-        LOG.debug("<" + instanceName + ">" + "Deleting node: " + node);
-        zooKeeper.delete(joinPath(this.rsZNode, node), -1);
-      }
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to delete " + rsZNode + " znodes in ZooKeeper: " + e);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to delete " + rsZNode + " znodes in ZooKeeper: " + e);
-    }
-  }
-
-  /**
-   * @return the number of region server znodes in the RS directory
-   */
-  public int getRSDirectoryCount() {
-    Stat stat = null;
-    try {
-      stat = zooKeeper.exists(rsZNode, false);
-    } catch (KeeperException e) {
-      LOG.warn("Problem getting stats for " + rsZNode, e);
-    } catch (InterruptedException e) {
-      LOG.warn("Problem getting stats for " + rsZNode, e);
-    }
-    return (stat != null) ? stat.getNumChildren() : 0;
-  }
-
-  private boolean checkExistenceOf(String path) {
-    Stat stat = null;
-    try {
-      stat = zooKeeper.exists(path, false);
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "checking existence of " + path, e);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "checking existence of " + path, e);
-    }
-
-    return stat != null;
-  }
-
-  /**
-   * Close this ZooKeeper session.
-   */
-  public void close() {
-    try {
-      zooKeeper.close();
-      INSTANCES.remove(instanceName);
-      LOG.debug("<" + instanceName + ">" + "Closed connection with ZooKeeper; " + this.rootRegionZNode);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to close connection with ZooKeeper");
-    }
-  }
-
-  public String getZNode(String parentZNode, String znodeName) {
-    return znodeName.charAt(0) == ZNODE_PATH_SEPARATOR ?
-        znodeName : joinPath(parentZNode, znodeName);
-  }
-
-  public String getZNodePathForHBase(String znodeName) {
-    return getZNode(parentZNode, znodeName);
-  }
-
-  private String joinPath(String parent, String child) {
-    return parent + ZNODE_PATH_SEPARATOR + child;
-  }
-
-  /**
-   * Get the path of the masterElectionZNode
-   * @return the path to masterElectionZNode
-   */
-  public String getMasterElectionZNode() {
-    return masterElectionZNode;
-  }
-
-  /**
-   * Get the path of the parent ZNode
-   * @return path of that znode
-   */
-  public String getParentZNode() {
-    return parentZNode;
-  }
-
-  /**
-   * Scan a directory of address data.
-   * @param znode The parent node
-   * @param watcher The watcher to put on the found znodes, if not null
-   * @return The directory contents
-   */
-  public List<HServerAddress> scanAddressDirectory(String znode,
-      Watcher watcher) {
-    List<HServerAddress> list = new ArrayList<HServerAddress>();
-    List<String> nodes = this.listZnodes(znode);
-    if(nodes == null) {
-      return list;
-    }
-    for (String node : nodes) {
-      String path = joinPath(znode, node);
-      list.add(readAddress(path, watcher));
-    }
-    return list;
-  }
-
-  /**
-   * List all znodes in the specified path
-   * @param znode path to list
-   * @return a list of all the znodes
-   */
-  public List<String> listZnodes(String znode) {
-    return listZnodes(znode, this);
-  }
-
-  /**
-   * List all znodes in the specified path and set a watcher on each
-   * @param znode path to list
-   * @param watcher watch to set, can be null
-   * @return a list of all the znodes
-   */
-  public List<String> listZnodes(String znode, Watcher watcher) {
-    List<String> nodes = null;
-    if (watcher == null) {
-      watcher = this;
-    }
-    try {
-      if (checkExistenceOf(znode)) {
-        nodes = zooKeeper.getChildren(znode, watcher);
-        for (String node : nodes) {
-          getDataAndWatch(znode, node, watcher);
-        }
-      }
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to read " + znode + " znode in ZooKeeper: " + e);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to read " + znode + " znode in ZooKeeper: " + e);
-    }
-    return nodes;
-  }
-
-  public byte[] getData(String parentZNode, String znode) {
-    return getDataAndWatch(parentZNode, znode, null);
-  }
-
-  public byte[] getDataAndWatch(String parentZNode,
-                                String znode, Watcher watcher) {
-    byte[] data = null;
-    try {
-      String path = joinPath(parentZNode, znode);
-      // TODO: ZK-REFACTOR: remove existance check?
-      if (checkExistenceOf(path)) {
-        data = zooKeeper.getData(path, watcher, null);
-      }
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to read " + znode + " znode in ZooKeeper: " + e);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to read " + znode + " znode in ZooKeeper: " + e);
-    }
-    return data;
-  }
-
-  /**
-   * Write a znode and fail if it already exists
-   * @param parentPath parent path to the new znode
-   * @param child name of the znode
-   * @param strData data to insert
-   * @throws InterruptedException
-   * @throws KeeperException
-   */
-  public void writeZNode(String parentPath, String child, String strData)
-      throws InterruptedException, KeeperException {
-    writeZNode(parentPath, child, strData, false);
-  }
-
-
-  /**
-   * Write (and optionally over-write) a znode
-   * @param parentPath parent path to the new znode
-   * @param child name of the znode
-   * @param strData data to insert
-   * @param failOnWrite true if an exception should be returned if the znode
-   * already exists, false if it should be overwritten
-   * @throws InterruptedException
-   * @throws KeeperException
-   */
-  public void writeZNode(String parentPath, String child, String strData,
-      boolean failOnWrite) throws InterruptedException, KeeperException {
-    String path = joinPath(parentPath, child);
-    if (!ensureExists(parentPath)) {
-      LOG.error("<" + instanceName + ">" + "unable to ensure parent exists: " + parentPath);
-    }
-    byte[] data = Bytes.toBytes(strData);
-    Stat stat = this.zooKeeper.exists(path, false);
-    if (failOnWrite || stat == null) {
-      this.zooKeeper.create(path, data,
-          Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-      LOG.debug("<" + instanceName + ">" + "Created " + path + " with data " + strData);
-    } else {
-      this.zooKeeper.setData(path, data, -1);
-      LOG.debug("<" + instanceName + ">" + "Updated " + path + " with data " + strData);
-    }
-  }
-
-  /**
-   * Get the key to the ZK ensemble for this configuration without
-   * adding a name at the end
-   * @param conf Configuration to use to build the key
-   * @return ensemble key without a name
-   */
-  public static String getZookeeperClusterKey(Configuration conf) {
-    return getZookeeperClusterKey(conf, null);
-  }
-
-  /**
-   * Get the key to the ZK ensemble for this configuration and append
-   * a name at the end
-   * @param conf Configuration to use to build the key
-   * @param name Name that should be appended at the end if not empty or null
-   * @return ensemble key with a name (if any)
-   */
-  public static String getZookeeperClusterKey(Configuration conf, String name) {
-    String quorum = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll(
-        "[\\t\\n\\x0B\\f\\r]", ""));
-    StringBuilder builder = new StringBuilder(quorum);
-    builder.append(":");
-    builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
-    if (name != null && !name.isEmpty()) {
-      builder.append(",");
-      builder.append(name);
-    }
-    return builder.toString();
-  }
-
-  /**
-   * Get the znode that has all the regions in transition.
-   * @return path to znode
-   */
-  public String getRegionInTransitionZNode() {
-    return this.rgnsInTransitZNode;
-  }
-
-  /**
-   * Get the path of this region server's znode
-   * @return path to znode
-   */
-  public String getRsZNode() {
-    return this.rsZNode;
-  }
-
-  public void deleteZNode(String zNodeName, int version) {
-    String fullyQualifiedZNodeName = getZNode(parentZNode, zNodeName);
-    try
-    {
-      zooKeeper.delete(fullyQualifiedZNodeName, version);
-    }
-    catch (InterruptedException e)
-    {
-      LOG.warn("<" + instanceName + ">" + "Failed to delete ZNode " + fullyQualifiedZNodeName + " in ZooKeeper", e);
-    }
-    catch (KeeperException e)
-    {
-      LOG.warn("<" + instanceName + ">" + "Failed to delete ZNode " + fullyQualifiedZNodeName + " in ZooKeeper", e);
-    }
-  }
-
-  public String createZNodeIfNotExists(String zNodeName) {
-    return createZNodeIfNotExists(zNodeName, null, CreateMode.PERSISTENT, true);
-  }
-
-  public void watchZNode(String zNodeName) {
-    String fullyQualifiedZNodeName = getZNode(parentZNode, zNodeName);
-
-    try {
-      zooKeeper.exists(fullyQualifiedZNodeName, this);
-      zooKeeper.getData(fullyQualifiedZNodeName, this, null);
-      zooKeeper.getChildren(fullyQualifiedZNodeName, this);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to create ZNode " + fullyQualifiedZNodeName + " in ZooKeeper", e);
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to create ZNode " + fullyQualifiedZNodeName + " in ZooKeeper", e);
-    }
-  }
-
-  public String createZNodeIfNotExists(String zNodeName, byte[] data, CreateMode createMode, boolean watch) {
-    String fullyQualifiedZNodeName = getZNode(parentZNode, zNodeName);
-
-    if (!ensureParentExists(fullyQualifiedZNodeName)) {
-      return null;
-    }
-
-    try {
-      // create the znode
-      zooKeeper.create(fullyQualifiedZNodeName, data, Ids.OPEN_ACL_UNSAFE, createMode);
-      LOG.debug("<" + instanceName + ">" + "Created ZNode " + fullyQualifiedZNodeName + " in ZooKeeper");
-      // watch the znode for deletion, data change, creation of children
-      if(watch) {
-        watchZNode(zNodeName);
-      }
-      return fullyQualifiedZNodeName;
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to create ZNode " + fullyQualifiedZNodeName + " in ZooKeeper", e);
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to create ZNode " + fullyQualifiedZNodeName + " in ZooKeeper", e);
-    }
-
-    return null;
-  }
-
-  public byte[] readZNode(String znodeName, Stat stat) throws IOException {
-    byte[] data;
-    try {
-      String fullyQualifiedZNodeName = getZNode(parentZNode, znodeName);
-      data = zooKeeper.getData(fullyQualifiedZNodeName, this, stat);
-    } catch (InterruptedException e) {
-      throw new IOException(e);
-    } catch (KeeperException e) {
-      throw new IOException(e);
-    }
-    return data;
-  }
-
-  // TODO: perhaps return the version number from this write?
-  public boolean writeZNode(String znodeName, byte[] data, int version, boolean watch) throws IOException {
-      try {
-        String fullyQualifiedZNodeName = getZNode(parentZNode, znodeName);
-        zooKeeper.setData(fullyQualifiedZNodeName, data, version);
-        if(watch) {
-          zooKeeper.getData(fullyQualifiedZNodeName, this, null);
-        }
-        return true;
-      } catch (InterruptedException e) {
-        LOG.warn("<" + instanceName + ">" + "Failed to write data to ZooKeeper", e);
-        throw new IOException(e);
-      } catch (KeeperException e) {
-        LOG.warn("<" + instanceName + ">" + "Failed to write data to ZooKeeper", e);
-        throw new IOException(e);
-      }
-    }
-
-  /**
-   * Given a region name and some data, this method creates a new the region
-   * znode data under the UNASSGINED znode with the data passed in. This method
-   * will not update data for existing znodes.
-   *
-   * @param regionName - encoded name of the region
-   * @param data - new serialized data to update the region znode
-   */
-  private void createUnassignedRegion(String regionName, byte[] data) {
-    String znode = getZNode(getRegionInTransitionZNode(), regionName);
-    if(LOG.isDebugEnabled()) {
-      // check if this node already exists -
-      //   - it should not exist
-      //   - if it does, it should be in the CLOSED state
-      if(exists(znode, true)) {
-        Stat stat = new Stat();
-        byte[] oldData = null;
-        try {
-          oldData = readZNode(znode, stat);
-        } catch (IOException e) {
-          LOG.error("Error reading data for " + znode);
-        }
-        if(oldData == null) {
-          LOG.debug("While creating UNASSIGNED region " + regionName + " exists with no data" );
-        }
-        else {
-          LOG.debug("While creating UNASSIGNED region " + regionName + " exists, state = " + (HBaseEventType.fromByte(oldData[0])));
-        }
-      }
-      else {
-        if(data == null) {
-          LOG.debug("Creating UNASSIGNED region " + regionName + " with no data" );
-        }
-        else {
-          LOG.debug("Creating UNASSIGNED region " + regionName + " in state = " + (HBaseEventType.fromByte(data[0])));
-        }
-      }
-    }
-    synchronized(unassignedZNodesWatched) {
-      unassignedZNodesWatched.add(znode);
-      createZNodeIfNotExists(znode, data, CreateMode.PERSISTENT, true);
-    }
-  }
-
-  /**
-   * Given a region name and some data, this method updates the region znode
-   * data under the UNASSGINED znode with the latest data. This method will
-   * update the znode data only if it already exists.
-   *
-   * @param regionName - encoded name of the region
-   * @param data - new serialized data to update the region znode
-   */
-  public void updateUnassignedRegion(String regionName, byte[] data) {
-    String znode = getZNode(getRegionInTransitionZNode(), regionName);
-    // this is an update - make sure the node already exists
-    if(!exists(znode, true)) {
-      LOG.error("Cannot update " + znode + " - node does not exist" );
-      return;
-    }
-
-    Stat stat = new Stat();
-    byte[] oldData = null;
-    try {
-      oldData = readZNode(znode, stat);
-    } catch (IOException e) {
-      LOG.error("Error reading data for " + znode);
-    }
-    // If there is no data in the ZNode, then update it
-    if(oldData == null) {
-      LOG.debug("While updating UNASSIGNED region " + regionName + " - node exists with no data" );
-    }
-    // If there is data in the ZNode, do not update if it is already correct
-    else {
-      HBaseEventType curState = HBaseEventType.fromByte(oldData[0]);
-      HBaseEventType newState = HBaseEventType.fromByte(data[0]);
-      // If the znode has the right state already, do not update it. Updating
-      // the znode again and again will bump up the zk version. This may cause
-      // the region server to fail. The RS expects that the znode is never
-      // updated by anyone else while it is opening/closing a region.
-      if(curState == newState) {
-        LOG.debug("No need to update UNASSIGNED region " + regionName +
-                  " as it already exists in state = " + curState);
-        return;
-      }
-
-      // If the ZNode is in another state, then update it
-      LOG.debug("UNASSIGNED region " + regionName + " is currently in state = " +
-                curState + ", updating it to " + newState);
-    }
-    // Update the ZNode
-    synchronized(unassignedZNodesWatched) {
-      unassignedZNodesWatched.add(znode);
-      try {
-        writeZNode(znode, data, -1, true);
-      } catch (IOException e) {
-        LOG.error("Error writing data for " + znode + ", could not update state to " + (HBaseEventType.fromByte(data[0])));
-      }
-    }
-  }
-
-  /**
-   * This method will create a new region in transition entry in ZK with the
-   * speficied data if none exists. If one already exists, it will update the
-   * data with whatever is passed in.
-   *
-   * @param regionName - encoded name of the region
-   * @param data - serialized data for the region znode
-   */
-  public void createOrUpdateUnassignedRegion(String regionName, byte[] data) {
-    String znode = getZNode(getRegionInTransitionZNode(), regionName);
-    if(exists(znode, true)) {
-      updateUnassignedRegion(regionName, data);
-    }
-    else {
-      createUnassignedRegion(regionName, data);
-    }
-  }
-
-  public void deleteUnassignedRegion(String regionName) {
-    String znode = getZNode(getRegionInTransitionZNode(), regionName);
-    try {
-      LOG.debug("Deleting ZNode " + znode + " in ZooKeeper as region is open...");
-      synchronized(unassignedZNodesWatched) {
-        unassignedZNodesWatched.remove(znode);
-        deleteZNode(znode);
-      }
-    } catch (KeeperException.SessionExpiredException e) {
-      LOG.error("Zookeeper session has expired", e);
-      // if the session has expired try to reconnect to ZK, then perform query
-      try {
-        // TODO: ZK-REFACTOR: should just quit on reconnect??
-        reconnectToZk();
-        synchronized(unassignedZNodesWatched) {
-          unassignedZNodesWatched.remove(znode);
-          deleteZNode(znode);
-        }
-      } catch (IOException e1) {
-        LOG.error("Error reconnecting to zookeeper", e1);
-        throw new RuntimeException("Error reconnecting to zookeeper", e1);
-      } catch (KeeperException.SessionExpiredException e1) {
-        LOG.error("Error reading after reconnecting to zookeeper", e1);
-        throw new RuntimeException("Error reading after reconnecting to zookeeper", e1);
-      } catch (KeeperException e1) {
-        LOG.error("Error reading after reconnecting to zookeeper", e1);
-      } catch (InterruptedException e1) {
-        LOG.error("Error reading after reconnecting to zookeeper", e1);
-      }
-    } catch (KeeperException e) {
-      LOG.error("Error deleting region " + regionName, e);
-    } catch (InterruptedException e) {
-      LOG.error("Error deleting region " + regionName, e);
-    }
-  }
-
-  /**
-   * Atomically adds a watch and reads data from the unwatched znodes in the
-   * UNASSGINED region. This works because the master is the only person
-   * deleting nodes.
-   * @param znode
-   * @return
-   */
-  public List<ZNodePathAndData> watchAndGetNewChildren(String znode) {
-    List<String> nodes = null;
-    List<ZNodePathAndData> newNodes = new ArrayList<ZNodePathAndData>();
-    try {
-      if (checkExistenceOf(znode)) {
-        synchronized(unassignedZNodesWatched) {
-          nodes = zooKeeper.getChildren(znode, this);
-          for (String node : nodes) {
-            String znodePath = joinPath(znode, node);
-            if(!unassignedZNodesWatched.contains(znodePath)) {
-              byte[] data = getDataAndWatch(znode, node, this);
-              newNodes.add(new ZNodePathAndData(znodePath, data));
-              unassignedZNodesWatched.add(znodePath);
-            }
-          }
-        }
-      }
-    } catch (KeeperException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to read " + znode + " znode in ZooKeeper: " + e);
-    } catch (InterruptedException e) {
-      LOG.warn("<" + instanceName + ">" + "Failed to read " + znode + " znode in ZooKeeper: " + e);
-    }
-    return newNodes;
-  }
-
-  public static class ZNodePathAndData {
-    private String zNodePath;
-    private byte[] data;
-
-    public ZNodePathAndData(String zNodePath, byte[] data) {
-      this.zNodePath = zNodePath;
-      this.data = data;
-    }
-
-    public String getzNodePath() {
-      return zNodePath;
-    }
-    public byte[] getData() {
-      return data;
-    }
-
-  }
-}
diff --git a/src/main/resources/hbase-webapps/master/master.jsp b/src/main/resources/hbase-webapps/master/master.jsp
index 2c01d467a22..37bd6e38c8e 100644
--- a/src/main/resources/hbase-webapps/master/master.jsp
+++ b/src/main/resources/hbase-webapps/master/master.jsp
@@ -6,17 +6,16 @@
   import="org.apache.hadoop.hbase.util.FSUtils"
   import="org.apache.hadoop.hbase.master.HMaster"
   import="org.apache.hadoop.hbase.HConstants"
-  import="org.apache.hadoop.hbase.master.MetaRegion"
   import="org.apache.hadoop.hbase.client.HBaseAdmin"
   import="org.apache.hadoop.hbase.HServerInfo"
   import="org.apache.hadoop.hbase.HServerAddress"
   import="org.apache.hadoop.hbase.HTableDescriptor" %><%
   HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
   Configuration conf = master.getConfiguration();
-  HServerAddress rootLocation = master.getRegionManager().getRootRegionLocation();
-  Map<byte [], MetaRegion> onlineRegions = master.getRegionManager().getOnlineMetaRegions();
+  HServerAddress rootLocation = master.getCatalogTracker().getRootLocation();
+  boolean metaOnline = master.getCatalogTracker().getMetaLocation() != null;
   Map<String, HServerInfo> serverToServerInfos =
-    master.getServerManager().getServersToServerInfo();
+    master.getServerManager().getOnlineServers();
   int interval = conf.getInt("hbase.regionserver.msginterval", 1000)/1000;
   if (interval == 0) {
       interval = 1;
@@ -24,7 +23,7 @@
   boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false);
   Map<String, Integer> frags = null;
   if (showFragmentation) {
-      frags = master.getTableFragmentation();
+      frags = FSUtils.getTableFragmentation(master);
   }
 %><?xml version="1.0" encoding="UTF-8" ?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" 
@@ -66,13 +65,12 @@
 <tr><td>HBase Compiled</td><td><%= org.apache.hadoop.hbase.util.VersionInfo.getDate() %>, <%= org.apache.hadoop.hbase.util.VersionInfo.getUser() %></td><td>When HBase version was compiled and by whom</td></tr>
 <tr><td>Hadoop Version</td><td><%= org.apache.hadoop.util.VersionInfo.getVersion() %>, r<%= org.apache.hadoop.util.VersionInfo.getRevision() %></td><td>Hadoop version and svn revision</td></tr>
 <tr><td>Hadoop Compiled</td><td><%= org.apache.hadoop.util.VersionInfo.getDate() %>, <%= org.apache.hadoop.util.VersionInfo.getUser() %></td><td>When Hadoop version was compiled and by whom</td></tr>
-<tr><td>HBase Root Directory</td><td><%= master.getRootDir().toString() %></td><td>Location of HBase home directory</td></tr>
+<tr><td>HBase Root Directory</td><td><%= FSUtils.getRootDir(master.getConfiguration()).toString() %></td><td>Location of HBase home directory</td></tr>
 <tr><td>Load average</td><td><%= master.getServerManager().getAverageLoad() %></td><td>Average number of regions per regionserver. Naive computation.</td></tr>
-<tr><td>Regions On FS</td><td><%= master.getRegionManager().countRegionsOnFS() %></td><td>Number of regions on FileSystem. Rough count.</td></tr>
 <%  if (showFragmentation) { %>
         <tr><td>Fragmentation</td><td><%= frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %></td><td>Overall fragmentation of all tables, including .META. and -ROOT-.</td></tr>
 <%  } %>
-<tr><td>Zookeeper Quorum</td><td><%= master.getZooKeeperWrapper().getQuorumServers() %></td><td>Addresses of all registered ZK servers. For more, see <a href="/zk.jsp">zk dump</a>.</td></tr>
+<tr><td>Zookeeper Quorum</td><td><%= master.getZooKeeperWatcher().getQuorum() %></td><td>Addresses of all registered ZK servers. For more, see <a href="/zk.jsp">zk dump</a>.</td></tr>
 </table>
 
 <h2>Catalog Tables</h2>
@@ -94,7 +92,7 @@
     <td>The -ROOT- table holds references to all .META. regions.</td>
 </tr>
 <%
-    if (onlineRegions != null && onlineRegions.size() > 0) { %>
+    if (metaOnline) { %>
 <tr>
     <td><a href="table.jsp?name=<%= Bytes.toString(HConstants.META_TABLE_NAME) %>"><%= Bytes.toString(HConstants.META_TABLE_NAME) %></a></td>
 <%  if (showFragmentation) { %>
diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index 35ad8a46a58..d69d417c560 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -9,8 +9,8 @@
   import="org.apache.hadoop.hbase.HServerInfo"
   import="org.apache.hadoop.hbase.io.ImmutableBytesWritable"
   import="org.apache.hadoop.hbase.master.HMaster" 
-  import="org.apache.hadoop.hbase.master.MetaRegion"
   import="org.apache.hadoop.hbase.util.Bytes"
+  import="org.apache.hadoop.hbase.util.FSUtils"
   import="java.util.Map"
   import="org.apache.hadoop.hbase.HConstants"%><%
   HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
@@ -19,11 +19,11 @@
   String tableName = request.getParameter("name");
   HTable table = new HTable(conf, tableName);
   String tableHeader = "<h2>Table Regions</h2><table><tr><th>Name</th><th>Region Server</th><th>Start Key</th><th>End Key</th></tr>";
-  HServerAddress rootLocation = master.getRegionManager().getRootRegionLocation();
+  HServerAddress rl = master.getCatalogTracker().getRootLocation();
   boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false);
   Map<String, Integer> frags = null;
   if (showFragmentation) {
-      frags = master.getTableFragmentation();
+      frags = FSUtils.getTableFragmentation(master);
   }
 %>
 
@@ -46,15 +46,19 @@
 <p><hr><p>
 <%
   if (action.equals("split")) {
+  /*
     if (key != null && key.length() > 0) {
       Writable[] arr = new Writable[1];
       arr[0] = new ImmutableBytesWritable(Bytes.toBytes(key));
       master.modifyTable(Bytes.toBytes(tableName), HConstants.Modify.TABLE_SPLIT, arr);
     } else {
-      master.modifyTable(Bytes.toBytes(tableName), HConstants.Modify.TABLE_SPLIT, null);
+      master.modifyTable(Bytes.toBytes(tableName), HConstants.Modify.TABLE_SPLIT);
     }
-    %> Split request accepted. <%
+    */
+    
+    %> Split request accepted -- BUT CURRENTLY A NOOP -- FIX!. <%
   } else if (action.equals("compact")) {
+  /*
     if (key != null && key.length() > 0) {
       Writable[] arr = new Writable[1];
       arr[0] = new ImmutableBytesWritable(Bytes.toBytes(key));
@@ -62,7 +66,8 @@
     } else {
       master.modifyTable(Bytes.toBytes(tableName), HConstants.Modify.TABLE_COMPACT, null);
     }
-    %> Compact request accepted. <%
+    */
+    %> Compact request accepted  -- BUT CURRENTLY A NOOP -- FIX! <%
   }
 %>
 <p>Reload.
@@ -84,12 +89,12 @@
 %>
 <%= tableHeader %>
 <%
-  int infoPort = master.getServerManager().getHServerInfo(rootLocation).getInfoPort();
-  String url = "http://" + rootLocation.getHostname() + ":" + infoPort + "/";
+  int infoPort = master.getServerManager().getHServerInfo(rl).getInfoPort();
+  String url = "http://" + rl.getHostname() + ":" + infoPort + "/";
 %>
 <tr>
   <td><%= tableName %></td>
-  <td><a href="<%= url %>"><%= rootLocation.getHostname() %>:<%= rootLocation.getPort() %></a></td>
+  <td><a href="<%= url %>"><%= rl.getHostname() %>:<%= rl.getPort() %></a></td>
   <td>-</td>
   <td></td>
   <td>-</td>
@@ -100,14 +105,16 @@
 %>
 <%= tableHeader %>
 <%
-  Map<byte [], MetaRegion> onlineRegions = master.getRegionManager().getOnlineMetaRegions();
-  for (MetaRegion meta: onlineRegions.values()) {
-    int infoPort = master.getServerManager().getHServerInfo(meta.getServer()).getInfoPort();
-    String url = "http://" + meta.getServer().getHostname() + ":" + infoPort + "/";
+  // NOTE: Presumes one meta region only.
+  HRegionInfo meta = HRegionInfo.FIRST_META_REGIONINFO;
+  HServerAddress metaLocation = master.getCatalogTracker().getMetaLocation();
+  for (int i = 0; i <= 1; i++) {
+    int infoPort = master.getServerManager().getHServerInfo(metaLocation).getInfoPort();
+    String url = "http://" + metaLocation.getHostname() + ":" + infoPort + "/";
 %>
 <tr>
-  <td><%= Bytes.toString(meta.getRegionName()) %></td>
-    <td><a href="<%= url %>"><%= meta.getServer().getHostname().toString() + ":" + infoPort %></a></td>
+  <td><%= meta.getRegionNameAsString() %></td>
+    <td><a href="<%= url %>"><%= metaLocation.getHostname().toString() + ":" + infoPort %></a></td>
     <td>-</td><td><%= Bytes.toString(meta.getStartKey()) %></td><td><%= Bytes.toString(meta.getEndKey()) %></td>
 </tr>
 <%  } %>
diff --git a/src/main/resources/hbase-webapps/master/zk.jsp b/src/main/resources/hbase-webapps/master/zk.jsp
index 3a0c9f7dddf..e7d3269dc5b 100644
--- a/src/main/resources/hbase-webapps/master/zk.jsp
+++ b/src/main/resources/hbase-webapps/master/zk.jsp
@@ -4,7 +4,8 @@
   import="org.apache.hadoop.hbase.client.HBaseAdmin"
   import="org.apache.hadoop.hbase.client.HConnection"
   import="org.apache.hadoop.hbase.HRegionInfo"
-  import="org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper"
+  import="org.apache.hadoop.hbase.zookeeper.ZKUtil"
+  import="org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher"
   import="org.apache.hadoop.hbase.HBaseConfiguration"
   import="org.apache.hadoop.hbase.master.HMaster" 
   import="org.apache.hadoop.hbase.HConstants"%><%
@@ -12,7 +13,7 @@
   Configuration conf = master.getConfiguration();
   HBaseAdmin hbadmin = new HBaseAdmin(conf);
   HConnection connection = hbadmin.getConnection();
-  ZooKeeperWrapper wrapper = connection.getZooKeeperWrapper();
+  ZooKeeperWatcher watcher = connection.getZooKeeperWatcher();
 %>
 
 <?xml version="1.0" encoding="UTF-8" ?>
@@ -29,7 +30,7 @@
 <p id="links_menu"><a href="/master.jsp">Master</a>, <a href="/logs/">Local logs</a>, <a href="/stacks">Thread Dump</a>, <a href="/logLevel">Log Level</a></p>
 <hr id="head_rule" />
 <pre>
-<%= wrapper.dump() %>
+<%= ZKUtil.dump(watcher) %>
 </pre>
 
 </body>
diff --git a/src/main/resources/hbase-webapps/regionserver/regionserver.jsp b/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
index efc509c3449..e50188adc95 100644
--- a/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
+++ b/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
@@ -18,7 +18,7 @@
     e.printStackTrace();
   }
   RegionServerMetrics metrics = regionServer.getMetrics();
-  Collection<HRegionInfo> onlineRegions = regionServer.getSortedOnlineRegionInfos();
+  Collection<HRegionInfo> onlineRegions = regionServer.getOnlineRegions();
   int interval = regionServer.getConfiguration().getInt("hbase.regionserver.msginterval", 3000)/1000;
 
 %><?xml version="1.0" encoding="UTF-8" ?>
@@ -42,7 +42,7 @@
 <tr><td>HBase Version</td><td><%= org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, r<%= org.apache.hadoop.hbase.util.VersionInfo.getRevision() %></td><td>HBase version and svn revision</td></tr>
 <tr><td>HBase Compiled</td><td><%= org.apache.hadoop.hbase.util.VersionInfo.getDate() %>, <%= org.apache.hadoop.hbase.util.VersionInfo.getUser() %></td><td>When HBase version was compiled and by whom</td></tr>
 <tr><td>Metrics</td><td><%= metrics.toString() %></td><td>RegionServer Metrics; file and heap sizes are in megabytes</td></tr>
-<tr><td>Zookeeper Quorum</td><td><%= regionServer.getZooKeeperWrapper().getQuorumServers() %></td><td>Addresses of all registered ZK servers</td></tr>
+<tr><td>Zookeeper Quorum</td><td><%= regionServer.getZooKeeper().getQuorum() %></td><td>Addresses of all registered ZK servers</td></tr>
 </table>
 
 <h2>Online Regions</h2>
@@ -50,7 +50,7 @@
 <table>
 <tr><th>Region Name</th><th>Start Key</th><th>End Key</th><th>Metrics</th></tr>
 <%   for (HRegionInfo r: onlineRegions) { 
-        HServerLoad.RegionLoad load = regionServer.createRegionLoad(r.getRegionName());
+        HServerLoad.RegionLoad load = regionServer.createRegionLoad(r.getEncodedName());
  %>
 <tr><td><%= r.getRegionNameAsString() %></td>
     <td><%= Bytes.toStringBinary(r.getStartKey()) %></td><td><%= Bytes.toStringBinary(r.getEndKey()) %></td>
diff --git a/src/test/java/org/apache/hadoop/hbase/BROKE_TODO_FIX_TestAcidGuarantees.java b/src/test/java/org/apache/hadoop/hbase/BROKE_TODO_FIX_TestAcidGuarantees.java
new file mode 100644
index 00000000000..6741acc701e
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/BROKE_TODO_FIX_TestAcidGuarantees.java
@@ -0,0 +1,330 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext;
+import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Test case that uses multiple threads to read and write multifamily rows
+ * into a table, verifying that reads never see partially-complete writes.
+ * 
+ * This can run as a junit test, or with a main() function which runs against
+ * a real cluster (eg for testing with failures, region movement, etc)
+ */
+public class BROKE_TODO_FIX_TestAcidGuarantees {
+  protected static final Log LOG = LogFactory.getLog(BROKE_TODO_FIX_TestAcidGuarantees.class);
+  public static final byte [] TABLE_NAME = Bytes.toBytes("TestAcidGuarantees");
+  public static final byte [] FAMILY_A = Bytes.toBytes("A");
+  public static final byte [] FAMILY_B = Bytes.toBytes("B");
+  public static final byte [] FAMILY_C = Bytes.toBytes("C");
+  public static final byte [] QUALIFIER_NAME = Bytes.toBytes("data");
+
+  public static final byte[][] FAMILIES = new byte[][] {
+    FAMILY_A, FAMILY_B, FAMILY_C };
+
+  private HBaseTestingUtility util;
+
+  public static int NUM_COLS_TO_CHECK = 50;
+
+  private void createTableIfMissing()
+    throws IOException {
+    try {
+      util.createTable(TABLE_NAME, FAMILIES);
+    } catch (TableExistsException tee) {
+    }
+  }
+
+  public BROKE_TODO_FIX_TestAcidGuarantees() {
+    // Set small flush size for minicluster so we exercise reseeking scanners
+    Configuration conf = HBaseConfiguration.create();
+    conf.set("hbase.hregion.memstore.flush.size", String.valueOf(128*1024));
+    util = new HBaseTestingUtility(conf);
+  }
+  
+  /**
+   * Thread that does random full-row writes into a table.
+   */
+  public static class AtomicityWriter extends RepeatingTestThread {
+    Random rand = new Random();
+    byte data[] = new byte[10];
+    byte targetRows[][];
+    byte targetFamilies[][];
+    HTable table;
+    AtomicLong numWritten = new AtomicLong();
+    
+    public AtomicityWriter(TestContext ctx, byte targetRows[][],
+                           byte targetFamilies[][]) throws IOException {
+      super(ctx);
+      this.targetRows = targetRows;
+      this.targetFamilies = targetFamilies;
+      table = new HTable(ctx.getConf(), TABLE_NAME);
+    }
+    public void doAnAction() throws Exception {
+      // Pick a random row to write into
+      byte[] targetRow = targetRows[rand.nextInt(targetRows.length)];
+      Put p = new Put(targetRow); 
+      rand.nextBytes(data);
+
+      for (byte[] family : targetFamilies) {
+        for (int i = 0; i < NUM_COLS_TO_CHECK; i++) {
+          byte qualifier[] = Bytes.toBytes("col" + i);
+          p.add(family, qualifier, data);
+        }
+      }
+      table.put(p);
+      numWritten.getAndIncrement();
+    }
+  }
+  
+  /**
+   * Thread that does single-row reads in a table, looking for partially
+   * completed rows.
+   */
+  public static class AtomicGetReader extends RepeatingTestThread {
+    byte targetRow[];
+    byte targetFamilies[][];
+    HTable table;
+    int numVerified = 0;
+    AtomicLong numRead = new AtomicLong();
+
+    public AtomicGetReader(TestContext ctx, byte targetRow[],
+                           byte targetFamilies[][]) throws IOException {
+      super(ctx);
+      this.targetRow = targetRow;
+      this.targetFamilies = targetFamilies;
+      table = new HTable(ctx.getConf(), TABLE_NAME);
+    }
+
+    public void doAnAction() throws Exception {
+      Get g = new Get(targetRow);
+      Result res = table.get(g);
+      byte[] gotValue = null;
+      if (res.getRow() == null) {
+        // Trying to verify but we didn't find the row - the writing
+        // thread probably just hasn't started writing yet, so we can
+        // ignore this action
+        return;
+      }
+      
+      for (byte[] family : targetFamilies) {
+        for (int i = 0; i < NUM_COLS_TO_CHECK; i++) {
+          byte qualifier[] = Bytes.toBytes("col" + i);
+          byte thisValue[] = res.getValue(family, qualifier);
+          if (gotValue != null && !Bytes.equals(gotValue, thisValue)) {
+            gotFailure(gotValue, res);
+          }
+          numVerified++;
+          gotValue = thisValue;
+        }
+      }
+      numRead.getAndIncrement();
+    }
+
+    private void gotFailure(byte[] expected, Result res) {
+      StringBuilder msg = new StringBuilder();
+      msg.append("Failed after ").append(numVerified).append("!");
+      msg.append("Expected=").append(Bytes.toStringBinary(expected));
+      msg.append("Got:\n");
+      for (KeyValue kv : res.list()) {
+        msg.append(kv.toString());
+        msg.append(" val= ");
+        msg.append(Bytes.toStringBinary(kv.getValue()));
+        msg.append("\n");
+      }
+      throw new RuntimeException(msg.toString());
+    }
+  }
+  
+  /**
+   * Thread that does full scans of the table looking for any partially completed
+   * rows.
+   */
+  public static class AtomicScanReader extends RepeatingTestThread {
+    byte targetFamilies[][];
+    HTable table;
+    AtomicLong numScans = new AtomicLong();
+    AtomicLong numRowsScanned = new AtomicLong();
+
+    public AtomicScanReader(TestContext ctx,
+                           byte targetFamilies[][]) throws IOException {
+      super(ctx);
+      this.targetFamilies = targetFamilies;
+      table = new HTable(ctx.getConf(), TABLE_NAME);
+    }
+
+    public void doAnAction() throws Exception {
+      Scan s = new Scan();
+      for (byte[] family : targetFamilies) {
+        s.addFamily(family);
+      }
+      ResultScanner scanner = table.getScanner(s);
+      
+      for (Result res : scanner) {
+        byte[] gotValue = null;
+  
+        for (byte[] family : targetFamilies) {
+          for (int i = 0; i < NUM_COLS_TO_CHECK; i++) {
+            byte qualifier[] = Bytes.toBytes("col" + i);
+            byte thisValue[] = res.getValue(family, qualifier);
+            if (gotValue != null && !Bytes.equals(gotValue, thisValue)) {
+              gotFailure(gotValue, res);
+            }
+            gotValue = thisValue;
+          }
+        }
+        numRowsScanned.getAndIncrement();
+      }
+      numScans.getAndIncrement();
+    }
+
+    private void gotFailure(byte[] expected, Result res) {
+      StringBuilder msg = new StringBuilder();
+      msg.append("Failed after ").append(numRowsScanned).append("!");
+      msg.append("Expected=").append(Bytes.toStringBinary(expected));
+      msg.append("Got:\n");
+      for (KeyValue kv : res.list()) {
+        msg.append(kv.toString());
+        msg.append(" val= ");
+        msg.append(Bytes.toStringBinary(kv.getValue()));
+        msg.append("\n");
+      }
+      throw new RuntimeException(msg.toString());
+    }
+  }
+
+
+  public void runTestAtomicity(long millisToRun,
+      int numWriters,
+      int numGetters,
+      int numScanners,
+      int numUniqueRows) throws Exception {
+    createTableIfMissing();
+    TestContext ctx = new TestContext(util.getConfiguration());
+    
+    byte rows[][] = new byte[numUniqueRows][];
+    for (int i = 0; i < numUniqueRows; i++) {
+      rows[i] = Bytes.toBytes("test_row_" + i);
+    }
+    
+    List<AtomicityWriter> writers = Lists.newArrayList();
+    for (int i = 0; i < numWriters; i++) {
+      AtomicityWriter writer = new AtomicityWriter(
+          ctx, rows, FAMILIES);
+      writers.add(writer);
+      ctx.addThread(writer);
+    }
+
+    List<AtomicGetReader> getters = Lists.newArrayList();
+    for (int i = 0; i < numGetters; i++) {
+      AtomicGetReader getter = new AtomicGetReader(
+          ctx, rows[i % numUniqueRows], FAMILIES);
+      getters.add(getter);
+      ctx.addThread(getter);
+    }
+    
+    List<AtomicScanReader> scanners = Lists.newArrayList();
+    for (int i = 0; i < numScanners; i++) {
+      AtomicScanReader scanner = new AtomicScanReader(ctx, FAMILIES);
+      scanners.add(scanner);
+      ctx.addThread(scanner);
+    }
+    
+    ctx.startThreads();
+    ctx.waitFor(millisToRun);
+    ctx.stop();
+    
+    LOG.info("Finished test. Writers:");
+    for (AtomicityWriter writer : writers) {
+      LOG.info("  wrote " + writer.numWritten.get());
+    }
+    LOG.info("Readers:");
+    for (AtomicGetReader reader : getters) {
+      LOG.info("  read " + reader.numRead.get());
+    }
+    LOG.info("Scanners:");
+    for (AtomicScanReader scanner : scanners) {
+      LOG.info("  scanned " + scanner.numScans.get());
+      LOG.info("  verified " + scanner.numRowsScanned.get() + " rows");
+    }
+  }
+
+  @Test
+  public void testGetAtomicity() throws Exception {
+    util.startMiniCluster(1);
+    try {
+      runTestAtomicity(20000, 5, 5, 0, 3);
+    } finally {
+      util.shutdownMiniCluster();
+    }    
+  }
+
+  @Test
+  @Ignore("Currently not passing - see HBASE-2670")
+  public void testScanAtomicity() throws Exception {
+    util.startMiniCluster(1);
+    try {
+      runTestAtomicity(20000, 5, 0, 5, 3);
+    } finally {
+      util.shutdownMiniCluster();
+    }    
+  }
+
+  @Test
+  @Ignore("Currently not passing - see HBASE-2670")
+  public void testMixedAtomicity() throws Exception {
+    util.startMiniCluster(1);
+    try {
+      runTestAtomicity(20000, 5, 2, 2, 3);
+    } finally {
+      util.shutdownMiniCluster();
+    }    
+  }
+
+  public static void main(String args[]) throws Exception {
+    Configuration c = HBaseConfiguration.create();
+    BROKE_TODO_FIX_TestAcidGuarantees test = new BROKE_TODO_FIX_TestAcidGuarantees();
+    test.setConf(c);
+    test.runTestAtomicity(5*60*1000, 5, 2, 2, 3);
+  }
+
+  private void setConf(Configuration c) {
+    util = new HBaseTestingUtility(c);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseClusterTestCase.java b/src/test/java/org/apache/hadoop/hbase/HBaseClusterTestCase.java
index 500ab708c5b..51e6cd920c1 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseClusterTestCase.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseClusterTestCase.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.util.ReflectionUtils;
 
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index fadee21eebc..9cc11681fb6 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -59,7 +59,9 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -72,9 +74,9 @@ import org.apache.zookeeper.ZooKeeper;
 import com.google.common.base.Preconditions;
 
 /**
- * Facility for testing HBase. Added as tool to abet junit4 testing.  Replaces
+ * Facility for testing HBase. Replacement for
  * old HBaseTestCase and HBaseCluserTestCase functionality.
- * Create an instance and keep it around doing HBase testing.  This class is
+ * Create an instance and keep it around testing HBase.  This class is
  * meant to be your one-stop shop for anything you might need testing.  Manages
  * one cluster at a time only.  Depends on log4j being on classpath and
  * hbase-site.xml for logging and test-run configuration.  It does not set
@@ -93,13 +95,14 @@ public class HBaseTestingUtility {
 
   /**
    * System property key to get test directory value.
+   * Name is as it is because mini dfs has hard-codings to put test data here.
    */
   public static final String TEST_DIRECTORY_KEY = "test.build.data";
 
   /**
-   * Default parent direccounttory for test output.
+   * Default parent directory for test output.
    */
-  public static final String DEFAULT_TEST_DIRECTORY = "target/build/data";
+  public static final String DEFAULT_TEST_DIRECTORY = "target/test-data";
 
   public HBaseTestingUtility() {
     this(HBaseConfiguration.create());
@@ -120,6 +123,8 @@ public class HBaseTestingUtility {
    * @return Where to write test data on local filesystem; usually
    * {@link #DEFAULT_TEST_DIRECTORY}
    * @see #setupClusterTestBuildDir()
+   * @see #clusterTestBuildDir()
+   * @see #getTestFileSystem()
    */
   public static Path getTestDir() {
     return new Path(System.getProperty(TEST_DIRECTORY_KEY,
@@ -131,22 +136,26 @@ public class HBaseTestingUtility {
    * @return Path to a subdirectory named <code>subdirName</code> under
    * {@link #getTestDir()}.
    * @see #setupClusterTestBuildDir()
+   * @see #clusterTestBuildDir(String)
+   * @see #getTestFileSystem()
    */
   public static Path getTestDir(final String subdirName) {
     return new Path(getTestDir(), subdirName);
   }
 
   /**
-   * Home our cluster in a dir under target/test.  Give it a random name
+   * Home our cluster in a dir under {@link #DEFAULT_TEST_DIRECTORY}.  Give it a
+   * random name
    * so can have many concurrent clusters running if we need to.  Need to
-   * amend the test.build.data System property.  Its what minidfscluster bases
+   * amend the {@link #TEST_DIRECTORY_KEY} System property.  Its what
+   * minidfscluster bases
    * it data dir on.  Moding a System property is not the way to do concurrent
    * instances -- another instance could grab the temporary
    * value unintentionally -- but not anything can do about it at moment;
    * single instance only is how the minidfscluster works.
    * @return The calculated cluster test build directory.
    */
-  File setupClusterTestBuildDir() {
+  public File setupClusterTestBuildDir() {
     String randomStr = UUID.randomUUID().toString();
     String dirStr = getTestDir(randomStr).toString();
     File dir = new File(dirStr).getAbsoluteFile();
@@ -158,8 +167,8 @@ public class HBaseTestingUtility {
   /**
    * @throws IOException If a cluster -- zk, dfs, or hbase -- already running.
    */
-  void isRunningCluster() throws IOException {
-    if (this.clusterTestBuildDir == null) return;
+  void isRunningCluster(String passedBuildPath) throws IOException {
+    if (this.clusterTestBuildDir == null || passedBuildPath != null) return;
     throw new IOException("Cluster already running at " +
       this.clusterTestBuildDir);
   }
@@ -190,8 +199,11 @@ public class HBaseTestingUtility {
     //     base_dir = new File(System.getProperty("test.build.data", "build/test/data"), "dfs/");
     // Some tests also do this:
     //  System.getProperty("test.cache.data", "build/test/cache");
-    if (dir == null) this.clusterTestBuildDir = setupClusterTestBuildDir();
-    else this.clusterTestBuildDir = dir;
+    if (dir == null) {
+      this.clusterTestBuildDir = setupClusterTestBuildDir();
+    } else {
+      this.clusterTestBuildDir = dir;
+    }
     System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestBuildDir.toString());
     System.setProperty("test.cache.data", this.clusterTestBuildDir.toString());
     this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
@@ -242,7 +254,10 @@ public class HBaseTestingUtility {
    * @see #startMiniZKCluster()
    */
   public void shutdownMiniZKCluster() throws IOException {
-    if (this.zkCluster != null) this.zkCluster.shutdown();
+    if (this.zkCluster != null) {
+      this.zkCluster.shutdown();
+      this.zkCluster = null;
+    }
   }
 
   /**
@@ -272,10 +287,15 @@ public class HBaseTestingUtility {
   throws Exception {
     LOG.info("Starting up minicluster");
     // If we already put up a cluster, fail.
-    isRunningCluster();
+    String testBuildPath = conf.get(TEST_DIRECTORY_KEY, null);
+    isRunningCluster(testBuildPath);
+    if(testBuildPath != null) {
+      LOG.info("Using passed path: " + testBuildPath);
+    }
     // Make a new random dir to home everything in.  Set it as system property.
     // minidfs reads home from system property.
-    this.clusterTestBuildDir = setupClusterTestBuildDir();
+    this.clusterTestBuildDir = testBuildPath == null?
+      setupClusterTestBuildDir() : new File(testBuildPath);
     System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestBuildDir.getPath());
     // Bring up mini dfs cluster. This spews a bunch of warnings about missing
     // scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'.
@@ -302,11 +322,30 @@ public class HBaseTestingUtility {
     // Don't leave here till we've done a successful scan of the .META.
     HTable t = new HTable(this.conf, HConstants.META_TABLE_NAME);
     ResultScanner s = t.getScanner(new Scan());
-    while (s.next() != null) continue;
+    while (s.next() != null) {
+      continue;
+    }
     LOG.info("Minicluster is up");
     return this.hbaseCluster;
   }
 
+  /**
+   * Starts the hbase cluster up again after shutting it down previously in a
+   * test.  Use this if you want to keep dfs/zk up and just stop/start hbase.
+   * @param servers number of region servers
+   * @throws IOException
+   */
+  public void restartHBaseCluster(int servers) throws IOException {
+    this.hbaseCluster = new MiniHBaseCluster(this.conf, servers);
+    // Don't leave here till we've done a successful scan of the .META.
+    HTable t = new HTable(this.conf, HConstants.META_TABLE_NAME);
+    ResultScanner s = t.getScanner(new Scan());
+    while (s.next() != null) {
+      continue;
+    }
+    LOG.info("HBase has been restarted");
+  }
+
   /**
    * @return Current mini hbase cluster. Only has something in it after a call
    * to {@link #startMiniCluster()}.
@@ -339,6 +378,7 @@ public class HBaseTestingUtility {
           new Path(this.clusterTestBuildDir.toString()))) {
         LOG.warn("Failed delete of " + this.clusterTestBuildDir.toString());
       }
+      this.clusterTestBuildDir = null;
     }
     LOG.info("Minicluster is down");
   }
@@ -609,6 +649,7 @@ public class HBaseTestingUtility {
     // and end key. Adding the custom regions below adds those blindly,
     // including the new start region from empty to "bbb". lg
     List<byte[]> rows = getMetaTableRows(htd.getName());
+    List<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(startKeys.length);
     // add custom ones
     int count = 0;
     for (int i = 0; i < startKeys.length; i++) {
@@ -620,6 +661,7 @@ public class HBaseTestingUtility {
         Writables.getBytes(hri));
       meta.put(put);
       LOG.info("createMultiRegions: inserted " + hri.toString());
+      newRegions.add(hri);
       count++;
     }
     // see comment above, remove "old" (or previous) single region
@@ -631,6 +673,10 @@ public class HBaseTestingUtility {
     // flush cache of regions
     HConnection conn = table.getConnection();
     conn.clearRegionCache();
+    // assign all the new regions
+    for(HRegionInfo hri : newRegions) {
+      hbaseCluster.getMaster().assignRegion(hri);
+    }
     return count;
   }
 
@@ -757,7 +803,7 @@ public class HBaseTestingUtility {
    */
   public void expireMasterSession() throws Exception {
     HMaster master = hbaseCluster.getMaster();
-    expireSession(master.getZooKeeperWrapper());
+    expireSession(master.getZooKeeper(), master);
   }
 
   /**
@@ -767,19 +813,16 @@ public class HBaseTestingUtility {
    */
   public void expireRegionServerSession(int index) throws Exception {
     HRegionServer rs = hbaseCluster.getRegionServer(index);
-    expireSession(rs.getZooKeeperWrapper());
+    expireSession(rs.getZooKeeper(), rs);
   }
 
-  public void expireSession(ZooKeeperWrapper nodeZK) throws Exception{
-    ZooKeeperWrapper zkw =
-        ZooKeeperWrapper.createInstance(conf,
-            ZooKeeperWrapper.class.getName());
-    zkw.registerListener(EmptyWatcher.instance);
-    String quorumServers = zkw.getQuorumServers();
+  public void expireSession(ZooKeeperWatcher nodeZK, Server server)
+  throws Exception {
+    String quorumServers = ZKConfig.getZKQuorumServersString(conf);
     int sessionTimeout = 5 * 1000; // 5 seconds
 
-    byte[] password = nodeZK.getSessionPassword();
-    long sessionID = nodeZK.getSessionID();
+    byte[] password = nodeZK.getZooKeeper().getSessionPasswd();
+    long sessionID = nodeZK.getZooKeeper().getSessionId();
 
     ZooKeeper zk = new ZooKeeper(quorumServers,
         sessionTimeout, EmptyWatcher.instance, sessionID, password);
@@ -805,9 +848,10 @@ public class HBaseTestingUtility {
    * Returns a HBaseAdmin instance.
    *
    * @return The HBaseAdmin instance.
-   * @throws MasterNotRunningException
+   * @throws IOException 
    */
-  public HBaseAdmin getHBaseAdmin() throws MasterNotRunningException {
+  public HBaseAdmin getHBaseAdmin()
+  throws IOException {
     if (hbaseAdmin == null) {
       hbaseAdmin = new HBaseAdmin(getConfiguration());
     }
@@ -832,7 +876,7 @@ public class HBaseTestingUtility {
    */
   public void closeRegion(byte[] regionName) throws IOException {
     HBaseAdmin admin = getHBaseAdmin();
-    admin.closeRegion(regionName, (Object[]) null);
+    admin.closeRegion(regionName, null);
   }
 
   /**
@@ -874,8 +918,34 @@ public class HBaseTestingUtility {
     return FileSystem.get(conf);
   }
 
-  public void cleanupTestDir() throws IOException {
-    getTestDir().getFileSystem(conf).delete(getTestDir(), true);
+  /**
+   * @return True if we removed the test dir
+   * @throws IOException
+   */
+  public boolean cleanupTestDir() throws IOException {
+    return deleteDir(getTestDir());
+  }
+
+  /**
+   * @param subdir Test subdir name.
+   * @return True if we removed the test dir
+   * @throws IOException
+   */
+  public boolean cleanupTestDir(final String subdir) throws IOException {
+    return deleteDir(getTestDir(subdir));
+  }
+
+  /**
+   * @param dir Directory to delete
+   * @return True if we deleted it.
+   * @throws IOException 
+   */
+  public boolean deleteDir(final Path dir) throws IOException {
+    FileSystem fs = getTestFileSystem();
+    if (fs.exists(dir)) {
+      return fs.delete(getTestDir(), true);
+    }
+    return false;
   }
 
   public void waitTableAvailable(byte[] table, long timeoutMillis)
@@ -1006,12 +1076,16 @@ public class HBaseTestingUtility {
       for (Result r = null; (r = s.next()) != null;) {
         byte [] b =
           r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-        if (b == null || b.length <= 0) break;
+        if (b == null || b.length <= 0) {
+          break;
+        }
         rows++;
       }
       s.close();
       // If I get to here and all rows have a Server, then all have been assigned.
-      if (rows == countOfRegions) break;
+      if (rows == countOfRegions) {
+        break;
+      }
       LOG.info("Found=" + rows);
       Threads.sleep(1000);
     }
diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index 9c49e36f868..b65274eaa39 100644
--- a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.security.UnixUserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.zookeeper.KeeperException;
 
 /**
  * This class creates a single process HBase cluster.
@@ -82,7 +83,7 @@ public class MiniHBaseCluster {
       new ConcurrentHashMap<HServerInfo, IOException>();
 
     public MiniHBaseClusterMaster(final Configuration conf)
-    throws IOException {
+    throws IOException, KeeperException, InterruptedException {
       super(conf);
     }
 
@@ -151,7 +152,7 @@ public class MiniHBaseCluster {
     private Thread shutdownThread = null;
 
     public MiniHBaseClusterRegionServer(Configuration conf)
-        throws IOException {
+        throws IOException, InterruptedException {
       super(setDifferentUser(conf));
     }
 
@@ -181,8 +182,8 @@ public class MiniHBaseCluster {
     }
 
     @Override
-    protected void init(MapWritable c) throws IOException {
-      super.init(c);
+    protected void handleReportForDutyResponse(MapWritable c) throws IOException {
+      super.handleReportForDutyResponse(c);
       // Run this thread to shutdown our filesystem on way out.
       this.shutdownThread = new SingleFileSystemShutdownThread(getFileSystem());
     }
@@ -303,7 +304,7 @@ public class MiniHBaseCluster {
     JVMClusterUtil.RegionServerThread server =
       hbaseCluster.getRegionServers().get(serverNumber);
     LOG.info("Stopping " + server.toString());
-    server.getRegionServer().stop();
+    server.getRegionServer().stop("Stopping rs " + serverNumber);
     return server;
   }
 
@@ -342,7 +343,7 @@ public class MiniHBaseCluster {
   public void flushcache() throws IOException {
     for (JVMClusterUtil.RegionServerThread t:
         this.hbaseCluster.getRegionServers()) {
-      for(HRegion r: t.getRegionServer().getOnlineRegions()) {
+      for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) {
         r.flushcache();
       }
     }
@@ -355,7 +356,7 @@ public class MiniHBaseCluster {
   public void flushcache(byte [] tableName) throws IOException {
     for (JVMClusterUtil.RegionServerThread t:
         this.hbaseCluster.getRegionServers()) {
-      for(HRegion r: t.getRegionServer().getOnlineRegions()) {
+      for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) {
         if(Bytes.equals(r.getTableDesc().getName(), tableName)) {
           r.flushcache();
         }
@@ -390,7 +391,7 @@ public class MiniHBaseCluster {
     List<HRegion> ret = new ArrayList<HRegion>();
     for (JVMClusterUtil.RegionServerThread rst : getRegionServerThreads()) {
       HRegionServer hrs = rst.getRegionServer();
-      for (HRegion region : hrs.getOnlineRegions()) {
+      for (HRegion region : hrs.getOnlineRegionsLocalContext()) {
         if (Bytes.equals(region.getTableDesc().getName(), tableName)) {
           ret.add(region);
         }
@@ -481,4 +482,4 @@ public class MiniHBaseCluster {
   throws IOException {
     ((MiniHBaseClusterMaster)getMaster()).addMessage(hrs.getHServerInfo(), msg);
   }
-}
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 1d7320c6ecb..3982effcd1d 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Hash;
 import org.apache.hadoop.hbase.util.MurmurHash;
 import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.NullWritable;
diff --git a/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java b/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
index aad84eeea31..9c530ebceaf 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
@@ -20,16 +20,19 @@
 
 package org.apache.hadoop.hbase;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import static org.junit.Assert.assertEquals;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.*;
-import static org.junit.Assert.assertEquals;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
 
 public class TestFullLogReconstruction {
 
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java
new file mode 100644
index 00000000000..1cd88d3783a
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java
@@ -0,0 +1,105 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test our testing utility class
+ */
+public class TestHBaseTestingUtility {
+  private final Log LOG = LogFactory.getLog(this.getClass());
+
+  private HBaseTestingUtility hbt;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    this.hbt = new HBaseTestingUtility();
+    this.hbt.cleanupTestDir();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+  }
+
+  @Test public void testMiniCluster() throws Exception {
+    MiniHBaseCluster cluster = this.hbt.startMiniCluster();
+    try {
+      assertEquals(1, cluster.getLiveRegionServerThreads().size());
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  @Test public void testMiniDFSCluster() throws Exception {
+    MiniDFSCluster cluster = this.hbt.startMiniDFSCluster(1);
+    FileSystem dfs = cluster.getFileSystem();
+    Path dir = new Path("dir");
+    Path qualifiedDir = dfs.makeQualified(dir);
+    LOG.info("dir=" + dir + ", qualifiedDir=" + qualifiedDir);
+    assertFalse(dfs.exists(qualifiedDir));
+    assertTrue(dfs.mkdirs(qualifiedDir));
+    assertTrue(dfs.delete(qualifiedDir, true));
+    try {
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  @Test public void testSetupClusterTestBuildDir() {
+    File testdir = this.hbt.setupClusterTestBuildDir();
+    LOG.info("uuid-subdir=" + testdir);
+    assertFalse(testdir.exists());
+    assertTrue(testdir.mkdirs());
+    assertTrue(testdir.exists());
+  }
+
+  @Test public void testTestDir() throws IOException {
+    Path testdir = HBaseTestingUtility.getTestDir();
+    LOG.info("testdir=" + testdir);
+    FileSystem fs = this.hbt.getTestFileSystem();
+    assertTrue(!fs.exists(testdir));
+    assertTrue(fs.mkdirs(testdir));
+    assertTrue(this.hbt.cleanupTestDir());
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestHMsg.java b/src/test/java/org/apache/hadoop/hbase/TestHMsg.java
index 2b75f06fc94..b55956fa712 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestHMsg.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestHMsg.java
@@ -35,7 +35,7 @@ public class TestHMsg extends TestCase {
     final int size = 10;
     for (int i = 0; i < size; i++) {
       byte [] b = Bytes.toBytes(i);
-      hmsg = new HMsg(HMsg.Type.MSG_REGION_OPEN,
+      hmsg = new HMsg(HMsg.Type.STOP_REGIONSERVER,
         new HRegionInfo(new HTableDescriptor(Bytes.toBytes("test")), b, b));
       msgs.add(hmsg);
     }
@@ -45,12 +45,12 @@ public class TestHMsg extends TestCase {
     msgs.remove(index);
     assertEquals(size - 1, msgs.size());
     byte [] other = Bytes.toBytes("other");
-    hmsg = new HMsg(HMsg.Type.MSG_REGION_OPEN,
+    hmsg = new HMsg(HMsg.Type.STOP_REGIONSERVER,
       new HRegionInfo(new HTableDescriptor(Bytes.toBytes("test")), other, other));
     assertEquals(-1, msgs.indexOf(hmsg));
     // Assert that two HMsgs are same if same content.
     byte [] b = Bytes.toBytes(1);
-    hmsg = new HMsg(HMsg.Type.MSG_REGION_OPEN,
+    hmsg = new HMsg(HMsg.Type.STOP_REGIONSERVER,
      new HRegionInfo(new HTableDescriptor(Bytes.toBytes("test")), b, b));
     assertNotSame(-1, msgs.indexOf(hmsg));
   }
@@ -64,7 +64,7 @@ public class TestHMsg extends TestCase {
       new HRegionInfo(new HTableDescriptor(Bytes.toBytes("parent")),
       parentbytes, parentbytes);
     // Assert simple HMsg serializes
-    HMsg hmsg = new HMsg(HMsg.Type.MSG_REGION_CLOSE, parent);
+    HMsg hmsg = new HMsg(HMsg.Type.STOP_REGIONSERVER, parent);
     byte [] bytes = Writables.getBytes(hmsg);
     HMsg close = (HMsg)Writables.getWritable(bytes, new HMsg());
     assertTrue(close.equals(hmsg));
@@ -73,8 +73,8 @@ public class TestHMsg extends TestCase {
       new HRegionInfo(new HTableDescriptor(Bytes.toBytes("a")), abytes, abytes);
     HRegionInfo daughterb =
       new HRegionInfo(new HTableDescriptor(Bytes.toBytes("b")), bbytes, bbytes);
-    HMsg splithmsg = new HMsg(HMsg.Type.MSG_REPORT_SPLIT_INCLUDES_DAUGHTERS,
-      parent, daughtera, daughterb, Bytes.toBytes("split"));
+    HMsg splithmsg = new HMsg(HMsg.Type.REGION_SPLIT,
+      parent, daughtera, daughterb, Bytes.toBytes("REGION_SPLIT"));
     bytes = Writables.getBytes(splithmsg);
     hmsg = (HMsg)Writables.getWritable(bytes, new HMsg());
     assertTrue(splithmsg.equals(hmsg));
diff --git a/src/test/java/org/apache/hadoop/hbase/TestMultiParallel.java b/src/test/java/org/apache/hadoop/hbase/TestMultiParallel.java
deleted file mode 100644
index d813a39fe46..00000000000
--- a/src/test/java/org/apache/hadoop/hbase/TestMultiParallel.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/*
- * Copyright 2009 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase;
-
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.ArrayList;
-
-public class TestMultiParallel extends MultiRegionTable {
-
-  private static final byte[] VALUE = Bytes.toBytes("value");
-  private static final byte[] QUALIFIER = Bytes.toBytes("qual");
-  private static final String FAMILY = "family";
-  private static final String TEST_TABLE = "multi_test_table";
-  private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY);
-  private static final byte[] ONE_ROW = Bytes.toBytes("xxx");
-
-  List<byte[]> keys = new ArrayList<byte[]>();
-
-  public TestMultiParallel() {
-    super(2, FAMILY);
-    desc = new HTableDescriptor(TEST_TABLE);
-    desc.addFamily(new HColumnDescriptor(FAMILY));
-    makeKeys();
-  }
-
-  private void makeKeys() {
-    // Create a "non-uniform" test set with the following characteristics:
-    // a) Unequal number of keys per region
-
-    // Don't use integer as a multiple, so that we have a number of keys that is
-    // not a multiple of the number of regions
-    int numKeys = (int) ((float) KEYS.length * 10.33F);
-
-    for (int i = 0; i < numKeys; i++) {
-      int kIdx = i % KEYS.length;
-      byte[] k = KEYS[kIdx];
-      byte[] cp = new byte[k.length + 1];
-      System.arraycopy(k, 0, cp, 0, k.length);
-      cp[k.length] = new Integer(i % 256).byteValue();
-      keys.add(cp);
-    }
-
-    // b) Same duplicate keys (showing multiple Gets/Puts to the same row, which
-    // should work)
-    // c) keys are not in sorted order (within a region), to ensure that the
-    // sorting code and index mapping doesn't break the functionality
-    for (int i = 0; i < 100; i++) {
-      int kIdx = i % KEYS.length;
-      byte[] k = KEYS[kIdx];
-      byte[] cp = new byte[k.length + 1];
-      System.arraycopy(k, 0, cp, 0, k.length);
-      cp[k.length] = new Integer(i % 256).byteValue();
-      keys.add(cp);
-    }
-  }
-
-  public void testBatchWithGet() throws Exception {
-    HTable table = new HTable(conf, TEST_TABLE);
-
-    // load test data
-    List<Row> puts = constructPutRequests();
-    table.batch(puts);
-
-    // create a list of gets and run it
-    List<Row> gets = new ArrayList<Row>();
-    for (byte[] k : keys) {
-      Get get = new Get(k);
-      get.addColumn(BYTES_FAMILY, QUALIFIER);
-      gets.add(get);
-    }
-    Result[] multiRes = new Result[gets.size()];
-    table.batch(gets, multiRes);
-
-    // Same gets using individual call API
-    List<Result> singleRes = new ArrayList<Result>();
-    for (Row get : gets) {
-      singleRes.add(table.get((Get) get));
-    }
-
-    // Compare results
-    assertEquals(singleRes.size(), multiRes.length);
-    for (int i = 0; i < singleRes.size(); i++) {
-      assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER));
-      KeyValue[] singleKvs = singleRes.get(i).raw();
-      KeyValue[] multiKvs = multiRes[i].raw();
-      for (int j = 0; j < singleKvs.length; j++) {
-        assertEquals(singleKvs[j], multiKvs[j]);
-        assertEquals(0, Bytes.compareTo(singleKvs[j].getValue(), multiKvs[j]
-            .getValue()));
-      }
-    }
-  }
-
-  /**
-   * Only run one Multi test with a forced RegionServer abort. Otherwise, the
-   * unit tests will take an unnecessarily long time to run.
-   * 
-   * @throws Exception
-   */
-  public void testFlushCommitsWithAbort() throws Exception {
-    doTestFlushCommits(true);
-  }
-
-  public void testFlushCommitsNoAbort() throws Exception {
-    doTestFlushCommits(false);
-  }
-
-  public void doTestFlushCommits(boolean doAbort) throws Exception {
-    // Load the data
-    HTable table = new HTable(conf, TEST_TABLE);
-    table.setAutoFlush(false);
-    table.setWriteBufferSize(10 * 1024 * 1024);
-
-    List<Row> puts = constructPutRequests();
-    for (Row put : puts) {
-      table.put((Put) put);
-    }
-    table.flushCommits();
-
-    if (doAbort) {
-      cluster.abortRegionServer(0);
-
-      // try putting more keys after the abort. same key/qual... just validating
-      // no exceptions thrown
-      puts = constructPutRequests();
-      for (Row put : puts) {
-        table.put((Put) put);
-      }
-
-      table.flushCommits();
-    }
-
-    validateLoadedData(table);
-
-    // Validate server and region count
-    HBaseAdmin admin = new HBaseAdmin(conf);
-    ClusterStatus cs = admin.getClusterStatus();
-    assertEquals((doAbort ? 1 : 2), cs.getServers());
-    for (HServerInfo info : cs.getServerInfo()) {
-      System.out.println(info);
-      assertTrue(info.getLoad().getNumberOfRegions() > 10);
-    }
-  }
-
-  public void testBatchWithPut() throws Exception {
-
-    HTable table = new HTable(conf, TEST_TABLE);
-
-    // put multiple rows using a batch
-    List<Row> puts = constructPutRequests();
-
-    Result[] results = table.batch(puts);
-    validateSizeAndEmpty(results, keys.size());
-
-    if (true) {
-      cluster.abortRegionServer(0);
-
-      puts = constructPutRequests();
-      results = table.batch(puts);
-      validateSizeAndEmpty(results, keys.size());
-    }
-
-    validateLoadedData(table);
-  }
-
-  public void testBatchWithDelete() throws Exception {
-
-    HTable table = new HTable(conf, TEST_TABLE);
-
-    // Load some data
-    List<Row> puts = constructPutRequests();
-    Result[] results = table.batch(puts);
-    validateSizeAndEmpty(results, keys.size());
-
-    // Deletes
-    List<Row> deletes = new ArrayList<Row>();
-    for (int i = 0; i < keys.size(); i++) {
-      Delete delete = new Delete(keys.get(i));
-      delete.deleteFamily(BYTES_FAMILY);
-      deletes.add(delete);
-    }
-    results = table.batch(deletes);
-    validateSizeAndEmpty(results, keys.size());
-
-    // Get to make sure ...
-    for (byte[] k : keys) {
-      Get get = new Get(k);
-      get.addColumn(BYTES_FAMILY, QUALIFIER);
-      assertFalse(table.exists(get));
-    }
-
-  }
-
-  public void testHTableDeleteWithList() throws Exception {
-
-    HTable table = new HTable(conf, TEST_TABLE);
-
-    // Load some data
-    List<Row> puts = constructPutRequests();
-    Result[] results = table.batch(puts);
-    validateSizeAndEmpty(results, keys.size());
-
-    // Deletes
-    ArrayList<Delete> deletes = new ArrayList<Delete>();
-    for (int i = 0; i < keys.size(); i++) {
-      Delete delete = new Delete(keys.get(i));
-      delete.deleteFamily(BYTES_FAMILY);
-      deletes.add(delete);
-    }
-    table.delete(deletes);
-    assertTrue(deletes.isEmpty());
-
-    // Get to make sure ...
-    for (byte[] k : keys) {
-      Get get = new Get(k);
-      get.addColumn(BYTES_FAMILY, QUALIFIER);
-      assertFalse(table.exists(get));
-    }
-
-  }
-
-  public void testBatchWithManyColsInOneRowGetAndPut() throws Exception {
-    HTable table = new HTable(conf, TEST_TABLE);
-
-    List<Row> puts = new ArrayList<Row>();
-    for (int i = 0; i < 100; i++) {
-      Put put = new Put(ONE_ROW);
-      byte[] qual = Bytes.toBytes("column" + i);
-      put.add(BYTES_FAMILY, qual, VALUE);
-      puts.add(put);
-    }
-    Result[] results = table.batch(puts);
-
-    // validate
-    validateSizeAndEmpty(results, 100);
-
-    // get the data back and validate that it is correct
-    List<Row> gets = new ArrayList<Row>();
-    for (int i = 0; i < 100; i++) {
-      Get get = new Get(ONE_ROW);
-      byte[] qual = Bytes.toBytes("column" + i);
-      get.addColumn(BYTES_FAMILY, qual);
-      gets.add(get);
-    }
-
-    Result[] multiRes = table.batch(gets);
-
-    int idx = 0;
-    for (Result r : multiRes) {
-      byte[] qual = Bytes.toBytes("column" + idx);
-      validateResult(r, qual, VALUE);
-      idx++;
-    }
-
-  }
-
-  public void testBatchWithMixedActions() throws Exception {
-    HTable table = new HTable(conf, TEST_TABLE);
-
-    // Load some data to start
-    Result[] results = table.batch(constructPutRequests());
-    validateSizeAndEmpty(results, keys.size());
-
-    // Batch: get, get, put(new col), delete, get, get of put, get of deleted,
-    // put
-    List<Row> actions = new ArrayList<Row>();
-
-    byte[] qual2 = Bytes.toBytes("qual2");
-    byte[] val2 = Bytes.toBytes("putvalue2");
-
-    // 0 get
-    Get get = new Get(keys.get(10));
-    get.addColumn(BYTES_FAMILY, QUALIFIER);
-    actions.add(get);
-
-    // 1 get
-    get = new Get(keys.get(11));
-    get.addColumn(BYTES_FAMILY, QUALIFIER);
-    actions.add(get);
-
-    // 2 put of new column
-    Put put = new Put(keys.get(10));
-    put.add(BYTES_FAMILY, qual2, val2);
-    actions.add(put);
-
-    // 3 delete
-    Delete delete = new Delete(keys.get(20));
-    delete.deleteFamily(BYTES_FAMILY);
-    actions.add(delete);
-
-    // 4 get
-    get = new Get(keys.get(30));
-    get.addColumn(BYTES_FAMILY, QUALIFIER);
-    actions.add(get);
-
-    // 5 get of the put in #2 (entire family)
-    get = new Get(keys.get(10));
-    get.addFamily(BYTES_FAMILY);
-    actions.add(get);
-
-    // 6 get of the delete from #3
-    get = new Get(keys.get(20));
-    get.addColumn(BYTES_FAMILY, QUALIFIER);
-    actions.add(get);
-
-    // 7 put of new column
-    put = new Put(keys.get(40));
-    put.add(BYTES_FAMILY, qual2, val2);
-    actions.add(put);
-
-    results = table.batch(actions);
-
-    // Validation
-
-    validateResult(results[0]);
-    validateResult(results[1]);
-    validateEmpty(results[2]);
-    validateEmpty(results[3]);
-    validateResult(results[4]);
-    validateResult(results[5]);
-    validateResult(results[5], qual2, val2); // testing second column in #5
-    validateEmpty(results[6]); // deleted
-    validateEmpty(results[7]);
-
-    // validate last put, externally from the batch
-    get = new Get(keys.get(40));
-    get.addColumn(BYTES_FAMILY, qual2);
-    Result r = table.get(get);
-    validateResult(r, qual2, val2);
-  }
-
-  // // Helper methods ////
-
-  private void validateResult(Result r) {
-    validateResult(r, QUALIFIER, VALUE);
-  }
-
-  private void validateResult(Result r, byte[] qual, byte[] val) {
-    assertTrue(r.containsColumn(BYTES_FAMILY, qual));
-    assertEquals(0, Bytes.compareTo(val, r.getValue(BYTES_FAMILY, qual)));
-  }
-
-  private List<Row> constructPutRequests() {
-    List<Row> puts = new ArrayList<Row>();
-    for (byte[] k : keys) {
-      Put put = new Put(k);
-      put.add(BYTES_FAMILY, QUALIFIER, VALUE);
-      puts.add(put);
-    }
-    return puts;
-  }
-
-  private void validateLoadedData(HTable table) throws IOException {
-    // get the data back and validate that it is correct
-    for (byte[] k : keys) {
-      Get get = new Get(k);
-      get.addColumn(BYTES_FAMILY, QUALIFIER);
-      Result r = table.get(get);
-      assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
-      assertEquals(0, Bytes.compareTo(VALUE, r
-          .getValue(BYTES_FAMILY, QUALIFIER)));
-    }
-  }
-
-  private void validateEmpty(Result result) {
-    assertTrue(result != null);
-    assertTrue(result.getRow() == null);
-    assertEquals(0, result.raw().length);
-  }
-
-  private void validateSizeAndEmpty(Result[] results, int expectedSize) {
-    // Validate got back the same number of Result objects, all empty
-    assertEquals(expectedSize, results.length);
-    for (Result result : results) {
-      validateEmpty(result);
-    }
-  }
-
-}
diff --git a/src/test/java/org/apache/hadoop/hbase/TestMultiParallelPut.java b/src/test/java/org/apache/hadoop/hbase/TestMultiParallelPut.java
index e69de29bb2d..0004380f3b1 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestMultiParallelPut.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestMultiParallelPut.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class TestMultiParallelPut extends MultiRegionTable {
+  final Log LOG = LogFactory.getLog(getClass());
+  private static final byte[] VALUE = Bytes.toBytes("value");
+  private static final byte[] QUALIFIER = Bytes.toBytes("qual");
+  private static final String FAMILY = "family";
+  private static final String TEST_TABLE = "test_table";
+  private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY);
+
+
+  public TestMultiParallelPut() {
+    super(2, FAMILY);
+    desc = new HTableDescriptor(TEST_TABLE);
+    desc.addFamily(new HColumnDescriptor(FAMILY));
+    makeKeys();
+  }
+
+  private void makeKeys() {
+    for (byte [] k : KEYS) {
+      byte [] cp = new byte[k.length+1];
+      System.arraycopy(k, 0, cp, 0, k.length);
+      cp[k.length] = 1;
+      keys.add(cp);
+    }
+  }
+
+  List<byte[]> keys = new ArrayList<byte[]>();
+
+  public void testParallelPut() throws Exception {
+    LOG.info("Starting testParallelPut");
+    doATest(false);
+  }
+
+  public void testParallelPutWithRSAbort() throws Exception {
+    LOG.info("Starting testParallelPutWithRSAbort");
+    doATest(true);
+  }
+
+  public void doATest(boolean doAbort) throws Exception {
+    conf.setInt("hbase.client.retries.number", 10);
+    HTable table = new HTable(conf, TEST_TABLE);
+    table.setAutoFlush(false);
+    table.setWriteBufferSize(10 * 1024 * 1024);
+    for ( byte [] k : keys ) {
+      Put put = new Put(k);
+      put.add(BYTES_FAMILY, QUALIFIER, VALUE);
+      table.put(put);
+    }
+    table.flushCommits();
+
+    if (doAbort) {
+      LOG.info("Aborting...");
+      cluster.abortRegionServer(0);
+      // try putting more keys after the abort.
+      for ( byte [] k : keys ) {
+        Put put = new Put(k);
+        put.add(BYTES_FAMILY, QUALIFIER, VALUE);
+        table.put(put);
+      }
+      table.flushCommits();
+    }
+
+    for (byte [] k : keys ) {
+      Get get = new Get(k);
+      get.addColumn(BYTES_FAMILY, QUALIFIER);
+      Result r = table.get(get);
+      assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
+      assertEquals(0,
+          Bytes.compareTo(VALUE,
+              r.getValue(BYTES_FAMILY, QUALIFIER)));
+    }
+
+    HBaseAdmin admin = new HBaseAdmin(conf);
+    ClusterStatus cs = admin.getClusterStatus();
+    int expectedServerCount = 2;
+    if (doAbort)  expectedServerCount = 1;
+    LOG.info("Clusterstatus servers count " + cs.getServers());
+    assertEquals(expectedServerCount, cs.getServers());
+    for ( HServerInfo info : cs.getServerInfo()) {
+      LOG.info("Info from clusterstatus=" + info);
+      assertTrue(info.getLoad().getNumberOfRegions() > 8);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
index 23ed1f6bb68..6908111b15c 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.apache.hadoop.hbase.util.Threads;
 
 /**
  * Test whether region rebalancing works. (HBASE-71)
@@ -103,31 +104,40 @@ public class TestRegionRebalancing extends HBaseClusterTestCase {
 
     LOG.debug("Adding 2nd region server.");
     // add a region server - total of 2
-    cluster.startRegionServer();
+    LOG.info("Started=" +
+      cluster.startRegionServer().getRegionServer().getServerName());
+    cluster.getMaster().balance();
     assertRegionsAreBalanced();
 
     // add a region server - total of 3
     LOG.debug("Adding 3rd region server.");
-    cluster.startRegionServer();
+    LOG.info("Started=" +
+      cluster.startRegionServer().getRegionServer().getServerName());
+    cluster.getMaster().balance();
     assertRegionsAreBalanced();
 
     // kill a region server - total of 2
     LOG.debug("Killing the 3rd region server.");
-    cluster.stopRegionServer(2, false);
+    LOG.info("Stopped=" + cluster.stopRegionServer(2, false));
     cluster.waitOnRegionServer(2);
+    cluster.getMaster().balance();
     assertRegionsAreBalanced();
 
     // start two more region servers - total of 4
     LOG.debug("Adding 3rd region server");
-    cluster.startRegionServer();
+    LOG.info("Started=" +
+      cluster.startRegionServer().getRegionServer().getServerName());
     LOG.debug("Adding 4th region server");
-    cluster.startRegionServer();
+    LOG.info("Started=" +
+      cluster.startRegionServer().getRegionServer().getServerName());
+    cluster.getMaster().balance();
     assertRegionsAreBalanced();
 
     for (int i = 0; i < 6; i++){
       LOG.debug("Adding " + (i + 5) + "th region server");
       cluster.startRegionServer();
     }
+    cluster.getMaster().balance();
     assertRegionsAreBalanced();
   }
 
@@ -157,7 +167,7 @@ public class TestRegionRebalancing extends HBaseClusterTestCase {
 
       int regionCount = getRegionCount();
       List<HRegionServer> servers = getOnlineRegionServers();
-      double avg = cluster.getMaster().getAverageLoad();
+      double avg = cluster.getMaster().getServerManager().getAverageLoad();
       int avgLoadPlusSlop = (int)Math.ceil(avg * (1 + slop));
       int avgLoadMinusSlop = (int)Math.floor(avg * (1 - slop)) - 1;
       LOG.debug("There are " + servers.size() + " servers and " + regionCount
@@ -166,10 +176,10 @@ public class TestRegionRebalancing extends HBaseClusterTestCase {
 
       for (HRegionServer server : servers) {
         int serverLoad = server.getOnlineRegions().size();
-        LOG.debug(server.hashCode() + " Avg: " + avg + " actual: " + serverLoad);
+        LOG.debug(server.getServerName() + " Avg: " + avg + " actual: " + serverLoad);
         if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop
             && serverLoad >= avgLoadMinusSlop)) {
-          LOG.debug(server.hashCode() + " Isn't balanced!!! Avg: " + avg +
+          LOG.debug(server.getServerName() + " Isn't balanced!!! Avg: " + avg +
               " actual: " + serverLoad + " slop: " + slop);
           success = false;
         }
diff --git a/src/test/java/org/apache/hadoop/hbase/TestSerialization.java b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
index 98103850888..befcdafb53e 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
@@ -20,6 +20,8 @@
 package org.apache.hadoop.hbase;
 
 
+import static org.junit.Assert.*;
+
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.util.ArrayList;
@@ -44,23 +46,14 @@ import org.apache.hadoop.hbase.io.TimeRange;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.io.DataInputBuffer;
+import org.junit.Test;
 
 /**
  * Test HBase Writables serializations
  */
-public class TestSerialization extends HBaseTestCase {
+public class TestSerialization {
 
-  @Override
-  protected void setUp() throws Exception {
-    super.setUp();
-  }
-
-  @Override
-  protected void tearDown() throws Exception {
-    super.tearDown();
-  }
-
-  public void testCompareFilter() throws Exception {
+  @Test public void testCompareFilter() throws Exception {
     Filter f = new RowFilter(CompareOp.EQUAL,
       new BinaryComparator(Bytes.toBytes("testRowOne-2")));
     byte [] bytes = Writables.getBytes(f);
@@ -68,10 +61,11 @@ public class TestSerialization extends HBaseTestCase {
     assertNotNull(ff);
   }
 
-  public void testKeyValue() throws Exception {
-    byte [] row = Bytes.toBytes(getName());
-    byte [] family = Bytes.toBytes(getName());
-    byte [] qualifier = Bytes.toBytes(getName());
+  @Test public void testKeyValue() throws Exception {
+    final String name = "testKeyValue";
+    byte [] row = Bytes.toBytes(name);
+    byte [] family = Bytes.toBytes(name);
+    byte [] qualifier = Bytes.toBytes(name);
     KeyValue original = new KeyValue(row, family, qualifier);
     byte [] bytes = Writables.getBytes(original);
     KeyValue newone = (KeyValue)Writables.getWritable(bytes, new KeyValue());
@@ -79,7 +73,7 @@ public class TestSerialization extends HBaseTestCase {
   }
 
   @SuppressWarnings("unchecked")
-  public void testHbaseMapWritable() throws Exception {
+  @Test public void testHbaseMapWritable() throws Exception {
     HbaseMapWritable<byte [], byte []> hmw =
       new HbaseMapWritable<byte[], byte[]>();
     hmw.put("key".getBytes(), "value".getBytes());
@@ -90,13 +84,14 @@ public class TestSerialization extends HBaseTestCase {
     assertTrue(Bytes.equals("value".getBytes(), hmw.get("key".getBytes())));
   }
 
-  public void testHMsg() throws Exception {
-    HMsg  m = new HMsg(HMsg.Type.MSG_REGIONSERVER_QUIESCE);
+  @Test public void testHMsg() throws Exception {
+    final String name = "testHMsg";
+    HMsg  m = new HMsg(HMsg.Type.STOP_REGIONSERVER);
     byte [] mb = Writables.getBytes(m);
     HMsg deserializedHMsg = (HMsg)Writables.getWritable(mb, new HMsg());
     assertTrue(m.equals(deserializedHMsg));
-    m = new HMsg(HMsg.Type.MSG_REGIONSERVER_QUIESCE,
-      new HRegionInfo(new HTableDescriptor(getName()),
+    m = new HMsg(HMsg.Type.STOP_REGIONSERVER,
+      new HRegionInfo(new HTableDescriptor(name),
         HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
         "Some message".getBytes());
     mb = Writables.getBytes(m);
@@ -104,8 +99,9 @@ public class TestSerialization extends HBaseTestCase {
     assertTrue(m.equals(deserializedHMsg));
   }
 
-  public void testTableDescriptor() throws Exception {
-    HTableDescriptor htd = createTableDescriptor(getName());
+  @Test public void testTableDescriptor() throws Exception {
+    final String name = "testTableDescriptor";
+    HTableDescriptor htd = createTableDescriptor(name);
     byte [] mb = Writables.getBytes(htd);
     HTableDescriptor deserializedHtd =
       (HTableDescriptor)Writables.getWritable(mb, new HTableDescriptor());
@@ -116,8 +112,9 @@ public class TestSerialization extends HBaseTestCase {
    * Test RegionInfo serialization
    * @throws Exception
    */
-  public void testRegionInfo() throws Exception {
-    HTableDescriptor htd = new HTableDescriptor(getName());
+  @Test public void testRegionInfo() throws Exception {
+    final String name = "testRegionInfo";
+    HTableDescriptor htd = new HTableDescriptor(name);
     String [] families = new String [] {"info", "anchor"};
     for (int i = 0; i < families.length; i++) {
       htd.addFamily(new HColumnDescriptor(families[i]));
@@ -136,7 +133,7 @@ public class TestSerialization extends HBaseTestCase {
    * Test ServerInfo serialization
    * @throws Exception
    */
-  public void testServerInfo() throws Exception {
+  @Test public void testServerInfo() throws Exception {
     HServerInfo hsi = new HServerInfo(new HServerAddress("0.0.0.0:123"), -1,
       1245, "default name");
     byte [] b = Writables.getBytes(hsi);
@@ -145,7 +142,7 @@ public class TestSerialization extends HBaseTestCase {
     assertTrue(hsi.equals(deserializedHsi));
   }
 
-  public void testPut() throws Exception{
+  @Test public void testPut() throws Exception{
     byte[] row = "row".getBytes();
     byte[] fam = "fam".getBytes();
     byte[] qf1 = "qf1".getBytes();
@@ -193,7 +190,7 @@ public class TestSerialization extends HBaseTestCase {
   }
 
 
-  public void testPut2() throws Exception{
+  @Test public void testPut2() throws Exception{
     byte[] row = "testAbort,,1243116656250".getBytes();
     byte[] fam = "historian".getBytes();
     byte[] qf1 = "creation".getBytes();
@@ -221,7 +218,7 @@ public class TestSerialization extends HBaseTestCase {
   }
 
 
-  public void testDelete() throws Exception{
+  @Test public void testDelete() throws Exception{
     byte[] row = "row".getBytes();
     byte[] fam = "fam".getBytes();
     byte[] qf1 = "qf1".getBytes();
@@ -248,7 +245,7 @@ public class TestSerialization extends HBaseTestCase {
     }
   }
 
-  public void testGet() throws Exception{
+  @Test public void testGet() throws Exception{
     byte[] row = "row".getBytes();
     byte[] fam = "fam".getBytes();
     byte[] qf1 = "qf1".getBytes();
@@ -289,7 +286,8 @@ public class TestSerialization extends HBaseTestCase {
   }
 
 
-  public void testScan() throws Exception{
+  @Test public void testScan() throws Exception {
+    
     byte[] startRow = "startRow".getBytes();
     byte[] stopRow  = "stopRow".getBytes();
     byte[] fam = "fam".getBytes();
@@ -323,7 +321,8 @@ public class TestSerialization extends HBaseTestCase {
 
       // Test filters are serialized properly.
       scan = new Scan(startRow);
-      byte [] prefix = Bytes.toBytes(getName());
+      final String name = "testScan";
+      byte [] prefix = Bytes.toBytes(name);
       scan.setFilter(new PrefixFilter(prefix));
       sb = Writables.getBytes(scan);
       desScan = (Scan)Writables.getWritable(sb, new Scan());
@@ -338,7 +337,7 @@ public class TestSerialization extends HBaseTestCase {
     assertEquals(tr.getMin(), desTr.getMin());
   }
 
-  public void testResultEmpty() throws Exception {
+  @Test public void testResultEmpty() throws Exception {
     List<KeyValue> keys = new ArrayList<KeyValue>();
     Result r = new Result(keys);
     assertTrue(r.isEmpty());
@@ -348,7 +347,7 @@ public class TestSerialization extends HBaseTestCase {
   }
 
 
-  public void testResult() throws Exception {
+  @Test public void testResult() throws Exception {
     byte [] rowA = Bytes.toBytes("rowA");
     byte [] famA = Bytes.toBytes("famA");
     byte [] qfA = Bytes.toBytes("qfA");
@@ -380,7 +379,7 @@ public class TestSerialization extends HBaseTestCase {
     assertEquals(r.size(), deserialized.size());
   }
 
-  public void testResultDynamicBuild() throws Exception {
+  @Test public void testResultDynamicBuild() throws Exception {
     byte [] rowA = Bytes.toBytes("rowA");
     byte [] famA = Bytes.toBytes("famA");
     byte [] qfA = Bytes.toBytes("qfA");
@@ -421,7 +420,7 @@ public class TestSerialization extends HBaseTestCase {
 
   }
 
-  public void testResultArray() throws Exception {
+  @Test public void testResultArray() throws Exception {
     byte [] rowA = Bytes.toBytes("rowA");
     byte [] famA = Bytes.toBytes("famA");
     byte [] qfA = Bytes.toBytes("qfA");
@@ -470,7 +469,7 @@ public class TestSerialization extends HBaseTestCase {
 
   }
 
-  public void testResultArrayEmpty() throws Exception {
+  @Test public void testResultArrayEmpty() throws Exception {
     List<KeyValue> keys = new ArrayList<KeyValue>();
     Result r = new Result(keys);
     Result [] results = new Result [] {r};
@@ -512,7 +511,7 @@ public class TestSerialization extends HBaseTestCase {
 
   }
 
-  public void testTimeRange(String[] args) throws Exception{
+  @Test public void testTimeRange() throws Exception{
     TimeRange tr = new TimeRange(0,5);
     byte [] mb = Writables.getBytes(tr);
     TimeRange deserializedTr =
@@ -523,8 +522,9 @@ public class TestSerialization extends HBaseTestCase {
 
   }
 
-  public void testKeyValue2() throws Exception {
-    byte[] row = getName().getBytes();
+  @Test public void testKeyValue2() throws Exception {
+    final String name = "testKeyValue2";
+    byte[] row = name.getBytes();
     byte[] fam = "fam".getBytes();
     byte[] qf = "qf".getBytes();
     long ts = System.currentTimeMillis();
@@ -539,4 +539,48 @@ public class TestSerialization extends HBaseTestCase {
     assertEquals(kv.getOffset(), deserializedKv.getOffset());
     assertEquals(kv.getLength(), deserializedKv.getLength());
   }
+
+  protected static final int MAXVERSIONS = 3;
+  protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
+  protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
+  protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
+  protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
+
+  /**
+   * Create a table of name <code>name</code> with {@link COLUMNS} for
+   * families.
+   * @param name Name to give table.
+   * @return Column descriptor.
+   */
+  protected HTableDescriptor createTableDescriptor(final String name) {
+    return createTableDescriptor(name, MAXVERSIONS);
+  }
+
+  /**
+   * Create a table of name <code>name</code> with {@link COLUMNS} for
+   * families.
+   * @param name Name to give table.
+   * @param versions How many versions to allow per column.
+   * @return Column descriptor.
+   */
+  protected HTableDescriptor createTableDescriptor(final String name,
+      final int versions) {
+    HTableDescriptor htd = new HTableDescriptor(name);
+    htd.addFamily(new HColumnDescriptor(fam1, versions,
+      HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
+      Integer.MAX_VALUE, HConstants.FOREVER,
+      HColumnDescriptor.DEFAULT_BLOOMFILTER,
+      HConstants.REPLICATION_SCOPE_LOCAL));
+    htd.addFamily(new HColumnDescriptor(fam2, versions,
+        HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
+        Integer.MAX_VALUE, HConstants.FOREVER,
+        HColumnDescriptor.DEFAULT_BLOOMFILTER,
+        HConstants.REPLICATION_SCOPE_LOCAL));
+    htd.addFamily(new HColumnDescriptor(fam3, versions,
+        HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
+        Integer.MAX_VALUE,  HConstants.FOREVER,
+        HColumnDescriptor.DEFAULT_BLOOMFILTER,
+        HConstants.REPLICATION_SCOPE_LOCAL));
+    return htd;
+  }
 }
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
index 4c59ea2bd37..a07155a970b 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
+++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
@@ -19,6 +19,13 @@
  */
 package org.apache.hadoop.hbase;
 
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -29,22 +36,16 @@ import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooKeeper;
-import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import java.io.IOException;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
 public class TestZooKeeper {
   private final Log LOG = LogFactory.getLog(this.getClass());
 
@@ -91,15 +92,12 @@ public class TestZooKeeper {
       throws IOException, InterruptedException {
     new HTable(conf, HConstants.META_TABLE_NAME);
 
-    ZooKeeperWrapper zkw =
-        ZooKeeperWrapper.createInstance(conf, TestZooKeeper.class.getName());
-    zkw.registerListener(EmptyWatcher.instance);
-    String quorumServers = zkw.getQuorumServers();
+    String quorumServers = ZKConfig.getZKQuorumServersString(conf);
     int sessionTimeout = 5 * 1000; // 5 seconds
     HConnection connection = HConnectionManager.getConnection(conf);
-    ZooKeeperWrapper connectionZK = connection.getZooKeeperWrapper();
-    long sessionID = connectionZK.getSessionID();
-    byte[] password = connectionZK.getSessionPassword();
+    ZooKeeperWatcher connectionZK = connection.getZooKeeperWatcher();
+    long sessionID = connectionZK.getZooKeeper().getSessionId();
+    byte[] password = connectionZK.getZooKeeper().getSessionPasswd();
 
     ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout,
         EmptyWatcher.instance, sessionID, password);
@@ -110,15 +108,18 @@ public class TestZooKeeper {
     System.err.println("ZooKeeper should have timed out");
     connection.relocateRegion(HConstants.ROOT_TABLE_NAME, HConstants.EMPTY_BYTE_ARRAY);
   }
+  
   @Test
-  public void testRegionServerSessionExpired() throws Exception{
+  public void testRegionServerSessionExpired() throws Exception {
     LOG.info("Starting testRegionServerSessionExpired");
     new HTable(conf, HConstants.META_TABLE_NAME);
-    TEST_UTIL.expireRegionServerSession(0);
+    int metaIndex = TEST_UTIL.getMiniHBaseCluster().getServerWithMeta();
+    TEST_UTIL.expireRegionServerSession(metaIndex);
     testSanity();
   }
-  @Test
-  public void testMasterSessionExpired() throws Exception {
+
+  //@Test
+  public void disabledTestMasterSessionExpired() throws Exception {
     LOG.info("Starting testMasterSessionExpired");
     new HTable(conf, HConstants.META_TABLE_NAME);
     TEST_UTIL.expireMasterSession();
@@ -133,13 +134,12 @@ public class TestZooKeeper {
 
     HBaseAdmin admin = new HBaseAdmin(conf);
     String tableName = "test"+System.currentTimeMillis();
-    HTableDescriptor desc =
-        new HTableDescriptor(tableName);
+    HTableDescriptor desc = new HTableDescriptor(tableName);
     HColumnDescriptor family = new HColumnDescriptor("fam");
     desc.addFamily(family);
     admin.createTable(desc);
 
-    HTable table = new HTable(tableName);
+    HTable table = new HTable(conf, tableName);
     Put put = new Put(Bytes.toBytes("testrow"));
     put.add(Bytes.toBytes("fam"),
         Bytes.toBytes("col"), Bytes.toBytes("testdata"));
@@ -160,12 +160,11 @@ public class TestZooKeeper {
       ipMeta.exists(new Get(HConstants.LAST_ROW));
 
       // make sure they aren't the same
-      assertFalse(HConnectionManager.getClientZooKeeperWatcher(conf)
-          .getZooKeeperWrapper() == HConnectionManager.getClientZooKeeperWatcher(
-          otherConf).getZooKeeperWrapper());
+      assertFalse(HConnectionManager.getConnection(conf).getZooKeeperWatcher()
+          == HConnectionManager.getConnection(otherConf).getZooKeeperWatcher());
       assertFalse(HConnectionManager.getConnection(conf)
-          .getZooKeeperWrapper().getQuorumServers().equals(HConnectionManager
-          .getConnection(otherConf).getZooKeeperWrapper().getQuorumServers()));
+          .getZooKeeperWatcher().getQuorum().equals(HConnectionManager
+              .getConnection(otherConf).getZooKeeperWatcher().getQuorum()));
     } catch (Exception e) {
       e.printStackTrace();
       fail();
@@ -179,19 +178,18 @@ public class TestZooKeeper {
    */
   @Test
   public void testZNodeDeletes() throws Exception {
-    ZooKeeperWrapper zkw =
-        ZooKeeperWrapper.createInstance(conf, TestZooKeeper.class.getName());
-    zkw.registerListener(EmptyWatcher.instance);
-    zkw.ensureExists("/l1/l2/l3/l4");
+    ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, 
+        TestZooKeeper.class.getName(), null);
+    ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4");
     try {
-      zkw.deleteZNode("/l1/l2");
+      ZKUtil.deleteNode(zkw, "/l1/l2");
       fail("We should not be able to delete if znode has childs");
     } catch (KeeperException ex) {
-      assertNotNull(zkw.getData("/l1/l2/l3", "l4"));
+      assertNotNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
     }
-    zkw.deleteZNode("/l1/l2", true);
-    assertNull(zkw.getData("/l1/l2/l3", "l4"));
-    zkw.deleteZNode("/l1");
-    assertNull(zkw.getData("/l1", "l2"));
+    ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
+    assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
+    ZKUtil.deleteNode(zkw, "/l1");
+    assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2", null));
   }
 }
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java b/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java
index 3e293b67517..e7c588edd54 100644
--- a/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java
@@ -22,30 +22,23 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hbase.HBaseClusterTestCase;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
 
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericArray;
 import org.apache.avro.generic.GenericData;
-
+import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.avro.generated.AColumn;
 import org.apache.hadoop.hbase.avro.generated.AColumnValue;
 import org.apache.hadoop.hbase.avro.generated.AFamilyDescriptor;
 import org.apache.hadoop.hbase.avro.generated.AGet;
 import org.apache.hadoop.hbase.avro.generated.APut;
-import org.apache.hadoop.hbase.avro.generated.AResult;
 import org.apache.hadoop.hbase.avro.generated.ATableDescriptor;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
 
 /**
  * Unit testing for AvroServer.HBaseImpl, a part of the
@@ -129,7 +122,7 @@ public class TestAvroServer {
 
     tableA.maxFileSize = 123456L;
     impl.modifyTable(tableAname, tableA);
-    assertEquals((long) impl.describeTable(tableAname).maxFileSize, 123456L);
+    assertEquals(123456L, (long) impl.describeTable(tableAname).maxFileSize);
 
     impl.enableTable(tableAname);
     assertTrue(impl.isTableEnabled(tableAname));
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
index c9b78b9cdfa..40309276579 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
@@ -21,12 +21,13 @@ package org.apache.hadoop.hbase.client;
 
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
@@ -37,9 +38,14 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.TableExistsException;
 import org.apache.hadoop.hbase.TableNotDisabledException;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -86,7 +92,173 @@ public class TestAdmin {
   }
 
   @Test
-  public void testCreateTableWithRegions() throws IOException {
+  public void testGetTableDescriptor() throws IOException {
+    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
+    HColumnDescriptor fam2 = new HColumnDescriptor("fam2");
+    HColumnDescriptor fam3 = new HColumnDescriptor("fam3");
+    HTableDescriptor htd = new HTableDescriptor("myTestTable");
+    htd.addFamily(fam1);
+    htd.addFamily(fam2);
+    htd.addFamily(fam3);
+    this.admin.createTable(htd);
+    HTable table = new HTable(TEST_UTIL.getConfiguration(), "myTestTable");
+    HTableDescriptor confirmedHtd = table.getTableDescriptor();
+    assertEquals(htd.compareTo(confirmedHtd), 0);
+  }
+
+  /**
+   * Verify schema modification takes.
+   * @throws IOException
+   */
+  @Test public void testChangeTableSchema() throws IOException {
+    final byte [] tableName = Bytes.toBytes("changeTableSchema");
+    HTableDescriptor [] tables = admin.listTables();
+    int numTables = tables.length;
+    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
+    tables = this.admin.listTables();
+    assertEquals(numTables + 1, tables.length);
+
+    // FIRST, do htabledescriptor changes.
+    HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
+    // Make a copy and assert copy is good.
+    HTableDescriptor copy = new HTableDescriptor(htd);
+    assertTrue(htd.equals(copy));
+    // Now amend the copy. Introduce differences.
+    long newFlushSize = htd.getMemStoreFlushSize() / 2;
+    copy.setMemStoreFlushSize(newFlushSize);
+    final String key = "anyoldkey";
+    assertTrue(htd.getValue(key) == null);
+    copy.setValue(key, key);
+    boolean expectedException = false;
+    try {
+      this.admin.modifyTable(tableName, copy);
+    } catch (TableNotDisabledException re) {
+      expectedException = true;
+    }
+    assertTrue(expectedException);
+    this.admin.disableTable(tableName);
+    assertTrue(this.admin.isTableDisabled(tableName));
+    modifyTable(tableName, copy);
+    HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName);
+    // Assert returned modifiedhcd is same as the copy.
+    assertFalse(htd.equals(modifiedHtd));
+    assertTrue(copy.equals(modifiedHtd));
+    assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize());
+    assertEquals(key, modifiedHtd.getValue(key));
+
+    // Reenable table to test it fails if not disabled.
+    this.admin.enableTable(tableName);
+    assertFalse(this.admin.isTableDisabled(tableName));
+
+    // Now work on column family changes.
+    int countOfFamilies = modifiedHtd.getFamilies().size();
+    assertTrue(countOfFamilies > 0);
+    HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next();
+    int maxversions = hcd.getMaxVersions();
+    final int newMaxVersions = maxversions + 1;
+    hcd.setMaxVersions(newMaxVersions);
+    final byte [] hcdName = hcd.getName();
+    expectedException = false;
+    try {
+      this.admin.modifyColumn(tableName, hcd);
+    } catch (TableNotDisabledException re) {
+      expectedException = true;
+    }
+    assertTrue(expectedException);
+    this.admin.disableTable(tableName);
+    assertTrue(this.admin.isTableDisabled(tableName));
+    // Modify Column is synchronous
+    this.admin.modifyColumn(tableName, hcd);
+    modifiedHtd = this.admin.getTableDescriptor(tableName);
+    HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName);
+    assertEquals(newMaxVersions, modifiedHcd.getMaxVersions());
+
+    // Try adding a column
+    // Reenable table to test it fails if not disabled.
+    this.admin.enableTable(tableName);
+    assertFalse(this.admin.isTableDisabled(tableName));
+    final String xtracolName = "xtracol";
+    HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName);
+    xtracol.setValue(xtracolName, xtracolName);
+    try {
+      this.admin.addColumn(tableName, xtracol);
+    } catch (TableNotDisabledException re) {
+      expectedException = true;
+    }
+    assertTrue(expectedException);
+    this.admin.disableTable(tableName);
+    assertTrue(this.admin.isTableDisabled(tableName));
+    this.admin.addColumn(tableName, xtracol);
+    modifiedHtd = this.admin.getTableDescriptor(tableName);
+    hcd = modifiedHtd.getFamily(xtracol.getName());
+    assertTrue(hcd != null);
+    assertTrue(hcd.getValue(xtracolName).equals(xtracolName));
+
+    // Delete the just-added column.
+    this.admin.deleteColumn(tableName, xtracol.getName());
+    modifiedHtd = this.admin.getTableDescriptor(tableName);
+    hcd = modifiedHtd.getFamily(xtracol.getName());
+    assertTrue(hcd == null);
+
+    // Delete the table
+    this.admin.deleteTable(tableName);
+    this.admin.listTables();
+    assertFalse(this.admin.tableExists(tableName));
+  }
+
+  /**
+   * Modify table is async so wait on completion of the table operation in master.
+   * @param tableName
+   * @param htd
+   * @throws IOException
+   */
+  private void modifyTable(final byte [] tableName, final HTableDescriptor htd)
+  throws IOException {
+    MasterServices services = TEST_UTIL.getMiniHBaseCluster().getMaster();
+    ExecutorService executor = services.getExecutorService();
+    AtomicBoolean done = new AtomicBoolean(false);
+    executor.registerListener(EventType.C2M_MODIFY_TABLE, new DoneListener(done));
+    this.admin.modifyTable(tableName, htd);
+    while (!done.get()) {
+      synchronized (done) {
+        try {
+          done.wait(1000);
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+        }
+      }
+    }
+    executor.unregisterListener(EventType.C2M_MODIFY_TABLE);
+  }
+
+  /**
+   * Listens for when an event is done in Master.
+   */
+  static class DoneListener implements EventHandler.EventHandlerListener {
+    private final AtomicBoolean done;
+
+    DoneListener(final AtomicBoolean done) {
+      super();
+      this.done = done;
+    }
+
+    @Override
+    public void afterProcess(EventHandler event) {
+      this.done.set(true);
+      synchronized (this.done) {
+        // Wake anyone waiting on this value to change.
+        this.done.notifyAll();
+      }
+    }
+
+    @Override
+    public void beforeProcess(EventHandler event) {
+      // continue
+    }
+  }
+
+  @Test
+  public void testCreateTableWithRegions() throws IOException, InterruptedException {
 
     byte[] tableName = Bytes.toBytes("testCreateTableWithRegions");
 
@@ -255,19 +427,24 @@ public class TestAdmin {
     Put put = new Put(row);
     put.add(HConstants.CATALOG_FAMILY, qualifier, value);
     ht.put(put);
+    Get get = new Get(row);
+    get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
+    ht.get(get);
 
     this.admin.disableTable(table);
 
     // Test that table is disabled
-    Get get = new Get(row);
+    get = new Get(row);
     get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
     boolean ok = false;
     try {
       ht.get(get);
+    } catch (NotServingRegionException e) {
+      ok = true;
     } catch (RetriesExhaustedException e) {
       ok = true;
     }
-    assertEquals(true, ok);
+    assertTrue(ok);
     this.admin.enableTable(table);
 
     //Test that table is enabled
@@ -276,7 +453,7 @@ public class TestAdmin {
     } catch (RetriesExhaustedException e) {
       ok = false;
     }
-    assertEquals(true, ok);
+    assertTrue(ok);
   }
 
   @Test
@@ -361,8 +538,8 @@ public class TestAdmin {
       }
     };
     t.start();
-    // tell the master to split the table
-    admin.split(Bytes.toString(tableName));
+    // Split the table
+    this.admin.split(Bytes.toString(tableName));
     t.join();
 
     // Verify row count
@@ -571,26 +748,13 @@ public class TestAdmin {
     for(int i = 0; i < times; i++) {
       String tableName = "table"+i;
       this.admin.disableTable(tableName);
+      byte [] tableNameBytes = Bytes.toBytes(tableName);
+      assertTrue(this.admin.isTableDisabled(tableNameBytes));
       this.admin.enableTable(tableName);
+      assertFalse(this.admin.isTableDisabled(tableNameBytes));
       this.admin.disableTable(tableName);
+      assertTrue(this.admin.isTableDisabled(tableNameBytes));
       this.admin.deleteTable(tableName);
     }
   }
-
-  @Test
-  public void testGetTableDescriptor() throws IOException {
-    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
-    HColumnDescriptor fam2 = new HColumnDescriptor("fam2");
-    HColumnDescriptor fam3 = new HColumnDescriptor("fam3");
-    HTableDescriptor htd = new HTableDescriptor("myTestTable");
-    htd.addFamily(fam1);
-    htd.addFamily(fam2);
-    htd.addFamily(fam3);
-    this.admin.createTable(htd);
-    HTable table = new HTable("myTestTable");
-    HTableDescriptor confirmedHtd = table.getTableDescriptor();
-
-    assertEquals(htd.compareTo(confirmedHtd), 0);
-  }
 }
-
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index fc7b2afa713..8ba0a94d4b2 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -25,9 +25,9 @@ import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import java.io.File;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -41,7 +41,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
@@ -51,6 +50,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.PrefixFilter;
@@ -59,7 +59,6 @@ import org.apache.hadoop.hbase.filter.RegexStringComparator;
 import org.apache.hadoop.hbase.filter.RowFilter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -262,9 +261,11 @@ public class TestFromClientSide {
    * logs to ensure that we're not scanning more regions that we're supposed to.
    * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
    * @throws IOException
+   * @throws InterruptedException 
    */
   @Test
-  public void testFilterAcrossMutlipleRegions() throws IOException {
+  public void testFilterAcrossMultipleRegions()
+  throws IOException, InterruptedException {
     byte [] name = Bytes.toBytes("testFilterAcrossMutlipleRegions");
     HTable t = TEST_UTIL.createTable(name, FAMILY);
     int rowCount = TEST_UTIL.loadTable(t, FAMILY);
@@ -314,32 +315,6 @@ public class TestFromClientSide {
       CompareFilter.CompareOp.GREATER_OR_EQUAL));
     assertEquals(rowCount - endKeyCount, countGreater);
   }
-  
-  /*
-   * Load table with rows from 'aaa' to 'zzz'.
-   * @param t
-   * @return Count of rows loaded.
-   * @throws IOException
-   */
-  private int loadTable(final HTable t) throws IOException {
-    // Add data to table.
-    byte[] k = new byte[3];
-    int rowCount = 0;
-    for (byte b1 = 'a'; b1 < 'z'; b1++) {
-      for (byte b2 = 'a'; b2 < 'z'; b2++) {
-        for (byte b3 = 'a'; b3 < 'z'; b3++) {
-          k[0] = b1;
-          k[1] = b2;
-          k[2] = b3;
-          Put put = new Put(k);
-          put.add(FAMILY, new byte[0], k);
-          t.put(put);
-          rowCount++;
-        }
-      }
-    }
-    return rowCount;
-  }
 
   /*
    * @param key
@@ -399,7 +374,7 @@ public class TestFromClientSide {
    * @throws IOException
    */
   private Map<HRegionInfo, HServerAddress> splitTable(final HTable t)
-  throws IOException {
+  throws IOException, InterruptedException {
     // Split this table in two.
     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
     admin.split(t.getTableName());
@@ -3376,7 +3351,7 @@ public class TestFromClientSide {
   }
 
   @Test
-  public void testListTables() throws IOException {
+  public void testListTables() throws IOException, InterruptedException {
     byte [] t1 = Bytes.toBytes("testListTables1");
     byte [] t2 = Bytes.toBytes("testListTables2");
     byte [] t3 = Bytes.toBytes("testListTables3");
@@ -3459,7 +3434,7 @@ public class TestFromClientSide {
     for (HColumnDescriptor c : desc.getFamilies())
       c.setValue(attrName, attrValue);
     // update metadata for all regions of this table
-    admin.modifyTable(tableAname, HConstants.Modify.TABLE_SET_HTD, desc);
+    admin.modifyTable(tableAname, desc);
     // enable the table
     admin.enableTable(tableAname);
 
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
index 3818892b122..386eb7b5e30 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
@@ -247,12 +247,12 @@ public class TestHFileOutputFormat  {
     }
     return ret;
   }
-  
+
   @Test
   public void testMRIncrementalLoad() throws Exception {
     doIncrementalLoadTest(false);
   }
-  
+
   @Test
   public void testMRIncrementalLoadWithSplit() throws Exception {
     doIncrementalLoadTest(true);
@@ -308,13 +308,12 @@ public class TestHFileOutputFormat  {
             
       // Cause regions to reopen
       admin.disableTable(TABLE_NAME);
-      while (table.getRegionsInfo().size() != 0) {
+      while (!admin.isTableDisabled(TABLE_NAME)) {
         Thread.sleep(1000);
         LOG.info("Waiting for table to disable"); 
       }
       admin.enableTable(TABLE_NAME);
       util.waitTableAvailable(TABLE_NAME, 30000);
-      
       assertEquals("Data should remain after reopening of regions",
           tableDigestBefore, util.checksumRows(table));
     } finally {
@@ -322,9 +321,7 @@ public class TestHFileOutputFormat  {
       util.shutdownMiniCluster();
     }
   }
-  
-  
-  
+
   private void runIncrementalPELoad(
       Configuration conf, HTable table, Path outDir)
   throws Exception {
diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index e02d11a2fbd..1f0eb94ddbd 100644
--- a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -112,7 +112,7 @@ public class TestLoadIncrementalHFiles {
       htd.addFamily(new HColumnDescriptor(FAMILY));
       admin.createTable(htd, SPLIT_KEYS);
 
-      HTable table = new HTable(TABLE);
+      HTable table = new HTable(util.getConfiguration(), TABLE);
       util.waitTableAvailable(TABLE, 30000);
       LoadIncrementalHFiles loader = new LoadIncrementalHFiles(
           util.getConfiguration());
diff --git a/src/test/java/org/apache/hadoop/hbase/master/OOMEHMaster.java b/src/test/java/org/apache/hadoop/hbase/master/OOMEHMaster.java
index 607a2f00478..145c38fab9a 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/OOMEHMaster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/OOMEHMaster.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HMsg;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.zookeeper.KeeperException;
 
 /**
  * An HMaster that runs out of memory.
@@ -37,7 +38,8 @@ import org.apache.hadoop.hbase.HServerInfo;
 public class OOMEHMaster extends HMaster {
   private List<byte []> retainer = new ArrayList<byte[]>();
 
-  public OOMEHMaster(HBaseConfiguration conf) throws IOException {
+  public OOMEHMaster(HBaseConfiguration conf)
+  throws IOException, KeeperException, InterruptedException {
     super(conf);
   }
 
@@ -52,8 +54,9 @@ public class OOMEHMaster extends HMaster {
 
   /**
    * @param args
+   * @throws IOException 
    */
-  public static void main(String[] args) {
+  public static void main(String[] args) throws IOException {
     doMain(args, OOMEHMaster.class);
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
new file mode 100644
index 00000000000..030bc127c4c
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
@@ -0,0 +1,234 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.Semaphore;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the {@link ActiveMasterManager}.
+ */
+public class TestActiveMasterManager {
+  private final static Log LOG = LogFactory.getLog(TestActiveMasterManager.class);
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniZKCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniZKCluster();
+  }
+
+  /**
+   * Unit tests that uses ZooKeeper but does not use the master-side methods
+   * but rather acts directly on ZK.
+   * @throws Exception
+   */
+  @Test
+  public void testActiveMasterManagerFromZK() throws Exception {
+
+    ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+        "testActiveMasterManagerFromZK", null);
+    ZKUtil.createAndFailSilent(zk, zk.baseZNode);
+    try {
+      ZKUtil.deleteNode(zk, zk.masterAddressZNode);
+    } catch(KeeperException.NoNodeException nne) {}
+
+    // Create the master node with a dummy address
+    HServerAddress firstMasterAddress = new HServerAddress("firstMaster", 1234);
+    HServerAddress secondMasterAddress = new HServerAddress("secondMaster", 1234);
+
+    // Should not have a master yet
+    DummyMaster ms1 = new DummyMaster();
+    ActiveMasterManager activeMasterManager = new ActiveMasterManager(zk,
+        firstMasterAddress, ms1);
+    zk.registerListener(activeMasterManager);
+    assertFalse(activeMasterManager.clusterHasActiveMaster.get());
+
+    // First test becoming the active master uninterrupted
+    activeMasterManager.blockUntilBecomingActiveMaster();
+    assertTrue(activeMasterManager.clusterHasActiveMaster.get());
+    assertMaster(zk, firstMasterAddress);
+
+    // New manager will now try to become the active master in another thread
+    WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
+    zk.registerListener(t.manager);
+    t.start();
+    // Wait for this guy to figure out there is another active master
+    // Wait for 1 second at most
+    int sleeps = 0;
+    while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
+      Thread.sleep(10);
+      sleeps++;
+    }
+
+    // Both should see that there is an active master
+    assertTrue(activeMasterManager.clusterHasActiveMaster.get());
+    assertTrue(t.manager.clusterHasActiveMaster.get());
+    // But secondary one should not be the active master
+    assertFalse(t.isActiveMaster);
+
+    // Close the first server and delete it's master node
+    ms1.stop("stopping first server");
+
+    // Use a listener to capture when the node is actually deleted
+    NodeDeletionListener listener = new NodeDeletionListener(zk, zk.masterAddressZNode);
+    zk.registerListener(listener);
+
+    LOG.info("Deleting master node");
+    ZKUtil.deleteNode(zk, zk.masterAddressZNode);
+
+    // Wait for the node to be deleted
+    LOG.info("Waiting for active master manager to be notified");
+    listener.waitForDeletion();
+    LOG.info("Master node deleted");
+
+    // Now we expect the secondary manager to have and be the active master
+    // Wait for 1 second at most
+    sleeps = 0;
+    while(!t.isActiveMaster && sleeps < 100) {
+      Thread.sleep(10);
+      sleeps++;
+    }
+    LOG.debug("Slept " + sleeps + " times");
+
+    assertTrue(t.manager.clusterHasActiveMaster.get());
+    assertTrue(t.isActiveMaster);
+  }
+
+  /**
+   * Assert there is an active master and that it has the specified address.
+   * @param zk
+   * @param thisMasterAddress
+   * @throws KeeperException
+   */
+  private void assertMaster(ZooKeeperWatcher zk,
+      HServerAddress expectedAddress) throws KeeperException {
+    HServerAddress readAddress = ZKUtil.getDataAsAddress(zk, zk.masterAddressZNode);
+    assertNotNull(readAddress);
+    assertTrue(expectedAddress.equals(readAddress));
+  }
+
+  public static class WaitToBeMasterThread extends Thread {
+
+    ActiveMasterManager manager;
+    boolean isActiveMaster;
+
+    public WaitToBeMasterThread(ZooKeeperWatcher zk,
+        HServerAddress address) {
+      this.manager = new ActiveMasterManager(zk, address,
+          new DummyMaster());
+      isActiveMaster = false;
+    }
+
+    @Override
+    public void run() {
+      manager.blockUntilBecomingActiveMaster();
+      LOG.info("Second master has become the active master!");
+      isActiveMaster = true;
+    }
+  }
+
+  public static class NodeDeletionListener extends ZooKeeperListener {
+    private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
+
+    private Semaphore lock;
+    private String node;
+
+    public NodeDeletionListener(ZooKeeperWatcher watcher, String node) {
+      super(watcher);
+      lock = new Semaphore(0);
+      this.node = node;
+    }
+
+    @Override
+    public void nodeDeleted(String path) {
+      if(path.equals(node)) {
+        LOG.debug("nodeDeleted(" + path + ")");
+        lock.release();
+      }
+    }
+
+    public void waitForDeletion() throws InterruptedException {
+      lock.acquire();
+    }
+  }
+
+  /**
+   * Dummy Master Implementation.
+   */
+  public static class DummyMaster implements Server {
+    private volatile boolean stopped;
+
+    @Override
+    public void abort(final String msg, final Throwable t) {}
+
+    @Override
+    public Configuration getConfiguration() {
+      return null;
+    }
+
+    @Override
+    public ZooKeeperWatcher getZooKeeper() {
+      return null;
+    }
+
+    @Override
+    public String getServerName() {
+      return null;
+    }
+
+    @Override
+    public boolean isStopped() {
+      return this.stopped;
+    }
+
+    @Override
+    public void stop(String why) {
+      this.stopped = true;
+    }
+
+    @Override
+    public CatalogTracker getCatalogTracker() {
+      return null;
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestServerManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java
similarity index 71%
rename from src/test/java/org/apache/hadoop/hbase/master/TestServerManager.java
rename to src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java
index d017e0397f9..03a55b96064 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestServerManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java
@@ -25,16 +25,17 @@ import java.util.Set;
 import org.junit.Test;
 
 
-public class TestServerManager {
+public class TestDeadServer {
   @Test public void testIsDead() {
     Set<String> deadServers = new HashSet<String>();
+    DeadServer ds = new DeadServer();
     final String hostname123 = "one,123,3";
-    assertFalse(ServerManager.isDead(deadServers, hostname123, false));
-    assertFalse(ServerManager.isDead(deadServers, hostname123, true));
+    assertFalse(ds.isDeadServer(hostname123, false));
+    assertFalse(ds.isDeadServer(hostname123, true));
     deadServers.add(hostname123);
-    assertTrue(ServerManager.isDead(deadServers, hostname123, false));
-    assertFalse(ServerManager.isDead(deadServers, "one:1", true));
-    assertFalse(ServerManager.isDead(deadServers, "one:1234", true));
-    assertTrue(ServerManager.isDead(deadServers, "one:123", true));
+    assertTrue(ds.isDeadServer(hostname123, false));
+    assertFalse(ds.isDeadServer("one:1", true));
+    assertFalse(ds.isDeadServer("one:1234", true));
+    assertTrue(ds.isDeadServer("one:123", true));
   }
 }
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestKillingServersFromMaster.java b/src/test/java/org/apache/hadoop/hbase/master/TestKillingServersFromMaster.java
index 8ced96d9339..f5fd2436c74 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestKillingServersFromMaster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestKillingServersFromMaster.java
@@ -20,7 +20,6 @@
 package org.apache.hadoop.hbase.master;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 
@@ -28,9 +27,8 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer;
 import org.apache.hadoop.hbase.YouAreDeadException;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java b/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java
new file mode 100644
index 00000000000..d05364cac15
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java
@@ -0,0 +1,379 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Random;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestLoadBalancer {
+  private static final Log LOG = LogFactory.getLog(TestLoadBalancer.class);
+
+  private static LoadBalancer loadBalancer;
+
+  private static Random rand;
+
+  @BeforeClass
+  public static void beforeAllTests() throws Exception {
+    loadBalancer = new LoadBalancer();
+    rand = new Random();
+  }
+
+  // int[testnum][servernumber] -> numregions
+  int [][] clusterStateMocks = new int [][] {
+      // 1 node
+      new int [] { 0 },
+      new int [] { 1 },
+      new int [] { 10 },
+      // 2 node
+      new int [] { 0, 0 },
+      new int [] { 2, 0 },
+      new int [] { 2, 1 },
+      new int [] { 2, 2 },
+      new int [] { 2, 3 },
+      new int [] { 2, 4 },
+      new int [] { 1, 1 },
+      new int [] { 0, 1 },
+      new int [] { 10, 1 },
+      new int [] { 14, 1432 },
+      new int [] { 47, 53 },
+      // 3 node
+      new int [] { 0, 1, 2 },
+      new int [] { 1, 2, 3 },
+      new int [] { 0, 2, 2 },
+      new int [] { 0, 3, 0 },
+      new int [] { 0, 4, 0 },
+      new int [] { 20, 20, 0 },
+      // 4 node
+      new int [] { 0, 1, 2, 3 },
+      new int [] { 4, 0, 0, 0 },
+      new int [] { 5, 0, 0, 0 },
+      new int [] { 6, 6, 0, 0 },
+      new int [] { 6, 2, 0, 0 },
+      new int [] { 6, 1, 0, 0 },
+      new int [] { 6, 0, 0, 0 },
+      new int [] { 4, 4, 4, 7 },
+      new int [] { 4, 4, 4, 8 },
+      new int [] { 0, 0, 0, 7 },
+      // 5 node
+      new int [] { 1, 1, 1, 1, 4 },
+      // more nodes
+      new int [] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
+      new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 },
+      new int [] { 6, 6, 5, 6, 6, 6, 6, 6, 6, 1 },
+      new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 54 },
+      new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 55 },
+      new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 },
+      new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 16 },
+      new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 8 },
+      new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 9 },
+      new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 10 },
+      new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 123 },
+      new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 155 },
+      new int [] { 0, 0, 144, 1, 1, 1, 1, 1123, 133, 138, 12, 1444 },
+      new int [] { 0, 0, 144, 1, 0, 4, 1, 1123, 133, 138, 12, 1444 }
+  };
+
+  int [][] regionsAndServersMocks = new int [][] {
+      // { num regions, num servers }
+      new int [] { 0, 0 },
+      new int [] { 0, 1 },
+      new int [] { 1, 1 },
+      new int [] { 2, 1 },
+      new int [] { 10, 1 },
+      new int [] { 1, 2 },
+      new int [] { 2, 2 },
+      new int [] { 3, 2 },
+      new int [] { 1, 3 },
+      new int [] { 2, 3 },
+      new int [] { 3, 3 },
+      new int [] { 25, 3 },
+      new int [] { 2, 10 },
+      new int [] { 2, 100 },
+      new int [] { 12, 10 },
+      new int [] { 12, 100 },
+  };
+
+  /**
+   * Test the load balancing algorithm.
+   *
+   * Invariant is that all servers should be hosting either
+   * floor(average) or ceiling(average)
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testBalanceCluster() throws Exception {
+
+    for(int [] mockCluster : clusterStateMocks) {
+      Map<HServerInfo,List<HRegionInfo>> servers = mockClusterServers(mockCluster);
+      LOG.info("Mock Cluster : " + printMock(servers) + " " + printStats(servers));
+      List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
+      List<HServerInfo> balancedCluster = reconcile(servers, plans);
+      LOG.info("Mock Balance : " + printMock(balancedCluster));
+      assertClusterAsBalanced(balancedCluster);
+      for(Map.Entry<HServerInfo, List<HRegionInfo>> entry : servers.entrySet()) {
+        returnRegions(entry.getValue());
+        returnServer(entry.getKey());
+      }
+    }
+
+  }
+
+  /**
+   * Invariant is that all servers have between floor(avg) and ceiling(avg)
+   * number of regions.
+   */
+  public void assertClusterAsBalanced(List<HServerInfo> servers) {
+    int numServers = servers.size();
+    int numRegions = 0;
+    int maxRegions = 0;
+    int minRegions = Integer.MAX_VALUE;
+    for(HServerInfo server : servers) {
+      int nr = server.getLoad().getNumberOfRegions();
+      if(nr > maxRegions) {
+        maxRegions = nr;
+      }
+      if(nr < minRegions) {
+        minRegions = nr;
+      }
+      numRegions += nr;
+    }
+    if(maxRegions - minRegions < 2) {
+      // less than 2 between max and min, can't balance
+      return;
+    }
+    int min = numRegions / numServers;
+    int max = numRegions % numServers == 0 ? min : min + 1;
+
+    for(HServerInfo server : servers) {
+      assertTrue(server.getLoad().getNumberOfRegions() <= max);
+      assertTrue(server.getLoad().getNumberOfRegions() >= min);
+    }
+  }
+
+  /**
+   * Tests immediate assignment.
+   *
+   * Invariant is that all regions have an assignment.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testImmediateAssignment() throws Exception {
+    for(int [] mock : regionsAndServersMocks) {
+      LOG.debug("testImmediateAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
+      List<HRegionInfo> regions = randomRegions(mock[0]);
+      List<HServerInfo> servers = randomServers(mock[1], 0);
+      Map<HRegionInfo,HServerInfo> assignments =
+        LoadBalancer.immediateAssignment(regions, servers);
+      assertImmediateAssignment(regions, servers, assignments);
+      returnRegions(regions);
+      returnServers(servers);
+    }
+  }
+
+  /**
+   * All regions have an assignment.
+   * @param regions
+   * @param servers
+   * @param assignments
+   */
+  private void assertImmediateAssignment(List<HRegionInfo> regions,
+      List<HServerInfo> servers, Map<HRegionInfo,HServerInfo> assignments) {
+    for(HRegionInfo region : regions) {
+      assertTrue(assignments.containsKey(region));
+    }
+  }
+
+  /**
+   * Tests the bulk assignment used during cluster startup.
+   *
+   * Round-robin.  Should yield a balanced cluster so same invariant as the load
+   * balancer holds, all servers holding either floor(avg) or ceiling(avg).
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testBulkAssignment() throws Exception {
+    for(int [] mock : regionsAndServersMocks) {
+      LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
+      List<HRegionInfo> regions = randomRegions(mock[0]);
+      List<HServerInfo> servers = randomServers(mock[1], 0);
+      Map<HServerInfo,List<HRegionInfo>> assignments =
+        LoadBalancer.bulkAssignment(regions, servers);
+      float average = (float)regions.size()/servers.size();
+      int min = (int)Math.floor(average);
+      int max = (int)Math.ceil(average);
+      if(assignments != null && !assignments.isEmpty()) {
+        for(List<HRegionInfo> regionList : assignments.values()) {
+          assertTrue(regionList.size() == min || regionList.size() == max);
+        }
+      }
+      returnRegions(regions);
+      returnServers(servers);
+    }
+  }
+
+  private String printStats(Map<HServerInfo, List<HRegionInfo>> servers) {
+    int numServers = servers.size();
+    int totalRegions = 0;
+    for(HServerInfo server : servers.keySet()) {
+      totalRegions += server.getLoad().getNumberOfRegions();
+    }
+    float average = (float)totalRegions / numServers;
+    int max = (int)Math.ceil(average);
+    int min = (int)Math.floor(average);
+    return "[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max + " min=" + min + "]";
+  }
+
+  private String printMock(Map<HServerInfo, List<HRegionInfo>> servers) {
+    return printMock(Arrays.asList(servers.keySet().toArray(new HServerInfo[servers.size()])));
+  }
+
+  private String printMock(List<HServerInfo> balancedCluster) {
+    SortedSet<HServerInfo> sorted = new TreeSet<HServerInfo>(balancedCluster);
+    HServerInfo [] arr = sorted.toArray(new HServerInfo[sorted.size()]);
+    StringBuilder sb = new StringBuilder(sorted.size() * 4 + 4);
+    sb.append("{ ");
+    for(int i=0;i<arr.length;i++) {
+      if(i != 0) {
+        sb.append(" , ");
+      }
+      sb.append(arr[i].getLoad().getNumberOfRegions());
+    }
+    sb.append(" }");
+    return sb.toString();
+  }
+
+  /**
+   * This assumes the RegionPlan HSI instances are the same ones in the map, so
+   * actually no need to even pass in the map, but I think it's clearer.
+   * @param servers
+   * @param plans
+   * @return
+   */
+  private List<HServerInfo> reconcile(
+      Map<HServerInfo, List<HRegionInfo>> servers, List<RegionPlan> plans) {
+    if(plans != null) {
+      for(RegionPlan plan : plans) {
+        plan.getSource().getLoad().setNumberOfRegions(
+            plan.getSource().getLoad().getNumberOfRegions() - 1);
+        plan.getDestination().getLoad().setNumberOfRegions(
+            plan.getDestination().getLoad().getNumberOfRegions() + 1);
+      }
+    }
+    return Arrays.asList(servers.keySet().toArray(new HServerInfo[servers.size()]));
+  }
+
+  private Map<HServerInfo, List<HRegionInfo>> mockClusterServers(
+      int [] mockCluster) {
+    int numServers = mockCluster.length;
+    Map<HServerInfo,List<HRegionInfo>> servers =
+      new TreeMap<HServerInfo,List<HRegionInfo>>();
+    for(int i=0;i<numServers;i++) {
+      int numRegions = mockCluster[i];
+      HServerInfo server = randomServer(numRegions);
+      List<HRegionInfo> regions = randomRegions(numRegions);
+      servers.put(server, regions);
+    }
+    return servers;
+  }
+
+  private Queue<HRegionInfo> regionQueue = new LinkedList<HRegionInfo>();
+
+  private List<HRegionInfo> randomRegions(int numRegions) {
+    List<HRegionInfo> regions = new ArrayList<HRegionInfo>(numRegions);
+    byte [] start = new byte[16];
+    byte [] end = new byte[16];
+    rand.nextBytes(start);
+    rand.nextBytes(end);
+    for(int i=0;i<numRegions;i++) {
+      if(!regionQueue.isEmpty()) {
+        regions.add(regionQueue.poll());
+        continue;
+      }
+      Bytes.putInt(start, 0, numRegions << 1);
+      Bytes.putInt(end, 0, (numRegions << 1) + 1);
+      HRegionInfo hri = new HRegionInfo(
+          new HTableDescriptor(Bytes.toBytes("table")), start, end);
+      regions.add(hri);
+    }
+    return regions;
+  }
+
+  private void returnRegions(List<HRegionInfo> regions) {
+    regionQueue.addAll(regions);
+  }
+
+  private Queue<HServerInfo> serverQueue = new LinkedList<HServerInfo>();
+
+  private HServerInfo randomServer(int numRegions) {
+    if(!serverQueue.isEmpty()) {
+      HServerInfo server = this.serverQueue.poll();
+      server.getLoad().setNumberOfRegions(numRegions);
+      return server;
+    }
+    String host = RandomStringUtils.random(16);
+    int port = rand.nextInt(60000);
+    long startCode = rand.nextLong();
+    HServerInfo hsi =
+      new HServerInfo(new HServerAddress(host, port), startCode, port, host);
+    hsi.getLoad().setNumberOfRegions(numRegions);
+    return hsi;
+  }
+
+  private List<HServerInfo> randomServers(int numServers, int numRegionsPerServer) {
+    List<HServerInfo> servers = new ArrayList<HServerInfo>(numServers);
+    for(int i=0;i<numServers;i++) {
+      servers.add(randomServer(numRegionsPerServer));
+    }
+    return servers;
+  }
+
+  private void returnServer(HServerInfo server) {
+    serverQueue.add(server);
+  }
+
+  private void returnServers(List<HServerInfo> servers) {
+    serverQueue.addAll(servers);
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java
index 8b7f60fa779..1a986fe4c68 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java
@@ -21,30 +21,27 @@ package org.apache.hadoop.hbase.master;
 
 import static org.junit.Assert.assertEquals;
 
+import java.net.URLEncoder;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.replication.ReplicationZookeeperWrapper;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.conf.Configuration;
-
-import java.net.URLEncoder;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 public class TestLogsCleaner {
 
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
 
-  private ReplicationZookeeperWrapper zkHelper;
+  private ReplicationZookeeper zkHelper;
 
   /**
    * @throws java.lang.Exception
@@ -68,9 +65,11 @@ public class TestLogsCleaner {
   @Before
   public void setUp() throws Exception {
     Configuration conf = TEST_UTIL.getConfiguration();
+    /* TODO REENABLE
     zkHelper = new ReplicationZookeeperWrapper(
         ZooKeeperWrapper.createInstance(conf, HRegionServer.class.getName()),
         conf, new AtomicBoolean(true), "test-cluster");
+        */
   }
 
   /**
@@ -83,13 +82,25 @@ public class TestLogsCleaner {
   @Test
   public void testLogCleaning() throws Exception{
     Configuration c = TEST_UTIL.getConfiguration();
-    Path oldLogDir = new Path(TEST_UTIL.getTestDir(),
+    Path oldLogDir = new Path(HBaseTestingUtility.getTestDir(),
         HConstants.HREGION_OLDLOGDIR_NAME);
     String fakeMachineName = URLEncoder.encode("regionserver:60020", "UTF8");
 
     FileSystem fs = FileSystem.get(c);
-    AtomicBoolean stop = new AtomicBoolean(false);
-    LogsCleaner cleaner = new LogsCleaner(1000, stop,c, fs, oldLogDir);
+    Stoppable stoppable = new Stoppable() {
+      private volatile boolean stopped = false;
+
+      @Override
+      public void stop(String why) {
+        this.stopped = true;
+      }
+
+      @Override
+      public boolean isStopped() {
+        return this.stopped;
+      }
+    };
+    LogCleaner cleaner  = new LogCleaner(1000, stoppable, c, fs, oldLogDir);
 
     // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files
     long now = System.currentTimeMillis();
@@ -113,7 +124,7 @@ public class TestLogsCleaner {
       // (TimeToLiveLogCleaner) but would be rejected by the second
       // (ReplicationLogCleaner)
       if (i % (30/3) == 0) {
-        zkHelper.addLogToList(fileName.getName(), fakeMachineName);
+// REENABLE        zkHelper.addLogToList(fileName.getName(), fakeMachineName);
         System.out.println("Replication log file: " + fileName);
       }
     }
@@ -144,5 +155,4 @@ public class TestLogsCleaner {
       System.out.println("Keeped log files: " + file.getPath().getName());
     }
   }
-
-}
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
index 01ae0d241eb..4976e5c882b 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
@@ -23,16 +23,15 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventHandlerListener;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
@@ -51,7 +50,7 @@ import static org.junit.Assert.*;
 
 public class TestMaster {
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private static final Log LOG = LogFactory.getLog(TestMasterWithDisabling.class);
+  private static final Log LOG = LogFactory.getLog(TestMaster.class);
   private static final byte[] TABLENAME = Bytes.toBytes("TestMaster");
   private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
 
@@ -73,10 +72,12 @@ public class TestMaster {
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
 
     TEST_UTIL.createTable(TABLENAME, FAMILYNAME);
-    TEST_UTIL.loadTable(new HTable(TABLENAME), FAMILYNAME);
+    TEST_UTIL.loadTable(new HTable(TEST_UTIL.getConfiguration(), TABLENAME),
+      FAMILYNAME);
 
     List<Pair<HRegionInfo, HServerAddress>> tableRegions =
-      m.getTableRegions(TABLENAME);
+      MetaReader.getTableRegionsAndLocations(m.getCatalogTracker(),
+          Bytes.toString(TABLENAME));
     LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
     assertEquals(1, tableRegions.size());
     assertArrayEquals(HConstants.EMPTY_START_ROW,
@@ -85,11 +86,12 @@ public class TestMaster {
         tableRegions.get(0).getFirst().getEndKey());
 
     // Now trigger a split and stop when the split is in progress
-    
+
     CountDownLatch aboutToOpen = new CountDownLatch(1);
     CountDownLatch proceed = new CountDownLatch(1);
     RegionOpenListener list = new RegionOpenListener(aboutToOpen, proceed);
-    HBaseEventHandler.registerListener(list);
+    cluster.getMaster().executorService.
+      registerListener(EventType.RS2ZK_REGION_OPENED, list);
 
     LOG.info("Splitting table");
     admin.split(TABLENAME);
@@ -97,7 +99,9 @@ public class TestMaster {
     aboutToOpen.await(60, TimeUnit.SECONDS);
     try {
       LOG.info("Making sure we can call getTableRegions while opening");
-      tableRegions = m.getTableRegions(TABLENAME);
+      tableRegions = MetaReader.getTableRegionsAndLocations(
+          m.getCatalogTracker(), Bytes.toString(TABLENAME));
+
       LOG.info("Regions: " + Joiner.on(',').join(tableRegions));
       // We have three regions because one is split-in-progress
       assertEquals(3, tableRegions.size());
@@ -105,14 +109,16 @@ public class TestMaster {
       Pair<HRegionInfo,HServerAddress> pair =
         m.getTableRegionForRow(TABLENAME, Bytes.toBytes("cde"));
       LOG.info("Result is: " + pair);
-      Pair<HRegionInfo, HServerAddress> tableRegionFromName = m.getTableRegionFromName(pair.getFirst().getRegionName());
+      Pair<HRegionInfo, HServerAddress> tableRegionFromName =
+        MetaReader.getRegion(m.getCatalogTracker(),
+            pair.getFirst().getRegionName());
       assertEquals(tableRegionFromName.getFirst(), pair.getFirst());
     } finally {
       proceed.countDown();
     }
   }
 
-  static class RegionOpenListener implements HBaseEventHandlerListener {
+  static class RegionOpenListener implements EventHandlerListener {
     CountDownLatch aboutToOpen, proceed;
 
     public RegionOpenListener(CountDownLatch aboutToOpen, CountDownLatch proceed)
@@ -122,8 +128,8 @@ public class TestMaster {
     }
 
     @Override
-    public void afterProcess(HBaseEventHandler event) {
-      if (event.getHBEvent() != HBaseEventType.RS2ZK_REGION_OPENED) {
+    public void afterProcess(EventHandler event) {
+      if (event.getEventType() != EventType.RS2ZK_REGION_OPENED) {
         return;
       }
       try {
@@ -136,8 +142,7 @@ public class TestMaster {
     }
 
     @Override
-    public void beforeProcess(HBaseEventHandler event) {
+    public void beforeProcess(EventHandler event) {
     }
   }
-
-}
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
index 8848b3df605..bb5076adebd 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
@@ -19,42 +19,25 @@
  */
 package org.apache.hadoop.hbase.master;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.net.BindException;
-import java.util.Collection;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArraySet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HMsg;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer;
-import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 /**
@@ -104,6 +87,7 @@ public class TestMasterTransitions {
    * the requeuing,  send over a close of a region on 'otherServer' so it comes
    * into a master that has its meta region marked as offline.
    */
+  /*
   static class HBase2428Listener implements RegionServerOperationListener {
     // Map of what we've delayed so we don't do do repeated delays.
     private final Set<RegionServerOperation> postponed =
@@ -164,12 +148,13 @@ public class TestMasterTransitions {
       if (isWantedCloseOperation(op) != null) return;
       this.done = true;
     }
-
+*/
     /*
      * @param op
      * @return Null if not the wanted ProcessRegionClose, else <code>op</code>
      * cast as a ProcessRegionClose.
      */
+  /*
     private ProcessRegionClose isWantedCloseOperation(final RegionServerOperation op) {
       // Count every time we get a close operation.
       if (op instanceof ProcessRegionClose) {
@@ -198,14 +183,15 @@ public class TestMasterTransitions {
       return true;
     }
   }
-
+*/
   /**
    * In 2428, the meta region has just been set offline and then a close comes
    * in.
    * @see <a href="https://issues.apache.org/jira/browse/HBASE-2428">HBASE-2428</a> 
    */
-  @Test (timeout=300000) public void testRegionCloseWhenNoMetaHBase2428()
+  @Ignore @Test  (timeout=300000) public void testRegionCloseWhenNoMetaHBase2428()
   throws Exception {
+    /*
     LOG.info("Running testRegionCloseWhenNoMetaHBase2428");
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     final HMaster master = cluster.getMaster();
@@ -249,6 +235,7 @@ public class TestMasterTransitions {
       master.getRegionServerOperationQueue().
         unregisterRegionServerOperationListener(listener);
     }
+    */
   }
 
   /**
@@ -257,8 +244,9 @@ public class TestMasterTransitions {
    * If confusion between old and new, purportedly meta never comes back.  Test
    * that meta gets redeployed.
    */
-  @Test (timeout=300000) public void testAddingServerBeforeOldIsDead2413()
+  @Ignore @Test (timeout=300000) public void testAddingServerBeforeOldIsDead2413()
   throws IOException {
+    /*
     LOG.info("Running testAddingServerBeforeOldIsDead2413");
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     int count = count();
@@ -298,9 +286,9 @@ public class TestMasterTransitions {
     } finally {
       c.set(HConstants.REGIONSERVER_PORT, oldPort);
     }
+    */
   }
 
-
   /**
    * HBase2482 is about outstanding region openings.  If any are outstanding
    * when a regionserver goes down, then they'll never deploy.  They'll be
@@ -309,6 +297,7 @@ public class TestMasterTransitions {
    * then we kill it.  It also looks out for a close message on the victim
    * server because that signifies start of the fireworks.
    */
+  /*
   static class HBase2482Listener implements RegionServerOperationListener {
     private final HRegionServer victim;
     private boolean abortSent = false;
@@ -367,7 +356,7 @@ public class TestMasterTransitions {
       }
     }
   }
-
+*/
   /**
    * In 2482, a RS with an opening region on it dies.  The said region is then
    * stuck in the master's regions-in-transition and never leaves it.  This
@@ -382,8 +371,9 @@ public class TestMasterTransitions {
    * done.
    * @see <a href="https://issues.apache.org/jira/browse/HBASE-2482">HBASE-2482</a> 
    */
-  @Test (timeout=300000) public void testKillRSWithOpeningRegion2482()
+  @Ignore @Test (timeout=300000) public void testKillRSWithOpeningRegion2482()
   throws Exception {
+    /*
     LOG.info("Running testKillRSWithOpeningRegion2482");
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     if (cluster.getLiveRegionServerThreads().size() < 2) {
@@ -413,7 +403,7 @@ public class TestMasterTransitions {
       // After all closes, add blocking message before the region opens start to
       // come in.
       cluster.addMessageToSendRegionServer(hrs,
-        new HMsg(HMsg.Type.TESTING_MSG_BLOCK_RS));
+        new HMsg(HMsg.Type.TESTING_BLOCK_REGIONSERVER));
       // Wait till one of the above close messages has an effect before we start
       // wait on all regions back online.
       while (!listener.closed) Threads.sleep(100);
@@ -427,11 +417,13 @@ public class TestMasterTransitions {
       m.getRegionServerOperationQueue().
         unregisterRegionServerOperationListener(listener);
     }
+    */
   }
 
   /*
    * @return Count of all non-catalog regions on the designated server
    */
+/*
   private int closeAllNonCatalogRegions(final MiniHBaseCluster cluster,
     final MiniHBaseCluster.MiniHBaseClusterRegionServer hrs)
   throws IOException {
@@ -460,6 +452,7 @@ public class TestMasterTransitions {
    * @return Count of regions in meta table.
    * @throws IOException
    */
+  /*
   private static int countOfMetaRegions()
   throws IOException {
     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
@@ -477,7 +470,7 @@ public class TestMasterTransitions {
     s.close();
     return rows;
   }
-
+*/
   /*
    * Add to each of the regions in .META. a value.  Key is the startrow of the
    * region (except its 'aaa' for first region).  Actual value is the row name.
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterWithDisabling.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterWithDisabling.java
deleted file mode 100644
index 91d4d080008..00000000000
--- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterWithDisabling.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import static org.junit.Assert.fail;
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-/**
- * Disabling is tricky. This class tests how the Master behaves during those
- */
-public class TestMasterWithDisabling {
-
-  private static final Log LOG = LogFactory.getLog(TestMasterWithDisabling.class);
-  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private static final byte[] TABLENAME = Bytes.toBytes("disabling");
-  private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
-
-  @BeforeClass
-  public static void beforeAllTests() throws Exception {
-    // Start a cluster of two regionservers.
-    TEST_UTIL.startMiniCluster(2);
-  }
-
-  @AfterClass
-  public static void afterAllTests() throws IOException {
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  @Test
-  public void testDisableBetweenSplit() throws IOException {
-    // Table that splits like crazy
-    HTableDescriptor htd = new HTableDescriptor(TABLENAME);
-    htd.setMaxFileSize(1024);
-    htd.setMemStoreFlushSize(1024);
-    HColumnDescriptor hcd = new HColumnDescriptor(FAMILYNAME);
-    htd.addFamily(hcd);
-    TEST_UTIL.getHBaseAdmin().createTable(htd);
-    HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
-    HBase2515Listener list = new HBase2515Listener(TEST_UTIL.getHBaseAdmin());
-    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
-    HMaster m = cluster.getMaster();
-    m.getRegionServerOperationQueue().
-      registerRegionServerOperationListener(list);
-    try {
-      TEST_UTIL.loadTable(t, FAMILYNAME);
-    } catch (IOException ex) {
-      // We disable the table during a split, we will end up here
-      LOG.info("Expected", ex);
-    }
-    // Check that there's no region in flight, HBASE-2515
-    assertEquals(0,cluster.getMaster().
-        getClusterStatus().getRegionsInTransition().size());
-  }
-
-  /**
-   * Simple listener that simulates a very long processing of a split. When
-   * we catch it, we first disable the table then let the processing go forward
-   */
-  static class HBase2515Listener implements RegionServerOperationListener {
-    HBaseAdmin admin;
-
-    public HBase2515Listener(HBaseAdmin admin) {
-      this.admin = admin;
-    }
-
-    @Override
-    public boolean process(HServerInfo serverInfo, HMsg incomingMsg) {
-      if (!incomingMsg.isType(HMsg.Type.MSG_REPORT_SPLIT_INCLUDES_DAUGHTERS)) {
-        return true;
-      }
-      try {
-        LOG.info("Disabling table");
-        admin.disableTable(TABLENAME);
-      } catch (IOException e) {
-        LOG.warn(e);
-        fail("Disable should always work");
-      }
-      return true;
-    }
-
-    @Override
-    public boolean process(RegionServerOperation op) throws IOException {
-      return true;
-    }
-
-    @Override
-    public void processed(RegionServerOperation op) {
-    }
-  }
-
-}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestROOTAssignment.java b/src/test/java/org/apache/hadoop/hbase/master/TestROOTAssignment.java
deleted file mode 100644
index 0125134221f..00000000000
--- a/src/test/java/org/apache/hadoop/hbase/master/TestROOTAssignment.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArraySet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Threads;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Test issues assigning ROOT.
- */
-public class TestROOTAssignment {
-  private static final Log LOG = LogFactory.getLog(TestROOTAssignment.class);
-  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private static final byte [] TABLENAME = Bytes.toBytes("root_assignments");
-  private static final byte [][] FAMILIES =
-    new byte [][] {Bytes.toBytes("family")};
-
-  /**
-   * Start up a mini cluster and put a small table of many empty regions into it.
-   * @throws Exception
-   */
-  @BeforeClass public static void beforeAllTests() throws Exception {
-    TEST_UTIL.getConfiguration().setInt("hbase.regions.percheckin", 2);
-    // Start a cluster of two regionservers.
-    TEST_UTIL.startMiniCluster(2);
-    // Create a table of three families.  This will assign a region.
-    TEST_UTIL.createTable(TABLENAME, FAMILIES);
-    HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
-    int countOfRegions = TEST_UTIL.createMultiRegions(t, FAMILIES[0]);
-    TEST_UTIL.waitUntilAllRegionsAssigned(countOfRegions);
-    HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
-    TEST_UTIL.loadTable(table, FAMILIES[0]);
-    table.close();
-  }
-
-  @AfterClass public static void afterAllTests() throws IOException {
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  @Before public void setup() throws IOException {
-    TEST_UTIL.ensureSomeRegionServersAvailable(2);
-  }
-
-  /**
-   * Interrupt processing of server shutdown so it gets put on delay queue.
-   */
-  static class PostponeShutdownProcessing implements RegionServerOperationListener {
-    // Map of what we've delayed so we don't do do repeated delays.
-    private final Set<RegionServerOperation> postponed =
-      new CopyOnWriteArraySet<RegionServerOperation>();
-    private boolean done = false;
-    private final HServerAddress rootServerAddress;
-    private final HMaster master;
- 
-    PostponeShutdownProcessing(final HMaster master,
-        final HServerAddress rootServerAddress) {
-      this.master = master;
-      this.rootServerAddress = rootServerAddress;
-    }
-
-    @Override
-    public boolean process(final RegionServerOperation op) throws IOException {
-      // If a regionserver shutdown and its of the root server, then we want to
-      // delay the processing of the shutdown
-      boolean result = true;
-      if (op instanceof ProcessServerShutdown) {
-        ProcessServerShutdown pss = (ProcessServerShutdown)op;
-        if (pss.getDeadServerAddress().equals(this.rootServerAddress)) {
-          // Don't postpone more than once.
-          if (!this.postponed.contains(pss)) {
-            this.postponed.add(pss);
-            Assert.assertNull(this.master.getRegionManager().getRootRegionLocation());
-            pss.setDelay(1 * 1000);
-            // Return false.  This will add this op to the delayed queue.
-            result = false;
-          }
-        }
-      }
-      return result;
-    }
-
-    @Override
-    public boolean process(HServerInfo serverInfo, HMsg incomingMsg) {
-      return true;
-    }
-
-    @Override
-    public void processed(RegionServerOperation op) {
-      if (op instanceof ProcessServerShutdown) {
-        ProcessServerShutdown pss = (ProcessServerShutdown)op;
-        if (pss.getDeadServerAddress().equals(this.rootServerAddress)) {
-          this.done = true;
-        }
-      }
-    }
-
-    public boolean isDone() {
-      return this.done;
-    }
-  }
-
-  /**
-   * If the split of the log for the regionserver hosting ROOT doesn't go off
-   * smoothly, if the process server shutdown gets added to the delayed queue
-   * of events to process, then ROOT was not being allocated, ever.
-   * @see <a href="https://issues.apache.org/jira/browse/HBASE-2707">HBASE-2707</a> 
-   */
-  @Test (timeout=300000) public void testROOTDeployedThoughProblemSplittingLog()
-  throws Exception {
-    LOG.info("Running testROOTDeployedThoughProblemSplittingLog");
-    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
-    final HMaster master = cluster.getMaster();
-    byte [] rootRegion = Bytes.toBytes("-ROOT-,,0");
-    int rootIndex = cluster.getServerWith(rootRegion);
-    final HRegionServer rootHRS = cluster.getRegionServer(rootIndex);
- 
-    // Add our RegionServerOperationsListener
-    PostponeShutdownProcessing listener = new PostponeShutdownProcessing(master,
-      rootHRS.getHServerInfo().getServerAddress());
-    master.getRegionServerOperationQueue().
-      registerRegionServerOperationListener(listener);
-    try {
-      // Now close the server carrying meta.
-      cluster.abortRegionServer(rootIndex);
-
-      // Wait for processing of the shutdown server.
-      while(!listener.isDone()) Threads.sleep(100);
-      master.getRegionManager().waitForRootRegionLocation();
-    } finally {
-      master.getRegionServerOperationQueue().
-        unregisterRegionServerOperationListener(listener);
-    }
-  }
-}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestRegionManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestRegionManager.java
deleted file mode 100644
index 7206e12a25b..00000000000
--- a/src/test/java/org/apache/hadoop/hbase/master/TestRegionManager.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright 2009 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.HBaseClusterTestCase;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.util.Bytes;
-
-public class TestRegionManager extends HBaseClusterTestCase {
-   public void testGetFirstMetaRegionForRegionAfterMetaSplit()
-   throws Exception {
-     HTable meta = new HTable(HConstants.META_TABLE_NAME);
-     HMaster master = this.cluster.getMaster();
-     HServerAddress address = master.getMasterAddress();
-     HTableDescriptor tableDesc = new HTableDescriptor(Bytes.toBytes("_MY_TABLE_"));
-     HTableDescriptor metaTableDesc = meta.getTableDescriptor();
-     // master.regionManager.onlineMetaRegions already contains first .META. region at key Bytes.toBytes("")
-     byte[] startKey0 = Bytes.toBytes("f");
-     byte[] endKey0 = Bytes.toBytes("h");
-     HRegionInfo regionInfo0 = new HRegionInfo(tableDesc, startKey0, endKey0);
-
-     // 1st .META. region will be something like .META.,,1253625700761
-     HRegionInfo metaRegionInfo0 = new HRegionInfo(metaTableDesc, Bytes.toBytes(""), regionInfo0.getRegionName());
-     MetaRegion meta0 = new MetaRegion(address, metaRegionInfo0);
-
-     byte[] startKey1 = Bytes.toBytes("j");
-     byte[] endKey1 = Bytes.toBytes("m");
-     HRegionInfo regionInfo1 = new HRegionInfo(tableDesc, startKey1, endKey1);
-     // 2nd .META. region will be something like .META.,_MY_TABLE_,f,1253625700761,1253625700761
-     HRegionInfo metaRegionInfo1 = new HRegionInfo(metaTableDesc, regionInfo0.getRegionName(), regionInfo1.getRegionName());
-     MetaRegion meta1 = new MetaRegion(address, metaRegionInfo1);
-
-
-     // 3rd .META. region will be something like .META.,_MY_TABLE_,j,1253625700761,1253625700761
-     HRegionInfo metaRegionInfo2 = new HRegionInfo(metaTableDesc, regionInfo1.getRegionName(), Bytes.toBytes(""));
-     MetaRegion meta2 = new MetaRegion(address, metaRegionInfo2);
-
-     byte[] startKeyX = Bytes.toBytes("h");
-     byte[] endKeyX = Bytes.toBytes("j");
-     HRegionInfo regionInfoX = new HRegionInfo(tableDesc, startKeyX, endKeyX);
-
-
-     master.getRegionManager().offlineMetaRegionWithStartKey(startKey0);
-     master.getRegionManager().putMetaRegionOnline(meta0);
-     master.getRegionManager().putMetaRegionOnline(meta1);
-     master.getRegionManager().putMetaRegionOnline(meta2);
-
-//    for (byte[] b : master.regionManager.getOnlineMetaRegions().keySet()) {
-//      System.out.println("FROM TEST KEY " + b +"  " +new String(b));
-//    }
-
-     assertEquals(metaRegionInfo1.getStartKey(),
-       master.getRegionManager().getFirstMetaRegionForRegion(regionInfoX).getStartKey());
-     assertEquals(metaRegionInfo1.getRegionName(),
-      master.getRegionManager().getFirstMetaRegionForRegion(regionInfoX).getRegionName());
-   }
-}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java b/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java
deleted file mode 100644
index bf63e1a8b2a..00000000000
--- a/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-import static org.junit.Assert.*;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.master.RegionServerOperationQueue.ProcessingResultCode;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test the queue used to manage RegionServerOperations.
- * Currently RegionServerOperationQueue is untestable because each
- * RegionServerOperation has a {@link HMaster} reference.  TOOD: Fix.
- */
-public class TestRegionServerOperationQueue {
-  private RegionServerOperationQueue queue;
-  private Configuration conf;
-  private AtomicBoolean closed;
-
-  @Before
-  public void setUp() throws Exception {
-    this.closed = new AtomicBoolean(false);
-    this.conf = new Configuration();
-    this.queue = new RegionServerOperationQueue(this.conf, this.closed);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-  }
-
-  @Test
-  public void testWeDoNotGetStuckInDelayQueue() throws Exception {
-    ProcessingResultCode code = this.queue.process();
-    assertTrue(ProcessingResultCode.NOOP == code);
-  }
-}
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java b/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
index 4584e520551..24886120220 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
@@ -19,72 +19,112 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.client.MetaScanner;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-import org.junit.AfterClass;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class TestRestartCluster {
   private static final Log LOG = LogFactory.getLog(TestRestartCluster.class);
-  private static Configuration conf;
-  private static HBaseTestingUtility utility;
-  private static ZooKeeperWrapper zkWrapper;
+  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static ZooKeeperWatcher zooKeeper;
   private static final byte[] TABLENAME = Bytes.toBytes("master_transitions");
   private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a")};
-  
-  @BeforeClass public static void beforeAllTests() throws Exception {
-    conf = HBaseConfiguration.create();
-    utility = new HBaseTestingUtility(conf);
+
+
+  private static final byte [][] TABLES = new byte[][] {
+      Bytes.toBytes("restartTableOne"),
+      Bytes.toBytes("restartTableTwo"),
+      Bytes.toBytes("restartTableThree")
+  };
+  private static final byte [] FAMILY = Bytes.toBytes("family");
+
+  @Before public void setup() throws Exception {
   }
 
-  @AfterClass public static void afterAllTests() throws IOException {
-    utility.shutdownMiniCluster();
+  @After public void teardown() throws IOException {
+    UTIL.shutdownMiniCluster();
   }
 
-  @Before public void setup() throws IOException {
-  }
-
-  @Test (timeout=300000) public void testRestartClusterAfterKill()throws Exception {
-    utility.startMiniZKCluster();
-    zkWrapper = ZooKeeperWrapper.createInstance(conf, "cluster1");
+  @Test (timeout=300000) public void testRestartClusterAfterKill()
+  throws Exception {
+    UTIL.startMiniZKCluster();
+    zooKeeper = new ZooKeeperWatcher(UTIL.getConfiguration(), "cluster1", null);
 
     // create the unassigned region, throw up a region opened state for META
-    String unassignedZNode = zkWrapper.getRegionInTransitionZNode();
-    zkWrapper.createZNodeIfNotExists(unassignedZNode);
-    byte[] data = null;
-    HBaseEventType hbEventType = HBaseEventType.RS2ZK_REGION_OPENED;
-    try {
-      data = Writables.getBytes(new RegionTransitionEventData(hbEventType, HMaster.MASTER));
-    } catch (IOException e) {
-      LOG.error("Error creating event data for " + hbEventType, e);
-    }
-    zkWrapper.createOrUpdateUnassignedRegion(
-        HRegionInfo.ROOT_REGIONINFO.getEncodedName(), data);
-    zkWrapper.createOrUpdateUnassignedRegion(
-        HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(), data);
-    LOG.debug("Created UNASSIGNED zNode for ROOT and META regions in state " + HBaseEventType.M2ZK_REGION_OFFLINE);
-    
+    String unassignedZNode = zooKeeper.assignmentZNode;
+    ZKUtil.createAndFailSilent(zooKeeper, unassignedZNode);
+
+    ZKAssign.createNodeOffline(zooKeeper, HRegionInfo.ROOT_REGIONINFO,
+      HMaster.MASTER);
+
+    ZKAssign.createNodeOffline(zooKeeper, HRegionInfo.FIRST_META_REGIONINFO,
+      HMaster.MASTER);
+
+    LOG.debug("Created UNASSIGNED zNode for ROOT and META regions in state " +
+        EventType.M2ZK_REGION_OFFLINE);
+
     // start the HB cluster
     LOG.info("Starting HBase cluster...");
-    utility.startMiniCluster(2);  
-    
-    utility.createTable(TABLENAME, FAMILIES);
-    LOG.info("Created a table, waiting for table to be available...");
-    utility.waitTableAvailable(TABLENAME, 60*1000);
+    UTIL.startMiniCluster(2);
 
-    LOG.info("Master deleted unassgined region and started up successfully.");
+    UTIL.createTable(TABLENAME, FAMILIES);
+    LOG.info("Created a table, waiting for table to be available...");
+    UTIL.waitTableAvailable(TABLENAME, 60*1000);
+
+    LOG.info("Master deleted unassigned region and started up successfully.");
+  }
+
+  @Test (timeout=300000)
+  public void testClusterRestart() throws Exception {
+    UTIL.startMiniCluster(3);
+    LOG.info("\n\nCreating tables");
+    for(byte [] TABLE : TABLES) {
+      UTIL.createTable(TABLE, FAMILY);
+      UTIL.waitTableAvailable(TABLE, 30000);
+    }
+    List<HRegionInfo> allRegions =
+      MetaScanner.listAllRegions(UTIL.getConfiguration());
+    assertEquals(3, allRegions.size());
+
+    LOG.info("\n\nShutting down cluster");
+    UTIL.getHBaseCluster().shutdown();
+    UTIL.getHBaseCluster().join();
+
+    LOG.info("\n\nSleeping a bit");
+    Thread.sleep(2000);
+
+    LOG.info("\n\nStarting cluster the second time");
+    UTIL.restartHBaseCluster(3);
+
+    allRegions = MetaScanner.listAllRegions(UTIL.getConfiguration());
+    assertEquals(3, allRegions.size());
+
+    LOG.info("\n\nWaiting for tables to be available");
+    for(byte [] TABLE: TABLES) {
+      try {
+        UTIL.createTable(TABLE, FAMILY);
+        assertTrue("Able to create table that should already exist", false);
+      } catch(TableExistsException tee) {
+        LOG.info("Table already exists as expected");
+      }
+      UTIL.waitTableAvailable(TABLE, 30000);
+    }
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedCloseRegion.java b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
similarity index 52%
rename from src/test/java/org/apache/hadoop/hbase/master/TestZKBasedCloseRegion.java
rename to src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
index d0a94039d5c..fa1c9f20f69 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedCloseRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
@@ -27,22 +27,19 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.ProcessRegionClose;
-import org.apache.hadoop.hbase.master.RegionServerOperation;
-import org.apache.hadoop.hbase.master.RegionServerOperationListener;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
@@ -53,10 +50,13 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-public class TestZKBasedCloseRegion {
-  private static final Log LOG = LogFactory.getLog(TestZKBasedCloseRegion.class);
+/**
+ * Test open and close of regions using zk.
+ */
+public class TestZKBasedOpenCloseRegion {
+  private static final Log LOG = LogFactory.getLog(TestZKBasedOpenCloseRegion.class);
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private static final String TABLENAME = "master_transitions";
+  private static final String TABLENAME = "TestZKBasedOpenCloseRegion";
   private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
     Bytes.toBytes("b"), Bytes.toBytes("c")};
 
@@ -82,7 +82,107 @@ public class TestZKBasedCloseRegion {
       // Need at least two servers.
       LOG.info("Started new server=" +
         TEST_UTIL.getHBaseCluster().startRegionServer());
-      
+
+    }
+  }
+
+  /**
+   * Test we reopen a region once closed.
+   * @throws Exception
+   */
+  @Test (timeout=300000) public void testReOpenRegion()
+  throws Exception {
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    LOG.info("Number of region servers = " +
+      cluster.getLiveRegionServerThreads().size());
+
+    int rsIdx = 0;
+    HRegionServer regionServer =
+      TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
+    HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
+    LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
+
+    AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
+    AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
+
+    EventHandlerListener closeListener =
+      new ReopenEventListener(hri.getRegionNameAsString(),
+          closeEventProcessed, EventType.RS2ZK_REGION_CLOSED);
+    cluster.getMaster().executorService.
+      registerListener(EventType.RS2ZK_REGION_CLOSED, closeListener);
+
+    EventHandlerListener openListener =
+      new ReopenEventListener(hri.getRegionNameAsString(),
+          reopenEventProcessed, EventType.RS2ZK_REGION_OPENED);
+    cluster.getMaster().executorService.
+      registerListener(EventType.RS2ZK_REGION_OPENED, openListener);
+
+    LOG.info("Unassign " + hri.getRegionNameAsString());
+    cluster.getMaster().assignmentManager.unassign(hri);
+
+    while (!closeEventProcessed.get()) {
+      Threads.sleep(100);
+    }
+
+    while (!reopenEventProcessed.get()) {
+      Threads.sleep(100);
+    }
+
+    LOG.info("Done with testReOpenRegion");
+  }
+
+  private HRegionInfo getNonMetaRegion(final Collection<HRegionInfo> regions) {
+    HRegionInfo hri = null;
+    for (HRegionInfo i: regions) {
+      LOG.info(i.getRegionNameAsString());
+      if (!i.isMetaRegion()) {
+        hri = i;
+        break;
+      }
+    }
+    return hri;
+  }
+
+  public static class ReopenEventListener implements EventHandlerListener {
+    private static final Log LOG = LogFactory.getLog(ReopenEventListener.class);
+    String regionName;
+    AtomicBoolean eventProcessed;
+    EventType eventType;
+
+    public ReopenEventListener(String regionName,
+        AtomicBoolean eventProcessed, EventType eventType) {
+      this.regionName = regionName;
+      this.eventProcessed = eventProcessed;
+      this.eventType = eventType;
+    }
+
+    @Override
+    public void beforeProcess(EventHandler event) {
+      if(event.getEventType() == eventType) {
+        LOG.info("Received " + eventType + " and beginning to process it");
+      }
+    }
+
+    @Override
+    public void afterProcess(EventHandler event) {
+      LOG.info("afterProcess(" + event + ")");
+      if(event.getEventType() == eventType) {
+        LOG.info("Finished processing " + eventType);
+        String regionName = "";
+        if(eventType == EventType.RS2ZK_REGION_OPENED) {
+          TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
+          regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
+        } else if(eventType == EventType.RS2ZK_REGION_CLOSED) {
+          TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
+          regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
+        }
+        if(this.regionName.equals(regionName)) {
+          eventProcessed.set(true);
+        }
+        synchronized(eventProcessed) {
+          eventProcessed.notifyAll();
+        }
+      }
     }
   }
 
@@ -94,72 +194,56 @@ public class TestZKBasedCloseRegion {
 
     int rsIdx = 0;
     HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
-    Collection<HRegion> regions = regionServer.getOnlineRegions();
-    HRegion region = regions.iterator().next();
-    LOG.debug("Asking RS to close region " + region.getRegionNameAsString());
+    HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
+    LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
 
     AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
-    RegionServerOperationListener listener = 
-      new CloseRegionEventListener(region.getRegionNameAsString(), closeEventProcessed);
-    HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
-    master.getRegionServerOperationQueue().registerRegionServerOperationListener(listener);
-    HMsg closeRegionMsg = new HMsg(HMsg.Type.MSG_REGION_CLOSE, 
-                                   region.getRegionInfo(),
-                                   Bytes.toBytes("Forcing close in test")
-                                  );
-    TEST_UTIL.getHBaseCluster().addMessageToSendRegionServer(rsIdx, closeRegionMsg);
-    
-    synchronized(closeEventProcessed) {
-      // wait for 3 minutes
-      closeEventProcessed.wait(3*60*1000);
-    }
-    if(!closeEventProcessed.get()) {
-      throw new Exception("Timed out, close event not called on master.");
-    }
-    else {
-      LOG.info("Done with test, RS informed master successfully.");
+    EventHandlerListener listener =
+      new CloseRegionEventListener(hri.getRegionNameAsString(),
+          closeEventProcessed);
+    cluster.getMaster().executorService.registerListener(EventType.RS2ZK_REGION_CLOSED, listener);
+
+    cluster.getMaster().assignmentManager.unassign(hri);
+
+    while (!closeEventProcessed.get()) {
+      Threads.sleep(100);
     }
+    LOG.info("Done with testCloseRegion");
   }
-  
-  public static class CloseRegionEventListener implements RegionServerOperationListener {
-    
+
+  public static class CloseRegionEventListener implements EventHandlerListener {
     private static final Log LOG = LogFactory.getLog(CloseRegionEventListener.class);
     String regionToClose;
     AtomicBoolean closeEventProcessed;
 
-    public CloseRegionEventListener(String regionToClose, AtomicBoolean closeEventProcessed) {
+    public CloseRegionEventListener(String regionToClose,
+        AtomicBoolean closeEventProcessed) {
       this.regionToClose = regionToClose;
       this.closeEventProcessed = closeEventProcessed;
     }
 
     @Override
-    public boolean process(HServerInfo serverInfo, HMsg incomingMsg) {
-      return true;
-    }
-
-    @Override
-    public boolean process(RegionServerOperation op) throws IOException {
-      return true;
-    }
-
-    @Override
-    public void processed(RegionServerOperation op) {
-      LOG.debug("Master processing object: " + op.getClass().getCanonicalName());
-      if(op instanceof ProcessRegionClose) {
-        ProcessRegionClose regionCloseOp = (ProcessRegionClose)op;
-        String region = regionCloseOp.getRegionInfo().getRegionNameAsString();
-        LOG.debug("Finished closing region " + region + ", expected to close region " + regionToClose);
-        if(regionToClose.equals(region)) {
+    public void afterProcess(EventHandler event) {
+      LOG.info("afterProcess(" + event + ")");
+      if(event.getEventType() == EventType.RS2ZK_REGION_CLOSED) {
+        LOG.info("Finished processing CLOSE REGION");
+        TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
+        if (regionToClose.equals(hriCarrier.getHRegionInfo().getRegionNameAsString())) {
+          LOG.info("Setting closeEventProcessed flag");
           closeEventProcessed.set(true);
-        }
-        synchronized(closeEventProcessed) {
-          closeEventProcessed.notifyAll();
+        } else {
+          LOG.info("Region to close didn't match");
         }
       }
     }
-    
+
+    @Override
+    public void beforeProcess(EventHandler event) {
+      if(event.getEventType() == EventType.M2RS_CLOSE_REGION) {
+        LOG.info("Received CLOSE RPC and beginning to process it");
+      }
+    }
   }
-  
 
   private static void waitUntilAllRegionsAssigned(final int countOfRegions)
   throws IOException {
@@ -173,14 +257,18 @@ public class TestZKBasedCloseRegion {
       for (Result r = null; (r = s.next()) != null;) {
         byte [] b =
           r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-        if (b == null || b.length <= 0) break;
+        if (b == null || b.length <= 0) {
+          break;
+        }
         rows++;
       }
       s.close();
       // If I get to here and all rows have a Server, then all have been assigned.
-      if (rows == countOfRegions) break;
+      if (rows == countOfRegions) {
+        break;
+      }
       LOG.info("Found=" + rows);
-      Threads.sleep(1000); 
+      Threads.sleep(1000);
     }
   }
 
@@ -202,7 +290,9 @@ public class TestZKBasedCloseRegion {
     for (Result r = null; (r = s.next()) != null;) {
       byte [] b =
         r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-      if (b == null || b.length <= 0) break;
+      if (b == null || b.length <= 0) {
+        break;
+      }
       HRegionInfo hri = Writables.getHRegionInfo(b);
       // If start key, add 'aaa'.
       byte [] row = getStartKey(hri);
@@ -228,14 +318,14 @@ public class TestZKBasedCloseRegion {
   private static byte [] getTestQualifier() {
     return getTestFamily();
   }
-  
+
   public static void main(String args[]) throws Exception {
-    TestZKBasedCloseRegion.beforeAllTests();
-    
-    TestZKBasedCloseRegion test = new TestZKBasedCloseRegion();
+    TestZKBasedOpenCloseRegion.beforeAllTests();
+
+    TestZKBasedOpenCloseRegion test = new TestZKBasedOpenCloseRegion();
     test.setup();
     test.testCloseRegion();
-    
-    TestZKBasedCloseRegion.afterAllTests();
+
+    TestZKBasedOpenCloseRegion.afterAllTests();
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedReopenRegion.java b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedReopenRegion.java
deleted file mode 100644
index 4fb4c6586d4..00000000000
--- a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedReopenRegion.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.ProcessRegionClose;
-import org.apache.hadoop.hbase.master.ProcessRegionOpen;
-import org.apache.hadoop.hbase.master.RegionServerOperation;
-import org.apache.hadoop.hbase.master.RegionServerOperationListener;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.hbase.util.Writables;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestZKBasedReopenRegion {
-  private static final Log LOG = LogFactory.getLog(TestZKBasedReopenRegion.class);
-  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private static final String TABLENAME = "master_transitions";
-  private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
-    Bytes.toBytes("b"), Bytes.toBytes("c")};
-
-  @BeforeClass public static void beforeAllTests() throws Exception {
-    Configuration c = TEST_UTIL.getConfiguration();
-    c.setBoolean("dfs.support.append", true);
-    c.setInt("hbase.regionserver.info.port", 0);
-    c.setInt("hbase.master.meta.thread.rescanfrequency", 5*1000);
-    TEST_UTIL.startMiniCluster(2);
-    TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
-    HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
-    int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
-    waitUntilAllRegionsAssigned(countOfRegions);
-    addToEachStartKey(countOfRegions);
-  }
-
-  @AfterClass public static void afterAllTests() throws IOException {
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  @Before public void setup() throws IOException {
-    if (TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size() < 2) {
-      // Need at least two servers.
-      LOG.info("Started new server=" +
-        TEST_UTIL.getHBaseCluster().startRegionServer());
-      
-    }
-  }
-
-  @Test (timeout=300000) public void testOpenRegion()
-  throws Exception {
-    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
-    LOG.info("Number of region servers = " + cluster.getLiveRegionServerThreads().size());
-
-    int rsIdx = 0;
-    HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
-    Collection<HRegion> regions = regionServer.getOnlineRegions();
-    HRegion region = regions.iterator().next();
-    LOG.debug("Asking RS to close region " + region.getRegionNameAsString());
-
-    AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
-    AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
-    RegionServerOperationListener listener = 
-      new ReopenRegionEventListener(region.getRegionNameAsString(), 
-                                    closeEventProcessed,
-                                    reopenEventProcessed);
-    HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
-    master.getRegionServerOperationQueue().registerRegionServerOperationListener(listener);
-    HMsg closeRegionMsg = new HMsg(HMsg.Type.MSG_REGION_CLOSE, 
-                                   region.getRegionInfo(),
-                                   Bytes.toBytes("Forcing close in test")
-                                  );
-    TEST_UTIL.getHBaseCluster().addMessageToSendRegionServer(rsIdx, closeRegionMsg);
-    
-    synchronized(closeEventProcessed) {
-      closeEventProcessed.wait(3*60*1000);
-    }
-    if(!closeEventProcessed.get()) {
-      throw new Exception("Timed out, close event not called on master.");
-    }
-
-    synchronized(reopenEventProcessed) {
-      reopenEventProcessed.wait(3*60*1000);
-    }
-    if(!reopenEventProcessed.get()) {
-      throw new Exception("Timed out, open event not called on master after region close.");
-    }    
-    
-    LOG.info("Done with test, RS informed master successfully.");
-  }
-  
-  public static class ReopenRegionEventListener implements RegionServerOperationListener {
-    
-    private static final Log LOG = LogFactory.getLog(ReopenRegionEventListener.class);
-    String regionToClose;
-    AtomicBoolean closeEventProcessed;
-    AtomicBoolean reopenEventProcessed;
-
-    public ReopenRegionEventListener(String regionToClose, 
-                                     AtomicBoolean closeEventProcessed,
-                                     AtomicBoolean reopenEventProcessed) {
-      this.regionToClose = regionToClose;
-      this.closeEventProcessed = closeEventProcessed;
-      this.reopenEventProcessed = reopenEventProcessed;
-    }
-
-    @Override
-    public boolean process(HServerInfo serverInfo, HMsg incomingMsg) {
-      return true;
-    }
-
-    @Override
-    public boolean process(RegionServerOperation op) throws IOException {
-      return true;
-    }
-
-    @Override
-    public void processed(RegionServerOperation op) {
-      LOG.debug("Master processing object: " + op.getClass().getCanonicalName());
-      if(op instanceof ProcessRegionClose) {
-        ProcessRegionClose regionCloseOp = (ProcessRegionClose)op;
-        String region = regionCloseOp.getRegionInfo().getRegionNameAsString();
-        LOG.debug("Finished closing region " + region + ", expected to close region " + regionToClose);
-        if(regionToClose.equals(region)) {
-          closeEventProcessed.set(true);
-        }
-        synchronized(closeEventProcessed) {
-          closeEventProcessed.notifyAll();
-        }
-      }
-      // Wait for open event AFTER we have closed the region
-      if(closeEventProcessed.get()) {
-        if(op instanceof ProcessRegionOpen) {
-          ProcessRegionOpen regionOpenOp = (ProcessRegionOpen)op;
-          String region = regionOpenOp.getRegionInfo().getRegionNameAsString();
-          LOG.debug("Finished closing region " + region + ", expected to close region " + regionToClose);
-          if(regionToClose.equals(region)) {
-            reopenEventProcessed.set(true);
-          }
-          synchronized(reopenEventProcessed) {
-            reopenEventProcessed.notifyAll();
-          }
-        }        
-      }
-      
-    }
-    
-  }
-  
-
-  private static void waitUntilAllRegionsAssigned(final int countOfRegions)
-  throws IOException {
-    HTable meta = new HTable(TEST_UTIL.getConfiguration(),
-      HConstants.META_TABLE_NAME);
-    while (true) {
-      int rows = 0;
-      Scan scan = new Scan();
-      scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-      ResultScanner s = meta.getScanner(scan);
-      for (Result r = null; (r = s.next()) != null;) {
-        byte [] b =
-          r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
-        if (b == null || b.length <= 0) break;
-        rows++;
-      }
-      s.close();
-      // If I get to here and all rows have a Server, then all have been assigned.
-      if (rows == countOfRegions) break;
-      LOG.info("Found=" + rows);
-      Threads.sleep(1000); 
-    }
-  }
-
-  /*
-   * Add to each of the regions in .META. a value.  Key is the startrow of the
-   * region (except its 'aaa' for first region).  Actual value is the row name.
-   * @param expected
-   * @return
-   * @throws IOException
-   */
-  private static int addToEachStartKey(final int expected) throws IOException {
-    HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
-    HTable meta = new HTable(TEST_UTIL.getConfiguration(),
-        HConstants.META_TABLE_NAME);
-    int rows = 0;
-    Scan scan = new Scan();
-    scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-    ResultScanner s = meta.getScanner(scan);
-    for (Result r = null; (r = s.next()) != null;) {
-      byte [] b =
-        r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-      if (b == null || b.length <= 0) break;
-      HRegionInfo hri = Writables.getHRegionInfo(b);
-      // If start key, add 'aaa'.
-      byte [] row = getStartKey(hri);
-      Put p = new Put(row);
-      p.add(getTestFamily(), getTestQualifier(), row);
-      t.put(p);
-      rows++;
-    }
-    s.close();
-    Assert.assertEquals(expected, rows);
-    return rows;
-  }
-
-  private static byte [] getStartKey(final HRegionInfo hri) {
-    return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
-        Bytes.toBytes("aaa"): hri.getStartKey();
-  }
-
-  private static byte [] getTestFamily() {
-    return FAMILIES[0];
-  }
-
-  private static byte [] getTestQualifier() {
-    return getTestFamily();
-  }
-  
-  public static void main(String args[]) throws Exception {
-    TestZKBasedReopenRegion.beforeAllTests();
-    
-    TestZKBasedReopenRegion test = new TestZKBasedReopenRegion();
-    test.setup();
-    test.testOpenRegion();
-    
-    TestZKBasedReopenRegion.afterAllTests();
-  }
-}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java
index e90df70405f..5b8b464004d 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java
@@ -130,7 +130,7 @@ public class DisabledTestRegionServerExit extends HBaseClusterTestCase {
     int server = -1;
     for (int i = 0; i < regionThreads.size() && server == -1; i++) {
       HRegionServer s = regionThreads.get(i).getRegionServer();
-      Collection<HRegion> regions = s.getOnlineRegions();
+      Collection<HRegion> regions = s.getOnlineRegionsLocalContext();
       for (HRegion r : regions) {
         if (Bytes.equals(r.getTableDesc().getName(),
             HConstants.META_TABLE_NAME)) {
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java b/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
index fd1d1bf90c6..c591632f73e 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.client.Put;
 public class OOMERegionServer extends HRegionServer {
   private List<Put> retainer = new ArrayList<Put>();
 
-  public OOMERegionServer(HBaseConfiguration conf) throws IOException {
+  public OOMERegionServer(HBaseConfiguration conf) throws IOException, InterruptedException {
     super(conf);
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
index ec6235a8470..895406c2ce2 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -161,8 +160,9 @@ public class TestFSErrorsExposed {
           fam, 1, HColumnDescriptor.DEFAULT_COMPRESSION,
           false, false, HConstants.FOREVER, "NONE"));
       admin.createTable(desc);
-
-      HTable table = new HTable(tableName);
+      // Make it fail faster.
+      util.getConfiguration().setInt("hbase.client.retries.number", 1);
+      HTable table = new HTable(util.getConfiguration(), tableName);
 
       // Load some data
       util.loadTable(table, fam);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index d0b84cc5150..8ed20b18695 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -1316,7 +1316,7 @@ public class TestHRegion extends HBaseTestCase {
     // the prepare call -- we are not ready to split just now.  Just return.
     if (!st.prepare()) return null;
     try {
-      result = st.execute(null);
+      result = st.execute(null, null);
     } catch (IOException ioe) {
       try {
         LOG.info("Running rollback of failed split of " +
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java
new file mode 100644
index 00000000000..72568991593
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java
@@ -0,0 +1,116 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.Semaphore;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.MasterAddressTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestMasterAddressManager {
+  private static final Log LOG = LogFactory.getLog(TestMasterAddressManager.class);
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniZKCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniZKCluster();
+  }
+  /**
+   * Unit tests that uses ZooKeeper but does not use the master-side methods
+   * but rather acts directly on ZK.
+   * @throws Exception
+   */
+  @Test
+  public void testMasterAddressManagerFromZK() throws Exception {
+
+    ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+        "testMasterAddressManagerFromZK", null);
+    ZKUtil.createAndFailSilent(zk, zk.baseZNode);
+
+    // Should not have a master yet
+    MasterAddressTracker addressManager = new MasterAddressTracker(zk, null);
+    addressManager.start();
+    assertFalse(addressManager.hasMaster());
+    zk.registerListener(addressManager);
+
+    // Use a listener to capture when the node is actually created
+    NodeCreationListener listener = new NodeCreationListener(zk, zk.masterAddressZNode);
+    zk.registerListener(listener);
+
+    // Create the master node with a dummy address
+    String host = "hostname";
+    int port = 1234;
+    HServerAddress dummyAddress = new HServerAddress(host, port);
+    LOG.info("Creating master node");
+    ZKUtil.setAddressAndWatch(zk, zk.masterAddressZNode, dummyAddress);
+
+    // Wait for the node to be created
+    LOG.info("Waiting for master address manager to be notified");
+    listener.waitForCreation();
+    LOG.info("Master node created");
+    assertTrue(addressManager.hasMaster());
+    HServerAddress pulledAddress = addressManager.getMasterAddress();
+    assertTrue(pulledAddress.equals(dummyAddress));
+
+  }
+
+  public static class NodeCreationListener extends ZooKeeperListener {
+    private static final Log LOG = LogFactory.getLog(NodeCreationListener.class);
+
+    private Semaphore lock;
+    private String node;
+
+    public NodeCreationListener(ZooKeeperWatcher watcher, String node) {
+      super(watcher);
+      lock = new Semaphore(0);
+      this.node = node;
+    }
+
+    @Override
+    public void nodeCreated(String path) {
+      if(path.equals(node)) {
+        LOG.debug("nodeCreated(" + path + ")");
+        lock.release();
+      }
+    }
+
+    public void waitForCreation() throws InterruptedException {
+      lock.acquire();
+    }
+  }
+}
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
index a245d974d1b..3bb0b58ecb1 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -44,6 +45,7 @@ import org.apache.hadoop.hbase.util.PairOfSameType;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 /**
  * Test the {@link SplitTransaction} class against an HRegion (as opposed to
@@ -67,8 +69,9 @@ public class TestSplitTransaction {
     this.fs.delete(this.testdir, true);
     this.wal = new HLog(fs, new Path(this.testdir, "logs"),
       new Path(this.testdir, "archive"),
-      TEST_UTIL.getConfiguration(), null);
+      TEST_UTIL.getConfiguration());
     this.parent = createRegion(this.testdir, this.wal);
+    TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true);
   }
 
   @After public void teardown() throws IOException {
@@ -128,7 +131,9 @@ public class TestSplitTransaction {
     SplitTransaction st = prepareGOOD_SPLIT_ROW();
 
     // Run the execute.  Look at what it returns.
-    PairOfSameType<HRegion> daughters = st.execute(null);
+    Server mockServer = Mockito.mock(Server.class);
+    when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
+    PairOfSameType<HRegion> daughters = st.execute(mockServer, null);
     // Do some assertions about execution.
     assertTrue(this.fs.exists(st.getSplitDir()));
     // Assert the parent region is closed.
@@ -150,7 +155,7 @@ public class TestSplitTransaction {
     int daughtersRowCount = 0;
     for (HRegion r: daughters) {
       // Open so can count its content.
-      HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(), this.testdir,
+      HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(),
         r.getLog(), r.getConf());
       try {
         int count = countRows(openRegion);
@@ -174,12 +179,14 @@ public class TestSplitTransaction {
     // Start transaction.
     SplitTransaction st = prepareGOOD_SPLIT_ROW();
     SplitTransaction spiedUponSt = spy(st);
-    when(spiedUponSt.createDaughterRegion(spiedUponSt.getSecondDaughter())).
+    when(spiedUponSt.createDaughterRegion(spiedUponSt.getSecondDaughter(), null)).
       thenThrow(new MockedFailedDaughterCreation());
     // Run the execute.  Look at what it returns.
     boolean expectedException = false;
+    Server mockServer = Mockito.mock(Server.class);
+    when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
     try {
-      spiedUponSt.execute(null);
+      spiedUponSt.execute(mockServer, null);
     } catch (MockedFailedDaughterCreation e) {
       expectedException = true;
     }
@@ -198,12 +205,12 @@ public class TestSplitTransaction {
 
     // Now retry the split but do not throw an exception this time.
     assertTrue(st.prepare());
-    PairOfSameType<HRegion> daughters = st.execute(null);
+    PairOfSameType<HRegion> daughters = st.execute(mockServer, null);
     // Count rows.
     int daughtersRowCount = 0;
     for (HRegion r: daughters) {
       // Open so can count its content.
-      HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(), this.testdir,
+      HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(),
         r.getLog(), r.getConf());
       try {
         int count = countRows(openRegion);
@@ -248,6 +255,6 @@ public class TestSplitTransaction {
     HColumnDescriptor hcd = new HColumnDescriptor(CF);
     htd.addFamily(hcd);
     HRegionInfo hri = new HRegionInfo(htd, STARTROW, ENDROW);
-    return HRegion.openHRegion(hri, testdir, wal, TEST_UTIL.getConfiguration());
+    return HRegion.openHRegion(hri, wal, TEST_UTIL.getConfiguration());
   }
 }
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
index 0b47975b399..377e6b14ce0 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
@@ -123,7 +123,7 @@ public class TestStore extends TestCase {
     HTableDescriptor htd = new HTableDescriptor(table);
     htd.addFamily(hcd);
     HRegionInfo info = new HRegionInfo(htd, null, null, false);
-    HLog hlog = new HLog(fs, logdir, oldLogDir, conf, null);
+    HLog hlog = new HLog(fs, logdir, oldLogDir, conf);
     HRegion region = new HRegion(basedir, hlog, fs, conf, info, null);
 
     store = new Store(basedir, region, hcd, fs, conf);
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java
index 7c740d386d4..41329c344d0 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java
@@ -29,7 +29,7 @@ public class InstrumentedSequenceFileLogWriter extends SequenceFileLogWriter {
   @Override
     public void append(HLog.Entry entry) throws IOException {
       super.append(entry);
-      if (activateFailure && Bytes.equals(entry.getKey().getRegionName(), "break".getBytes())) {
+      if (activateFailure && Bytes.equals(entry.getKey().getEncodedRegionName(), "break".getBytes())) {
         System.out.println(getClass().getName() + ": I will throw an exception now...");
         throw(new IOException("This exception is instrumented and should only be thrown for testing"));
       }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index ad8f9e5e0a2..185aff25ab9 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -133,7 +133,7 @@ public class TestHLog  {
     final byte [] tableName = Bytes.toBytes(getName());
     final byte [] rowName = tableName;
     Path logdir = new Path(dir, HConstants.HREGION_LOGDIR_NAME);
-    HLog log = new HLog(fs, logdir, oldLogDir, conf, null);
+    HLog log = new HLog(fs, logdir, oldLogDir, conf);
     final int howmany = 3;
     HRegionInfo[] infos = new HRegionInfo[3];
     for(int i = 0; i < howmany; i++) {
@@ -192,7 +192,7 @@ public class TestHLog  {
     out.close();
     in.close();
     Path subdir = new Path(dir, "hlogdir");
-    HLog wal = new HLog(fs, subdir, oldLogDir, conf, null);
+    HLog wal = new HLog(fs, subdir, oldLogDir, conf);
     final int total = 20;
 
     HRegionInfo info = new HRegionInfo(new HTableDescriptor(bytes),
@@ -295,7 +295,7 @@ public class TestHLog  {
         HLog.Entry entry = new HLog.Entry();
         while((entry = reader.next(entry)) != null) {
           HLogKey key = entry.getKey();
-          String region = Bytes.toString(key.getRegionName());
+          String region = Bytes.toString(key.getEncodedRegionName());
           // Assert that all edits are for same region.
           if (previousRegion != null) {
             assertEquals(previousRegion, region);
@@ -325,7 +325,7 @@ public class TestHLog  {
         HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
     Path subdir = new Path(dir, "hlogdir");
     Path archdir = new Path(dir, "hlogdir_archive");
-    HLog wal = new HLog(fs, subdir, archdir, conf, null);
+    HLog wal = new HLog(fs, subdir, archdir, conf);
     final int total = 20;
 
     for (int i = 0; i < total; i++) {
@@ -429,7 +429,7 @@ public class TestHLog  {
     final byte [] tableName = Bytes.toBytes("tablename");
     final byte [] row = Bytes.toBytes("row");
     HLog.Reader reader = null;
-    HLog log = new HLog(fs, dir, oldLogDir, conf, null);
+    HLog log = new HLog(fs, dir, oldLogDir, conf);
     try {
       // Write columns named 1, 2, 3, etc. and then values of single byte
       // 1, 2, 3...
@@ -442,10 +442,9 @@ public class TestHLog  {
       }
       HRegionInfo info = new HRegionInfo(new HTableDescriptor(tableName),
         row,Bytes.toBytes(Bytes.toString(row) + "1"), false);
-      final byte [] regionName = info.getRegionName();
       log.append(info, tableName, cols, System.currentTimeMillis());
       long logSeqId = log.startCacheFlush();
-      log.completeCacheFlush(regionName, tableName, logSeqId, info.isMetaRegion());
+      log.completeCacheFlush(info.getEncodedNameAsBytes(), tableName, logSeqId, info.isMetaRegion());
       log.close();
       Path filename = log.computeFilename();
       log = null;
@@ -458,7 +457,7 @@ public class TestHLog  {
         if (entry == null) break;
         HLogKey key = entry.getKey();
         WALEdit val = entry.getEdit();
-        assertTrue(Bytes.equals(regionName, key.getRegionName()));
+        assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName()));
         assertTrue(Bytes.equals(tableName, key.getTablename()));
         KeyValue kv = val.getKeyValues().get(0);
         assertTrue(Bytes.equals(row, kv.getRow()));
@@ -470,7 +469,7 @@ public class TestHLog  {
         HLogKey key = entry.getKey();
         WALEdit val = entry.getEdit();
         // Assert only one more row... the meta flushed row.
-        assertTrue(Bytes.equals(regionName, key.getRegionName()));
+        assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName()));
         assertTrue(Bytes.equals(tableName, key.getTablename()));
         KeyValue kv = val.getKeyValues().get(0);
         assertTrue(Bytes.equals(HLog.METAROW, kv.getRow()));
@@ -498,7 +497,7 @@ public class TestHLog  {
     final byte [] tableName = Bytes.toBytes("tablename");
     final byte [] row = Bytes.toBytes("row");
     Reader reader = null;
-    HLog log = new HLog(fs, dir, oldLogDir, conf, null);
+    HLog log = new HLog(fs, dir, oldLogDir, conf);
     try {
       // Write columns named 1, 2, 3, etc. and then values of single byte
       // 1, 2, 3...
@@ -513,7 +512,7 @@ public class TestHLog  {
           HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
       log.append(hri, tableName, cols, System.currentTimeMillis());
       long logSeqId = log.startCacheFlush();
-      log.completeCacheFlush(hri.getRegionName(), tableName, logSeqId, false);
+      log.completeCacheFlush(hri.getEncodedNameAsBytes(), tableName, logSeqId, false);
       log.close();
       Path filename = log.computeFilename();
       log = null;
@@ -524,7 +523,7 @@ public class TestHLog  {
       int idx = 0;
       for (KeyValue val : entry.getEdit().getKeyValues()) {
         assertTrue(Bytes.equals(hri.getRegionName(),
-          entry.getKey().getRegionName()));
+          entry.getKey().getEncodedRegionName()));
         assertTrue(Bytes.equals(tableName, entry.getKey().getTablename()));
         assertTrue(Bytes.equals(row, val.getRow()));
         assertEquals((byte)(idx + '0'), val.getValue()[0]);
@@ -537,7 +536,7 @@ public class TestHLog  {
       assertEquals(1, entry.getEdit().size());
       for (KeyValue val : entry.getEdit().getKeyValues()) {
         assertTrue(Bytes.equals(hri.getRegionName(),
-          entry.getKey().getRegionName()));
+          entry.getKey().getEncodedRegionName()));
         assertTrue(Bytes.equals(tableName, entry.getKey().getTablename()));
         assertTrue(Bytes.equals(HLog.METAROW, val.getRow()));
         assertTrue(Bytes.equals(HLog.METAFAMILY, val.getFamily()));
@@ -564,9 +563,9 @@ public class TestHLog  {
     final int COL_COUNT = 10;
     final byte [] tableName = Bytes.toBytes("tablename");
     final byte [] row = Bytes.toBytes("row");
-    HLog log = new HLog(fs, dir, oldLogDir, conf, null);
-    DumbLogEntriesVisitor visitor = new DumbLogEntriesVisitor();
-    log.addLogEntryVisitor(visitor);
+    HLog log = new HLog(fs, dir, oldLogDir, conf);
+    DumbWALObserver visitor = new DumbWALObserver();
+    log.registerWALActionsListener(visitor);
     long timestamp = System.currentTimeMillis();
     HRegionInfo hri = new HRegionInfo(new HTableDescriptor(tableName),
         HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
@@ -578,7 +577,7 @@ public class TestHLog  {
       log.append(hri, tableName, cols, System.currentTimeMillis());
     }
     assertEquals(COL_COUNT, visitor.increments);
-    log.removeLogEntryVisitor(visitor);
+    log.unregisterWALActionsListener(visitor);
     WALEdit cols = new WALEdit();
     cols.add(new KeyValue(row, Bytes.toBytes("column"),
         Bytes.toBytes(Integer.toString(11)),
@@ -587,8 +586,7 @@ public class TestHLog  {
     assertEquals(COL_COUNT, visitor.increments);
   }
 
-  static class DumbLogEntriesVisitor implements LogEntryVisitor {
-
+  static class DumbWALObserver implements WALObserver {
     int increments = 0;
 
     @Override
@@ -596,5 +594,17 @@ public class TestHLog  {
                                          WALEdit logEdit) {
       increments++;
     }
+
+    @Override
+    public void logRolled(Path newFile) {
+      // TODO Auto-generated method stub
+      
+    }
+
+    @Override
+    public void logRollRequested() {
+      // TODO Auto-generated method stub
+      
+    }
   }
 }
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
index 20ea8ee615f..287f1fb9213 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
@@ -154,6 +154,8 @@ public class TestLogRolling  {
   private void startAndWriteData() throws IOException {
     // When the META table can be opened, the region servers are running
     new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
+    this.server = cluster.getRegionServerThreads().get(0).getRegionServer();
+    this.log = server.getWAL();
 
     // Create the test table and open it
     HTableDescriptor desc = new HTableDescriptor(tableName);
@@ -162,7 +164,7 @@ public class TestLogRolling  {
     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
 
     server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName));
-    this.log = server.getLog();
+    this.log = server.getWAL();
     for (int i = 1; i <= 256; i++) {    // 256 writes should cause 8 log rolls
       Put put = new Put(Bytes.toBytes("row" + String.format("%1$04d", i)));
       put.add(HConstants.CATALOG_FAMILY, null, value);
@@ -192,7 +194,7 @@ public class TestLogRolling  {
       // flush all regions
 
       List<HRegion> regions =
-        new ArrayList<HRegion>(server.getOnlineRegions());
+        new ArrayList<HRegion>(server.getOnlineRegionsLocalContext());
       for (HRegion r: regions) {
         r.flushcache();
       }
@@ -226,7 +228,6 @@ public class TestLogRolling  {
   /**
    * Give me the HDFS pipeline for this log file
    */
-  @SuppressWarnings("null")
   DatanodeInfo[] getPipeline(HLog log) throws IllegalArgumentException,
       IllegalAccessException, InvocationTargetException {
     OutputStream stm = log.getOutputStream();
@@ -258,10 +259,15 @@ public class TestLogRolling  {
   public void testLogRollOnDatanodeDeath() throws IOException,
       InterruptedException, IllegalArgumentException, IllegalAccessException,
       InvocationTargetException {
-    assertTrue("This test requires HLog file replication.", fs
-        .getDefaultReplication() > 1);
+    assertTrue("This test requires HLog file replication.",
+      fs.getDefaultReplication() > 1);
+    LOG.info("Replication=" + fs.getDefaultReplication());
     // When the META table can be opened, the region servers are running
     new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
+
+    this.server = cluster.getRegionServer(0);
+    this.log = server.getWAL();
+    
     // Create the test table and open it
     String tableName = getName();
     HTableDescriptor desc = new HTableDescriptor(tableName);
@@ -275,7 +281,7 @@ public class TestLogRolling  {
     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
 
     server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName));
-    this.log = server.getLog();
+    this.log = server.getWAL();
 
     assertTrue("Need HDFS-826 for this test", log.canGetCurReplicas());
     // don't run this test without append support (HDFS-200 & HDFS-142)
@@ -297,8 +303,7 @@ public class TestLogRolling  {
     assertTrue("Log should have a timestamp older than now",
         curTime > oldFilenum && oldFilenum != -1);
 
-    assertTrue("The log shouldn't have rolled yet", oldFilenum == log
-        .getFilenum());
+    assertTrue("The log shouldn't have rolled yet", oldFilenum == log.getFilenum());
     DatanodeInfo[] pipeline = getPipeline(log);
     assertTrue(pipeline.length == fs.getDefaultReplication());
 
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALObserver.java
similarity index 79%
rename from src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
rename to src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALObserver.java
index 6b03c4cc010..a1a18811622 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALObserver.java
@@ -19,6 +19,9 @@
  */
 package org.apache.hadoop.hbase.regionserver.wal;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -39,10 +42,8 @@ import static org.junit.Assert.*;
 /**
  * Test that the actions are called while playing with an HLog
  */
-public class TestLogActionsListener {
-
-  protected static final Log LOG =
-      LogFactory.getLog(TestLogActionsListener.class);
+public class TestWALObserver {
+  protected static final Log LOG = LogFactory.getLog(TestWALObserver.class);
 
   private final static HBaseTestingUtility TEST_UTIL =
       new HBaseTestingUtility();
@@ -82,9 +83,11 @@ public class TestLogActionsListener {
    */
   @Test
   public void testActionListener() throws Exception {
-    DummyLogActionsListener list = new DummyLogActionsListener();
-    DummyLogActionsListener laterList = new DummyLogActionsListener();
-    HLog hlog = new HLog(fs, logDir, oldLogDir, conf, null, list, null);
+    DummyWALObserver observer = new DummyWALObserver();
+    List<WALObserver> list = new ArrayList<WALObserver>();
+    list.add(observer);
+    DummyWALObserver laterobserver = new DummyWALObserver();
+    HLog hlog = new HLog(fs, logDir, oldLogDir, conf, list, null);
     HRegionInfo hri = new HRegionInfo(new HTableDescriptor(SOME_BYTES),
         SOME_BYTES, SOME_BYTES, false);
 
@@ -96,26 +99,37 @@ public class TestLogActionsListener {
       HLogKey key = new HLogKey(b,b, 0, 0);
       hlog.append(hri, key, edit);
       if (i == 10) {
-        hlog.addLogActionsListerner(laterList);
+        hlog.registerWALActionsListener(laterobserver);
       }
       if (i % 2 == 0) {
         hlog.rollWriter();
       }
     }
-    assertEquals(11, list.logRollCounter);
-    assertEquals(5, laterList.logRollCounter);
+    assertEquals(11, observer.logRollCounter);
+    assertEquals(5, laterobserver.logRollCounter);
   }
 
   /**
    * Just counts when methods are called
    */
-  static class DummyLogActionsListener implements LogActionsListener {
-
+  static class DummyWALObserver implements WALObserver {
     public int logRollCounter = 0;
 
     @Override
     public void logRolled(Path newFile) {
       logRollCounter++;
     }
+
+    @Override
+    public void logRollRequested() {
+      // Not interested
+    }
+
+    @Override
+    public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey,
+        WALEdit logEdit) {
+      // Not interested
+      
+    }
   }
-}
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index c982662c70f..6c11eaf3596 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.regionserver.FlushRequester;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -183,7 +184,7 @@ public class TestWALReplay {
     Path basedir = new Path(this.hbaseRootDir, tableNameStr);
     deleteDir(basedir);
     HLog wal = createWAL(this.conf);
-    HRegion region = HRegion.openHRegion(hri, basedir, wal, this.conf);
+    HRegion region = HRegion.openHRegion(hri, wal, this.conf);
     Path f =  new Path(basedir, "hfile");
     HFile.Writer writer = new HFile.Writer(this.fs, f);
     byte [] family = hri.getTableDesc().getFamilies().iterator().next().getName();
@@ -327,7 +328,7 @@ public class TestWALReplay {
     HLog wal = createWAL(this.conf);
     final byte[] tableName = Bytes.toBytes(tableNameStr);
     final byte[] rowName = tableName;
-    final byte[] regionName = hri.getRegionName();
+    final byte[] regionName = hri.getEncodedNameAsBytes();
 
     // Add 1k to each family.
     final int countPerFamily = 1000;
@@ -358,7 +359,6 @@ public class TestWALReplay {
     // Set down maximum recovery so we dfsclient doesn't linger retrying something
     // long gone.
     HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
-
     // Make a new conf and a new fs for the splitter to run on so we can take
     // over old wal.
     Configuration newConf = HBaseTestingUtility.setDifferentUser(this.conf,
@@ -396,6 +396,23 @@ public class TestWALReplay {
     }
   }
 
+  // Flusher used in this test.  Keep count of how often we are called and
+  // actually run the flush inside here.
+  class TestFlusher implements FlushRequester {
+    private int count = 0;
+    private HRegion r;
+
+    @Override
+    public void requestFlush(HRegion region) {
+      count++;
+      try {
+        r.flushcache();
+      } catch (IOException e) {
+        throw new RuntimeException("Exception flushing", e);
+      }
+    }
+  }
+
   private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
       final byte [] rowName, final byte [] family, 
       final int count, EnvironmentEdge ee, final HLog wal)
@@ -464,7 +481,7 @@ public class TestWALReplay {
    * @throws IOException
    */
   private HLog createWAL(final Configuration c) throws IOException {
-    HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c, null);
+    HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c);
     // Set down maximum recovery so we dfsclient doesn't linger retrying something
     // long gone.
     HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java b/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
index e1f4f98a640..3ba7b6764ec 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
@@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.replication;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface;
 import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager;
 
@@ -39,7 +40,7 @@ public class ReplicationSourceDummy implements ReplicationSourceInterface {
 
   @Override
   public void init(Configuration conf, FileSystem fs,
-                   ReplicationSourceManager manager, AtomicBoolean stopper,
+                   ReplicationSourceManager manager, Stoppable stopper,
                    AtomicBoolean replicating, String peerClusterId)
       throws IOException {
     this.manager = manager;
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
index 31cc6805c8d..55d5702f3ae 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MiniZooKeeperCluster;
+// import org.apache.hadoop.hbase.MiniZooKeeperCluster;
 import org.apache.hadoop.hbase.UnknownScannerException;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
@@ -39,7 +39,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+// import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -56,9 +56,10 @@ public class TestReplication {
 
   private static Configuration conf1;
   private static Configuration conf2;
-
+/*
   private static ZooKeeperWrapper zkw1;
   private static ZooKeeperWrapper zkw2;
+  */
 
   private static HTable htable1;
   private static HTable htable2;
@@ -96,6 +97,7 @@ public class TestReplication {
 
     utility1 = new HBaseTestingUtility(conf1);
     utility1.startMiniZKCluster();
+    /* REENALBE
     MiniZooKeeperCluster miniZK = utility1.getZkCluster();
     zkw1 = ZooKeeperWrapper.createInstance(conf1, "cluster1");
     zkw1.writeZNode("/1", "replication", "");
@@ -103,7 +105,7 @@ public class TestReplication {
         conf1.get(HConstants.ZOOKEEPER_QUORUM)+":" +
             conf1.get("hbase.zookeeper.property.clientPort")+":/1");
     setIsReplication(true);
-
+*/
     LOG.info("Setup first Zk");
 
     conf2 = HBaseConfiguration.create();
@@ -112,7 +114,7 @@ public class TestReplication {
     conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
     conf2.setBoolean("dfs.support.append", true);
     conf2.setLong("hbase.regions.percheckin", 1);
-
+/* REENALBE
     utility2 = new HBaseTestingUtility(conf2);
     utility2.setZkCluster(miniZK);
     zkw2 = ZooKeeperWrapper.createInstance(conf2, "cluster2");
@@ -124,7 +126,7 @@ public class TestReplication {
     zkw1.writeZNode("/1/replication/peers", "1",
         conf2.get(HConstants.ZOOKEEPER_QUORUM)+":" +
             conf2.get("hbase.zookeeper.property.clientPort")+":/2");
-
+*/
     LOG.info("Setup second Zk");
 
     utility1.startMiniCluster(2);
@@ -149,7 +151,7 @@ public class TestReplication {
 
   private static void setIsReplication(boolean rep) throws Exception {
     LOG.info("Set rep " + rep);
-    zkw1.writeZNode("/1/replication", "state", Boolean.toString(rep));
+   // REENALBE  zkw1.writeZNode("/1/replication", "state", Boolean.toString(rep));
     // Takes some ms for ZK to fire the watcher
     Thread.sleep(SLEEP_TIME);
   }
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java
index dc4f458c8d4..2277ecf77b7 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
@@ -69,9 +70,22 @@ public class TestReplicationSink {
   private static final byte[] FAM_NAME1 = Bytes.toBytes("info1");
   private static final byte[] FAM_NAME2 = Bytes.toBytes("info2");
 
-  private static final AtomicBoolean STOPPER = new AtomicBoolean(false);
-
   private static HTable table1;
+  private static Stoppable STOPPABLE = new Stoppable() {
+    final AtomicBoolean stop = new AtomicBoolean(false);
+
+    @Override
+    public boolean isStopped() {
+      return this.stop.get();
+    }
+
+    @Override
+    public void stop(String why) {
+      LOG.info("STOPPING BECAUSE: " + why);
+      this.stop.set(true);
+    }
+    
+  };
 
   private static HTable table2;
 
@@ -85,7 +99,7 @@ public class TestReplicationSink {
         HConstants.REPLICATION_ENABLE_KEY, true);
     TEST_UTIL.startMiniCluster(3);
     conf.setBoolean("dfs.support.append", true);
-    SINK = new ReplicationSink(conf,STOPPER);
+    SINK = new ReplicationSink(conf, STOPPABLE);
     table1 = TEST_UTIL.createTable(TABLE_NAME1, FAM_NAME1);
     table2 = TEST_UTIL.createTable(TABLE_NAME2, FAM_NAME2);
   }
@@ -95,7 +109,7 @@ public class TestReplicationSink {
    */
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
-    STOPPER.set(true);
+    STOPPABLE.stop("Shutting down");
     TEST_UTIL.shutdownMiniCluster();
   }
 
diff --git a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index b497b9a81bc..b904fd0b005 100644
--- a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -35,10 +35,11 @@ import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.regionserver.wal.WALObserver;
 import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
-import org.apache.hadoop.hbase.replication.ReplicationZookeeperWrapper;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+// REENABLE import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -46,6 +47,8 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.junit.Assert.assertEquals;
@@ -59,13 +62,11 @@ public class TestReplicationSourceManager {
 
   private static HBaseTestingUtility utility;
 
-  private static final AtomicBoolean STOPPER = new AtomicBoolean(false);
-
   private static final AtomicBoolean REPLICATING = new AtomicBoolean(false);
 
   private static ReplicationSourceManager manager;
 
-  private static ZooKeeperWrapper zkw;
+  // REENALBE private static ZooKeeperWrapper zkw;
 
   private static HTableDescriptor htd;
 
@@ -98,19 +99,18 @@ public class TestReplicationSourceManager {
     utility = new HBaseTestingUtility(conf);
     utility.startMiniZKCluster();
 
-    zkw = ZooKeeperWrapper.createInstance(conf, "test");
-    zkw.writeZNode("/hbase", "replication", "");
-    zkw.writeZNode("/hbase/replication", "master",
-        conf.get(HConstants.ZOOKEEPER_QUORUM)+":" +
-    conf.get("hbase.zookeeper.property.clientPort")+":/1");
-    zkw.writeZNode("/hbase/replication/peers", "1",
-          conf.get(HConstants.ZOOKEEPER_QUORUM)+":" +
-          conf.get("hbase.zookeeper.property.clientPort")+":/1");
+    // REENABLE
+//    zkw = ZooKeeperWrapper.createInstance(conf, "test");
+//    zkw.writeZNode("/hbase", "replication", "");
+//    zkw.writeZNode("/hbase/replication", "master",
+//        conf.get(HConstants.ZOOKEEPER_QUORUM)+":" +
+//    conf.get("hbase.zookeeper.property.clientPort")+":/1");
+//    zkw.writeZNode("/hbase/replication/peers", "1",
+//          conf.get(HConstants.ZOOKEEPER_QUORUM)+":" +
+//          conf.get("hbase.zookeeper.property.clientPort")+":/1");
 
     HRegionServer server = new HRegionServer(conf);
-    ReplicationZookeeperWrapper helper = new ReplicationZookeeperWrapper(
-        server.getZooKeeperWrapper(), conf,
-        REPLICATING, "123456789");
+    ReplicationZookeeper helper = new ReplicationZookeeper(server, REPLICATING);
     fs = FileSystem.get(conf);
     oldLogDir = new Path(utility.getTestDir(),
         HConstants.HREGION_OLDLOGDIR_NAME);
@@ -118,7 +118,7 @@ public class TestReplicationSourceManager {
         HConstants.HREGION_LOGDIR_NAME);
 
     manager = new ReplicationSourceManager(helper,
-        conf, STOPPER, fs, REPLICATING, logDir, oldLogDir);
+        conf, server, fs, REPLICATING, logDir, oldLogDir);
     manager.addSource("1");
 
     htd = new HTableDescriptor(test);
@@ -136,7 +136,7 @@ public class TestReplicationSourceManager {
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
-    manager.join();
+// REENABLE   manager.join();
     utility.shutdownMiniCluster();
   }
 
@@ -159,8 +159,9 @@ public class TestReplicationSourceManager {
     KeyValue kv = new KeyValue(r1, f1, r1);
     WALEdit edit = new WALEdit();
     edit.add(kv);
-
-    HLog hlog = new HLog(fs, logDir, oldLogDir, conf, null, manager,
+    List<WALObserver> listeners = new ArrayList<WALObserver>();
+// REENABLE    listeners.add(manager);
+    HLog hlog = new HLog(fs, logDir, oldLogDir, conf, listeners,
       URLEncoder.encode("regionserver:60020", "UTF8"));
 
     manager.init();
@@ -193,14 +194,14 @@ public class TestReplicationSourceManager {
 
     hlog.rollWriter();
 
-    manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
-        "1", 0, false);
+ // REENABLE     manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
+ // REENABLE        "1", 0, false);
 
     HLogKey key = new HLogKey(hri.getRegionName(),
           test, seq++, System.currentTimeMillis());
     hlog.append(hri, key, edit);
 
-    assertEquals(1, manager.getHLogs().size());
+ // REENABLE     assertEquals(1, manager.getHLogs().size());
 
 
     // TODO Need a case with only 2 HLogs and we only want to delete the first one
diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
index 2813db7cdf0..235b24ea5e6 100644
--- a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
@@ -77,7 +77,7 @@ public class TestRemoteTable extends HBaseRESTClusterTestBase {
       htd.addFamily(new HColumnDescriptor(COLUMN_2));
       htd.addFamily(new HColumnDescriptor(COLUMN_3));
       admin.createTable(htd);
-      HTable table = new HTable(TABLE);
+      HTable table = new HTable(conf, TABLE);
       Put put = new Put(ROW_1);
       put.add(COLUMN_1, QUALIFIER_1, TS_2, VALUE_1);
       table.put(put);
diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
index 7b7831f1bd8..f39a0c28a4c 100644
--- a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
+++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java
@@ -72,7 +72,7 @@ public class TestThriftServer extends HBaseClusterTestCase {
    * @throws Exception
    */
   public void doTestTableCreateDrop() throws Exception {
-    ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+    ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(this.conf);
 
     // Create/enable/disable/delete tables, ensure methods act correctly
     assertEquals(handler.getTableNames().size(), 0);
@@ -103,7 +103,7 @@ public class TestThriftServer extends HBaseClusterTestCase {
    */
   public void doTestTableMutations() throws Exception {
     // Setup
-    ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+    ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(this.conf);
     handler.createTable(tableAname, getColumnDescriptors());
 
     // Apply a few Mutations to rowA
@@ -167,7 +167,7 @@ public class TestThriftServer extends HBaseClusterTestCase {
    */
   public void doTestTableTimestampsAndColumns() throws Exception {
     // Setup
-    ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+    ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(this.conf);
     handler.createTable(tableAname, getColumnDescriptors());
 
     // Apply timestamped Mutations to rowA
@@ -245,7 +245,7 @@ public class TestThriftServer extends HBaseClusterTestCase {
    */
   public void doTestTableScanners() throws Exception {
     // Setup
-    ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+    ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(this.conf);
     handler.createTable(tableAname, getColumnDescriptors());
 
     // Apply timestamped Mutations to rowA
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java b/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
index 418fc1e8d1b..e70135fc60e 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
@@ -93,6 +93,14 @@ public class TestBytes extends TestCase {
     // If split more than once, this should fail
     parts = Bytes.split(low, high, 2);
     assertTrue("Returned split but should have failed", parts == null);
+
+    // Split 0 times should throw IAE
+    try {
+      parts = Bytes.split(low, high, 0);
+      assertTrue("Should not be able to split 0 times", false);
+    } catch(IllegalArgumentException iae) {
+      // Correct
+    }
   }
 
   public void testToLong() throws Exception {
diff --git a/src/test/java/org/apache/hadoop/hbase/TestMergeMeta.java b/src/test/java/org/apache/hadoop/hbase/util/TestMergeMeta.java
similarity index 87%
rename from src/test/java/org/apache/hadoop/hbase/TestMergeMeta.java
rename to src/test/java/org/apache/hadoop/hbase/util/TestMergeMeta.java
index 611589b917d..2f729dee364 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestMergeMeta.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestMergeMeta.java
@@ -17,10 +17,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase;
+package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
 
+import org.apache.hadoop.hbase.AbstractMergeTestBase;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.HMerge;
+
 /** Tests region merging */
 public class TestMergeMeta extends AbstractMergeTestBase {
 
@@ -39,6 +43,6 @@ public class TestMergeMeta extends AbstractMergeTestBase {
    */
   public void testMergeMeta() throws IOException {
     assertNotNull(dfsCluster);
-    HMerge.merge(conf, dfsCluster.getFileSystem(), HConstants.META_TABLE_NAME);
+    HMerge.merge(conf, dfsCluster.getFileSystem(), HConstants.META_TABLE_NAME, false);
   }
 }
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/TestMergeTable.java b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
similarity index 81%
rename from src/test/java/org/apache/hadoop/hbase/TestMergeTable.java
rename to src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
index ec37f9bd74e..c29dc502696 100644
--- a/src/test/java/org/apache/hadoop/hbase/TestMergeTable.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
@@ -17,10 +17,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase;
+package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
 
+import org.apache.hadoop.hbase.AbstractMergeTestBase;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.HMerge;
+
 /**
  * Tests merging a normal table's regions
  */
@@ -32,6 +36,8 @@ public class TestMergeTable extends AbstractMergeTestBase {
    */
   public void testMergeTable() throws IOException {
     assertNotNull(dfsCluster);
+    HBaseAdmin admin = new HBaseAdmin(conf);
+    admin.disableTable(desc.getName());
     HMerge.merge(conf, dfsCluster.getFileSystem(), desc.getName());
   }
 }
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
index 3d6f0ecafa2..6aaa79da090 100644
--- a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
@@ -181,8 +181,7 @@ public class TestMergeTool extends HBaseTestCase {
 
     // Now verify that we can read all the rows from regions 0, 1
     // in the new merged region.
-    HRegion merged =
-      HRegion.openHRegion(mergedInfo, this.testDir, log, this.conf);
+    HRegion merged = HRegion.openHRegion(mergedInfo, log, this.conf);
     verifyMerge(merged, upperbound);
     merged.close();
     LOG.info("Verified " + msg);
@@ -249,7 +248,7 @@ public class TestMergeTool extends HBaseTestCase {
       System.currentTimeMillis());
     LOG.info("Creating log " + logPath.toString());
     Path oldLogDir = new Path("/tmp", HConstants.HREGION_OLDLOGDIR_NAME);
-    HLog log = new HLog(this.fs, logPath, oldLogDir, this.conf, null);
+    HLog log = new HLog(this.fs, logPath, oldLogDir, this.conf);
     try {
        // Merge Region 0 and Region 1
       HRegion merged = mergeAndVerify("merging regions 0 and 1",
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java
index f0d90c09c6c..07ab3a9f6a5 100644
--- a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java
@@ -20,59 +20,62 @@
 package org.apache.hadoop.hbase.zookeeper;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.util.Map;
 import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
 import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
+import org.junit.Before;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.*;
 
 /**
  * Test for HQuorumPeer.
  */
-public class TestHQuorumPeer extends HBaseTestCase {
+public class TestHQuorumPeer {
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private Path dataDir;
 
-  @Override
-  protected void setUp() throws Exception {
-    super.setUp();
-    String userName = System.getProperty("user.name");
-    dataDir = new Path("/tmp/hbase-" + userName, "zookeeper");
-    if (fs.exists(dataDir)) {
-      if (!fs.isDirectory(dataDir)) {
-        fail();
-      }
-    } else {
-      if (!fs.mkdirs(dataDir)) {
-        fail();
+
+  @Before public void setup() throws IOException {
+    // Set it to a non-standard port.
+    TEST_UTIL.getConfiguration().setInt("hbase.zookeeper.property.clientPort",
+      21810);
+    this.dataDir = HBaseTestingUtility.getTestDir(this.getClass().getName());
+    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
+    if (fs.exists(this.dataDir)) {
+      if (!fs.delete(this.dataDir, true)) {
+        throw new IOException("Failed cleanup of " + this.dataDir);
       }
     }
+    if (!fs.mkdirs(this.dataDir)) {
+      throw new IOException("Failed create of " + this.dataDir);
+    }
   }
 
-  @Override
-  protected void tearDown() throws Exception {
-    if (fs.exists(dataDir) && !fs.delete(dataDir, true)) {
-      fail();
-    }
-    super.tearDown();
-  }
-
-  /** */
-  public void testMakeZKProps() {
-    Properties properties = HQuorumPeer.makeZKProps(conf);
-    assertEquals(dataDir.toString(), properties.get("dataDir"));
-    assertEquals(Integer.valueOf(21810), Integer.valueOf(properties.getProperty("clientPort")));
+  @Test public void testMakeZKProps() {
+    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
+    conf.set("hbase.zookeeper.property.dataDir", this.dataDir.toString());
+    Properties properties = ZKConfig.makeZKProps(conf);
+    assertEquals(dataDir.toString(), (String)properties.get("dataDir"));
+    assertEquals(Integer.valueOf(21810),
+      Integer.valueOf(properties.getProperty("clientPort")));
     assertEquals("localhost:2888:3888", properties.get("server.0"));
     assertEquals(null, properties.get("server.1"));
 
     String oldValue = conf.get(HConstants.ZOOKEEPER_QUORUM);
     conf.set(HConstants.ZOOKEEPER_QUORUM, "a.foo.bar,b.foo.bar,c.foo.bar");
-    properties = HQuorumPeer.makeZKProps(conf);
+    properties = ZKConfig.makeZKProps(conf);
     assertEquals(dataDir.toString(), properties.get("dataDir"));
     assertEquals(Integer.valueOf(21810), Integer.valueOf(properties.getProperty("clientPort")));
     assertEquals("a.foo.bar:2888:3888", properties.get("server.0"));
@@ -82,25 +85,26 @@ public class TestHQuorumPeer extends HBaseTestCase {
     conf.set(HConstants.ZOOKEEPER_QUORUM, oldValue);
   }
 
-  /** @throws Exception */
-  public void testConfigInjection() throws Exception {
+  @Test public void testConfigInjection() throws Exception {
     String s =
-      "dataDir=${hbase.tmp.dir}/zookeeper\n" +
+      "dataDir=" + this.dataDir.toString() + "\n" +
       "clientPort=2181\n" +
       "server.0=${hbase.master.hostname}:2888:3888\n";
 
     System.setProperty("hbase.master.hostname", "localhost");
     InputStream is = new ByteArrayInputStream(s.getBytes());
-    Properties properties = HQuorumPeer.parseZooCfg(conf, is);
+    Configuration conf = TEST_UTIL.getConfiguration();
+    Properties properties = ZKConfig.parseZooCfg(conf, is);
 
-    assertEquals(dataDir.toString(), properties.get("dataDir"));
-    assertEquals(Integer.valueOf(2181), Integer.valueOf(properties.getProperty("clientPort")));
+    assertEquals(this.dataDir.toString(), properties.get("dataDir"));
+    assertEquals(Integer.valueOf(2181),
+      Integer.valueOf(properties.getProperty("clientPort")));
     assertEquals("localhost:2888:3888", properties.get("server.0"));
 
     QuorumPeerConfig config = new QuorumPeerConfig();
     config.parseProperties(properties);
 
-    assertEquals(dataDir.toString(), config.getDataDir());
+    assertEquals(this.dataDir.toString(), config.getDataDir());
     assertEquals(2181, config.getClientPortAddress().getPort());
     Map<Long,QuorumServer> servers = config.getServers();
     assertEquals(1, servers.size());
@@ -111,7 +115,7 @@ public class TestHQuorumPeer extends HBaseTestCase {
     // Override with system property.
     System.setProperty("hbase.master.hostname", "foo.bar");
     is = new ByteArrayInputStream(s.getBytes());
-    properties = HQuorumPeer.parseZooCfg(conf, is);
+    properties = ZKConfig.parseZooCfg(conf, is);
     assertEquals("foo.bar:2888:3888", properties.get("server.0"));
 
     config.parseProperties(properties);
@@ -124,11 +128,11 @@ public class TestHQuorumPeer extends HBaseTestCase {
   /**
    * Test Case for HBASE-2305
    */
-  public void testShouldAssignDefaultZookeeperClientPort() {
+  @Test public void testShouldAssignDefaultZookeeperClientPort() {
     Configuration config = HBaseConfiguration.create();
     config.clear();
-    Properties p = HQuorumPeer.makeZKProps(config);
+    Properties p = ZKConfig.makeZKProps(config);
     assertNotNull(p);
     assertEquals(2181, p.get("hbase.zookeeper.property.clientPort"));
   }
-}
+}
\ No newline at end of file
diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperNodeTracker.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperNodeTracker.java
new file mode 100644
index 00000000000..62786f847de
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperNodeTracker.java
@@ -0,0 +1,281 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Random;
+import java.util.concurrent.Semaphore;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.master.TestActiveMasterManager.NodeDeletionListener;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+public class TestZooKeeperNodeTracker {
+  private static final Log LOG = LogFactory.getLog(TestZooKeeperNodeTracker.class);
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  private final static Random rand = new Random();
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniZKCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniZKCluster();
+  }
+
+  @Test
+  public void testNodeTracker() throws Exception {
+
+    Abortable abortable = new StubAbortable();
+    ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+        "testNodeTracker", abortable);
+    ZKUtil.createAndFailSilent(zk, zk.baseZNode);
+
+    final String node =
+      ZKUtil.joinZNode(zk.baseZNode, new Long(rand.nextLong()).toString());
+
+    final byte [] dataOne = Bytes.toBytes("dataOne");
+    final byte [] dataTwo = Bytes.toBytes("dataTwo");
+
+    // Start a ZKNT with no node currently available
+    TestTracker localTracker = new TestTracker(zk, node, abortable);
+    localTracker.start();
+    zk.registerListener(localTracker);
+
+    // Make sure we don't have a node
+    assertNull(localTracker.getData());
+
+    // Spin up a thread with another ZKNT and have it block
+    WaitToGetDataThread thread = new WaitToGetDataThread(zk, node);
+    thread.start();
+
+    // Verify the thread doesn't have a node
+    assertFalse(thread.hasData);
+
+    // Put up an additional zk listener so we know when zk event is done
+    TestingZKListener zkListener = new TestingZKListener(zk, node);
+    zk.registerListener(zkListener);
+    assertEquals(0, zkListener.createdLock.availablePermits());
+
+    // Create a completely separate zk connection for test triggers and avoid
+    // any weird watcher interactions from the test
+    final ZooKeeper zkconn = new ZooKeeper(
+        ZKConfig.getZKQuorumServersString(TEST_UTIL.getConfiguration()), 60000,
+        new StubWatcher());
+
+    // Add the node with data one
+    zkconn.create(node, dataOne, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+
+    // Wait for the zk event to be processed
+    zkListener.waitForCreation();
+    thread.join();
+
+    // Both trackers should have the node available with data one
+    assertNotNull(localTracker.getData());
+    assertNotNull(localTracker.blockUntilAvailable());
+    assertTrue(Bytes.equals(localTracker.getData(), dataOne));
+    assertTrue(thread.hasData);
+    assertTrue(Bytes.equals(thread.tracker.getData(), dataOne));
+    LOG.info("Successfully got data one");
+
+    // Now, start a new ZKNT with the node already available
+    TestTracker secondTracker = new TestTracker(zk, node, null);
+    secondTracker.start();
+    zk.registerListener(secondTracker);
+
+    // Make sure it's available and with the expected data
+    assertNotNull(secondTracker.getData());
+    assertNotNull(secondTracker.blockUntilAvailable());
+    assertTrue(Bytes.equals(secondTracker.getData(), dataOne));
+    LOG.info("Successfully got data one with the second tracker");
+
+    // Drop the node
+    zkconn.delete(node, -1);
+    zkListener.waitForDeletion();
+
+    // Create a new thread but with the existing thread's tracker to wait
+    TestTracker threadTracker = thread.tracker;
+    thread = new WaitToGetDataThread(zk, node, threadTracker);
+    thread.start();
+
+    // Verify other guys don't have data
+    assertFalse(thread.hasData);
+    assertNull(secondTracker.getData());
+    assertNull(localTracker.getData());
+    LOG.info("Successfully made unavailable");
+
+    // Create with second data
+    zkconn.create(node, dataTwo, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+
+    // Wait for the zk event to be processed
+    zkListener.waitForCreation();
+    thread.join();
+
+    // All trackers should have the node available with data two
+    assertNotNull(localTracker.getData());
+    assertNotNull(localTracker.blockUntilAvailable());
+    assertTrue(Bytes.equals(localTracker.getData(), dataTwo));
+    assertNotNull(secondTracker.getData());
+    assertNotNull(secondTracker.blockUntilAvailable());
+    assertTrue(Bytes.equals(secondTracker.getData(), dataTwo));
+    assertTrue(thread.hasData);
+    assertTrue(Bytes.equals(thread.tracker.getData(), dataTwo));
+    LOG.info("Successfully got data two on all trackers and threads");
+
+    // Change the data back to data one
+    zkconn.setData(node, dataOne, -1);
+
+    // Wait for zk event to be processed
+    zkListener.waitForDataChange();
+
+    // All trackers should have the node available with data one
+    assertNotNull(localTracker.getData());
+    assertNotNull(localTracker.blockUntilAvailable());
+    assertTrue(Bytes.equals(localTracker.getData(), dataOne));
+    assertNotNull(secondTracker.getData());
+    assertNotNull(secondTracker.blockUntilAvailable());
+    assertTrue(Bytes.equals(secondTracker.getData(), dataOne));
+    assertTrue(thread.hasData);
+    assertTrue(Bytes.equals(thread.tracker.getData(), dataOne));
+    LOG.info("Successfully got data one following a data change on all trackers and threads");
+  }
+
+  public static class WaitToGetDataThread extends Thread {
+
+    TestTracker tracker;
+    boolean hasData;
+
+    public WaitToGetDataThread(ZooKeeperWatcher zk, String node) {
+      tracker = new TestTracker(zk, node, null);
+      tracker.start();
+      zk.registerListener(tracker);
+      hasData = false;
+    }
+
+    public WaitToGetDataThread(ZooKeeperWatcher zk, String node,
+        TestTracker tracker) {
+      this.tracker = tracker;
+      hasData = false;
+    }
+
+    @Override
+    public void run() {
+      LOG.info("Waiting for data to be available in WaitToGetDataThread");
+      try {
+        tracker.blockUntilAvailable();
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+      LOG.info("Data now available in tracker from WaitToGetDataThread");
+      hasData = true;
+    }
+  }
+
+  public static class TestTracker extends ZooKeeperNodeTracker {
+
+    public TestTracker(ZooKeeperWatcher watcher, String node,
+        Abortable abortable) {
+      super(watcher, node, abortable);
+    }
+  }
+
+  public static class TestingZKListener extends ZooKeeperListener {
+    private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
+
+    private Semaphore deletedLock;
+    private Semaphore createdLock;
+    private Semaphore changedLock;
+    private String node;
+
+    public TestingZKListener(ZooKeeperWatcher watcher, String node) {
+      super(watcher);
+      deletedLock = new Semaphore(0);
+      createdLock = new Semaphore(0);
+      changedLock = new Semaphore(0);
+      this.node = node;
+    }
+
+    @Override
+    public void nodeDeleted(String path) {
+      if(path.equals(node)) {
+        LOG.debug("nodeDeleted(" + path + ")");
+        deletedLock.release();
+      }
+    }
+
+    @Override
+    public void nodeCreated(String path) {
+      if(path.equals(node)) {
+        LOG.debug("nodeCreated(" + path + ")");
+        createdLock.release();
+      }
+    }
+
+    @Override
+    public void nodeDataChanged(String path) {
+      if(path.equals(node)) {
+        LOG.debug("nodeDataChanged(" + path + ")");
+        changedLock.release();
+      }
+    }
+
+    public void waitForDeletion() throws InterruptedException {
+      deletedLock.acquire();
+    }
+
+    public void waitForCreation() throws InterruptedException {
+      createdLock.acquire();
+    }
+
+    public void waitForDataChange() throws InterruptedException {
+      changedLock.acquire();
+    }
+  }
+
+  public static class StubAbortable implements Abortable {
+    @Override
+    public void abort(final String msg, final Throwable t) {}
+  }
+
+  public static class StubWatcher implements Watcher {
+    @Override
+    public void process(WatchedEvent event) {}
+  }
+}