HBASE-2449 Local HBase does not stop properly

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@945836 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -332,6 +332,7 @@ Release 0.21.0 - Unreleased
    HBASE-2457  RS gets stuck compacting region ad infinitum
    HBASE-2562  bin/hbase doesn't work in-situ in maven
                (Todd Lipcon via Stack)
+   HBASE-2449  Local HBase does not stop properly
 
   IMPROVEMENTS
    HBASE-1760  Cleanup TODOs in HTable

bin/start-hbase.sh

@@ -41,12 +41,12 @@ fi
 distMode=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool hbase.cluster.distributed`
 
 
-if [ $distMode == 'false' ] 
+if [ "$distMode" == 'false' ] 
 then
   "$bin"/hbase-daemon.sh start master
 else
-"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper
-"$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master 
-"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
-  --hosts "${HBASE_REGIONSERVERS}" start regionserver
+  "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper
+  "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master 
+  "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
+    --hosts "${HBASE_REGIONSERVERS}" start regionserver
 fi

bin/stop-hbase.sh

@@ -30,4 +30,8 @@ bin=`cd "$bin"; pwd`
 . "$bin"/hbase-config.sh
 
 "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master
-"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper
+distMode=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool hbase.cluster.distributed`
+if [ "$distMode" == 'true' ] 
+then
+  "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper
+fi

core/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -1158,6 +1159,33 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
     }
   }
 
+  /*
+   * Version of master that will shutdown the passed zk cluster on its way out.
+   */
+  static class LocalHMaster extends HMaster {
+    private MiniZooKeeperCluster zkcluster = null;
+
+    public LocalHMaster(Configuration conf) throws IOException {
+      super(conf);
+    }
+
+    @Override
+    public void run() {
+      super.run();
+      if (this.zkcluster != null) {
+        try {
+          this.zkcluster.shutdown();
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+      }
+    }
+
+    void setZKCluster(final MiniZooKeeperCluster zkcluster) {
+      this.zkcluster = zkcluster;
+    }
+  }
+
   protected static void doMain(String [] args,
       Class<? extends HMaster> masterClass) {
     if (args.length < 1) {
@@ -1185,14 +1213,13 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
           // If 'local', defer to LocalHBaseCluster instance.  Starts master
           // and regionserver both in the one JVM.
           if (LocalHBaseCluster.isLocal(conf)) {
-            // TODO make zookeepercluster a field and do an orderly shutdown
-            MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster();
+            final MiniZooKeeperCluster zooKeeperCluster =
+              new MiniZooKeeperCluster();
             File zkDataPath = new File(conf.get("hbase.zookeeper.property.dataDir"));
             int zkClientPort = conf.getInt("hbase.zookeeper.property.clientPort", 0);
             if (zkClientPort == 0) {
               throw new IOException("No config value for hbase.zookeeper.property.clientPort");
             }
-
             zooKeeperCluster.setTickTime(conf.getInt("hbase.zookeeper.property.tickTime", 3000));
             zooKeeperCluster.setClientPort(zkClientPort);
             int clientPort = zooKeeperCluster.startup(zkDataPath);
@@ -1203,8 +1230,14 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
               System.err.println(errorMsg);
               throw new IOException(errorMsg);
             }
-            conf.set("hbase.zookeeper.property.clientPort", Integer.toString(clientPort));
-            (new LocalHBaseCluster(conf)).startup();
+            conf.set("hbase.zookeeper.property.clientPort",
+              Integer.toString(clientPort));
+            // Need to have the zk cluster shutdown when master is shutdown.
+            // Run a subclass that does the zk cluster shutdown on its way out.
+            LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1,
+              LocalHMaster.class, HRegionServer.class);
+            ((LocalHMaster)cluster.getMaster()).setZKCluster(zooKeeperCluster);
+            cluster.startup();
           } else {
             HMaster master = constructMaster(masterClass, conf);
             if (master.shutdownRequested.get()) {

core/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java

@@ -797,7 +797,7 @@ public class ServerManager implements HConstants {
         LOG.info("Waiting on following regionserver(s) to go down " +
           this.serversToServerInfo.values());
         try {
-          this.serversToServerInfo.wait(1);
+          this.serversToServerInfo.wait(500);
         } catch (InterruptedException e) {
           // continue
         }

core/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

@@ -231,10 +231,6 @@ public class HRegionServer implements HConstants, HRegionInterface,
   // The main region server thread.
   private Thread regionServerThread;
 
-  // Run HDFS shutdown on exit if this is set. We clear this out when
-  // doing a restart() to prevent closing of HDFS.
-  public final AtomicBoolean shutdownHDFS = new AtomicBoolean(true);
-
   private final String machineName;
 
   /**
@@ -293,7 +289,6 @@ public class HRegionServer implements HConstants, HRegionInterface,
   private void reinitialize() throws IOException {
     this.abortRequested = false;
     this.stopRequested.set(false);
-    this.shutdownHDFS.set(true);
 
     // Server to handle client requests
     this.server = HBaseRPC.getServer(this, address.getBindAddress(),
@@ -361,7 +356,7 @@ public class HRegionServer implements HConstants, HRegionInterface,
       type + ", path: " + event.getPath());
 
     // Ignore events if we're shutting down.
-    if (stopRequested.get()) {
+    if (this.stopRequested.get()) {
       LOG.debug("Ignoring ZooKeeper event while shutting down");
       return;
     }
@@ -394,7 +389,6 @@ public class HRegionServer implements HConstants, HRegionInterface,
 
   private void restart() {
     LOG.info("Restarting Region Server");
-    shutdownHDFS.set(false);
     abort();
     Threads.shutdown(regionServerThread);
     boolean done = false;
@@ -670,8 +664,8 @@ public class HRegionServer implements HConstants, HRegionInterface,
     }
 
     if (!killed) {
-      join();
       this.zooKeeperWrapper.close();
+      join();
     }
     LOG.info(Thread.currentThread().getName() + " exiting");
   }
@@ -750,12 +744,6 @@ public class HRegionServer implements HConstants, HRegionInterface,
       this.conf.set("fs.defaultFS", this.conf.get("hbase.rootdir"));
       this.conf.setBoolean("fs.automatic.close", false);
       this.fs = FileSystem.get(this.conf);
-
-      // Register shutdown hook for HRegionServer, runs an orderly shutdown
-      // when a kill signal is recieved.  Shuts down hdfs too if its supposed.
-      Runtime.getRuntime().addShutdownHook(new ShutdownThread(this,
-        Thread.currentThread(), this.shutdownHDFS));
-
       this.rootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
       this.hlog = setupHLog();
       // Init in here rather than in constructor after thread name has been set
@@ -770,12 +758,6 @@ public class HRegionServer implements HConstants, HRegionInterface,
     }
   }
 
-  public void setShutdownHDFS(final boolean b) {
-    this.shutdownHDFS.set(b);
-  }
-
-  public boolean getShutdownHDFS() {return this.shutdownHDFS.get();}
-
   /*
    * @param r Region to get RegionLoad for.
    * @return RegionLoad instance.
@@ -897,47 +879,6 @@ public class HRegionServer implements HConstants, HRegionInterface,
     return this.fsOk;
   }
 
-  /*
-   * Thread to shutdown the region server in an orderly manner.  This thread
-   * is registered as a shutdown hook in the HRegionServer constructor and is
-   * only called when the HRegionServer receives a kill signal.
-   */
-  private static class ShutdownThread extends Thread {
-    private final HRegionServer instance;
-    private final Thread mainThread;
-    private final AtomicBoolean shutdownHDFS;
-
-    /**
-     * @param instance
-     * @param mainThread
-     * @param shutdownHDFS
-     */
-    public ShutdownThread(final HRegionServer instance, final Thread mainThread,
-        final AtomicBoolean shutdownHDFS) {
-      this.instance = instance;
-      this.mainThread = mainThread;
-      this.shutdownHDFS = shutdownHDFS;
-    }
-
-    @Override
-    public void run() {
-      LOG.info("Starting shutdown thread");
-
-      // tell the region server to stop
-      this.instance.stop();
-
-      // Wait for main thread to exit.
-      Threads.shutdown(this.mainThread);
-      try {
-        if (this.shutdownHDFS.get()) FileSystem.closeAll();
-      } catch (IOException e) {
-        e.printStackTrace();
-      }
-
-      LOG.info("Shutdown thread complete");
-    }
-  }
-
   /*
    * Inner class that runs on a long period checking if regions need major
    * compaction.

core/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java

@@ -625,7 +625,7 @@ public class ZooKeeperWrapper implements HConstants {
   public void close() {
     try {
       zooKeeper.close();
-      LOG.debug("Closed connection with ZooKeeper");
+      LOG.debug("Closed connection with ZooKeeper; " + this.rootRegionZNode);
     } catch (InterruptedException e) {
       LOG.warn("Failed to close connection with ZooKeeper");
     }

src/assembly/bin.xml

@@ -53,29 +53,6 @@
         <unpack>false</unpack>
       </binaries>
     </moduleSet>
-    <moduleSet>
-      <includes>
-        <include>org.apache.hbase:hbase-contrib-stargate*</include>
-      </includes>
-      <binaries>
-        <outputDirectory>contrib/stargate/</outputDirectory>
-        <unpack>false</unpack>
-        <dependencySets>
-          <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-          </dependencySet>
-        </dependencySets>
-      </binaries>
-    </moduleSet>
-    <moduleSet>
-      <includes>
-        <include>org.apache.hbase:hbase-contrib-transactional</include>
-      </includes>
-      <binaries>
-        <outputDirectory>contrib/transactional/</outputDirectory>
-        <unpack>false</unpack>
-      </binaries>
-    </moduleSet>
   </moduleSets>
   <fileSets>
     <fileSet>