diff --git a/CHANGES.txt b/CHANGES.txt index 3ab7edcc3f0..3d46e8ab80d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -325,6 +325,8 @@ Release 0.92.0 - Unreleased RegionServerStoppedException (Ming Ma) HBASE-4476 Compactions must fail if column tracker gets columns out of order (Mikhail Bautin) + HBASE-4209 The HBase hbase-daemon.sh SIGKILLs master when stopping it + (Roman Shaposhnik) TESTS HBASE-4450 test for number of blocks read: to serve as baseline for expected diff --git a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java index 5bc3bb02d25..54b493913cb 100644 --- a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java +++ b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java @@ -412,7 +412,7 @@ public class LocalHBaseCluster { /** * Start the cluster. */ - public void startup() { + public void startup() throws IOException { JVMClusterUtil.startup(this.masterThreads, this.regionThreads); } diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java index b25e575112c..4ccf68e7de5 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java @@ -21,6 +21,8 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,7 +36,7 @@ import org.apache.hadoop.hbase.util.Threads; * Manage regionserver shutdown hooks. * @see #install(Configuration, FileSystem, Stoppable, Thread) */ -class ShutdownHook { +public class ShutdownHook { private static final Log LOG = LogFactory.getLog(ShutdownHook.class); private static final String CLIENT_FINALIZER_DATA_METHOD = "clientFinalizer"; @@ -49,6 +51,13 @@ class ShutdownHook { */ public static final String FS_SHUTDOWN_HOOK_WAIT = "hbase.fs.shutdown.hook.wait"; + /** + * A place for keeping track of all the filesystem shutdown hooks that need + * to be executed after the last regionserver referring to a given filesystem + * stops. We keep track of the # of regionserver references in values of the map. + */ + private final static Map fsShutdownHooks = new HashMap(); + /** * Install a shutdown hook that calls stop on the passed Stoppable * and then thread joins against the passed threadToJoin. @@ -68,7 +77,7 @@ class ShutdownHook { * @param threadToJoin After calling stop on stop will then * join this thread. */ - static void install(final Configuration conf, final FileSystem fs, + public static void install(final Configuration conf, final FileSystem fs, final Stoppable stop, final Thread threadToJoin) { Thread fsShutdownHook = suppressHdfsShutdownHook(fs); Thread t = new ShutdownHookThread(conf, stop, threadToJoin, fsShutdownHook); @@ -103,10 +112,18 @@ class ShutdownHook { this.stop.stop("Shutdown hook"); Threads.shutdown(this.threadToJoin); if (this.fsShutdownHook != null) { - LOG.info("Starting fs shutdown hook thread."); - this.fsShutdownHook.start(); - Threads.shutdown(this.fsShutdownHook, - this.conf.getLong(FS_SHUTDOWN_HOOK_WAIT, 30000)); + synchronized (fsShutdownHooks) { + int refs = fsShutdownHooks.get(fsShutdownHook); + if (refs == 1) { + LOG.info("Starting fs shutdown hook thread."); + this.fsShutdownHook.start(); + Threads.shutdown(this.fsShutdownHook, + this.conf.getLong(FS_SHUTDOWN_HOOK_WAIT, 30000)); + } + if (refs > 0) { + fsShutdownHooks.put(fsShutdownHook, refs - 1); + } + } } } LOG.info("Shutdown hook finished."); @@ -171,10 +188,15 @@ class ShutdownHook { if (hdfsClientFinalizer == null) { throw new RuntimeException("Client finalizer is null, can't suppress!"); } - if (!Runtime.getRuntime().removeShutdownHook(hdfsClientFinalizer)) { + if (!fsShutdownHooks.containsKey(hdfsClientFinalizer) && + !Runtime.getRuntime().removeShutdownHook(hdfsClientFinalizer)) { throw new RuntimeException("Failed suppression of fs shutdown hook: " + hdfsClientFinalizer); } + synchronized (fsShutdownHooks) { + Integer refs = fsShutdownHooks.get(hdfsClientFinalizer); + fsShutdownHooks.put(hdfsClientFinalizer, refs == null ? 1 : refs + 1); + } return hdfsClientFinalizer; } catch (NoSuchFieldException nsfe) { LOG.fatal("Couldn't find field 'clientFinalizer' in FileSystem!", nsfe); @@ -231,4 +253,4 @@ class ShutdownHook { donothing.start(); ShutdownHook.install(conf, fs, new DoNothingStoppable(), donothing); } -} \ No newline at end of file +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java index b3ada90cda8..7f4b2a63a62 100644 --- a/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java +++ b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java @@ -26,8 +26,10 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.ShutdownHook; /** * Utility used running a cluster all in the one JVM. @@ -171,7 +173,7 @@ public class JVMClusterUtil { * @return Address to use contacting primary master. */ public static String startup(final List masters, - final List regionservers) { + final List regionservers) throws IOException { if (masters != null) { for (JVMClusterUtil.MasterThread t : masters) { t.start(); @@ -179,6 +181,9 @@ public class JVMClusterUtil { } if (regionservers != null) { for (JVMClusterUtil.RegionServerThread t: regionservers) { + HRegionServer hrs = t.getRegionServer(); + ShutdownHook.install(hrs.getConfiguration(), FileSystem.get(hrs + .getConfiguration()), hrs, t); t.start(); } }