diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java index 79115be6fed..5452b32789a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java @@ -128,7 +128,7 @@ public abstract class EventHandler implements Runnable, Comparable { process(); if (getListener() != null) getListener().afterProcess(this); } catch(Throwable t) { - LOG.error("Caught throwable while processing event " + eventType, t); + handleException(t); } finally { chunk.close(); } @@ -217,4 +217,12 @@ public abstract class EventHandler implements Runnable, Comparable { public String getInformativeName() { return this.getClass().toString(); } + + /** + * Event exception handler, may be overridden + * @param t Throwable object + */ + protected void handleException(Throwable t) { + LOG.error("Caught throwable while processing event " + eventType, t); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java index 44a933fc2d6..03acccfd6cf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java @@ -33,14 +33,23 @@ import org.apache.hadoop.hbase.executor.EventType; import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.DeadServer; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.util.Threads; import org.apache.zookeeper.KeeperException; +import com.google.common.annotations.VisibleForTesting; + +import java.util.concurrent.atomic.AtomicInteger; + /** * Shutdown handler for the server hosting hbase:meta */ @InterfaceAudience.Private public class MetaServerShutdownHandler extends ServerShutdownHandler { private static final Log LOG = LogFactory.getLog(MetaServerShutdownHandler.class); + private AtomicInteger eventExceptionCount = new AtomicInteger(0); + @VisibleForTesting + static final int SHOW_STRACKTRACE_FREQUENCY = 100; + public MetaServerShutdownHandler(final Server server, final MasterServices services, final DeadServer deadServers, final ServerName serverName) { @@ -115,8 +124,10 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler { this.deadServers.finish(serverName); } } - + super.process(); + // Clear this counter on successful handling. + this.eventExceptionCount.set(0); } @Override @@ -195,4 +206,20 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler { } return getClass().getSimpleName() + "-" + name + "-" + getSeqid(); } -} + + @Override + protected void handleException(Throwable t) { + int count = eventExceptionCount.getAndIncrement(); + if (count < 0) count = eventExceptionCount.getAndSet(0); + if (count > SHOW_STRACKTRACE_FREQUENCY) { // Too frequent, let's slow reporting + Threads.sleep(1000); + } + if (count % SHOW_STRACKTRACE_FREQUENCY == 0) { + LOG.error("Caught " + eventType + ", count=" + this.eventExceptionCount, t); + } else { + LOG.error("Caught " + eventType + ", count=" + this.eventExceptionCount + + "; " + t.getMessage() + "; stack trace shows every " + SHOW_STRACKTRACE_FREQUENCY + + "th time."); + } + } +} \ No newline at end of file