HBASE-10476 HBase Master log grows very fast after stopped hadoop (due to connection exception) (Demai Ni)
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1577340 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
889ff7cbed
commit
98cf3e27ee
|
@ -128,7 +128,7 @@ public abstract class EventHandler implements Runnable, Comparable<Runnable> {
|
||||||
process();
|
process();
|
||||||
if (getListener() != null) getListener().afterProcess(this);
|
if (getListener() != null) getListener().afterProcess(this);
|
||||||
} catch(Throwable t) {
|
} catch(Throwable t) {
|
||||||
LOG.error("Caught throwable while processing event " + eventType, t);
|
handleException(t);
|
||||||
} finally {
|
} finally {
|
||||||
chunk.close();
|
chunk.close();
|
||||||
}
|
}
|
||||||
|
@ -217,4 +217,12 @@ public abstract class EventHandler implements Runnable, Comparable<Runnable> {
|
||||||
public String getInformativeName() {
|
public String getInformativeName() {
|
||||||
return this.getClass().toString();
|
return this.getClass().toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Event exception handler, may be overridden
|
||||||
|
* @param t Throwable object
|
||||||
|
*/
|
||||||
|
protected void handleException(Throwable t) {
|
||||||
|
LOG.error("Caught throwable while processing event " + eventType, t);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -33,14 +33,23 @@ import org.apache.hadoop.hbase.executor.EventType;
|
||||||
import org.apache.hadoop.hbase.master.AssignmentManager;
|
import org.apache.hadoop.hbase.master.AssignmentManager;
|
||||||
import org.apache.hadoop.hbase.master.DeadServer;
|
import org.apache.hadoop.hbase.master.DeadServer;
|
||||||
import org.apache.hadoop.hbase.master.MasterServices;
|
import org.apache.hadoop.hbase.master.MasterServices;
|
||||||
|
import org.apache.hadoop.hbase.util.Threads;
|
||||||
import org.apache.zookeeper.KeeperException;
|
import org.apache.zookeeper.KeeperException;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Shutdown handler for the server hosting <code>hbase:meta</code>
|
* Shutdown handler for the server hosting <code>hbase:meta</code>
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class MetaServerShutdownHandler extends ServerShutdownHandler {
|
public class MetaServerShutdownHandler extends ServerShutdownHandler {
|
||||||
private static final Log LOG = LogFactory.getLog(MetaServerShutdownHandler.class);
|
private static final Log LOG = LogFactory.getLog(MetaServerShutdownHandler.class);
|
||||||
|
private AtomicInteger eventExceptionCount = new AtomicInteger(0);
|
||||||
|
@VisibleForTesting
|
||||||
|
static final int SHOW_STRACKTRACE_FREQUENCY = 100;
|
||||||
|
|
||||||
public MetaServerShutdownHandler(final Server server,
|
public MetaServerShutdownHandler(final Server server,
|
||||||
final MasterServices services,
|
final MasterServices services,
|
||||||
final DeadServer deadServers, final ServerName serverName) {
|
final DeadServer deadServers, final ServerName serverName) {
|
||||||
|
@ -117,6 +126,8 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler {
|
||||||
}
|
}
|
||||||
|
|
||||||
super.process();
|
super.process();
|
||||||
|
// Clear this counter on successful handling.
|
||||||
|
this.eventExceptionCount.set(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -195,4 +206,20 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler {
|
||||||
}
|
}
|
||||||
return getClass().getSimpleName() + "-" + name + "-" + getSeqid();
|
return getClass().getSimpleName() + "-" + name + "-" + getSeqid();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void handleException(Throwable t) {
|
||||||
|
int count = eventExceptionCount.getAndIncrement();
|
||||||
|
if (count < 0) count = eventExceptionCount.getAndSet(0);
|
||||||
|
if (count > SHOW_STRACKTRACE_FREQUENCY) { // Too frequent, let's slow reporting
|
||||||
|
Threads.sleep(1000);
|
||||||
|
}
|
||||||
|
if (count % SHOW_STRACKTRACE_FREQUENCY == 0) {
|
||||||
|
LOG.error("Caught " + eventType + ", count=" + this.eventExceptionCount, t);
|
||||||
|
} else {
|
||||||
|
LOG.error("Caught " + eventType + ", count=" + this.eventExceptionCount +
|
||||||
|
"; " + t.getMessage() + "; stack trace shows every " + SHOW_STRACKTRACE_FREQUENCY +
|
||||||
|
"th time.");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
Loading…
Reference in New Issue