diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSource.java index a25df56e7c1..558a8633bb1 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSource.java @@ -19,11 +19,12 @@ package org.apache.hadoop.hbase.thrift; import org.apache.hadoop.hbase.metrics.BaseSource; +import org.apache.hadoop.hbase.metrics.JvmPauseMonitorSource; /** * Interface of a class that will export metrics about Thrift to hadoop's metrics2. */ -public interface MetricsThriftServerSource extends BaseSource { +public interface MetricsThriftServerSource extends BaseSource, JvmPauseMonitorSource { String BATCH_GET_KEY = "batchGet"; String BATCH_MUTATE_KEY = "batchMutate"; diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java index f5d83b1e0ea..71f67eb9a12 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.thrift; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.metrics2.lib.MutableHistogram; @@ -42,11 +43,25 @@ public class MetricsThriftServerSourceImpl extends BaseSourceImpl implements private MutableGaugeLong callQueueLenGauge; + // pause monitor metrics + private final MutableFastCounter infoPauseThresholdExceeded; + private final MutableFastCounter warnPauseThresholdExceeded; + private final MetricHistogram pausesWithGc; + private final MetricHistogram pausesWithoutGc; + public MetricsThriftServerSourceImpl(String metricsName, String metricsDescription, String metricsContext, String metricsJmxContext) { super(metricsName, metricsDescription, metricsContext, metricsJmxContext); + + // pause monitor metrics + infoPauseThresholdExceeded = getMetricsRegistry().newCounter(INFO_THRESHOLD_COUNT_KEY, + INFO_THRESHOLD_COUNT_DESC, 0L); + warnPauseThresholdExceeded = getMetricsRegistry().newCounter(WARN_THRESHOLD_COUNT_KEY, + WARN_THRESHOLD_COUNT_DESC, 0L); + pausesWithGc = getMetricsRegistry().newTimeHistogram(PAUSE_TIME_WITH_GC_KEY); + pausesWithoutGc = getMetricsRegistry().newTimeHistogram(PAUSE_TIME_WITHOUT_GC_KEY); } @Override @@ -97,4 +112,23 @@ public class MetricsThriftServerSourceImpl extends BaseSourceImpl implements thriftSlowCallStat.add(time); } + @Override + public void incInfoThresholdExceeded(int count) { + infoPauseThresholdExceeded.incr(count); + } + + @Override + public void incWarnThresholdExceeded(int count) { + warnPauseThresholdExceeded.incr(count); + } + + @Override + public void updatePauseTimeWithGc(long t) { + pausesWithGc.add(t); + } + + @Override + public void updatePauseTimeWithoutGc(long t) { + pausesWithoutGc.add(t); + } } diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java index f2a662cb44a..3b5aac382ca 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java @@ -99,6 +99,7 @@ import org.apache.hadoop.hbase.thrift.generated.TScan; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ConnectionCache; import org.apache.hadoop.hbase.util.DNS; +import org.apache.hadoop.hbase.util.JvmPauseMonitor; import org.apache.hadoop.hbase.util.Strings; import org.apache.hadoop.security.SaslRpcServer.SaslGssCallbackHandler; import org.apache.hadoop.security.UserGroupInformation; @@ -202,6 +203,8 @@ public class ThriftServerRunner implements Runnable { private final boolean securityEnabled; private final boolean doAsEnabled; + private final JvmPauseMonitor pauseMonitor; + /** An enum of server implementation selections */ enum ImplType { HS_HA("hsha", true, THsHaServer.class, true), @@ -315,6 +318,7 @@ public class ThriftServerRunner implements Runnable { this.conf = HBaseConfiguration.create(conf); this.listenPort = conf.getInt(PORT_CONF_KEY, DEFAULT_LISTEN_PORT); this.metrics = new ThriftMetrics(conf, ThriftMetrics.ThriftServerType.ONE); + this.pauseMonitor = new JvmPauseMonitor(conf, this.metrics.getSource()); this.hbaseHandler = new HBaseHandler(conf, userProvider); this.hbaseHandler.initMetrics(metrics); this.handler = HbaseHandlerMetricsProxy.newInstance( @@ -344,6 +348,7 @@ public class ThriftServerRunner implements Runnable { @Override public Object run() { try { + pauseMonitor.start(); if (conf.getBoolean(USE_HTTP_CONF_KEY, false)) { setupHTTPServer(); httpServer.start(); @@ -364,6 +369,9 @@ public class ThriftServerRunner implements Runnable { } public void shutdown() { + if (pauseMonitor != null) { + pauseMonitor.stop(); + } if (tserver != null) { tserver.stop(); tserver = null; diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java index a2b4f03749d..9ba268bc587 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.thrift.CallQueue.Call; import org.apache.hadoop.hbase.thrift.ThriftMetrics; import org.apache.hadoop.hbase.thrift2.generated.THBaseService; import org.apache.hadoop.hbase.util.DNS; +import org.apache.hadoop.hbase.util.JvmPauseMonitor; import org.apache.hadoop.hbase.util.Strings; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.SaslRpcServer.SaslGssCallbackHandler; @@ -427,6 +428,7 @@ public class ThriftServer { boolean hsha = cmd.hasOption("hsha"); ThriftMetrics metrics = new ThriftMetrics(conf, ThriftMetrics.ThriftServerType.TWO); + final JvmPauseMonitor pauseMonitor = new JvmPauseMonitor(conf, metrics.getSource()); String implType = "threadpool"; if (nonblocking) { @@ -532,8 +534,13 @@ public class ThriftServer { new PrivilegedAction() { @Override public Object run() { - tserver.serve(); - return null; + pauseMonitor.start(); + try { + tserver.serve(); + return null; + } finally { + pauseMonitor.stop(); + } } }); }