From ce5c7a43275869a080b807a5f486a1e708e01518 Mon Sep 17 00:00:00 2001 From: Xiaolin Ha Date: Thu, 24 Mar 2022 19:03:22 +0800 Subject: [PATCH] HBASE-26175 MetricsHBaseServer should record all kinds of Exceptions (#4248) Signed-off-by: Pankaj Kumar --- .../hbase/metrics/ExceptionTrackingSource.java | 4 ++++ .../metrics/ExceptionTrackingSourceImpl.java | 16 ++++++++++++++++ .../hadoop/hbase/ipc/MetricsHBaseServer.java | 10 ++++++++-- .../apache/hadoop/hbase/ipc/TestRpcMetrics.java | 17 +++++++++++++++-- .../hadoop/hbase/thrift/ThriftMetrics.java | 10 ++++++++-- 5 files changed, 51 insertions(+), 6 deletions(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSource.java index 6d72d85289a..801dc7ac837 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSource.java @@ -43,6 +43,8 @@ public interface ExceptionTrackingSource extends BaseSource { String EXCEPTIONS_CALL_QUEUE_TOO_BIG_DESC = "Call queue is full"; String EXCEPTIONS_QUOTA_EXCEEDED = "exceptions.quotaExceeded"; String EXCEPTIONS_RPC_THROTTLING = "exceptions.rpcThrottling"; + String EXCEPTIONS_REQUEST_TOO_BIG = "exceptions.requestTooBig"; + String OTHER_EXCEPTIONS = "exceptions.otherExceptions"; void exception(); @@ -60,4 +62,6 @@ public interface ExceptionTrackingSource extends BaseSource { void callQueueTooBigException(); void quotaExceededException(); void rpcThrottlingException(); + void requestTooBigException(); + void otherExceptions(); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSourceImpl.java index 23dafad02a2..c6306f99ee0 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSourceImpl.java @@ -40,6 +40,8 @@ public class ExceptionTrackingSourceImpl extends BaseSourceImpl protected MutableFastCounter exceptionsCallQueueTooBig; protected MutableFastCounter exceptionsQuotaExceeded; protected MutableFastCounter exceptionsRpcThrottling; + protected MutableFastCounter exceptionRequestTooBig; + protected MutableFastCounter otherExceptions; public ExceptionTrackingSourceImpl(String metricsName, String metricsDescription, String metricsContext, String metricsJmxContext) { @@ -72,6 +74,10 @@ public class ExceptionTrackingSourceImpl extends BaseSourceImpl .newCounter(EXCEPTIONS_QUOTA_EXCEEDED, EXCEPTIONS_TYPE_DESC, 0L); this.exceptionsRpcThrottling = this.getMetricsRegistry() .newCounter(EXCEPTIONS_RPC_THROTTLING, EXCEPTIONS_TYPE_DESC, 0L); + this.exceptionRequestTooBig = this.getMetricsRegistry() + .newCounter(EXCEPTIONS_REQUEST_TOO_BIG, EXCEPTIONS_TYPE_DESC, 0L); + this.otherExceptions = this.getMetricsRegistry() + .newCounter(OTHER_EXCEPTIONS, EXCEPTIONS_TYPE_DESC, 0L); } @Override @@ -133,4 +139,14 @@ public class ExceptionTrackingSourceImpl extends BaseSourceImpl public void rpcThrottlingException() { exceptionsRpcThrottling.incr(); } + + @Override + public void requestTooBigException() { + exceptionRequestTooBig.incr(); + } + + @Override + public void otherExceptions() { + otherExceptions.incr(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServer.java index 11773338869..e522320cf7e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServer.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.MultiActionResultTooLarge; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.RegionTooBusyException; import org.apache.hadoop.hbase.UnknownScannerException; +import org.apache.hadoop.hbase.exceptions.RequestTooBigException; import org.apache.hadoop.hbase.quotas.QuotaExceededException; import org.apache.hadoop.hbase.quotas.RpcThrottlingException; import org.apache.yetus.audience.InterfaceAudience; @@ -126,8 +127,13 @@ public class MetricsHBaseServer { source.quotaExceededException(); } else if (throwable instanceof RpcThrottlingException) { source.rpcThrottlingException(); - } else if (LOG.isDebugEnabled()) { - LOG.debug("Unknown exception type", throwable); + } else if (throwable instanceof RequestTooBigException) { + source.requestTooBigException(); + } else { + source.otherExceptions(); + if (LOG.isDebugEnabled()) { + LOG.debug("Unknown exception type", throwable); + } } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java index 2f99d2b6592..0169a5d2486 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java @@ -17,15 +17,17 @@ */ package org.apache.hadoop.hbase.ipc; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.RegionTooBusyException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException; import org.apache.hadoop.hbase.exceptions.RegionMovedException; +import org.apache.hadoop.hbase.exceptions.RequestTooBigException; import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.RPCTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -143,11 +145,22 @@ public class TestRpcMetrics { mrpc.exception(new RegionTooBusyException("Some region")); mrpc.exception(new OutOfOrderScannerNextException()); mrpc.exception(new NotServingRegionException()); + mrpc.exception(new RequestTooBigException()); + mrpc.exception(new FakeException()); HELPER.assertCounter("exceptions.RegionMovedException", 1, serverSource); HELPER.assertCounter("exceptions.RegionTooBusyException", 1, serverSource); HELPER.assertCounter("exceptions.OutOfOrderScannerNextException", 1, serverSource); HELPER.assertCounter("exceptions.NotServingRegionException", 1, serverSource); - HELPER.assertCounter("exceptions", 5, serverSource); + HELPER.assertCounter("exceptions.requestTooBig", 1, serverSource); + HELPER.assertCounter("exceptions.otherExceptions", 1, serverSource); + HELPER.assertCounter("exceptions", 7, serverSource); + } + + private class FakeException extends DoNotRetryIOException { + + public FakeException() { + super(); + } } @Test diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java index e362817a97d..028f2afa275 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil; import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException; import org.apache.hadoop.hbase.exceptions.RegionMovedException; +import org.apache.hadoop.hbase.exceptions.RequestTooBigException; import org.apache.hadoop.hbase.exceptions.ScannerResetException; import org.apache.hadoop.hbase.quotas.QuotaExceededException; import org.apache.hadoop.hbase.quotas.RpcThrottlingException; @@ -152,8 +153,13 @@ public class ThriftMetrics { source.quotaExceededException(); } else if (throwable instanceof RpcThrottlingException) { source.rpcThrottlingException(); - } else if (LOG.isDebugEnabled()) { - LOG.debug("Unknown exception type", throwable); + } else if (throwable instanceof RequestTooBigException) { + source.requestTooBigException(); + } else { + source.otherExceptions(); + if (LOG.isDebugEnabled()) { + LOG.debug("Unknown exception type", throwable); + } } } }