From 2450ffcc90689d4a0fef72b53910d4fdfd4ce7a0 Mon Sep 17 00:00:00 2001 From: Richard Marscher Date: Wed, 29 Dec 2021 11:02:50 -0500 Subject: [PATCH] HubSpot Backport: HBASE-26623 Report CallDroppedException in exception metrics (#3980) `CallDroppedException` can be thrown with `CallRunner.drop()` by queue implementations that decide to drop calls to groom the RPC call backlog. The LifoCoDel queue does this I believe and with Pluggable queue it's possible for 3rd party queue implementations to be using `drop()` for similar reasons. It would be nice for the server to be tracking these exceptions in metrics since otherwise you might have to do some extra lifting on the client side. Signed-off-by: Duo Zhang Reviewed-by: Bryan Beaudreault --- .../hadoop/hbase/metrics/ExceptionTrackingSource.java | 2 ++ .../hadoop/hbase/metrics/ExceptionTrackingSourceImpl.java | 8 ++++++++ .../org/apache/hadoop/hbase/ipc/MetricsHBaseServer.java | 3 +++ .../java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java | 5 ++++- .../org/apache/hadoop/hbase/thrift/ThriftMetrics.java | 3 +++ .../hadoop/hbase/thrift/ErrorThrowingGetObserver.java | 6 +++++- 6 files changed, 25 insertions(+), 2 deletions(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSource.java index 6d72d85289a..2407a733701 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSource.java @@ -43,6 +43,7 @@ public interface ExceptionTrackingSource extends BaseSource { String EXCEPTIONS_CALL_QUEUE_TOO_BIG_DESC = "Call queue is full"; String EXCEPTIONS_QUOTA_EXCEEDED = "exceptions.quotaExceeded"; String EXCEPTIONS_RPC_THROTTLING = "exceptions.rpcThrottling"; + String EXCEPTIONS_CALL_DROPPED = "exceptions.callDropped"; void exception(); @@ -60,4 +61,5 @@ public interface ExceptionTrackingSource extends BaseSource { void callQueueTooBigException(); void quotaExceededException(); void rpcThrottlingException(); + void callDroppedException(); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSourceImpl.java index 23dafad02a2..d6dabadd8e8 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSourceImpl.java @@ -40,6 +40,7 @@ public class ExceptionTrackingSourceImpl extends BaseSourceImpl protected MutableFastCounter exceptionsCallQueueTooBig; protected MutableFastCounter exceptionsQuotaExceeded; protected MutableFastCounter exceptionsRpcThrottling; + protected MutableFastCounter exceptionsCallDropped; public ExceptionTrackingSourceImpl(String metricsName, String metricsDescription, String metricsContext, String metricsJmxContext) { @@ -72,6 +73,8 @@ public class ExceptionTrackingSourceImpl extends BaseSourceImpl .newCounter(EXCEPTIONS_QUOTA_EXCEEDED, EXCEPTIONS_TYPE_DESC, 0L); this.exceptionsRpcThrottling = this.getMetricsRegistry() .newCounter(EXCEPTIONS_RPC_THROTTLING, EXCEPTIONS_TYPE_DESC, 0L); + this.exceptionsCallDropped = this.getMetricsRegistry() + .newCounter(EXCEPTIONS_CALL_DROPPED, EXCEPTIONS_TYPE_DESC, 0L); } @Override @@ -133,4 +136,9 @@ public class ExceptionTrackingSourceImpl extends BaseSourceImpl public void rpcThrottlingException() { exceptionsRpcThrottling.incr(); } + + @Override + public void callDroppedException() { + exceptionsCallDropped.incr(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServer.java index 11773338869..d79c8eb4f66 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServer.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.ipc; +import org.apache.hadoop.hbase.CallDroppedException; import org.apache.hadoop.hbase.CallQueueTooBigException; import org.apache.hadoop.hbase.MultiActionResultTooLarge; import org.apache.hadoop.hbase.NotServingRegionException; @@ -126,6 +127,8 @@ public class MetricsHBaseServer { source.quotaExceededException(); } else if (throwable instanceof RpcThrottlingException) { source.rpcThrottlingException(); + } else if (throwable instanceof CallDroppedException) { + source.callDroppedException(); } else if (LOG.isDebugEnabled()) { LOG.debug("Unknown exception type", throwable); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java index 2f99d2b6592..c50414d01ad 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.ipc; import static org.junit.Assert.*; +import org.apache.hadoop.hbase.CallDroppedException; import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.NotServingRegionException; @@ -143,11 +144,13 @@ public class TestRpcMetrics { mrpc.exception(new RegionTooBusyException("Some region")); mrpc.exception(new OutOfOrderScannerNextException()); mrpc.exception(new NotServingRegionException()); + mrpc.exception(new CallDroppedException()); HELPER.assertCounter("exceptions.RegionMovedException", 1, serverSource); HELPER.assertCounter("exceptions.RegionTooBusyException", 1, serverSource); HELPER.assertCounter("exceptions.OutOfOrderScannerNextException", 1, serverSource); HELPER.assertCounter("exceptions.NotServingRegionException", 1, serverSource); - HELPER.assertCounter("exceptions", 5, serverSource); + HELPER.assertCounter("exceptions.callDropped", 1, serverSource); + HELPER.assertCounter("exceptions", 6, serverSource); } @Test diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java index e362817a97d..cf290bcb708 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.thrift; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CallDroppedException; import org.apache.hadoop.hbase.CallQueueTooBigException; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.MultiActionResultTooLarge; @@ -152,6 +153,8 @@ public class ThriftMetrics { source.quotaExceededException(); } else if (throwable instanceof RpcThrottlingException) { source.rpcThrottlingException(); + } else if (throwable instanceof CallDroppedException) { + source.callDroppedException(); } else if (LOG.isDebugEnabled()) { LOG.debug("Unknown exception type", throwable); } diff --git a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift/ErrorThrowingGetObserver.java b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift/ErrorThrowingGetObserver.java index 98f190b27c5..683884d0efd 100644 --- a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift/ErrorThrowingGetObserver.java +++ b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift/ErrorThrowingGetObserver.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import org.apache.hadoop.hbase.CallDroppedException; import org.apache.hadoop.hbase.CallQueueTooBigException; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.DoNotRetryIOException; @@ -85,6 +86,8 @@ public class ErrorThrowingGetObserver implements RegionCoprocessor, RegionObserv throw new QuotaExceededException("Failing for test"); case RPC_THROTTLING: throw new RpcThrottlingException("Failing for test"); + case CALL_DROPPED: + throw new CallDroppedException("Failing for test"); default: throw new DoNotRetryIOException("Failing for test"); } @@ -102,7 +105,8 @@ public class ErrorThrowingGetObserver implements RegionCoprocessor, RegionObserv REGION_TOO_BUSY(ExceptionTrackingSource.EXCEPTIONS_BUSY_NAME), OUT_OF_ORDER_SCANNER_NEXT(ExceptionTrackingSource.EXCEPTIONS_OOO_NAME), QUOTA_EXCEEDED(ExceptionTrackingSource.EXCEPTIONS_QUOTA_EXCEEDED), - RPC_THROTTLING(ExceptionTrackingSource.EXCEPTIONS_RPC_THROTTLING); + RPC_THROTTLING(ExceptionTrackingSource.EXCEPTIONS_RPC_THROTTLING), + CALL_DROPPED(ExceptionTrackingSource.EXCEPTIONS_CALL_DROPPED); private final String metricName;