From 2c65f039eddb6733ad2ac2fb89a8f5427495e243 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Wed, 27 Dec 2017 14:14:07 -0800 Subject: [PATCH] HBASE-19647 Logging cleanups; emit regionname when RegionTooBusyException inside RetriesExhausted... make netty connect/disconnect TRACE-level; ADDENDUM --- .../hadoop/hbase/RegionTooBusyException.java | 11 +++++------ .../RetriesExhaustedWithDetailsException.java | 2 ++ .../hadoop/hbase/regionserver/HRegion.java | 19 ++++++++++--------- .../hadoop/hbase/client/TestMetaCache.java | 2 +- .../hadoop/hbase/ipc/TestRpcMetrics.java | 2 +- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java index 7b03b45bb9d..49431b635d7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java @@ -24,21 +24,20 @@ import org.apache.yetus.audience.InterfaceAudience; /** * Thrown by a region server if it will block and wait to serve a request. * For example, the client wants to insert something to a region while the - * region is compacting. + * region is compacting. Keep variance in the passed 'msg' low because its msg is used as a key + * over in {@link org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException} + * grouping failure types. */ @InterfaceAudience.Public public class RegionTooBusyException extends IOException { private static final long serialVersionUID = 1728345723728342L; - /** default constructor */ - public RegionTooBusyException() { - super(); - } - /** * Constructor * @param msg message */ + // Be careful. Keep variance in the passed 'msg' low because its msg is used as a key over in + // RetriesExhaustedWithDetailsException grouping failure types. public RegionTooBusyException(final String msg) { super(msg); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java index e7eda2afccd..0cce728f114 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java @@ -157,6 +157,8 @@ extends RetriesExhaustedException { if (t instanceof DoNotRetryIOException || t instanceof RegionTooBusyException) { // If RegionTooBusyException, print message since it has Region name in it. + // RegionTooBusyException message was edited to remove variance. Has regionname, server, + // and why the exception; no longer has duration it waited on lock nor current memsize. name = t.getMessage(); } else { name = t.getClass().getSimpleName(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index cde6874549a..ca1bfd3bd4f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -4170,12 +4170,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi if (this.memstoreDataSize.get() > this.blockingMemStoreSize) { blockedRequestsCount.increment(); requestFlush(); - throw new RegionTooBusyException("Over memstore limit, " + - "regionName=" + (this.getRegionInfo() == null ? "unknown" : - this.getRegionInfo().getRegionNameAsString()) + - ", server=" + (this.getRegionServerServices() == null ? "unknown" : + // Don't print current limit because it will vary too much. The message is used as a key + // over in RetriesExhaustedWithDetailsException processing. + throw new RegionTooBusyException("Over memstore limit; regionName=" + + (this.getRegionInfo() == null? "unknown": this.getRegionInfo().getRegionNameAsString()) + + ", server=" + (this.getRegionServerServices() == null ? "unknown": this.getRegionServerServices().getServerName()) + - ", memstoreSize=" + memstoreDataSize.get() + ", blockingMemStoreSize=" + blockingMemStoreSize); } } @@ -8183,11 +8183,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi final long waitTime = Math.min(maxBusyWaitDuration, busyWaitDuration * Math.min(multiplier, maxBusyWaitMultiplier)); if (!lock.tryLock(waitTime, TimeUnit.MILLISECONDS)) { - throw new RegionTooBusyException( - "failed to get a lock in " + waitTime + " ms. " + - "regionName=" + (this.getRegionInfo() == null ? "unknown" : + // Don't print millis. Message is used as a key over in + // RetriesExhaustedWithDetailsException processing. + throw new RegionTooBusyException("Failed to obtain lock; regionName=" + + (this.getRegionInfo() == null? "unknown": this.getRegionInfo().getRegionNameAsString()) + - ", server=" + (this.getRegionServerServices() == null ? "unknown" : + ", server=" + (this.getRegionServerServices() == null? "unknown": this.getRegionServerServices().getServerName())); } } catch (InterruptedException ie) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java index b1fda41ef9e..ecdd6859211 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java @@ -187,7 +187,7 @@ public class TestMetaCache { public static List metaCachePreservingExceptions() { return new ArrayList() {{ add(new RegionOpeningException(" ")); - add(new RegionTooBusyException()); + add(new RegionTooBusyException("Some old message")); add(new ThrottlingException(" ")); add(new MultiActionResultTooLarge(" ")); add(new RetryImmediatelyException(" ")); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java index a0769dabe30..cfcfb21e6f4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java @@ -133,7 +133,7 @@ public class TestRpcMetrics { HELPER.assertCounter("exceptions", 1, serverSource); mrpc.exception(new RegionMovedException(ServerName.parseServerName("localhost:60020"), 100)); - mrpc.exception(new RegionTooBusyException()); + mrpc.exception(new RegionTooBusyException("Some region")); mrpc.exception(new OutOfOrderScannerNextException()); mrpc.exception(new NotServingRegionException()); HELPER.assertCounter("exceptions.RegionMovedException", 1, serverSource);