From 7d778e892cfbd7426337e7a92aa1d335ef2a72fa Mon Sep 17 00:00:00 2001 From: Mikhail Antonov Date: Mon, 22 Jun 2015 23:34:47 -0700 Subject: [PATCH] HBASE-13906 Improve handling of NeedUnmanagedConnectionException --- .../org/apache/hadoop/hbase/client/AsyncProcess.java | 3 ++- .../org/apache/hadoop/hbase/client/ClientScanner.java | 2 +- .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 3 ++- .../hbase/client/PreemptiveFastFailInterceptor.java | 3 +++ .../hbase/client/RegionAdminServiceCallable.java | 2 ++ .../client/RetriesExhaustedWithDetailsException.java | 3 ++- .../apache/hadoop/hbase/client/RpcRetryingCaller.java | 10 ++++++++-- .../client/RpcRetryingCallerWithReadReplicas.java | 6 ++++++ .../apache/hadoop/hbase/rest/RowResultGenerator.java | 3 ++- .../java/org/apache/hadoop/hbase/ipc/RpcServer.java | 4 +++- .../main/java/org/apache/hadoop/hbase/tool/Canary.java | 3 ++- 11 files changed, 33 insertions(+), 9 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java index 7b90168c1b4..9c5e48dfa1a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java @@ -1087,7 +1087,8 @@ class AsyncProcess { public Retry manageError(int originalIndex, Row row, Retry canRetry, Throwable throwable, ServerName server) { if (canRetry == Retry.YES - && throwable != null && throwable instanceof DoNotRetryIOException) { + && throwable != null && (throwable instanceof DoNotRetryIOException || + throwable instanceof NeedUnmanagedConnectionException)) { canRetry = Retry.NO_NOT_RETRIABLE; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java index 562cb56aff8..05cd92d6ca1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java @@ -415,7 +415,7 @@ public class ClientScanner extends AbstractClientScanner { continue; } retryAfterOutOfOrderException = true; - } catch (DoNotRetryIOException e) { + } catch (DoNotRetryIOException | NeedUnmanagedConnectionException e) { // An exception was thrown which makes any partial results that we were collecting // invalid. The scanner will need to be reset to the beginning of a row. clearPartialResults(); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 63d0b7bc65a..ab73f50c869 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -4148,7 +4148,8 @@ public class HBaseAdmin implements Admin { LOG.warn("failed to get the procedure result procId=" + procId, serviceEx); // Not much to do, if we have a DoNotRetryIOException - if (serviceEx instanceof DoNotRetryIOException) { + if (serviceEx instanceof DoNotRetryIOException || + serviceEx instanceof NeedUnmanagedConnectionException) { // TODO: looks like there is no way to unwrap this exception and get the proper // UnsupportedOperationException aside from looking at the message. // anyway, if we fail here we just failover to the compatibility side diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailInterceptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailInterceptor.java index 64cd03dfab6..ce21ee9df8b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailInterceptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailInterceptor.java @@ -203,6 +203,9 @@ class PreemptiveFastFailInterceptor extends RetryingCallerInterceptor { if (t instanceof DoNotRetryIOException) { throw (DoNotRetryIOException) t; } + if (t instanceof NeedUnmanagedConnectionException) { + throw new DoNotRetryIOException(t); + } if (t instanceof Error) { throw (Error) t; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionAdminServiceCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionAdminServiceCallable.java index 0d1fa020495..d31477711c1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionAdminServiceCallable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionAdminServiceCallable.java @@ -170,6 +170,8 @@ public abstract class RegionAdminServiceCallable implements RetryingCallable< rl = connection.locateRegion(tableName, row, useCache, true, replicaId); } catch (DoNotRetryIOException e) { throw e; + } catch (NeedUnmanagedConnectionException e) { + throw new DoNotRetryIOException(e); } catch (RetriesExhaustedException e) { throw e; } catch (InterruptedIOException e) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java index 253ff8b1133..5c179a1fe1a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java @@ -88,7 +88,8 @@ extends RetriesExhaustedException { // If all of the exceptions are DNRIOE not exception for (Throwable t : exceptions) { - if ( !(t instanceof DoNotRetryIOException)) { + if ( !(t instanceof DoNotRetryIOException || + t instanceof NeedUnmanagedConnectionException)) { res = true; } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java index 896222c8bdb..bb19d5fbf34 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java @@ -231,8 +231,12 @@ public class RpcRetryingCaller { if (t instanceof ServiceException) { ServiceException se = (ServiceException)t; Throwable cause = se.getCause(); - if (cause != null && cause instanceof DoNotRetryIOException) { - throw (DoNotRetryIOException)cause; + if (cause != null) { + if (cause instanceof DoNotRetryIOException) { + throw (DoNotRetryIOException)cause; + } else if (cause instanceof NeedUnmanagedConnectionException) { + throw new DoNotRetryIOException(cause); + } } // Don't let ServiceException out; its rpc specific. t = cause; @@ -240,6 +244,8 @@ public class RpcRetryingCaller { translateException(t); } else if (t instanceof DoNotRetryIOException) { throw (DoNotRetryIOException)t; + } else if (t instanceof NeedUnmanagedConnectionException) { + throw new DoNotRetryIOException(t); } return t; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java index a2bedb4c2b3..025daa02c94 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java @@ -262,6 +262,10 @@ public class RpcRetryingCallerWithReadReplicas { throw (DoNotRetryIOException) t; } + if (t instanceof NeedUnmanagedConnectionException) { + throw new DoNotRetryIOException(t); + } + RetriesExhaustedException.ThrowableWithExtraContext qt = new RetriesExhaustedException.ThrowableWithExtraContext(t, EnvironmentEdgeManager.currentTime(), null); @@ -302,6 +306,8 @@ public class RpcRetryingCallerWithReadReplicas { } } catch (DoNotRetryIOException e) { throw e; + } catch (NeedUnmanagedConnectionException e) { + throw new DoNotRetryIOException(e); } catch (RetriesExhaustedException e) { throw e; } catch (InterruptedIOException e) { diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java index 74afcd9417a..d07d81bc621 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.NeedUnmanagedConnectionException; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.Filter; @@ -70,7 +71,7 @@ public class RowResultGenerator extends ResultGenerator { if (result != null && !result.isEmpty()) { valuesI = result.listCells().iterator(); } - } catch (DoNotRetryIOException e) { + } catch (DoNotRetryIOException | NeedUnmanagedConnectionException e) { // Warn here because Stargate will return 404 in the case if multiple // column families were specified but one did not exist -- currently // HBase will fail the whole Get. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java index 461a8f09bc1..07a3033decc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.NeedUnmanagedConnectionException; import org.apache.hadoop.hbase.client.Operation; import org.apache.hadoop.hbase.codec.Codec; import org.apache.hadoop.hbase.exceptions.RegionMovedException; @@ -390,7 +391,8 @@ public class RpcServer implements RpcServerInterface { ExceptionResponse.Builder exceptionBuilder = ExceptionResponse.newBuilder(); exceptionBuilder.setExceptionClassName(t.getClass().getName()); exceptionBuilder.setStackTrace(errorMsg); - exceptionBuilder.setDoNotRetry(t instanceof DoNotRetryIOException); + exceptionBuilder.setDoNotRetry(t instanceof DoNotRetryIOException || + t instanceof NeedUnmanagedConnectionException); if (t instanceof RegionMovedException) { // Special casing for this exception. This is only one carrying a payload. // Do this instead of build a generic system for allowing exceptions carry diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java index 24bff16a78a..2cf34b386fb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.NeedUnmanagedConnectionException; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; @@ -272,7 +273,7 @@ public final class Canary implements Tool { } catch (TableNotEnabledException tnee) { // This is considered a success since we got a response. LOG.debug("The targeted table was disabled. Assuming success."); - } catch (DoNotRetryIOException dnrioe) { + } catch (DoNotRetryIOException | NeedUnmanagedConnectionException dnrioe) { sink.publishReadFailure(tableName.getNameAsString(), serverName); LOG.error(dnrioe); } catch (IOException e) {