HBASE-13906 Improve handling of NeedUnmanagedConnectionException

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java

@@ -1087,7 +1087,8 @@ class AsyncProcess {
     public Retry manageError(int originalIndex, Row row, Retry canRetry,
                                 Throwable throwable, ServerName server) {
       if (canRetry == Retry.YES
-          && throwable != null && throwable instanceof DoNotRetryIOException) {
+          && throwable != null && (throwable instanceof DoNotRetryIOException ||
+          throwable instanceof NeedUnmanagedConnectionException)) {
         canRetry = Retry.NO_NOT_RETRIABLE;
       }
 

hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java

@@ -415,7 +415,7 @@ public class ClientScanner extends AbstractClientScanner {
           continue;
         }
         retryAfterOutOfOrderException = true;
-      } catch (DoNotRetryIOException e) {
+      } catch (DoNotRetryIOException | NeedUnmanagedConnectionException e) {
         // An exception was thrown which makes any partial results that we were collecting
         // invalid. The scanner will need to be reset to the beginning of a row.
         clearPartialResults();

hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java

@@ -4148,7 +4148,8 @@ public class HBaseAdmin implements Admin {
           LOG.warn("failed to get the procedure result procId=" + procId, serviceEx);
 
           // Not much to do, if we have a DoNotRetryIOException
-          if (serviceEx instanceof DoNotRetryIOException) {
+          if (serviceEx instanceof DoNotRetryIOException ||
+              serviceEx instanceof NeedUnmanagedConnectionException) {
             // TODO: looks like there is no way to unwrap this exception and get the proper
             // UnsupportedOperationException aside from looking at the message.
             // anyway, if we fail here we just failover to the compatibility side

hbase-client/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailInterceptor.java

@@ -203,6 +203,9 @@ class PreemptiveFastFailInterceptor extends RetryingCallerInterceptor {
     if (t instanceof DoNotRetryIOException) {
       throw (DoNotRetryIOException) t;
     }
+    if (t instanceof NeedUnmanagedConnectionException) {
+      throw new DoNotRetryIOException(t);
+    }
     if (t instanceof Error) {
       throw (Error) t;
     }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionAdminServiceCallable.java

@@ -170,6 +170,8 @@ public abstract class RegionAdminServiceCallable<T> implements RetryingCallable<
       rl = connection.locateRegion(tableName, row, useCache, true, replicaId);
     } catch (DoNotRetryIOException e) {
       throw e;
+    } catch (NeedUnmanagedConnectionException e) {
+      throw new DoNotRetryIOException(e);
     } catch (RetriesExhaustedException e) {
       throw e;
     } catch (InterruptedIOException e) {

hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java

@@ -88,7 +88,8 @@ extends RetriesExhaustedException {
 
     // If all of the exceptions are DNRIOE not exception
     for (Throwable t : exceptions) {
-      if ( !(t instanceof DoNotRetryIOException)) {
+      if ( !(t instanceof DoNotRetryIOException ||
+             t instanceof NeedUnmanagedConnectionException)) {
         res = true;
       }
     }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java

@@ -231,8 +231,12 @@ public class RpcRetryingCaller<T> {
     if (t instanceof ServiceException) {
       ServiceException se = (ServiceException)t;
       Throwable cause = se.getCause();
-      if (cause != null && cause instanceof DoNotRetryIOException) {
+      if (cause != null) {
+        if (cause instanceof DoNotRetryIOException) {
           throw (DoNotRetryIOException)cause;
+        } else if (cause instanceof NeedUnmanagedConnectionException) {
+          throw new DoNotRetryIOException(cause);
+        }
       }
       // Don't let ServiceException out; its rpc specific.
       t = cause;
@@ -240,6 +244,8 @@ public class RpcRetryingCaller<T> {
       translateException(t);
     } else if (t instanceof DoNotRetryIOException) {
       throw (DoNotRetryIOException)t;
+    } else if (t instanceof NeedUnmanagedConnectionException) {
+      throw new DoNotRetryIOException(t);
     }
     return t;
   }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java

@@ -262,6 +262,10 @@ public class RpcRetryingCallerWithReadReplicas {
       throw (DoNotRetryIOException) t;
     }
 
+    if (t instanceof NeedUnmanagedConnectionException) {
+      throw new DoNotRetryIOException(t);
+    }
+
     RetriesExhaustedException.ThrowableWithExtraContext qt =
         new RetriesExhaustedException.ThrowableWithExtraContext(t,
             EnvironmentEdgeManager.currentTime(), null);
@@ -302,6 +306,8 @@ public class RpcRetryingCallerWithReadReplicas {
       }
     } catch (DoNotRetryIOException e) {
       throw e;
+    } catch (NeedUnmanagedConnectionException e) {
+      throw new DoNotRetryIOException(e);
     } catch (RetriesExhaustedException e) {
       throw e;
     } catch (InterruptedIOException e) {

hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java

@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.NeedUnmanagedConnectionException;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.filter.Filter;
@@ -70,7 +71,7 @@ public class RowResultGenerator extends ResultGenerator {
       if (result != null && !result.isEmpty()) {
         valuesI = result.listCells().iterator();
       }
-    } catch (DoNotRetryIOException e) {
+    } catch (DoNotRetryIOException | NeedUnmanagedConnectionException e) {
       // Warn here because Stargate will return 404 in the case if multiple
       // column families were specified but one did not exist -- currently
       // HBase will fail the whole Get.

hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java

@@ -78,6 +78,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.NeedUnmanagedConnectionException;
 import org.apache.hadoop.hbase.client.Operation;
 import org.apache.hadoop.hbase.codec.Codec;
 import org.apache.hadoop.hbase.exceptions.RegionMovedException;
@@ -390,7 +391,8 @@ public class RpcServer implements RpcServerInterface {
           ExceptionResponse.Builder exceptionBuilder = ExceptionResponse.newBuilder();
           exceptionBuilder.setExceptionClassName(t.getClass().getName());
           exceptionBuilder.setStackTrace(errorMsg);
-          exceptionBuilder.setDoNotRetry(t instanceof DoNotRetryIOException);
+          exceptionBuilder.setDoNotRetry(t instanceof DoNotRetryIOException ||
+            t instanceof NeedUnmanagedConnectionException);
           if (t instanceof RegionMovedException) {
             // Special casing for this exception.  This is only one carrying a payload.
             // Do this instead of build a generic system for allowing exceptions carry

hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java

@@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.NeedUnmanagedConnectionException;
 import org.apache.hadoop.hbase.client.RegionLocator;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
@@ -272,7 +273,7 @@ public final class Canary implements Tool {
       } catch (TableNotEnabledException tnee) {
         // This is considered a success since we got a response.
         LOG.debug("The targeted table was disabled.  Assuming success.");
-      } catch (DoNotRetryIOException dnrioe) {
+      } catch (DoNotRetryIOException | NeedUnmanagedConnectionException dnrioe) {
         sink.publishReadFailure(tableName.getNameAsString(), serverName);
         LOG.error(dnrioe);
       } catch (IOException e) {