From a5f2691b7175d091c7c6c24d449045b3e2286eee Mon Sep 17 00:00:00 2001
From: youchan <youcan199802@gmail.com>
Date: Thu, 23 Mar 2023 20:12:21 +0800
Subject: [PATCH] HBASE-22041 master should resolve regionserver's ip again
 when ConnectionException

---
 .../hadoop/hbase/client/AsyncConnectionImpl.java    |  4 ++++
 .../hadoop/hbase/client/AsyncRegionServerAdmin.java |  4 ++++
 .../master/procedure/RSProcedureDispatcher.java     | 13 +++++++++++--
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
index 3af574cfc0b..04af83b5e32 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
@@ -301,6 +301,10 @@ public class AsyncConnectionImpl implements AsyncConnection {
       () -> createAdminServerStub(serverName));
   }
 
+  void removeAdminStub(ServerName serverName) {
+    adminStubs.remove(getStubKey(AdminService.getDescriptor().getName(), serverName));
+  }
+
   CompletableFuture<MasterService.Interface> getMasterStub() {
     return ConnectionUtils.getOrFetch(masterStub, masterStubMakeFuture, false, () -> {
       CompletableFuture<MasterService.Interface> future = new CompletableFuture<>();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
index f5fcc02e918..528b4e8ca43 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
@@ -216,4 +216,8 @@ public class AsyncRegionServerAdmin {
     executeProcedures(ExecuteProceduresRequest request) {
     return call((stub, controller, done) -> stub.executeProcedures(controller, request, done));
   }
+
+  public void removeRsStub(ServerName serverName) {
+    conn.removeAdminStub(serverName);
+  }
 }
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index af22fba2729..e0d1f04fa92 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.procedure;
 
 import java.io.IOException;
 import java.lang.Thread.UncaughtExceptionHandler;
+import java.net.ConnectException;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -257,7 +258,7 @@ public class RSProcedureDispatcher extends RemoteProcedureDispatcher<MasterProce
         DEFAULT_RS_RPC_RETRY_INTERVAL);
     }
 
-    private AsyncRegionServerAdmin getRsAdmin() throws IOException {
+    private AsyncRegionServerAdmin getRsAdmin() {
       return master.getAsyncClusterConnection().getRegionServerAdmin(serverName);
     }
 
@@ -306,6 +307,14 @@ public class RSProcedureDispatcher extends RemoteProcedureDispatcher<MasterProce
           serverName, e.toString(), numberOfAttemptsSoFar);
         return false;
       }
+      //This situation may be that the master resolves to the wrong ip address.
+      // removing the cache so that the master can resolve the ip for regionserver again.
+      if (e instanceof ConnectException) {
+        getRsAdmin().removeRsStub(serverName);
+        LOG.warn("Request to {} failed due to {}, try={} retry get new rs admin... ",
+          serverName, e.toString(), numberOfAttemptsSoFar);
+      }
+
       if (e instanceof RegionServerAbortedException || e instanceof RegionServerStoppedException) {
         // A better way is to return true here to let the upper layer quit, and then schedule a
         // background task to check whether the region server is dead. And if it is dead, call
@@ -313,7 +322,7 @@ public class RSProcedureDispatcher extends RemoteProcedureDispatcher<MasterProce
         // result, but waste some resources.
         LOG.warn("{} is aborted or stopped, for safety we still need to"
           + " wait until it is fully dead, try={}", serverName, numberOfAttemptsSoFar);
-      } else {
+      } else if (!(e instanceof ConnectException)) {
         LOG.warn("request to {} failed due to {}, try={}, retrying...", serverName, e.toString(),
           numberOfAttemptsSoFar);
       }