From 8f3fc72b944bab98e79cefe0f34fc135d3ac0c10 Mon Sep 17 00:00:00 2001
From: Devaraj Das <ddas@apache.org>
Date: Mon, 23 Feb 2015 22:15:03 -0800
Subject: [PATCH] HBASE-13067. Fix caching of stubs to allow IP address changes
 of restarted remote servers.

---
 .../hbase/client/ConnectionManager.java       | 25 +++++++++++++++----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
index 17d13781ab8..eb3651e69cb 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
@@ -20,10 +20,13 @@ package org.apache.hadoop.hbase.client;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.lang.reflect.UndeclaredThrowableException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashSet;
@@ -45,6 +48,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.BlockingRpcChannel;
 import com.google.protobuf.RpcController;
 import com.google.protobuf.ServiceException;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -1472,7 +1476,7 @@ class ConnectionManager {
             throw new MasterNotRunningException(sn + " is dead.");
           }
           // Use the security info interface name as our stub key
-          String key = getStubKey(getServiceName(), sn.getHostAndPort());
+          String key = getStubKey(getServiceName(), sn.getHostname(), sn.getPort());
           connectionLock.putIfAbsent(key, key);
           Object stub = null;
           synchronized (connectionLock.get(key)) {
@@ -1561,7 +1565,7 @@ class ConnectionManager {
         throw new RegionServerStoppedException(serverName + " is dead.");
       }
       String key = getStubKey(AdminService.BlockingInterface.class.getName(),
-        serverName.getHostAndPort());
+          serverName.getHostname(), serverName.getPort());
       this.connectionLock.putIfAbsent(key, key);
       AdminService.BlockingInterface stub = null;
       synchronized (this.connectionLock.get(key)) {
@@ -1582,7 +1586,8 @@ class ConnectionManager {
       if (isDeadServer(sn)) {
         throw new RegionServerStoppedException(sn + " is dead.");
       }
-      String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostAndPort());
+      String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostname(),
+          sn.getPort());
       this.connectionLock.putIfAbsent(key, key);
       ClientService.BlockingInterface stub = null;
       synchronized (this.connectionLock.get(key)) {
@@ -1599,8 +1604,18 @@ class ConnectionManager {
       return stub;
     }
 
-    static String getStubKey(final String serviceName, final String rsHostnamePort) {
-      return serviceName + "@" + rsHostnamePort;
+    static String getStubKey(final String serviceName, final String rsHostname, int port) {
+      // Sometimes, servers go down and they come back up with the same hostname but a different
+      // IP address. Force a resolution of the rsHostname by trying to instantiate an
+      // InetSocketAddress, and this way we will rightfully get a new stubKey.
+      // Also, include the hostname in the key so as to take care of those cases where the
+      // DNS name is different but IP address remains the same.
+      InetAddress i =  new InetSocketAddress(rsHostname, port).getAddress();
+      String address = rsHostname;
+      if (i != null) {
+        address = i.getHostAddress() + "-" + rsHostname;
+      }
+      return serviceName + "@" + address + ":" + port;
     }
 
     private ZooKeeperKeepAliveConnection keepAliveZookeeper;