HBASE-13067. Fix caching of stubs to allow IP address changes of restarted remote servers.

This commit is contained in:
Devaraj Das 2015-02-23 22:15:03 -08:00
parent 540266a7a0
commit 8f3fc72b94
1 changed files with 20 additions and 5 deletions

View File

@ -20,10 +20,13 @@ package org.apache.hadoop.hbase.client;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.io.InterruptedIOException; import java.io.InterruptedIOException;
import java.lang.reflect.UndeclaredThrowableException; import java.lang.reflect.UndeclaredThrowableException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.HashSet; import java.util.HashSet;
@ -45,6 +48,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingRpcChannel; import com.google.protobuf.BlockingRpcChannel;
import com.google.protobuf.RpcController; import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException; import com.google.protobuf.ServiceException;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -1472,7 +1476,7 @@ class ConnectionManager {
throw new MasterNotRunningException(sn + " is dead."); throw new MasterNotRunningException(sn + " is dead.");
} }
// Use the security info interface name as our stub key // Use the security info interface name as our stub key
String key = getStubKey(getServiceName(), sn.getHostAndPort()); String key = getStubKey(getServiceName(), sn.getHostname(), sn.getPort());
connectionLock.putIfAbsent(key, key); connectionLock.putIfAbsent(key, key);
Object stub = null; Object stub = null;
synchronized (connectionLock.get(key)) { synchronized (connectionLock.get(key)) {
@ -1561,7 +1565,7 @@ class ConnectionManager {
throw new RegionServerStoppedException(serverName + " is dead."); throw new RegionServerStoppedException(serverName + " is dead.");
} }
String key = getStubKey(AdminService.BlockingInterface.class.getName(), String key = getStubKey(AdminService.BlockingInterface.class.getName(),
serverName.getHostAndPort()); serverName.getHostname(), serverName.getPort());
this.connectionLock.putIfAbsent(key, key); this.connectionLock.putIfAbsent(key, key);
AdminService.BlockingInterface stub = null; AdminService.BlockingInterface stub = null;
synchronized (this.connectionLock.get(key)) { synchronized (this.connectionLock.get(key)) {
@ -1582,7 +1586,8 @@ class ConnectionManager {
if (isDeadServer(sn)) { if (isDeadServer(sn)) {
throw new RegionServerStoppedException(sn + " is dead."); throw new RegionServerStoppedException(sn + " is dead.");
} }
String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostAndPort()); String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostname(),
sn.getPort());
this.connectionLock.putIfAbsent(key, key); this.connectionLock.putIfAbsent(key, key);
ClientService.BlockingInterface stub = null; ClientService.BlockingInterface stub = null;
synchronized (this.connectionLock.get(key)) { synchronized (this.connectionLock.get(key)) {
@ -1599,8 +1604,18 @@ class ConnectionManager {
return stub; return stub;
} }
static String getStubKey(final String serviceName, final String rsHostnamePort) { static String getStubKey(final String serviceName, final String rsHostname, int port) {
return serviceName + "@" + rsHostnamePort; // Sometimes, servers go down and they come back up with the same hostname but a different
// IP address. Force a resolution of the rsHostname by trying to instantiate an
// InetSocketAddress, and this way we will rightfully get a new stubKey.
// Also, include the hostname in the key so as to take care of those cases where the
// DNS name is different but IP address remains the same.
InetAddress i = new InetSocketAddress(rsHostname, port).getAddress();
String address = rsHostname;
if (i != null) {
address = i.getHostAddress() + "-" + rsHostname;
}
return serviceName + "@" + address + ":" + port;
} }
private ZooKeeperKeepAliveConnection keepAliveZookeeper; private ZooKeeperKeepAliveConnection keepAliveZookeeper;