diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2205f4ae444..c7d57fe1556 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -367,6 +367,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10211. Enable RPC protocol to negotiate SASL-QOP values between clients and servers. (Benoy Antony via Arpit Agarwal) + HADOOP-10386. Log proxy hostname in various exceptions being thrown in a HA + setup. (wheat9) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/DefaultFailoverProxyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/DefaultFailoverProxyProvider.java index ae37d0bed4a..4e97314931f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/DefaultFailoverProxyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/DefaultFailoverProxyProvider.java @@ -43,8 +43,8 @@ public class DefaultFailoverProxyProvider implements FailoverProxyProvider } @Override - public T getProxy() { - return proxy; + public ProxyInfo getProxy() { + return new ProxyInfo(proxy, null); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java index c267d028798..5acb936aad5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java @@ -30,6 +30,18 @@ import org.apache.hadoop.classification.InterfaceStability; */ @InterfaceStability.Evolving public interface FailoverProxyProvider extends Closeable { + public static final class ProxyInfo { + public final T proxy; + /* + * The information (e.g., the IP address) of the current proxy object. It + * provides information for debugging purposes. + */ + public final String proxyInfo; + public ProxyInfo(T proxy, String proxyInfo) { + this.proxy = proxy; + this.proxyInfo = proxyInfo; + } + } /** * Get the proxy object which should be used until the next failover event @@ -37,14 +49,14 @@ public interface FailoverProxyProvider extends Closeable { * * @return the proxy object to invoke methods upon */ - public T getProxy(); + public ProxyInfo getProxy(); /** * Called whenever the associated {@link RetryPolicy} determines that an error * warrants failing over. * - * @param currentProxy the proxy object which was being used before this - * failover event + * @param currentProxy + * the proxy object which was being used before this failover event */ public void performFailover(T currentProxy); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java index 51dd46a8f9e..35188261315 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java @@ -28,6 +28,7 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.retry.FailoverProxyProvider.ProxyInfo; import org.apache.hadoop.io.retry.RetryPolicy.RetryAction; import org.apache.hadoop.ipc.Client; import org.apache.hadoop.ipc.Client.ConnectionId; @@ -56,7 +57,7 @@ public class RetryInvocationHandler implements RpcInvocationHandler { private final RetryPolicy defaultPolicy; private final Map methodNameToPolicyMap; - private T currentProxy; + private ProxyInfo currentProxy; protected RetryInvocationHandler(FailoverProxyProvider proxyProvider, RetryPolicy retryPolicy) { @@ -82,7 +83,7 @@ public class RetryInvocationHandler implements RpcInvocationHandler { // The number of times this method invocation has been failed over. int invocationFailoverCount = 0; - final boolean isRpc = isRpcInvocation(currentProxy); + final boolean isRpc = isRpcInvocation(currentProxy.proxy); final int callId = isRpc? Client.nextCallId(): RpcConstants.INVALID_CALL_ID; int retries = 0; while (true) { @@ -114,9 +115,9 @@ public class RetryInvocationHandler implements RpcInvocationHandler { invocationFailoverCount, isIdempotentOrAtMostOnce); if (action.action == RetryAction.RetryDecision.FAIL) { if (action.reason != null) { - LOG.warn("Exception while invoking " + - currentProxy.getClass() + "." + method.getName() + - ". Not retrying because " + action.reason, e); + LOG.warn("Exception while invoking " + currentProxy.proxy.getClass() + + "." + method.getName() + " over " + currentProxy.proxyInfo + + ". Not retrying because " + action.reason, e); } throw e; } else { // retry or failover @@ -129,7 +130,9 @@ public class RetryInvocationHandler implements RpcInvocationHandler { if (action.action == RetryAction.RetryDecision.FAILOVER_AND_RETRY && worthLogging) { String msg = "Exception while invoking " + method.getName() - + " of class " + currentProxy.getClass().getSimpleName(); + + " of class " + currentProxy.proxy.getClass().getSimpleName() + + " over " + currentProxy.proxyInfo; + if (invocationFailoverCount > 0) { msg += " after " + invocationFailoverCount + " fail over attempts"; } @@ -140,8 +143,9 @@ public class RetryInvocationHandler implements RpcInvocationHandler { } else { if(LOG.isDebugEnabled()) { LOG.debug("Exception while invoking " + method.getName() - + " of class " + currentProxy.getClass().getSimpleName() + - ". Retrying " + formatSleepMessage(action.delayMillis), e); + + " of class " + currentProxy.proxy.getClass().getSimpleName() + + " over " + currentProxy.proxyInfo + ". Retrying " + + formatSleepMessage(action.delayMillis), e); } } @@ -154,7 +158,7 @@ public class RetryInvocationHandler implements RpcInvocationHandler { // single actual fail over. synchronized (proxyProvider) { if (invocationAttemptFailoverCount == proxyProviderFailoverCount) { - proxyProvider.performFailover(currentProxy); + proxyProvider.performFailover(currentProxy.proxy); proxyProviderFailoverCount++; currentProxy = proxyProvider.getProxy(); } else { @@ -182,7 +186,7 @@ public class RetryInvocationHandler implements RpcInvocationHandler { if (!method.isAccessible()) { method.setAccessible(true); } - return method.invoke(currentProxy, args); + return method.invoke(currentProxy.proxy, args); } catch (InvocationTargetException e) { throw e.getCause(); } @@ -207,7 +211,7 @@ public class RetryInvocationHandler implements RpcInvocationHandler { @Override //RpcInvocationHandler public ConnectionId getConnectionId() { - return RPC.getConnectionIdForProxy(currentProxy); + return RPC.getConnectionIdForProxy(currentProxy.proxy); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestFailoverProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestFailoverProxy.java index 3b4ebae4812..7d55fe1c13c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestFailoverProxy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestFailoverProxy.java @@ -48,8 +48,8 @@ public class TestFailoverProxy { } @Override - public T getProxy() { - return currentlyActive; + public ProxyInfo getProxy() { + return new ProxyInfo(currentlyActive, currentlyActive.toString()); } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java index 5b77698b100..ce9c16ea602 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java @@ -22,7 +22,7 @@ import java.io.IOException; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; -public class UnreliableImplementation implements UnreliableInterface { +class UnreliableImplementation implements UnreliableInterface { private int failsOnceInvocationCount, failsOnceWithValueInvocationCount, @@ -154,6 +154,11 @@ public class UnreliableImplementation implements UnreliableInterface { } } + @Override + public String toString() { + return getClass().getSimpleName() + "[" + identifier + "]"; + } + private static void throwAppropriateException(TypeOfExceptionToFailWith eType, String message) throws UnreliableException, StandbyException, IOException { switch (eType) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java index eab36481e1a..ebe6ef87a5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java @@ -116,10 +116,9 @@ public class ConfiguredFailoverProxyProvider implements /** * Lazily initialize the RPC proxy object. */ - @SuppressWarnings("unchecked") @Override - public synchronized T getProxy() { - AddressRpcProxyPair current = proxies.get(currentProxyIndex); + public synchronized ProxyInfo getProxy() { + AddressRpcProxyPair current = proxies.get(currentProxyIndex); if (current.namenode == null) { try { current.namenode = NameNodeProxies.createNonHAProxy(conf, @@ -129,7 +128,7 @@ public class ConfiguredFailoverProxyProvider implements throw new RuntimeException(e); } } - return (T)current.namenode; + return new ProxyInfo((T)current.namenode, current.address.toString()); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java index ef56edd4293..5577d207754 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java @@ -83,14 +83,14 @@ public class ConfiguredRMFailoverProxyProvider } @Override - public synchronized T getProxy() { + public synchronized ProxyInfo getProxy() { String rmId = rmServiceIds[currentProxyIndex]; T current = proxies.get(rmId); if (current == null) { current = getProxyInternal(); proxies.put(rmId, current); } - return current; + return new ProxyInfo(current, rmId); } @Override