diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java index 5668ad1ca30..3c9cda8e8fc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java @@ -134,7 +134,17 @@ public class RetryPolicies { Map, RetryPolicy> exceptionToPolicyMap) { return new RemoteExceptionDependentRetry(defaultPolicy, exceptionToPolicyMap); } - + + /** + * A retry policy for exceptions other than RemoteException. + */ + public static final RetryPolicy retryOtherThanRemoteException( + RetryPolicy defaultPolicy, + Map, RetryPolicy> exceptionToPolicyMap) { + return new OtherThanRemoteExceptionDependentRetry(defaultPolicy, + exceptionToPolicyMap); + } + public static final RetryPolicy failoverOnNetworkException(int maxFailovers) { return failoverOnNetworkException(TRY_ONCE_THEN_FAIL, maxFailovers); } @@ -483,7 +493,37 @@ public class RetryPolicies { return policy.shouldRetry(e, retries, failovers, isIdempotentOrAtMostOnce); } } - + + static class OtherThanRemoteExceptionDependentRetry implements RetryPolicy { + + private RetryPolicy defaultPolicy; + private Map, RetryPolicy> exceptionToPolicyMap; + + public OtherThanRemoteExceptionDependentRetry(RetryPolicy defaultPolicy, + Map, + RetryPolicy> exceptionToPolicyMap) { + this.defaultPolicy = defaultPolicy; + this.exceptionToPolicyMap = exceptionToPolicyMap; + } + + @Override + public RetryAction shouldRetry(Exception e, int retries, int failovers, + boolean isIdempotentOrAtMostOnce) throws Exception { + RetryPolicy policy = null; + // ignore Remote Exception + if (e instanceof RemoteException) { + // do nothing + } else { + policy = exceptionToPolicyMap.get(e.getClass()); + } + if (policy == null) { + policy = defaultPolicy; + } + return policy.shouldRetry( + e, retries, failovers, isIdempotentOrAtMostOnce); + } + } + static class ExponentialBackoffRetry extends RetryLimited { public ExponentialBackoffRetry( diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java index 79ea1b912d6..4ea93a8f288 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java @@ -22,12 +22,14 @@ import static org.apache.hadoop.io.retry.RetryPolicies.RETRY_FOREVER; import static org.apache.hadoop.io.retry.RetryPolicies.TRY_ONCE_THEN_FAIL; import static org.apache.hadoop.io.retry.RetryPolicies.retryByException; import static org.apache.hadoop.io.retry.RetryPolicies.retryByRemoteException; +import static org.apache.hadoop.io.retry.RetryPolicies.retryOtherThanRemoteException; import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumCountWithFixedSleep; import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumCountWithProportionalSleep; import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumTimeWithFixedSleep; import static org.apache.hadoop.io.retry.RetryPolicies.exponentialBackoffRetry; import static org.junit.Assert.*; +import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.concurrent.Callable; @@ -202,8 +204,29 @@ public class TestRetryProxy { } catch (RemoteException e) { // expected } - } - + } + + @Test + public void testRetryOtherThanRemoteException() throws Throwable { + Map, RetryPolicy> exceptionToPolicyMap = + Collections., RetryPolicy>singletonMap( + IOException.class, RETRY_FOREVER); + + UnreliableInterface unreliable = (UnreliableInterface) + RetryProxy.create(UnreliableInterface.class, unreliableImpl, + retryOtherThanRemoteException(TRY_ONCE_THEN_FAIL, + exceptionToPolicyMap)); + // should retry with local IOException. + unreliable.failsOnceWithIOException(); + try { + // won't get retry on remote exception + unreliable.failsOnceWithRemoteException(); + fail("Should fail"); + } catch (RemoteException e) { + // expected + } + } + @Test public void testRetryInterruptible() throws Throwable { final UnreliableInterface unreliable = (UnreliableInterface) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java index ce9c16ea602..9387772464e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java @@ -26,6 +26,8 @@ class UnreliableImplementation implements UnreliableInterface { private int failsOnceInvocationCount, failsOnceWithValueInvocationCount, + failsOnceIOExceptionInvocationCount, + failsOnceRemoteExceptionInvocationCount, failsTenTimesInvocationCount, succeedsOnceThenFailsCount, succeedsOnceThenFailsIdempotentCount, @@ -89,6 +91,21 @@ class UnreliableImplementation implements UnreliableInterface { return true; } + @Override + public void failsOnceWithIOException() throws IOException { + if (failsOnceIOExceptionInvocationCount++ == 0) { + throw new IOException("test exception for failsOnceWithIOException"); + } + } + + @Override + public void failsOnceWithRemoteException() throws RemoteException { + if (failsOnceRemoteExceptionInvocationCount++ == 0) { + throw new RemoteException(IOException.class.getName(), + "test exception for failsOnceWithRemoteException"); + } + } + @Override public void failsTenTimesThenSucceeds() throws UnreliableException { if (failsTenTimesInvocationCount++ < 10) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java index 3fbe11a1c25..6c9c15313f5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java @@ -54,6 +54,9 @@ public interface UnreliableInterface { void alwaysFailsWithFatalException() throws FatalException; void alwaysFailsWithRemoteFatalException() throws RemoteException; + void failsOnceWithIOException() throws IOException; + void failsOnceWithRemoteException() throws RemoteException; + void failsOnceThenSucceeds() throws UnreliableException; boolean failsOnceThenSucceedsWithReturnValue() throws UnreliableException; diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 374848f52b5..f4ee662555f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -148,6 +148,9 @@ Release 2.7.3 - UNRELEASED YARN-5206. RegistrySecurity includes id:pass in exception text if considered invalid (Steve Loughran via jlowe) + YARN-4288. Fixed RMProxy to retry on IOException from local host. + (Junping Du via jianhe) + Release 2.7.2 - 2016-01-25 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java index 28628f32a47..f381b330330 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java @@ -249,8 +249,10 @@ public class RMProxy { exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy); exceptionToPolicyMap.put(RetriableException.class, retryPolicy); exceptionToPolicyMap.put(SocketException.class, retryPolicy); - - return RetryPolicies.retryByException( + // YARN-4288: local IOException is also possible. + exceptionToPolicyMap.put(IOException.class, retryPolicy); + // Not retry on remote IO exception. + return RetryPolicies.retryOtherThanRemoteException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); } }