diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt index fecb9c8c82f..11a2b6b00d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt @@ -51,3 +51,5 @@ HDFS-2627. Determine DN's view of which NN is active based on heartbeat response HDFS-2634. Standby needs to ingest latest edit logs before transitioning to active (todd) HDFS-2671. NN should throw StandbyException in response to RPCs in STANDBY state (todd) + +HDFS-2680. DFSClient should construct failover proxy with exponential backoff (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 43af62c46cc..756899945d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -147,6 +147,9 @@ public class DFSClient implements java.io.Closeable { * DFSClient configuration */ static class Conf { + final int maxFailoverAttempts; + final int failoverSleepBaseMillis; + final int failoverSleepMaxMillis; final int maxBlockAcquireFailures; final int confTime; final int ioBufferSize; @@ -168,6 +171,16 @@ public class DFSClient implements java.io.Closeable { final boolean useLegacyBlockReader; Conf(Configuration conf) { + maxFailoverAttempts = conf.getInt( + DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, + DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT); + failoverSleepBaseMillis = conf.getInt( + DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY, + DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT); + failoverSleepMaxMillis = conf.getInt( + DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY, + DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT); + maxBlockAcquireFailures = conf.getInt( DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY, DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT); @@ -306,7 +319,12 @@ public class DFSClient implements java.io.Closeable { FailoverProxyProvider failoverProxyProvider = (FailoverProxyProvider) ReflectionUtils.newInstance(failoverProxyProviderClass, conf); this.namenode = (ClientProtocol)RetryProxy.create(ClientProtocol.class, - failoverProxyProvider, RetryPolicies.failoverOnNetworkException(1)); + failoverProxyProvider, + RetryPolicies.failoverOnNetworkException( + RetryPolicies.TRY_ONCE_THEN_FAIL, + dfsClientConf.maxFailoverAttempts, + dfsClientConf.failoverSleepBaseMillis, + dfsClientConf.failoverSleepMaxMillis)); nnAddress = null; } else if (nameNodeUri != null && rpcNamenode == null) { this.namenode = DFSUtil.createNamenode(NameNode.getAddress(nameNodeUri), conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 1c9ed581927..b0a57863552 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -49,6 +49,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY = "dfs.client.socketcache.capacity"; public static final int DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT = 16; public static final String DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX = "dfs.client.failover.proxy.provider"; + public static final String DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY = "dfs.client.failover.max.attempts"; + public static final int DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT = 15; + public static final String DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY = "dfs.client.failover.sleep.base.millis"; + public static final int DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT = 500; + public static final String DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY = "dfs.client.failover.sleep.max.millis"; + public static final int DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT = 15000; public static final String DFS_NAMENODE_BACKUP_ADDRESS_KEY = "dfs.namenode.backup.address"; public static final String DFS_NAMENODE_BACKUP_ADDRESS_DEFAULT = "localhost:50100";