Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-16514. Reduce the failover sleep time if multiple namenode are configured

This commit is contained in:
liubingxing 2022-04-15 16:45:49 +08:00
parent a6ebc42671
commit d37473e47c
2 changed files with 21 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
View File

@ -181,6 +181,13 @@ public class RetryPolicies {
maxRetries, delayMillis, maxDelayBase);
}
public static final RetryPolicy failoverOnNetworkException(
RetryPolicy fallbackPolicy, int maxFailovers, int maxRetries,
long delayMillis, long maxDelayBase, int nnSize) {
return new FailoverOnNetworkExceptionRetry(fallbackPolicy, maxFailovers,
maxRetries, delayMillis, maxDelayBase, nnSize);
}
static class TryOnceThenFail implements RetryPolicy {
@Override
public RetryAction shouldRetry(Exception e, int retries, int failovers,
@ -620,12 +627,13 @@ public class RetryPolicies {
* Fall back on underlying retry policy otherwise.
*/
static class FailoverOnNetworkExceptionRetry implements RetryPolicy {
private static final int minNnSize = 2;
private RetryPolicy fallbackPolicy;
private int maxFailovers;
private int maxRetries;
private long delayMillis;
private long maxDelayBase;
private int nnSize;
public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy,
int maxFailovers) {
@ -639,11 +647,18 @@ public class RetryPolicies {
public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy,
int maxFailovers, int maxRetries, long delayMillis, long maxDelayBase) {
this(fallbackPolicy, maxFailovers, maxRetries, delayMillis, maxDelayBase, minNnSize);
}
public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy,
int maxFailovers, int maxRetries, long delayMillis, long maxDelayBase, int nnSize) {
this.fallbackPolicy = fallbackPolicy;
this.maxFailovers = maxFailovers;
this.maxRetries = maxRetries;
this.delayMillis = delayMillis;
this.maxDelayBase = maxDelayBase;
// set the nn size to reduce the failover sleep time.
this.nnSize = nnSize;
}
/**
@ -651,7 +666,7 @@ public class RetryPolicies {
* sleep exponentially otherwise
*/
private long getFailoverOrRetrySleepTime(int times) {
return times == 0 ? 0 :
return times < (nnSize - 1) ? 0 :
calculateExponentialTime(delayMillis, times, maxDelayBase);
}

5
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
View File

@ -316,13 +316,16 @@ public class NameNodeProxiesClient {
Configuration conf, URI nameNodeUri, Class<T> xface,
AbstractNNFailoverProxyProvider<T> failoverProxyProvider) {
Preconditions.checkNotNull(failoverProxyProvider);
Map<String, Map<String, InetSocketAddress>> map =
DFSUtilClient.getAddresses(conf, null, HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
Map<String, InetSocketAddress> nnMap = map.get(nameNodeUri.getHost());
// HA case
DfsClientConf config = new DfsClientConf(conf);
T proxy = (T) RetryProxy.create(xface, failoverProxyProvider,
RetryPolicies.failoverOnNetworkException(
RetryPolicies.TRY_ONCE_THEN_FAIL, config.getMaxFailoverAttempts(),
config.getMaxRetryAttempts(), config.getFailoverSleepBaseMillis(),
config.getFailoverSleepMaxMillis()));
config.getFailoverSleepMaxMillis(), nnMap.size()));
Text dtService;
if (failoverProxyProvider.useLogicalURI()) {