diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 405e821110e..c1caf5f76d4 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -615,6 +615,9 @@ Release 2.7.0 - UNRELEASED HADOOP-11522. Update S3A Documentation. (Thomas Demoor via stevel) + HADOOP-11521. Make connection timeout configurable in s3a. + (Thomas Demoor via stevel) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 5eb301a244b..41121f18f80 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -757,6 +757,12 @@ for ldap providers in the same way as above does. How many times we should retry commands on transient errors. + + fs.s3a.connection.establish.timeout + 5000 + Socket connection setup timeout in seconds. + + fs.s3a.connection.timeout 50000 diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index b6863bb162f..1d4f67b837c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -42,6 +42,10 @@ public class Constants { // number of times we should retry errors public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum"; public static final int DEFAULT_MAX_ERROR_RETRIES = 10; + + // seconds until we give up trying to establish a connection to s3 + public static final String ESTABLISH_TIMEOUT = "fs.s3a.connection.establish.timeout"; + public static final int DEFAULT_ESTABLISH_TIMEOUT = 50000; // seconds until we give up on a connection to s3 public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 22350bc40c2..4de5c13b703 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -174,6 +174,8 @@ public class S3AFileSystem extends FileSystem { awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES)); + awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT, + DEFAULT_ESTABLISH_TIMEOUT)); awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT)); diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 8f511dd276c..1e44864a203 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -210,6 +210,12 @@ If you do any of these: change your credentials immediately! How many times we should retry commands on transient errors. + + fs.s3a.connection.establish.timeout + 5000 + Socket connection setup timeout in seconds. + + fs.s3a.connection.timeout 50000