Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-16610. Make fsck read timeout configurable (#4384)

Browse Source 
(cherry picked from commit 34a973a90ef89b633c9b5c13a79aa1ac11c92eb5)

 Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

(cherry picked from commit 7d6b133af3f7ee4b986bdd2b266b83693614b6ab)
This commit is contained in:
Stephen O'Donnell 2022-06-01 20:36:01 +01:00 committed by S O'Donnell
parent 52e405390e
commit 58513d367e
3 changed files with 39 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
View File

@ -205,6 +205,14 @@ public interface HdfsClientConfigKeys {
String DFS_LEASE_HARDLIMIT_KEY = "dfs.namenode.lease-hard-limit-sec";
long DFS_LEASE_HARDLIMIT_DEFAULT = 20 * 60;
String DFS_CLIENT_FSCK_CONNECT_TIMEOUT =
"dfs.client.fsck.connect.timeout";
int DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT = 60 * 1000;
String DFS_CLIENT_FSCK_READ_TIMEOUT =
"dfs.client.fsck.read.timeout";
int DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT = 60 * 1000;
/**
* These are deprecated config keys to client code.
*/

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
View File

@ -27,6 +27,7 @@
import java.net.URLConnection;
import java.net.URLEncoder;
import java.security.PrivilegedExceptionAction;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@ -37,6 +38,7 @@
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.security.UserGroupInformation;
@ -132,8 +134,17 @@ public DFSck(Configuration conf, PrintStream out) throws IOException {
super(conf);
this.ugi = UserGroupInformation.getCurrentUser();
this.out = out;
int connectTimeout = (int) conf.getTimeDuration(
HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT,
HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
int readTimeout = (int) conf.getTimeDuration(
HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT,
HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
this.connectionFactory = URLConnectionFactory
.newDefaultURLConnectionFactory(conf);
.newDefaultURLConnectionFactory(connectTimeout, readTimeout, conf);
this.isSpnegoEnabled = UserGroupInformation.isSecurityEnabled();
}

19
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -5508,4 +5508,23 @@
Determines the namenode automatic lease recovery interval in seconds.
</description>
</property>
<property>
<name>dfs.client.fsck.connect.timeout</name>
<value>60000ms</value>
<description>
The amount of time the fsck client will wait to connect to the namenode
before timing out.
</description>
</property>
<property>
<name>dfs.client.fsck.read.timeout</name>
<value>60000ms</value>
<description>
The amount of time the fsck client will wait to read from the namenode
before timing out. If the namenode does not report progress more
frequently than this time, the client will give up waiting.
</description>
</property>
</configuration>