Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-16610. Make fsck read timeout configurable (#4384)

This commit is contained in:
Stephen O'Donnell 2022-06-01 20:36:01 +01:00 committed by GitHub
parent 4496e2daf6
commit 34a973a90e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 37 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
View File

@ -273,6 +273,14 @@ public interface HdfsClientConfigKeys {
String DFS_LEASE_HARDLIMIT_KEY = "dfs.namenode.lease-hard-limit-sec";
long DFS_LEASE_HARDLIMIT_DEFAULT = 20 * 60;
String DFS_CLIENT_FSCK_CONNECT_TIMEOUT =
"dfs.client.fsck.connect.timeout";
int DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT = 60 * 1000;
String DFS_CLIENT_FSCK_READ_TIMEOUT =
"dfs.client.fsck.read.timeout";
int DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT = 60 * 1000;
/**
* These are deprecated config keys to client code.
*/

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
View File

@ -27,6 +27,7 @@
import java.net.URLConnection;
import java.net.URLEncoder;
import java.security.PrivilegedExceptionAction;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@ -37,6 +38,7 @@
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.security.UserGroupInformation;
@ -137,8 +139,17 @@ public DFSck(Configuration conf, PrintStream out) throws IOException {
super(conf);
this.ugi = UserGroupInformation.getCurrentUser();
this.out = out;
int connectTimeout = (int) conf.getTimeDuration(
HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT,
HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
int readTimeout = (int) conf.getTimeDuration(
HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT,
HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
this.connectionFactory = URLConnectionFactory
.newDefaultURLConnectionFactory(conf);
.newDefaultURLConnectionFactory(connectTimeout, readTimeout, conf);
this.isSpnegoEnabled = UserGroupInformation.isSecurityEnabled();
}

17
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -6429,4 +6429,21 @@
problem. In produce default set false, because it's have little performance loss.
</description>
</property>
<property>
<name>dfs.client.fsck.connect.timeout</name>
<value>60000ms</value>
<description>
The amount of time the fsck client will wait to connect to the namenode
before timing out.
</description>
</property>
<property>
<name>dfs.client.fsck.read.timeout</name>
<value>60000ms</value>
<description>
The amount of time the fsck client will wait to read from the namenode
before timing out. If the namenode does not report progress more
frequently than this time, the client will give up waiting.
</description>
</property>
</configuration>