Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11060. make DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED configurable. Contributed by Lantao Jin. 

(cherry picked from commit e95c5e9f62)
(cherry picked from commit 964f3454d1)
(cherry picked from commit 0af04ad89f)

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
This commit is contained in:
Wei-Chiu Chuang 2018-07-25 11:04:18 -07:00
parent 3aa17bd737
commit 5ef8fd5119
4 changed files with 20 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -220,6 +220,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.namenode.maintenance.replication.min"; "dfs.namenode.maintenance.replication.min";
public static final int DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT public static final int DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT
= 1; = 1;
public static final String DFS_NAMENODE_MAX_CORRUPT_FILE_BLOCKS_RETURNED_KEY = "dfs.namenode.max-corrupt-file-blocks-returned";
public static final int DFS_NAMENODE_MAX_CORRUPT_FILE_BLOCKS_RETURNED_DEFAULT = 100;
public static final String DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY = public static final String DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY =
HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY; HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY;
public static final int DFS_NAMENODE_REPLICATION_MAX_STREAMS_DEFAULT = 2; public static final int DFS_NAMENODE_REPLICATION_MAX_STREAMS_DEFAULT = 2;

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -393,7 +393,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
public static final Log auditLog = LogFactory.getLog( public static final Log auditLog = LogFactory.getLog(
FSNamesystem.class.getName() + ".audit"); FSNamesystem.class.getName() + ".audit");
static final int DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED = 100; private final int maxCorruptFileBlocksReturn;
static int BLOCK_DELETION_INCREMENT = 1000; static int BLOCK_DELETION_INCREMENT = 1000;
private final boolean isPermissionEnabled; private final boolean isPermissionEnabled;
private final UserGroupInformation fsOwner; private final UserGroupInformation fsOwner;
@ -796,6 +796,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
this.supportAppends = conf.getBoolean(DFS_SUPPORT_APPEND_KEY, DFS_SUPPORT_APPEND_DEFAULT); this.supportAppends = conf.getBoolean(DFS_SUPPORT_APPEND_KEY, DFS_SUPPORT_APPEND_DEFAULT);
LOG.info("Append Enabled: " + supportAppends); LOG.info("Append Enabled: " + supportAppends);
this.maxCorruptFileBlocksReturn = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_MAX_CORRUPT_FILE_BLOCKS_RETURNED_KEY,
DFSConfigKeys.DFS_NAMENODE_MAX_CORRUPT_FILE_BLOCKS_RETURNED_DEFAULT);
this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf); this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf);
this.standbyShouldCheckpoint = conf.getBoolean( this.standbyShouldCheckpoint = conf.getBoolean(
@ -5135,7 +5139,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (src.startsWith(path)){ if (src.startsWith(path)){
corruptFiles.add(new CorruptFileBlockInfo(src, blk)); corruptFiles.add(new CorruptFileBlockInfo(src, blk));
count++; count++;
if (count >= DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED) if (count >= maxCorruptFileBlocksReturn)
break; break;
} }
} }

9
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -591,6 +591,15 @@
</description> </description>
</property> </property>
<property>
<name>dfs.namenode.max-corrupt-file-blocks-returned</name>
<value>100</value>
<description>
The maximum number of corrupt file blocks listed by NameNode Web UI,
JMX and other client request.
</description>
</property>
<property> <property>
<name>dfs.blocksize</name> <name>dfs.blocksize</name>
<value>134217728</value> <value>134217728</value>

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java
View File

@ -452,7 +452,7 @@ public class TestListCorruptFileBlocks {
cluster = new MiniDFSCluster.Builder(conf).build(); cluster = new MiniDFSCluster.Builder(conf).build();
FileSystem fs = cluster.getFileSystem(); FileSystem fs = cluster.getFileSystem();
final int maxCorruptFileBlocks = final int maxCorruptFileBlocks =
FSNamesystem.DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED; conf.getInt(DFSConfigKeys.DFS_NAMENODE_MAX_CORRUPT_FILE_BLOCKS_RETURNED_KEY, 100);
// create 110 files with one block each // create 110 files with one block each
DFSTestUtil util = new DFSTestUtil.Builder().setName("testMaxCorruptFiles"). DFSTestUtil util = new DFSTestUtil.Builder().setName("testMaxCorruptFiles").