Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-13831. Make block increment deletion number configurable. Contributed by Ryan Wu.

This commit is contained in:
Wei-Chiu Chuang 2018-08-28 16:36:30 -07:00
parent 2d1f819979
commit 72e1e1904b
4 changed files with 21 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -360,6 +360,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY = "dfs.namenode.startup.delay.block.deletion.sec"; public static final String DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY = "dfs.namenode.startup.delay.block.deletion.sec";
public static final long DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT = 0L; public static final long DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT = 0L;
/** Block deletion increment. */
public static final String DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY =
"dfs.namenode.block.deletion.increment";
public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000;
public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES = public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES =
HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES; HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES;
public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT = public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT =

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -419,18 +419,18 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
FSNamesystem.class.getName() + ".audit"); FSNamesystem.class.getName() + ".audit");
private final int maxCorruptFileBlocksReturn; private final int maxCorruptFileBlocksReturn;
static int BLOCK_DELETION_INCREMENT = 1000;
private final boolean isPermissionEnabled; private final boolean isPermissionEnabled;
private final UserGroupInformation fsOwner; private final UserGroupInformation fsOwner;
private final String supergroup; private final String supergroup;
private final boolean standbyShouldCheckpoint; private final boolean standbyShouldCheckpoint;
private final int blockDeletionIncrement;
/** Interval between each check of lease to release. */ /** Interval between each check of lease to release. */
private final long leaseRecheckIntervalMs; private final long leaseRecheckIntervalMs;
/** Maximum time the lock is hold to release lease. */ /** Maximum time the lock is hold to release lease. */
private final long maxLockHoldToReleaseLeaseMs; private final long maxLockHoldToReleaseLeaseMs;
// Batch size for open files response // Batch size for open files respons
private final int maxListOpenFilesResponses; private final int maxListOpenFilesResponses;
// Scan interval is not configurable. // Scan interval is not configurable.
@ -895,6 +895,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
DFSConfigKeys.DFS_NAMENODE_LIST_OPENFILES_NUM_RESPONSES + DFSConfigKeys.DFS_NAMENODE_LIST_OPENFILES_NUM_RESPONSES +
" must be a positive integer." " must be a positive integer."
); );
this.blockDeletionIncrement = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY,
DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT);
Preconditions.checkArgument(blockDeletionIncrement > 0,
DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY +
" must be a positive integer.");
} catch(IOException e) { } catch(IOException e) {
LOG.error(getClass().getSimpleName() + " initialization failed.", e); LOG.error(getClass().getSimpleName() + " initialization failed.", e);
close(); close();
@ -3005,7 +3012,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
while (iter.hasNext()) { while (iter.hasNext()) {
writeLock(); writeLock();
try { try {
for (int i = 0; i < BLOCK_DELETION_INCREMENT && iter.hasNext(); i++) { for (int i = 0; i < blockDeletionIncrement && iter.hasNext(); i++) {
blockManager.removeBlock(iter.next()); blockManager.removeBlock(iter.next());
} }
} finally { } finally {

10
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -4814,12 +4814,12 @@
</property> </property>
<property> <property>
<name>dfs.reformat.disabled</name> <name>dfs.namenode.block.deletion.increment</name>
<value>false</value> <value>1000</value>
<description> <description>
Disable reformat of NameNode. If it's value is set to "true" The number of block deletion increment.
and metadata directories already exist then attempt to format NameNode This setting will control the block increment deletion rate to
will throw NameNodeFormatException. ensure that other waiters on the lock can get in.
</description> </description>
</property> </property>
</configuration> </configuration>

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
View File

@ -49,6 +49,7 @@ public class TestLargeDirectoryDelete {
static { static {
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1); CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1);
CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1); CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
CONF.setInt(DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY, 1);
} }
/** create a file with a length of <code>filelen</code> */ /** create a file with a length of <code>filelen</code> */
@ -137,7 +138,6 @@ public class TestLargeDirectoryDelete {
threads[1].start(); threads[1].start();
final long start = Time.now(); final long start = Time.now();
FSNamesystem.BLOCK_DELETION_INCREMENT = 1;
mc.getFileSystem().delete(new Path("/root"), true); // recursive delete mc.getFileSystem().delete(new Path("/root"), true); // recursive delete
final long end = Time.now(); final long end = Time.now();
threads[0].endThread(); threads[0].endThread();