From 7a662bea89a89eecf90b0216b1405a376c4b81de Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Wed, 2 Nov 2016 13:01:54 -0700 Subject: [PATCH] HDFS-11095. BlockManagerSafeMode should respect extension period default config value (30s). Contributed by Mingliang Liu (cherry picked from commit 9b0c17f85232950ecd927427e5ad9d28dd9d277f) --- .../blockmanagement/BlockManagerSafeMode.java | 11 ++++++++-- .../src/main/resources/hdfs-default.xml | 5 +++-- .../TestBlockManagerSafeMode.java | 20 +++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java index 1249274aec5..760ef8ef858 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java @@ -35,12 +35,16 @@ import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.util.Daemon; +import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.atomic.AtomicLong; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY; @@ -96,7 +100,8 @@ enum BMSafeModeStatus { private long blockReplQueueThreshold; /** How long (in ms) is the extension period. */ - private final int extension; + @VisibleForTesting + final long extension; /** Timestamp of the first time when thresholds are met. */ private final AtomicLong reachedTime = new AtomicLong(); /** Timestamp of the safe mode initialized. */ @@ -143,7 +148,9 @@ enum BMSafeModeStatus { this.replQueueThreshold = conf.getFloat(DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY, (float) threshold); - this.extension = conf.getInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); + this.extension = conf.getTimeDuration(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, + DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT, + MILLISECONDS); this.inRollBack = isInRollBackMode(NameNode.getStartupOption(conf)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index faa26b8ac8f..b5ab96347d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -823,8 +823,9 @@ dfs.namenode.safemode.extension 30000 - Determines extension of safe mode in milliseconds - after the threshold level is reached. + Determines extension of safe mode in milliseconds after the threshold level + is reached. Support multiple time unit suffix (case insensitive), as + described in dfs.heartbeat.interval. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java index 8826f1f27d6..b7e8f6dd526 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java @@ -38,6 +38,7 @@ import java.io.InterruptedIOException; import java.util.concurrent.TimeoutException; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -386,6 +387,25 @@ public void testStayInSafeModeWhenBytesInFuture() throws Exception { "are cleared.", bmSafeMode.leaveSafeMode(false)); } + @Test(timeout = 10000) + public void testExtensionConfig() { + final Configuration conf = new HdfsConfiguration(); + bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf); + assertEquals(DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT, bmSafeMode.extension); + + conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, "30000"); + bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf); + assertEquals(30000, bmSafeMode.extension); + + conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, "20s"); + bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf); + assertEquals(20 * 1000, bmSafeMode.extension); + + conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, "7m"); + bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf); + assertEquals(7 * 60 * 1000, bmSafeMode.extension); + } + /** * Test get safe mode tip. */