From 3283268d8bff839247d3a8170630b0bcfaffb72f Mon Sep 17 00:00:00 2001 From: cnauroth Date: Thu, 30 Oct 2014 22:26:35 -0700 Subject: [PATCH] HDFS-6385. Show when block deletion will start after NameNode startup in WebUI. Contributed by Chris Nauroth. (cherry picked from commit c637d6d9d5cb36b8cd5180787c5d25b68f2aa370) (cherry picked from commit caded2545c3c773cf6e9cd21e49dcb6858a4de04) --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/blockmanagement/BlockManager.java | 9 +++++++-- .../hadoop/hdfs/server/namenode/FSNamesystem.java | 5 +++++ .../server/namenode/metrics/FSNamesystemMBean.java | 6 ++++++ .../hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html | 1 + .../blockmanagement/TestPendingInvalidateBlock.java | 11 +++++++++++ 6 files changed, 33 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bb3823053f4..4cec5535988 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -294,6 +294,9 @@ Release 2.6.0 - UNRELEASED HDFS-7230. Add rolling downgrade documentation. (szetszwo via jing9) + HDFS-6385. Show when block deletion will start after NameNode startup in + WebUI. (cnauroth) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index f25ebe1ae47..3e738cd7884 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -121,6 +121,7 @@ public class BlockManager { private volatile long scheduledReplicationBlocksCount = 0L; private final AtomicLong excessBlocksCount = new AtomicLong(0L); private final AtomicLong postponedMisreplicatedBlocksCount = new AtomicLong(0L); + private final long startupDelayBlockDeletionInMs; /** Used by metrics */ public long getPendingReplicationBlocksCount() { @@ -143,6 +144,10 @@ public long getPendingDeletionBlocksCount() { return invalidateBlocks.numBlocks(); } /** Used by metrics */ + public long getStartupDelayBlockDeletionInMs() { + return startupDelayBlockDeletionInMs; + } + /** Used by metrics */ public long getExcessBlocksCount() { return excessBlocksCount.get(); } @@ -266,11 +271,11 @@ public BlockManager(final Namesystem namesystem, final FSClusterStats stats, datanodeManager = new DatanodeManager(this, namesystem, conf); heartbeatManager = datanodeManager.getHeartbeatManager(); - final long pendingPeriod = conf.getLong( + startupDelayBlockDeletionInMs = conf.getLong( DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; invalidateBlocks = new InvalidateBlocks( - datanodeManager.blockInvalidateLimit, pendingPeriod); + datanodeManager.blockInvalidateLimit, startupDelayBlockDeletionInMs); // Compute the map capacity by allocating 2% of total memory blocksMap = new BlocksMap( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 3b62ecbf789..95d25f24b03 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6575,6 +6575,11 @@ public long getPendingDeletionBlocks() { return blockManager.getPendingDeletionBlocksCount(); } + @Override + public long getBlockDeletionStartTime() { + return startTime + blockManager.getStartupDelayBlockDeletionInMs(); + } + @Metric public long getExcessBlocks() { return blockManager.getExcessBlocksCount(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java index 587746df154..708591b45ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java @@ -152,6 +152,12 @@ public interface FSNamesystemMBean { */ long getPendingDeletionBlocks(); + /** + * Time when block deletions will begin + * @return time when block deletions will begin + */ + long getBlockDeletionStartTime(); + /** * Number of content stale storages. * @return number of content stale storages diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 1e2813c3f07..87add23e162 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -166,6 +166,7 @@ Decommissioning Nodes{NumDecommissioningDataNodes} Number of Under-Replicated Blocks{UnderReplicatedBlocks} Number of Blocks Pending Deletion{PendingDeletionBlocks} + Block Deletion Start Time{BlockDeletionStartTime|date_tostring} {/fs} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java index fd3b815b492..e10d8243ab9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import java.text.SimpleDateFormat; + import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -88,6 +90,15 @@ public void testPendingDeletion() throws Exception { Thread.sleep(6000); Assert.assertEquals(0, cluster.getNamesystem().getBlocksTotal()); Assert.assertEquals(0, cluster.getNamesystem().getPendingDeletionBlocks()); + String nnStartedStr = cluster.getNamesystem().getNNStarted(); + long nnStarted = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy") + .parse(nnStartedStr).getTime(); + long blockDeletionStartTime = cluster.getNamesystem() + .getBlockDeletionStartTime(); + Assert.assertTrue(String.format( + "Expect blockDeletionStartTime = %d > nnStarted = %d/nnStartedStr = %s.", + blockDeletionStartTime, nnStarted, nnStartedStr), + blockDeletionStartTime > nnStarted); } /**