From c637d6d9d5cb36b8cd5180787c5d25b68f2aa370 Mon Sep 17 00:00:00 2001 From: cnauroth Date: Thu, 30 Oct 2014 22:26:35 -0700 Subject: [PATCH] HDFS-6385. Show when block deletion will start after NameNode startup in WebUI. Contributed by Chris Nauroth. --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/blockmanagement/BlockManager.java | 9 +++++++-- .../hadoop/hdfs/server/namenode/FSNamesystem.java | 5 +++++ .../server/namenode/metrics/FSNamesystemMBean.java | 6 ++++++ .../hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html | 1 + .../blockmanagement/TestPendingInvalidateBlock.java | 11 +++++++++++ 6 files changed, 33 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 438ed6663e4..47eea752df5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -671,6 +671,9 @@ Release 2.6.0 - UNRELEASED HDFS-7230. Add rolling downgrade documentation. (szetszwo via jing9) + HDFS-6385. Show when block deletion will start after NameNode startup in + WebUI. (cnauroth) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index ecc0a4a1e35..94e1c20521f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -121,6 +121,7 @@ public class BlockManager { private volatile long scheduledReplicationBlocksCount = 0L; private final AtomicLong excessBlocksCount = new AtomicLong(0L); private final AtomicLong postponedMisreplicatedBlocksCount = new AtomicLong(0L); + private final long startupDelayBlockDeletionInMs; /** Used by metrics */ public long getPendingReplicationBlocksCount() { @@ -143,6 +144,10 @@ public class BlockManager { return invalidateBlocks.numBlocks(); } /** Used by metrics */ + public long getStartupDelayBlockDeletionInMs() { + return startupDelayBlockDeletionInMs; + } + /** Used by metrics */ public long getExcessBlocksCount() { return excessBlocksCount.get(); } @@ -266,11 +271,11 @@ public class BlockManager { datanodeManager = new DatanodeManager(this, namesystem, conf); heartbeatManager = datanodeManager.getHeartbeatManager(); - final long pendingPeriod = conf.getLong( + startupDelayBlockDeletionInMs = conf.getLong( DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; invalidateBlocks = new InvalidateBlocks( - datanodeManager.blockInvalidateLimit, pendingPeriod); + datanodeManager.blockInvalidateLimit, startupDelayBlockDeletionInMs); // Compute the map capacity by allocating 2% of total memory blocksMap = new BlocksMap( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 8f56a99e66e..93ff63e1b05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6581,6 +6581,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats, return blockManager.getPendingDeletionBlocksCount(); } + @Override + public long getBlockDeletionStartTime() { + return startTime + blockManager.getStartupDelayBlockDeletionInMs(); + } + @Metric public long getExcessBlocks() { return blockManager.getExcessBlocksCount(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java index 587746df154..708591b45ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java @@ -152,6 +152,12 @@ public interface FSNamesystemMBean { */ long getPendingDeletionBlocks(); + /** + * Time when block deletions will begin + * @return time when block deletions will begin + */ + long getBlockDeletionStartTime(); + /** * Number of content stale storages. * @return number of content stale storages diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 7798cba65f3..4971e90e093 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -170,6 +170,7 @@ Decommissioning Nodes{NumDecommissioningDataNodes} Number of Under-Replicated Blocks{UnderReplicatedBlocks} Number of Blocks Pending Deletion{PendingDeletionBlocks} + Block Deletion Start Time{BlockDeletionStartTime|date_tostring} {/fs} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java index fd3b815b492..e10d8243ab9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import java.text.SimpleDateFormat; + import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -88,6 +90,15 @@ public class TestPendingInvalidateBlock { Thread.sleep(6000); Assert.assertEquals(0, cluster.getNamesystem().getBlocksTotal()); Assert.assertEquals(0, cluster.getNamesystem().getPendingDeletionBlocks()); + String nnStartedStr = cluster.getNamesystem().getNNStarted(); + long nnStarted = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy") + .parse(nnStartedStr).getTime(); + long blockDeletionStartTime = cluster.getNamesystem() + .getBlockDeletionStartTime(); + Assert.assertTrue(String.format( + "Expect blockDeletionStartTime = %d > nnStarted = %d/nnStartedStr = %s.", + blockDeletionStartTime, nnStarted, nnStartedStr), + blockDeletionStartTime > nnStarted); } /**