From c22cf004425daa9c350df5e365b0db85b1628b40 Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Wed, 21 Jun 2017 16:34:19 -0700 Subject: [PATCH] Confusion/name conflict between NameNodeActivity#BlockReportNumOps and RpcDetailedActivity#BlockReportNumOps. Contributed by Erik Krogen. --- .../server/blockmanagement/BlockManager.java | 2 +- .../server/namenode/NameNodeRpcServer.java | 1 - .../namenode/metrics/NameNodeMetrics.java | 25 ++++++++----------- .../org/apache/hadoop/hdfs/TestSafeMode.java | 4 +-- .../namenode/metrics/TestNameNodeMetrics.java | 4 +-- 5 files changed, 15 insertions(+), 21 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index acd4c474256..a0c4698421d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2436,7 +2436,7 @@ public class BlockManager implements BlockStatsMXBean { // Log the block report processing stats from Namenode perspective final NameNodeMetrics metrics = NameNode.getNameNodeMetrics(); if (metrics != null) { - metrics.addBlockReport((int) (endTime - startTime)); + metrics.addStorageBlockReport((int) (endTime - startTime)); } blockLog.info("BLOCK* processReport 0x{}: from storage {} node {}, " + "blocks: {}, hasStaleStorage: {}, processing time: {} msecs, " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 599178a893d..fd5a05be46b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -1486,7 +1486,6 @@ public class NameNodeRpcServer implements NamenodeProtocols { blocks, context); } }); - metrics.incrStorageBlockReportOps(); } bm.removeBRLeaseIfNeeded(nodeReg, context); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java index c4cfa6adbad..cb81f5a3766 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java @@ -74,8 +74,6 @@ public class NameNodeMetrics { MutableCounterLong snapshotDiffReportOps; @Metric("Number of blockReceivedAndDeleted calls") MutableCounterLong blockReceivedAndDeletedOps; - @Metric("Number of blockReports from individual storages") - MutableCounterLong storageBlockReportOps; @Metric("Number of blockReports and blockReceivedAndDeleted queued") MutableGaugeInt blockOpsQueued; @Metric("Number of blockReports and blockReceivedAndDeleted batch processed") @@ -111,8 +109,9 @@ public class NameNodeMetrics { final MutableQuantiles[] syncsQuantiles; @Metric("Journal transactions batched in sync") MutableCounterLong transactionsBatchedInSync; - @Metric("Block report") MutableRate blockReport; - final MutableQuantiles[] blockReportQuantiles; + @Metric("Number of blockReports from individual storages") + MutableRate storageBlockReport; + final MutableQuantiles[] storageBlockReportQuantiles; @Metric("Cache report") MutableRate cacheReport; final MutableQuantiles[] cacheReportQuantiles; @Metric("Generate EDEK time") private MutableRate generateEDEKTime; @@ -143,7 +142,7 @@ public class NameNodeMetrics { final int len = intervals.length; syncsQuantiles = new MutableQuantiles[len]; - blockReportQuantiles = new MutableQuantiles[len]; + storageBlockReportQuantiles = new MutableQuantiles[len]; cacheReportQuantiles = new MutableQuantiles[len]; generateEDEKTimeQuantiles = new MutableQuantiles[len]; warmUpEDEKTimeQuantiles = new MutableQuantiles[len]; @@ -154,9 +153,9 @@ public class NameNodeMetrics { syncsQuantiles[i] = registry.newQuantiles( "syncs" + interval + "s", "Journal syncs", "ops", "latency", interval); - blockReportQuantiles[i] = registry.newQuantiles( - "blockReport" + interval + "s", - "Block report", "ops", "latency", interval); + storageBlockReportQuantiles[i] = registry.newQuantiles( + "storageBlockReport" + interval + "s", + "Storage block report", "ops", "latency", interval); cacheReportQuantiles[i] = registry.newQuantiles( "cacheReport" + interval + "s", "Cache report", "ops", "latency", interval); @@ -284,10 +283,6 @@ public class NameNodeMetrics { public void incrBlockReceivedAndDeletedOps() { blockReceivedAndDeletedOps.incr(); } - - public void incrStorageBlockReportOps() { - storageBlockReportOps.incr(); - } public void setBlockOpsQueued(int size) { blockOpsQueued.set(size); @@ -316,9 +311,9 @@ public class NameNodeMetrics { fsImageLoadTime.set((int) elapsed); } - public void addBlockReport(long latency) { - blockReport.add(latency); - for (MutableQuantiles q : blockReportQuantiles) { + public void addStorageBlockReport(long latency) { + storageBlockReport.add(latency); + for (MutableQuantiles q : storageBlockReportQuantiles) { q.add(latency); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java index 62dac74aea7..f03b440b456 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java @@ -216,8 +216,8 @@ public class TestSafeMode { GenericTestUtils.waitFor(new Supplier() { @Override public Boolean get() { - return getLongCounter("StorageBlockReportOps", getMetrics(NN_METRICS)) == - cluster.getStoragesPerDatanode(); + return getLongCounter("StorageBlockReportNumOps", + getMetrics(NN_METRICS)) == cluster.getStoragesPerDatanode(); } }, 10, 10000); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index c84f8e81aa4..f598d8c6adf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -781,7 +781,7 @@ public class TestNameNodeMetrics { // We have one sync when the cluster starts up, just opening the journal assertCounter("SyncsNumOps", 3L, rb); // Each datanode reports in when the cluster comes up - assertCounter("BlockReportNumOps", + assertCounter("StorageBlockReportNumOps", (long) DATANODE_COUNT * cluster.getStoragesPerDatanode(), rb); // Sleep for an interval+slop to let the percentiles rollover @@ -789,7 +789,7 @@ public class TestNameNodeMetrics { // Check that the percentiles were updated assertQuantileGauges("Syncs1s", rb); - assertQuantileGauges("BlockReport1s", rb); + assertQuantileGauges("StorageBlockReport1s", rb); } /**