Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Confusion/name conflict between NameNodeActivity#BlockReportNumOps and RpcDetailedActivity#BlockReportNumOps. Contributed by Erik Krogen.

This commit is contained in:
Zhe Zhang 2017-06-21 16:34:19 -07:00
parent e806c6e0ce
commit c22cf00442
5 changed files with 15 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -2436,7 +2436,7 @@ public class BlockManager implements BlockStatsMXBean {
// Log the block report processing stats from Namenode perspective // Log the block report processing stats from Namenode perspective
final NameNodeMetrics metrics = NameNode.getNameNodeMetrics(); final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
if (metrics != null) { if (metrics != null) {
metrics.addBlockReport((int) (endTime - startTime)); metrics.addStorageBlockReport((int) (endTime - startTime));
} }
blockLog.info("BLOCK* processReport 0x{}: from storage {} node {}, " + blockLog.info("BLOCK* processReport 0x{}: from storage {} node {}, " +
"blocks: {}, hasStaleStorage: {}, processing time: {} msecs, " + "blocks: {}, hasStaleStorage: {}, processing time: {} msecs, " +

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
View File

@ -1486,7 +1486,6 @@ public class NameNodeRpcServer implements NamenodeProtocols {
blocks, context); blocks, context);
} }
}); });
metrics.incrStorageBlockReportOps();
} }
bm.removeBRLeaseIfNeeded(nodeReg, context); bm.removeBRLeaseIfNeeded(nodeReg, context);

25
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
View File

@ -74,8 +74,6 @@ public class NameNodeMetrics {
MutableCounterLong snapshotDiffReportOps; MutableCounterLong snapshotDiffReportOps;
@Metric("Number of blockReceivedAndDeleted calls") @Metric("Number of blockReceivedAndDeleted calls")
MutableCounterLong blockReceivedAndDeletedOps; MutableCounterLong blockReceivedAndDeletedOps;
@Metric("Number of blockReports from individual storages")
MutableCounterLong storageBlockReportOps;
@Metric("Number of blockReports and blockReceivedAndDeleted queued") @Metric("Number of blockReports and blockReceivedAndDeleted queued")
MutableGaugeInt blockOpsQueued; MutableGaugeInt blockOpsQueued;
@Metric("Number of blockReports and blockReceivedAndDeleted batch processed") @Metric("Number of blockReports and blockReceivedAndDeleted batch processed")
@ -111,8 +109,9 @@ public class NameNodeMetrics {
final MutableQuantiles[] syncsQuantiles; final MutableQuantiles[] syncsQuantiles;
@Metric("Journal transactions batched in sync") @Metric("Journal transactions batched in sync")
MutableCounterLong transactionsBatchedInSync; MutableCounterLong transactionsBatchedInSync;
@Metric("Block report") MutableRate blockReport; @Metric("Number of blockReports from individual storages")
final MutableQuantiles[] blockReportQuantiles; MutableRate storageBlockReport;
final MutableQuantiles[] storageBlockReportQuantiles;
@Metric("Cache report") MutableRate cacheReport; @Metric("Cache report") MutableRate cacheReport;
final MutableQuantiles[] cacheReportQuantiles; final MutableQuantiles[] cacheReportQuantiles;
@Metric("Generate EDEK time") private MutableRate generateEDEKTime; @Metric("Generate EDEK time") private MutableRate generateEDEKTime;
@ -143,7 +142,7 @@ public class NameNodeMetrics {
final int len = intervals.length; final int len = intervals.length;
syncsQuantiles = new MutableQuantiles[len]; syncsQuantiles = new MutableQuantiles[len];
blockReportQuantiles = new MutableQuantiles[len]; storageBlockReportQuantiles = new MutableQuantiles[len];
cacheReportQuantiles = new MutableQuantiles[len]; cacheReportQuantiles = new MutableQuantiles[len];
generateEDEKTimeQuantiles = new MutableQuantiles[len]; generateEDEKTimeQuantiles = new MutableQuantiles[len];
warmUpEDEKTimeQuantiles = new MutableQuantiles[len]; warmUpEDEKTimeQuantiles = new MutableQuantiles[len];
@ -154,9 +153,9 @@ public class NameNodeMetrics {
syncsQuantiles[i] = registry.newQuantiles( syncsQuantiles[i] = registry.newQuantiles(
"syncs" + interval + "s", "syncs" + interval + "s",
"Journal syncs", "ops", "latency", interval); "Journal syncs", "ops", "latency", interval);
blockReportQuantiles[i] = registry.newQuantiles( storageBlockReportQuantiles[i] = registry.newQuantiles(
"blockReport" + interval + "s", "storageBlockReport" + interval + "s",
"Block report", "ops", "latency", interval); "Storage block report", "ops", "latency", interval);
cacheReportQuantiles[i] = registry.newQuantiles( cacheReportQuantiles[i] = registry.newQuantiles(
"cacheReport" + interval + "s", "cacheReport" + interval + "s",
"Cache report", "ops", "latency", interval); "Cache report", "ops", "latency", interval);
@ -285,10 +284,6 @@ public class NameNodeMetrics {
blockReceivedAndDeletedOps.incr(); blockReceivedAndDeletedOps.incr();
} }
public void incrStorageBlockReportOps() {
storageBlockReportOps.incr();
}
public void setBlockOpsQueued(int size) { public void setBlockOpsQueued(int size) {
blockOpsQueued.set(size); blockOpsQueued.set(size);
} }
@ -316,9 +311,9 @@ public class NameNodeMetrics {
fsImageLoadTime.set((int) elapsed); fsImageLoadTime.set((int) elapsed);
} }
public void addBlockReport(long latency) { public void addStorageBlockReport(long latency) {
blockReport.add(latency); storageBlockReport.add(latency);
for (MutableQuantiles q : blockReportQuantiles) { for (MutableQuantiles q : storageBlockReportQuantiles) {
q.add(latency); q.add(latency);
} }
} }

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
View File

@ -216,8 +216,8 @@ public class TestSafeMode {
GenericTestUtils.waitFor(new Supplier<Boolean>() { GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override @Override
public Boolean get() { public Boolean get() {
return getLongCounter("StorageBlockReportOps", getMetrics(NN_METRICS)) == return getLongCounter("StorageBlockReportNumOps",
cluster.getStoragesPerDatanode(); getMetrics(NN_METRICS)) == cluster.getStoragesPerDatanode();
} }
}, 10, 10000); }, 10, 10000);

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
View File

@ -781,7 +781,7 @@ public class TestNameNodeMetrics {
// We have one sync when the cluster starts up, just opening the journal // We have one sync when the cluster starts up, just opening the journal
assertCounter("SyncsNumOps", 3L, rb); assertCounter("SyncsNumOps", 3L, rb);
// Each datanode reports in when the cluster comes up // Each datanode reports in when the cluster comes up
assertCounter("BlockReportNumOps", assertCounter("StorageBlockReportNumOps",
(long) DATANODE_COUNT * cluster.getStoragesPerDatanode(), rb); (long) DATANODE_COUNT * cluster.getStoragesPerDatanode(), rb);
// Sleep for an interval+slop to let the percentiles rollover // Sleep for an interval+slop to let the percentiles rollover
@ -789,7 +789,7 @@ public class TestNameNodeMetrics {
// Check that the percentiles were updated // Check that the percentiles were updated
assertQuantileGauges("Syncs1s", rb); assertQuantileGauges("Syncs1s", rb);
assertQuantileGauges("BlockReport1s", rb); assertQuantileGauges("StorageBlockReport1s", rb);
} }
/** /**