HDFS-9882. Add heartbeatsTotal in Datanode metrics. (Contributed by Hua Liu)

hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md

@@ -290,6 +290,8 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
 | `ReplaceBlockOpAvgTime` | Average time of block replace operations in milliseconds |
 | `HeartbeatsNumOps` | Total number of heartbeats |
 | `HeartbeatsAvgTime` | Average heartbeat time in milliseconds |
+| `HeartbeatsTotalNumOps` | Total number of heartbeats which is a duplicate of HeartbeatsNumOps |
+| `HeartbeatsTotalAvgTime` | Average total heartbeat time in milliseconds |
 | `LifelinesNumOps` | Total number of lifeline messages |
 | `LifelinesAvgTime` | Average lifeline message processing time in milliseconds |
 | `BlockReportsNumOps` | Total number of block report operations |

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java

@@ -607,6 +607,11 @@ class BPServiceActor implements Runnable {
           processCommand(new DatanodeCommand[]{ cmd });
         }
 
+        if (sendHeartbeat) {
+          dn.getMetrics().addHeartbeatTotal(
+              scheduler.monotonicNow() - startTime);
+        }
+
         // There is no work to do;  sleep until hearbeat timer elapses, 
         // or work arrives, and then iterate again.
         ibrManager.waitTillNextIBR(scheduler.getHeartbeatWaitTime());

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java

@@ -107,6 +107,7 @@ public class DataNodeMetrics {
   @Metric MutableRate copyBlockOp;
   @Metric MutableRate replaceBlockOp;
   @Metric MutableRate heartbeats;
+  @Metric MutableRate heartbeatsTotal;
   @Metric MutableRate lifelines;
   @Metric MutableRate blockReports;
   @Metric MutableRate incrementalBlockReports;
@@ -200,6 +201,10 @@ public class DataNodeMetrics {
     heartbeats.add(latency);
   }
 
+  public void addHeartbeatTotal(long latency) {
+    heartbeatsTotal.add(latency);
+  }
+
   public void addLifeline(long latency) {
     lifelines.add(latency);
   }