HBASE-18520 Add jmx value to determine true Master Start time

This is to determine how long it took in total for the master to start and finish initializing.

Signed-off-by: tedyu <yuzhihong@gmail.com>

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java

@@ -49,6 +49,7 @@ public interface MetricsMasterSource extends BaseSource {
   // Strings used for exporting to metrics system.
   String MASTER_ACTIVE_TIME_NAME = "masterActiveTime";
   String MASTER_START_TIME_NAME = "masterStartTime";
+  String MASTER_FINISHED_INITIALIZATION_TIME_NAME = "masterFinishedInitializationTime";
   String AVERAGE_LOAD_NAME = "averageLoad";
   String LIVE_REGION_SERVERS_NAME = "liveRegionServers";
   String DEAD_REGION_SERVERS_NAME = "deadRegionServers";
@@ -64,6 +65,7 @@ public interface MetricsMasterSource extends BaseSource {
   String CLUSTER_REQUESTS_NAME = "clusterRequests";
   String MASTER_ACTIVE_TIME_DESC = "Master Active Time";
   String MASTER_START_TIME_DESC = "Master Start Time";
+  String MASTER_FINISHED_INITIALIZATION_TIME_DESC = "Timestamp when Master has finished initializing";
   String AVERAGE_LOAD_DESC = "AverageLoad";
   String LIVE_REGION_SERVERS_DESC = "Names of live RegionServers";
   String NUMBER_OF_REGION_SERVERS_DESC = "Number of RegionServers";

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java

@@ -134,4 +134,9 @@ public interface MetricsMasterWrapper {
    * Gets the space usage and limit for each namespace.
    */
   Map<String,Entry<Long,Long>> getNamespaceSpaceUtilization();
+
+  /**
+   * Get the time in Millis when the master finished initializing/becoming the active master
+   */
+  long getMasterInitializationTime();
 }

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java

@@ -92,6 +92,8 @@ public class MetricsMasterSourceImpl
               MASTER_ACTIVE_TIME_DESC), masterWrapper.getActiveTime())
           .addGauge(Interns.info(MASTER_START_TIME_NAME,
               MASTER_START_TIME_DESC), masterWrapper.getStartTime())
+          .addGauge(Interns.info(MASTER_FINISHED_INITIALIZATION_TIME_NAME, MASTER_FINISHED_INITIALIZATION_TIME_DESC),
+              masterWrapper.getMasterInitializationTime())
           .addGauge(Interns.info(AVERAGE_LOAD_NAME, AVERAGE_LOAD_DESC),
               masterWrapper.getAverageLoad())
           .tag(Interns.info(LIVE_REGION_SERVERS_NAME, LIVE_REGION_SERVERS_DESC),

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -370,6 +370,9 @@ public class HMaster extends HRegionServer implements MasterServices {
   // Time stamps for when a hmaster became active
   private long masterActiveTime;
 
+  // Time stamp for when HMaster finishes becoming Active Master
+  private long masterFinishedInitializationTime;
+
   //should we check the compression codec type at master side, default true, HBASE-6370
   private final boolean masterCheckCompression;
 
@@ -873,6 +876,7 @@ public class HMaster extends HRegionServer implements MasterServices {
     status.markComplete("Initialization successful");
     LOG.info(String.format("Master has completed initialization %.3fsec",
        (System.currentTimeMillis() - masterActiveTime) / 1000.0f));
+    this.masterFinishedInitializationTime = System.currentTimeMillis();
     configurationManager.registerObserver(this.balancer);
     configurationManager.registerObserver(this.hfileCleaner);
 
@@ -2504,6 +2508,13 @@ public class HMaster extends HRegionServer implements MasterServices {
     return masterActiveTime;
   }
 
+  /**
+   * @return timestamp in millis when HMaster finished becoming the active master
+   */
+  public long getMasterFinishedInitializationTime() {
+    return masterFinishedInitializationTime;
+  }
+
   public int getNumWALFiles() {
     return procedureStore != null ? procedureStore.getActiveLogs().size() : 0;
   }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java

@@ -58,6 +58,11 @@ public class MetricsMasterWrapperImpl implements MetricsMasterWrapper {
     return master.getMergePlanCount();
   }
 
+  @Override
+  public long getMasterInitializationTime() {
+    return master.getMasterFinishedInitializationTime();
+  }
+
   @Override
   public String getClusterId() {
     return master.getClusterId();

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

@@ -545,6 +545,7 @@ public class HRegionServer extends HasThread implements
    */
   public HRegionServer(Configuration conf, CoordinatedStateManager csm) throws IOException {
     super("RegionServer");  // thread name
+    this.startcode = System.currentTimeMillis();
     this.fsOk = true;
     this.conf = conf;
     // initialize netty event loop group at the very beginning as we may use it to start rpc server,
@@ -590,7 +591,6 @@ public class HRegionServer extends HasThread implements
     this.stopped = false;
 
     rpcServices = createRpcServices();
-    this.startcode = System.currentTimeMillis();
     if (this instanceof HMaster) {
       useThisHostnameInstead = conf.get(MASTER_HOSTNAME_KEY);
     } else {