HBASE-25636 Expose HBCK report as metrics (#3031)

Signed-off-by: zhangduo <zhangduo@apache.org>

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java

@@ -61,6 +61,27 @@ public interface MetricsAssignmentManagerSource extends BaseSource {
   String RIT_DURATION_DESC =
       "Total durations in milliseconds for all Regions in Transition (Histogram).";
 
+  // HBCK report metrics
+  String ORPHAN_REGIONS_ON_RS = "orphanRegionsOnRS";
+  String ORPHAN_REGIONS_ON_FS = "orphanRegionsOnFS";
+  String INCONSISTENT_REGIONS = "inconsistentRegions";
+
+  String ORPHAN_REGIONS_ON_RS_DESC = "Current number of Orphan Regions on RS (Gauge).";
+  String ORPHAN_REGIONS_ON_FS_DESC = "Current number of Orphan Regions on FS (Gauge).";
+  String INCONSISTENT_REGIONS_DESC = "Current number of Inconsistent Regions (Gauge).";
+
+  // CatalogJanitor Consistency report metrics
+  String HOLES = "holes";
+  String OVERLAPS = "overlaps";
+  String UNKNOWN_SERVER_REGIONS = "unknownServerRegions";
+  String EMPTY_REGION_INFO_REGIONS = "emptyRegionInfoRegions";
+
+  String HOLES_DESC = "Current number of Holes (Gauge).";
+  String OVERLAPS_DESC = "Current number of Overlaps (Gauge).";
+  String UNKNOWN_SERVER_REGIONS_DESC = "Current number of Unknown Server Regions (Gauge).";
+  String EMPTY_REGION_INFO_REGIONS_DESC =
+      "Current number of Regions with Empty Region Info (Gauge).";
+
   String ASSIGN_METRIC_PREFIX = "assign";
   String UNASSIGN_METRIC_PREFIX = "unassign";
   String MOVE_METRIC_PREFIX = "move";
@@ -99,6 +120,55 @@ public interface MetricsAssignmentManagerSource extends BaseSource {
 
   void updateUnknownServerOpenRegions(int unknownRegions);
 
+  /**
+   * Set the number of orphan regions on RS.
+   *
+   * @param orphanRegionsOnRs count of the orphan regions on RS in HBCK chore report.
+   */
+  void setOrphanRegionsOnRs(int orphanRegionsOnRs);
+
+  /**
+   * Set the number of orphan regions on FS.
+   *
+   * @param orphanRegionsOnFs count of the orphan regions on FS in HBCK chore report.
+   */
+  void setOrphanRegionsOnFs(int orphanRegionsOnFs);
+
+  /**
+   * Set the number of inconsistent regions.
+   *
+   * @param inconsistentRegions count of the inconsistent regions in HBCK chore report.
+   */
+  void setInconsistentRegions(int inconsistentRegions);
+
+  /**
+   * Set the number of holes.
+   *
+   * @param holes count of the holes in CatalogJanitor Consistency report.
+   */
+  void setHoles(int holes);
+
+  /**
+   * Set the number of overlaps.
+   *
+   * @param overlaps count of the overlaps in CatalogJanitor Consistency report.
+   */
+  void setOverlaps(int overlaps);
+
+  /**
+   * Set the number of unknown server regions.
+   * @param unknownServerRegions count of the unknown server regions in CatalogJanitor Consistency
+   *          report.
+   */
+  void setUnknownServerRegions(int unknownServerRegions);
+
+  /**
+   * Set the number of regions with empty region info.
+   * @param emptyRegionInfoRegions count of the regions with empty region info in CatalogJanitor
+   *          Consistency report.
+   */
+  void setEmptyRegionInfoRegions(int emptyRegionInfoRegions);
+
   /**
    * TODO: Remove. This may not be needed now as assign and unassign counts are tracked separately
    * Increment the count of operations (assign/unassign).

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java

@@ -39,6 +39,15 @@ public class MetricsAssignmentManagerSourceImpl
   private MutableGaugeLong deadServerOpenRegions;
   private MutableGaugeLong unknownServerOpenRegions;
 
+  private MutableGaugeLong orphanRegionsOnRsGauge;
+  private MutableGaugeLong orphanRegionsOnFsGauge;
+  private MutableGaugeLong inconsistentRegionsGauge;
+
+  private MutableGaugeLong holesGauge;
+  private MutableGaugeLong overlapsGauge;
+  private MutableGaugeLong unknownServerRegionsGauge;
+  private MutableGaugeLong emptyRegionInfoRegionsGauge;
+
   private MutableFastCounter operationCounter;
 
   private OperationMetrics assignMetrics;
@@ -70,6 +79,20 @@ public class MetricsAssignmentManagerSourceImpl
     deadServerOpenRegions = metricsRegistry.newGauge(DEAD_SERVER_OPEN_REGIONS, "", 0);
     unknownServerOpenRegions = metricsRegistry.newGauge(UNKNOWN_SERVER_OPEN_REGIONS, "", 0);
 
+    orphanRegionsOnRsGauge =
+        metricsRegistry.newGauge(ORPHAN_REGIONS_ON_RS, ORPHAN_REGIONS_ON_RS_DESC, 0L);
+    orphanRegionsOnFsGauge =
+        metricsRegistry.newGauge(ORPHAN_REGIONS_ON_FS, ORPHAN_REGIONS_ON_FS_DESC, 0L);
+    inconsistentRegionsGauge =
+        metricsRegistry.newGauge(INCONSISTENT_REGIONS, INCONSISTENT_REGIONS_DESC, 0L);
+
+    holesGauge = metricsRegistry.newGauge(HOLES, HOLES_DESC, 0L);
+    overlapsGauge = metricsRegistry.newGauge(OVERLAPS, OVERLAPS_DESC, 0L);
+    unknownServerRegionsGauge =
+        metricsRegistry.newGauge(UNKNOWN_SERVER_REGIONS, UNKNOWN_SERVER_REGIONS_DESC, 0L);
+    emptyRegionInfoRegionsGauge =
+        metricsRegistry.newGauge(EMPTY_REGION_INFO_REGIONS, EMPTY_REGION_INFO_REGIONS_DESC, 0L);
+
     /**
      * NOTE: Please refer to HBASE-9774 and HBASE-14282. Based on these two issues, HBase is
      * moving away from using Hadoop's metric2 to having independent HBase specific Metrics. Use
@@ -120,6 +143,41 @@ public class MetricsAssignmentManagerSourceImpl
     unknownServerOpenRegions.set(unknownRegions);
   }
 
+  @Override
+  public void setOrphanRegionsOnRs(int orphanRegionsOnRs) {
+    orphanRegionsOnRsGauge.set(orphanRegionsOnRs);
+  }
+
+  @Override
+  public void setOrphanRegionsOnFs(int orphanRegionsOnFs) {
+    orphanRegionsOnFsGauge.set(orphanRegionsOnFs);
+  }
+
+  @Override
+  public void setInconsistentRegions(int inconsistentRegions) {
+    inconsistentRegionsGauge.set(inconsistentRegions);
+  }
+
+  @Override
+  public void setHoles(int holes) {
+    holesGauge.set(holes);
+  }
+
+  @Override
+  public void setOverlaps(int overlaps) {
+    overlapsGauge.set(overlaps);
+  }
+
+  @Override
+  public void setUnknownServerRegions(int unknownServerRegions) {
+    unknownServerRegionsGauge.set(unknownServerRegions);
+  }
+
+  @Override
+  public void setEmptyRegionInfoRegions(int emptyRegionInfoRegions) {
+    emptyRegionInfoRegionsGauge.set(emptyRegionInfoRegions);
+  }
+
   @Override
   public OperationMetrics getAssignMetrics() {
     return assignMetrics;

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java

@@ -144,6 +144,7 @@ public class HbckChore extends ScheduledChore {
       LOG.warn("Unexpected", t);
     }
     running = false;
+    updateAssignmentManagerMetrics();
   }
 
   // This function does the sanity checks of making sure the chore is not run when it is
@@ -310,6 +311,15 @@ public class HbckChore extends ScheduledChore {
         tableDirs.size(), numRegions, orphanRegionsOnFS.size());
   }
 
+  private void updateAssignmentManagerMetrics() {
+    master.getAssignmentManager().getAssignmentManagerMetrics()
+        .updateOrphanRegionsOnRs(getOrphanRegionsOnRS().size());
+    master.getAssignmentManager().getAssignmentManagerMetrics()
+        .updateOrphanRegionsOnFs(getOrphanRegionsOnFS().size());
+    master.getAssignmentManager().getAssignmentManagerMetrics()
+        .updateInconsistentRegions(getInconsistentRegions().size());
+  }
+
   /**
    * When running, the HBCK report may be changed later.
    */

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java

@@ -104,6 +104,34 @@ public class MetricsAssignmentManager {
     assignmentManagerSource.updateUnknownServerOpenRegions(unknownRegions);
   }
 
+  public void updateOrphanRegionsOnRs(int orphanRegionsOnRs) {
+    assignmentManagerSource.setOrphanRegionsOnRs(orphanRegionsOnRs);
+  }
+
+  public void updateOrphanRegionsOnFs(int orphanRegionsOnFs) {
+    assignmentManagerSource.setOrphanRegionsOnFs(orphanRegionsOnFs);
+  }
+
+  public void updateInconsistentRegions(int inconsistentRegions) {
+    assignmentManagerSource.setInconsistentRegions(inconsistentRegions);
+  }
+
+  public void updateHoles(int holes) {
+    assignmentManagerSource.setHoles(holes);
+  }
+
+  public void updateOverlaps(int overlaps) {
+    assignmentManagerSource.setOverlaps(overlaps);
+  }
+
+  public void updateUnknownServerRegions(int unknownServerRegions) {
+    assignmentManagerSource.setUnknownServerRegions(unknownServerRegions);
+  }
+
+  public void updateEmptyRegionInfoRegions(int emptyRegionInfoRegions) {
+    assignmentManagerSource.setEmptyRegionInfoRegions(emptyRegionInfoRegions);
+  }
+
   /**
    * @return Set of common metrics for assign procedure
    */

hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java

@@ -166,6 +166,7 @@ public class CatalogJanitor extends ScheduledChore {
       if (!this.lastReport.isEmpty()) {
         LOG.warn(this.lastReport.toString());
       }
+      updateAssignmentManagerMetrics();
 
       if (isRIT(this.services.getAssignmentManager())) {
         LOG.warn("Playing-it-safe skipping merge/split gc'ing of regions from hbase:meta while " +
@@ -405,6 +406,17 @@ public class CatalogJanitor extends ScheduledChore {
     return this.services.getTableDescriptors().get(tableName);
   }
 
+  private void updateAssignmentManagerMetrics() {
+    services.getAssignmentManager().getAssignmentManagerMetrics()
+        .updateHoles(lastReport.getHoles().size());
+    services.getAssignmentManager().getAssignmentManagerMetrics()
+        .updateOverlaps(lastReport.getOverlaps().size());
+    services.getAssignmentManager().getAssignmentManagerMetrics()
+        .updateUnknownServerRegions(lastReport.getUnknownServers().size());
+    services.getAssignmentManager().getAssignmentManagerMetrics()
+        .updateEmptyRegionInfoRegions(lastReport.getEmptyRegionInfo().size());
+  }
+
   private static void checkLog4jProperties() {
     String filename = "log4j.properties";
     try (final InputStream inStream =