HBASE-24586 Add table level locality in table.jsp

Closes #1926

Signed-off-by: Viraj Jasani <vjasani@apache.org>

hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java

@@ -166,4 +166,20 @@ public interface RegionMetrics {
    * @return the data locality for ssd of region in the regionserver
    */
   float getDataLocalityForSsd();
+
+  /**
+   * @return the data at local weight of this region in the regionserver
+   */
+  long getBlocksLocalWeight();
+
+  /**
+   * Different from blocksLocalWeight,this metric's numerator only include the data stored on ssd
+   * @return the data at local with ssd weight of this region in the regionserver
+   */
+  long getBlocksLocalWithSsdWeight();
+
+  /**
+   * @return the block total weight of this region
+   */
+  long getBlocksTotalWeight();
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java

@@ -53,6 +53,11 @@ public final class RegionMetricsBuilder {
         .setDataLocality(regionLoadPB.hasDataLocality() ? regionLoadPB.getDataLocality() : 0.0f)
         .setDataLocalityForSsd(regionLoadPB.hasDataLocalityForSsd() ?
           regionLoadPB.getDataLocalityForSsd() : 0.0f)
+        .setBlocksLocalWeight(regionLoadPB.hasBlocksLocalWeight() ?
+          regionLoadPB.getBlocksLocalWeight() : 0)
+        .setBlocksLocalWithSsdWeight(regionLoadPB.hasBlocksLocalWithSsdWeight() ?
+          regionLoadPB.getBlocksLocalWithSsdWeight() : 0)
+        .setBlocksTotalWeight(regionLoadPB.getBlocksTotalWeight())
         .setFilteredReadRequestCount(regionLoadPB.getFilteredReadRequestsCount())
         .setStoreFileUncompressedDataIndexSize(new Size(regionLoadPB.getTotalStaticIndexSizeKB(),
           Size.Unit.KILOBYTE))
@@ -151,6 +156,9 @@ public final class RegionMetricsBuilder {
   private float dataLocality;
   private long lastMajorCompactionTimestamp;
   private float dataLocalityForSsd;
+  private long blocksLocalWeight;
+  private long blocksLocalWithSsdWeight;
+  private long blocksTotalWeight;
   private RegionMetricsBuilder(byte[] name) {
     this.name = name;
   }
@@ -243,7 +251,18 @@ public final class RegionMetricsBuilder {
     this.dataLocalityForSsd = value;
     return this;
   }
-
+  public RegionMetricsBuilder setBlocksLocalWeight(long value) {
+    this.blocksLocalWeight = value;
+    return this;
+  }
+  public RegionMetricsBuilder setBlocksLocalWithSsdWeight(long value) {
+    this.blocksLocalWithSsdWeight = value;
+    return this;
+  }
+  public RegionMetricsBuilder setBlocksTotalWeight(long value) {
+    this.blocksTotalWeight = value;
+    return this;
+  }
   public RegionMetrics build() {
     return new RegionMetricsImpl(name,
         storeCount,
@@ -267,7 +286,10 @@ public final class RegionMetricsBuilder {
         storeSequenceIds,
         dataLocality,
         lastMajorCompactionTimestamp,
-        dataLocalityForSsd);
+        dataLocalityForSsd,
+        blocksLocalWeight,
+        blocksLocalWithSsdWeight,
+        blocksTotalWeight);
   }
 
   private static class RegionMetricsImpl implements RegionMetrics {
@@ -294,6 +316,9 @@ public final class RegionMetricsBuilder {
     private final float dataLocality;
     private final long lastMajorCompactionTimestamp;
     private final float dataLocalityForSsd;
+    private final long blocksLocalWeight;
+    private final long blocksLocalWithSsdWeight;
+    private final long blocksTotalWeight;
     RegionMetricsImpl(byte[] name,
         int storeCount,
         int storeFileCount,
@@ -316,7 +341,10 @@ public final class RegionMetricsBuilder {
         Map<byte[], Long> storeSequenceIds,
         float dataLocality,
         long lastMajorCompactionTimestamp,
-        float dataLocalityForSsd) {
+        float dataLocalityForSsd,
+        long blocksLocalWeight,
+        long blocksLocalWithSsdWeight,
+        long blocksTotalWeight) {
       this.name = Preconditions.checkNotNull(name);
       this.storeCount = storeCount;
       this.storeFileCount = storeFileCount;
@@ -340,6 +368,9 @@ public final class RegionMetricsBuilder {
       this.dataLocality = dataLocality;
       this.lastMajorCompactionTimestamp = lastMajorCompactionTimestamp;
       this.dataLocalityForSsd = dataLocalityForSsd;
+      this.blocksLocalWeight = blocksLocalWeight;
+      this.blocksLocalWithSsdWeight = blocksLocalWithSsdWeight;
+      this.blocksTotalWeight = blocksTotalWeight;
     }
 
     @Override
@@ -457,6 +488,21 @@ public final class RegionMetricsBuilder {
       return dataLocalityForSsd;
     }
 
+    @Override
+    public long getBlocksLocalWeight() {
+      return blocksLocalWeight;
+    }
+
+    @Override
+    public long getBlocksLocalWithSsdWeight() {
+      return blocksLocalWithSsdWeight;
+    }
+
+    @Override
+    public long getBlocksTotalWeight() {
+      return blocksTotalWeight;
+    }
+
     @Override
     public String toString() {
       StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "storeCount",
@@ -510,6 +556,12 @@ public final class RegionMetricsBuilder {
           this.getDataLocality());
       Strings.appendKeyValue(sb, "dataLocalityForSsd",
           this.getDataLocalityForSsd());
+      Strings.appendKeyValue(sb, "blocksLocalWeight",
+        blocksLocalWeight);
+      Strings.appendKeyValue(sb, "blocksLocalWithSsdWeight",
+        blocksLocalWithSsdWeight);
+      Strings.appendKeyValue(sb, "blocksTotalWeight",
+        blocksTotalWeight);
       return sb.toString();
     }
   }

hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto

@@ -158,6 +158,15 @@ message RegionLoad {
 
   /** The current data locality for ssd for region in the regionserver */
   optional float data_locality_for_ssd = 23;
+
+  /** The current blocks local weight for region in the regionserver */
+  optional uint64 blocks_local_weight = 24;
+
+  /** The current blocks local weight with ssd for region in the regionserver */
+  optional uint64 blocks_local_with_ssd_weight = 25;
+
+  /** The current blocks total weight for region in the regionserver */
+  optional uint64 blocks_total_weight = 26;
 }
 
 message UserLoad {

hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java

@@ -51,7 +51,7 @@ public class HDFSBlocksDistribution {
    */
   public static class HostAndWeight {
 
-    private String host;
+    private final String host;
     private long weight;
     private long weightForSsd;
 
@@ -228,7 +228,7 @@ public class HDFSBlocksDistribution {
    * Implementations 'visit' hostAndWeight.
    */
   public interface Visitor {
-    float visit(final HostAndWeight hostAndWeight);
+    long visit(final HostAndWeight hostAndWeight);
   }
 
   /**
@@ -236,8 +236,12 @@ public class HDFSBlocksDistribution {
    * @return the locality index of the given host
    */
   public float getBlockLocalityIndex(String host) {
-    return getBlockLocalityIndexInternal(host,
-      e -> (float) e.weight / (float) uniqueBlocksTotalWeight);
+    if (uniqueBlocksTotalWeight == 0) {
+      return 0.0f;
+    } else {
+      return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight)
+        / (float) uniqueBlocksTotalWeight;
+    }
   }
 
   /**
@@ -245,16 +249,36 @@ public class HDFSBlocksDistribution {
    * @return the locality index with ssd of the given host
    */
   public float getBlockLocalityIndexForSsd(String host) {
-    return getBlockLocalityIndexInternal(host,
-      e -> (float) e.weightForSsd / (float) uniqueBlocksTotalWeight);
+    if (uniqueBlocksTotalWeight == 0) {
+      return 0.0f;
+    } else {
+      return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd)
+        / (float) uniqueBlocksTotalWeight;
+    }
+  }
+
+  /**
+   * @param host the host name
+   * @return the blocks local weight of the given host
+   */
+  public long getBlocksLocalWeight(String host) {
+    return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight);
+  }
+
+  /**
+   * @param host the host name
+   * @return the blocks local with ssd weight of the given host
+   */
+  public long getBlocksLocalWithSsdWeight(String host) {
+    return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd);
   }
 
   /**
    * @param host the host name
    * @return the locality index of the given host
    */
-  private float getBlockLocalityIndexInternal(String host, Visitor visitor) {
-    float localityIndex = 0;
+  private long getBlocksLocalityWeightInternal(String host, Visitor visitor) {
+    long localityIndex = 0;
     HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
     if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) {
       localityIndex = visitor.visit(hostAndWeight);

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

@@ -1696,6 +1696,9 @@ public class HRegionServer extends Thread implements
     HDFSBlocksDistribution hdfsBd = r.getHDFSBlocksDistribution();
     float dataLocality = hdfsBd.getBlockLocalityIndex(serverName.getHostname());
     float dataLocalityForSsd = hdfsBd.getBlockLocalityIndexForSsd(serverName.getHostname());
+    long blocksTotalWeight = hdfsBd.getUniqueBlocksTotalWeight();
+    long blocksLocalWeight = hdfsBd.getBlocksLocalWeight(serverName.getHostname());
+    long blocksLocalWithSsdWeight = hdfsBd.getBlocksLocalWithSsdWeight(serverName.getHostname());
     if (regionLoadBldr == null) {
       regionLoadBldr = RegionLoad.newBuilder();
     }
@@ -1724,6 +1727,9 @@ public class HRegionServer extends Thread implements
       .setCurrentCompactedKVs(currentCompactedKVs)
       .setDataLocality(dataLocality)
       .setDataLocalityForSsd(dataLocalityForSsd)
+      .setBlocksLocalWeight(blocksLocalWeight)
+      .setBlocksLocalWithSsdWeight(blocksLocalWithSsdWeight)
+      .setBlocksTotalWeight(blocksTotalWeight)
       .setLastMajorCompactionTs(r.getOldestHfileTs(true));
     r.setCompleteSequenceId(regionLoadBldr);
 

hbase-server/src/main/resources/hbase-webapps/master/table.jsp

@@ -808,9 +808,14 @@
   long totalMemSize = 0;
   long totalCompactingCells = 0;
   long totalCompactedCells = 0;
+  long totalBlocksTotalWeight = 0;
+  long totalBlocksLocalWeight = 0;
+  long totalBlocksLocalWithSsdWeight = 0;
   String totalCompactionProgress = "";
   String totalMemSizeStr = ZEROMB;
   String totalSizeStr = ZEROMB;
+  String totalLocality = "";
+  String totalLocalityForSsd = "";
   Map<ServerName, Integer> regDistribution = new TreeMap<>();
   Map<ServerName, Integer> primaryRegDistribution = new TreeMap<>();
   List<HRegionLocation> regions = r.getAllRegionLocations();
@@ -835,6 +840,9 @@
           totalStoreFileSizeMB += regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE);
           totalCompactingCells += regionMetrics.getCompactingCellCount();
           totalCompactedCells += regionMetrics.getCompactedCellCount();
+          totalBlocksTotalWeight += regionMetrics.getBlocksTotalWeight();
+          totalBlocksLocalWeight += regionMetrics.getBlocksLocalWeight();
+          totalBlocksLocalWithSsdWeight += regionMetrics.getBlocksLocalWithSsdWeight();
         } else {
           RegionMetrics load0 = getEmptyRegionMetrics(regionInfo);
           regionsToLoad.put(regionInfo, load0);
@@ -858,7 +866,12 @@
     totalCompactionProgress = String.format("%.2f", 100 *
       ((float) totalCompactedCells / totalCompactingCells)) + "%";
   }
-
+  if (totalBlocksTotalWeight > 0) {
+    totalLocality = String.format("%.1f",
+      ((float) totalBlocksLocalWeight / totalBlocksTotalWeight));
+    totalLocalityForSsd = String.format("%.1f",
+      ((float) totalBlocksLocalWithSsdWeight / totalBlocksTotalWeight));
+  }
   if(regions != null && regions.size() > 0) { %>
 <h2>Table Regions</h2>
 <div class="tabbable">
@@ -963,8 +976,8 @@
           <tr>
             <th>Name(<%= String.format("%,1d", regions.size())%>)</th>
             <th>Region Server</th>
-            <th>Locality</th>
-            <th>LocalityForSsd</th>
+            <th>Locality<br>(<%= totalLocality %>)</th>
+            <th>LocalityForSsd<br>(<%= totalLocalityForSsd %>)</th>
           </tr>
         </thead>
         <tbody>

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java

@@ -497,6 +497,21 @@ public class TestRegionsRecoveryChore {
       public float getDataLocalityForSsd() {
         return 0;
       }
+
+      @Override
+      public long getBlocksLocalWeight() {
+        return 0;
+      }
+
+      @Override
+      public long getBlocksLocalWithSsdWeight() {
+        return 0;
+      }
+
+      @Override
+      public long getBlocksTotalWeight() {
+        return 0;
+      }
     };
     return regionMetrics;
   }