diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java index 27ae3e82b47..7d732607ae3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java @@ -166,4 +166,20 @@ public interface RegionMetrics { * @return the data locality for ssd of region in the regionserver */ float getDataLocalityForSsd(); + + /** + * @return the data at local weight of this region in the regionserver + */ + long getBlocksLocalWeight(); + + /** + * Different from blocksLocalWeight,this metric's numerator only include the data stored on ssd + * @return the data at local with ssd weight of this region in the regionserver + */ + long getBlocksLocalWithSsdWeight(); + + /** + * @return the block total weight of this region + */ + long getBlocksTotalWeight(); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java index 89801b39fc7..c3839662ac2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java @@ -53,6 +53,11 @@ public final class RegionMetricsBuilder { .setDataLocality(regionLoadPB.hasDataLocality() ? regionLoadPB.getDataLocality() : 0.0f) .setDataLocalityForSsd(regionLoadPB.hasDataLocalityForSsd() ? regionLoadPB.getDataLocalityForSsd() : 0.0f) + .setBlocksLocalWeight(regionLoadPB.hasBlocksLocalWeight() ? + regionLoadPB.getBlocksLocalWeight() : 0) + .setBlocksLocalWithSsdWeight(regionLoadPB.hasBlocksLocalWithSsdWeight() ? + regionLoadPB.getBlocksLocalWithSsdWeight() : 0) + .setBlocksTotalWeight(regionLoadPB.getBlocksTotalWeight()) .setFilteredReadRequestCount(regionLoadPB.getFilteredReadRequestsCount()) .setStoreFileUncompressedDataIndexSize(new Size(regionLoadPB.getTotalStaticIndexSizeKB(), Size.Unit.KILOBYTE)) @@ -151,6 +156,9 @@ public final class RegionMetricsBuilder { private float dataLocality; private long lastMajorCompactionTimestamp; private float dataLocalityForSsd; + private long blocksLocalWeight; + private long blocksLocalWithSsdWeight; + private long blocksTotalWeight; private RegionMetricsBuilder(byte[] name) { this.name = name; } @@ -243,7 +251,18 @@ public final class RegionMetricsBuilder { this.dataLocalityForSsd = value; return this; } - + public RegionMetricsBuilder setBlocksLocalWeight(long value) { + this.blocksLocalWeight = value; + return this; + } + public RegionMetricsBuilder setBlocksLocalWithSsdWeight(long value) { + this.blocksLocalWithSsdWeight = value; + return this; + } + public RegionMetricsBuilder setBlocksTotalWeight(long value) { + this.blocksTotalWeight = value; + return this; + } public RegionMetrics build() { return new RegionMetricsImpl(name, storeCount, @@ -267,7 +286,10 @@ public final class RegionMetricsBuilder { storeSequenceIds, dataLocality, lastMajorCompactionTimestamp, - dataLocalityForSsd); + dataLocalityForSsd, + blocksLocalWeight, + blocksLocalWithSsdWeight, + blocksTotalWeight); } private static class RegionMetricsImpl implements RegionMetrics { @@ -294,6 +316,9 @@ public final class RegionMetricsBuilder { private final float dataLocality; private final long lastMajorCompactionTimestamp; private final float dataLocalityForSsd; + private final long blocksLocalWeight; + private final long blocksLocalWithSsdWeight; + private final long blocksTotalWeight; RegionMetricsImpl(byte[] name, int storeCount, int storeFileCount, @@ -316,7 +341,10 @@ public final class RegionMetricsBuilder { Map storeSequenceIds, float dataLocality, long lastMajorCompactionTimestamp, - float dataLocalityForSsd) { + float dataLocalityForSsd, + long blocksLocalWeight, + long blocksLocalWithSsdWeight, + long blocksTotalWeight) { this.name = Preconditions.checkNotNull(name); this.storeCount = storeCount; this.storeFileCount = storeFileCount; @@ -340,6 +368,9 @@ public final class RegionMetricsBuilder { this.dataLocality = dataLocality; this.lastMajorCompactionTimestamp = lastMajorCompactionTimestamp; this.dataLocalityForSsd = dataLocalityForSsd; + this.blocksLocalWeight = blocksLocalWeight; + this.blocksLocalWithSsdWeight = blocksLocalWithSsdWeight; + this.blocksTotalWeight = blocksTotalWeight; } @Override @@ -457,6 +488,21 @@ public final class RegionMetricsBuilder { return dataLocalityForSsd; } + @Override + public long getBlocksLocalWeight() { + return blocksLocalWeight; + } + + @Override + public long getBlocksLocalWithSsdWeight() { + return blocksLocalWithSsdWeight; + } + + @Override + public long getBlocksTotalWeight() { + return blocksTotalWeight; + } + @Override public String toString() { StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "storeCount", @@ -510,6 +556,12 @@ public final class RegionMetricsBuilder { this.getDataLocality()); Strings.appendKeyValue(sb, "dataLocalityForSsd", this.getDataLocalityForSsd()); + Strings.appendKeyValue(sb, "blocksLocalWeight", + blocksLocalWeight); + Strings.appendKeyValue(sb, "blocksLocalWithSsdWeight", + blocksLocalWithSsdWeight); + Strings.appendKeyValue(sb, "blocksTotalWeight", + blocksTotalWeight); return sb.toString(); } } diff --git a/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto b/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto index c62a7ba7424..0c8e89d185d 100644 --- a/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto +++ b/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto @@ -158,6 +158,15 @@ message RegionLoad { /** The current data locality for ssd for region in the regionserver */ optional float data_locality_for_ssd = 23; + + /** The current blocks local weight for region in the regionserver */ + optional uint64 blocks_local_weight = 24; + + /** The current blocks local weight with ssd for region in the regionserver */ + optional uint64 blocks_local_with_ssd_weight = 25; + + /** The current blocks total weight for region in the regionserver */ + optional uint64 blocks_total_weight = 26; } message UserLoad { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java index 734a27ed60e..9d677f8053b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java @@ -51,7 +51,7 @@ public class HDFSBlocksDistribution { */ public static class HostAndWeight { - private String host; + private final String host; private long weight; private long weightForSsd; @@ -228,7 +228,7 @@ public class HDFSBlocksDistribution { * Implementations 'visit' hostAndWeight. */ public interface Visitor { - float visit(final HostAndWeight hostAndWeight); + long visit(final HostAndWeight hostAndWeight); } /** @@ -236,8 +236,12 @@ public class HDFSBlocksDistribution { * @return the locality index of the given host */ public float getBlockLocalityIndex(String host) { - return getBlockLocalityIndexInternal(host, - e -> (float) e.weight / (float) uniqueBlocksTotalWeight); + if (uniqueBlocksTotalWeight == 0) { + return 0.0f; + } else { + return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight) + / (float) uniqueBlocksTotalWeight; + } } /** @@ -245,16 +249,36 @@ public class HDFSBlocksDistribution { * @return the locality index with ssd of the given host */ public float getBlockLocalityIndexForSsd(String host) { - return getBlockLocalityIndexInternal(host, - e -> (float) e.weightForSsd / (float) uniqueBlocksTotalWeight); + if (uniqueBlocksTotalWeight == 0) { + return 0.0f; + } else { + return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd) + / (float) uniqueBlocksTotalWeight; + } + } + + /** + * @param host the host name + * @return the blocks local weight of the given host + */ + public long getBlocksLocalWeight(String host) { + return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight); + } + + /** + * @param host the host name + * @return the blocks local with ssd weight of the given host + */ + public long getBlocksLocalWithSsdWeight(String host) { + return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd); } /** * @param host the host name * @return the locality index of the given host */ - private float getBlockLocalityIndexInternal(String host, Visitor visitor) { - float localityIndex = 0; + private long getBlocksLocalityWeightInternal(String host, Visitor visitor) { + long localityIndex = 0; HostAndWeight hostAndWeight = this.hostAndWeights.get(host); if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) { localityIndex = visitor.visit(hostAndWeight); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 43fd908c9df..1e38e8ab002 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1696,6 +1696,9 @@ public class HRegionServer extends Thread implements HDFSBlocksDistribution hdfsBd = r.getHDFSBlocksDistribution(); float dataLocality = hdfsBd.getBlockLocalityIndex(serverName.getHostname()); float dataLocalityForSsd = hdfsBd.getBlockLocalityIndexForSsd(serverName.getHostname()); + long blocksTotalWeight = hdfsBd.getUniqueBlocksTotalWeight(); + long blocksLocalWeight = hdfsBd.getBlocksLocalWeight(serverName.getHostname()); + long blocksLocalWithSsdWeight = hdfsBd.getBlocksLocalWithSsdWeight(serverName.getHostname()); if (regionLoadBldr == null) { regionLoadBldr = RegionLoad.newBuilder(); } @@ -1724,6 +1727,9 @@ public class HRegionServer extends Thread implements .setCurrentCompactedKVs(currentCompactedKVs) .setDataLocality(dataLocality) .setDataLocalityForSsd(dataLocalityForSsd) + .setBlocksLocalWeight(blocksLocalWeight) + .setBlocksLocalWithSsdWeight(blocksLocalWithSsdWeight) + .setBlocksTotalWeight(blocksTotalWeight) .setLastMajorCompactionTs(r.getOldestHfileTs(true)); r.setCompleteSequenceId(regionLoadBldr); diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index 1a34b7fe8d0..23a66c887f1 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -808,9 +808,14 @@ long totalMemSize = 0; long totalCompactingCells = 0; long totalCompactedCells = 0; + long totalBlocksTotalWeight = 0; + long totalBlocksLocalWeight = 0; + long totalBlocksLocalWithSsdWeight = 0; String totalCompactionProgress = ""; String totalMemSizeStr = ZEROMB; String totalSizeStr = ZEROMB; + String totalLocality = ""; + String totalLocalityForSsd = ""; Map regDistribution = new TreeMap<>(); Map primaryRegDistribution = new TreeMap<>(); List regions = r.getAllRegionLocations(); @@ -835,6 +840,9 @@ totalStoreFileSizeMB += regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE); totalCompactingCells += regionMetrics.getCompactingCellCount(); totalCompactedCells += regionMetrics.getCompactedCellCount(); + totalBlocksTotalWeight += regionMetrics.getBlocksTotalWeight(); + totalBlocksLocalWeight += regionMetrics.getBlocksLocalWeight(); + totalBlocksLocalWithSsdWeight += regionMetrics.getBlocksLocalWithSsdWeight(); } else { RegionMetrics load0 = getEmptyRegionMetrics(regionInfo); regionsToLoad.put(regionInfo, load0); @@ -858,7 +866,12 @@ totalCompactionProgress = String.format("%.2f", 100 * ((float) totalCompactedCells / totalCompactingCells)) + "%"; } - + if (totalBlocksTotalWeight > 0) { + totalLocality = String.format("%.1f", + ((float) totalBlocksLocalWeight / totalBlocksTotalWeight)); + totalLocalityForSsd = String.format("%.1f", + ((float) totalBlocksLocalWithSsdWeight / totalBlocksTotalWeight)); + } if(regions != null && regions.size() > 0) { %>

Table Regions

@@ -963,8 +976,8 @@ Name(<%= String.format("%,1d", regions.size())%>) Region Server - Locality - LocalityForSsd + Locality
(<%= totalLocality %>) + LocalityForSsd
(<%= totalLocalityForSsd %>) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java index 38d60c481c5..50361edd6d8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java @@ -497,6 +497,21 @@ public class TestRegionsRecoveryChore { public float getDataLocalityForSsd() { return 0; } + + @Override + public long getBlocksLocalWeight() { + return 0; + } + + @Override + public long getBlocksLocalWithSsdWeight() { + return 0; + } + + @Override + public long getBlocksTotalWeight() { + return 0; + } }; return regionMetrics; }