Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-24586 Add table level locality in table.jsp 

Closes #1926

Signed-off-by: Viraj Jasani <vjasani@apache.org>
This commit is contained in:
Zheng Wang 2020-07-10 21:54:11 +05:30 committed by Viraj Jasani
parent 6c9534c804
commit b2321b3583
No known key found for this signature in database
GPG Key ID: B3D6C0B41C8ADFD5
7 changed files with 149 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
View File

@ -166,4 +166,20 @@ public interface RegionMetrics {
* @return the data locality for ssd of region in the regionserver
*/
float getDataLocalityForSsd();
/**
* @return the data at local weight of this region in the regionserver
*/
long getBlocksLocalWeight();
/**
* Different from blocksLocalWeight,this metric's numerator only include the data stored on ssd
* @return the data at local with ssd weight of this region in the regionserver
*/
long getBlocksLocalWithSsdWeight();
/**
* @return the block total weight of this region
*/
long getBlocksTotalWeight();
}

58
hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
View File

@ -53,6 +53,11 @@ public final class RegionMetricsBuilder {
.setDataLocality(regionLoadPB.hasDataLocality() ? regionLoadPB.getDataLocality() : 0.0f)
.setDataLocalityForSsd(regionLoadPB.hasDataLocalityForSsd() ?
regionLoadPB.getDataLocalityForSsd() : 0.0f)
.setBlocksLocalWeight(regionLoadPB.hasBlocksLocalWeight() ?
regionLoadPB.getBlocksLocalWeight() : 0)
.setBlocksLocalWithSsdWeight(regionLoadPB.hasBlocksLocalWithSsdWeight() ?
regionLoadPB.getBlocksLocalWithSsdWeight() : 0)
.setBlocksTotalWeight(regionLoadPB.getBlocksTotalWeight())
.setFilteredReadRequestCount(regionLoadPB.getFilteredReadRequestsCount())
.setStoreFileUncompressedDataIndexSize(new Size(regionLoadPB.getTotalStaticIndexSizeKB(),
Size.Unit.KILOBYTE))
@ -151,6 +156,9 @@ public final class RegionMetricsBuilder {
private float dataLocality;
private long lastMajorCompactionTimestamp;
private float dataLocalityForSsd;
private long blocksLocalWeight;
private long blocksLocalWithSsdWeight;
private long blocksTotalWeight;
private RegionMetricsBuilder(byte[] name) {
this.name = name;
}
@ -243,7 +251,18 @@ public final class RegionMetricsBuilder {
this.dataLocalityForSsd = value;
return this;
}
public RegionMetricsBuilder setBlocksLocalWeight(long value) {
this.blocksLocalWeight = value;
return this;
}
public RegionMetricsBuilder setBlocksLocalWithSsdWeight(long value) {
this.blocksLocalWithSsdWeight = value;
return this;
}
public RegionMetricsBuilder setBlocksTotalWeight(long value) {
this.blocksTotalWeight = value;
return this;
}
public RegionMetrics build() {
return new RegionMetricsImpl(name,
storeCount,
@ -267,7 +286,10 @@ public final class RegionMetricsBuilder {
storeSequenceIds,
dataLocality,
lastMajorCompactionTimestamp,
dataLocalityForSsd);
dataLocalityForSsd,
blocksLocalWeight,
blocksLocalWithSsdWeight,
blocksTotalWeight);
}
private static class RegionMetricsImpl implements RegionMetrics {
@ -294,6 +316,9 @@ public final class RegionMetricsBuilder {
private final float dataLocality;
private final long lastMajorCompactionTimestamp;
private final float dataLocalityForSsd;
private final long blocksLocalWeight;
private final long blocksLocalWithSsdWeight;
private final long blocksTotalWeight;
RegionMetricsImpl(byte[] name,
int storeCount,
int storeFileCount,
@ -316,7 +341,10 @@ public final class RegionMetricsBuilder {
Map<byte[], Long> storeSequenceIds,
float dataLocality,
long lastMajorCompactionTimestamp,
float dataLocalityForSsd) {
float dataLocalityForSsd,
long blocksLocalWeight,
long blocksLocalWithSsdWeight,
long blocksTotalWeight) {
this.name = Preconditions.checkNotNull(name);
this.storeCount = storeCount;
this.storeFileCount = storeFileCount;
@ -340,6 +368,9 @@ public final class RegionMetricsBuilder {
this.dataLocality = dataLocality;
this.lastMajorCompactionTimestamp = lastMajorCompactionTimestamp;
this.dataLocalityForSsd = dataLocalityForSsd;
this.blocksLocalWeight = blocksLocalWeight;
this.blocksLocalWithSsdWeight = blocksLocalWithSsdWeight;
this.blocksTotalWeight = blocksTotalWeight;
}
@Override
@ -457,6 +488,21 @@ public final class RegionMetricsBuilder {
return dataLocalityForSsd;
}
@Override
public long getBlocksLocalWeight() {
return blocksLocalWeight;
}
@Override
public long getBlocksLocalWithSsdWeight() {
return blocksLocalWithSsdWeight;
}
@Override
public long getBlocksTotalWeight() {
return blocksTotalWeight;
}
@Override
public String toString() {
StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "storeCount",
@ -510,6 +556,12 @@ public final class RegionMetricsBuilder {
this.getDataLocality());
Strings.appendKeyValue(sb, "dataLocalityForSsd",
this.getDataLocalityForSsd());
Strings.appendKeyValue(sb, "blocksLocalWeight",
blocksLocalWeight);
Strings.appendKeyValue(sb, "blocksLocalWithSsdWeight",
blocksLocalWithSsdWeight);
Strings.appendKeyValue(sb, "blocksTotalWeight",
blocksTotalWeight);
return sb.toString();
}
}

9
hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
View File

@ -158,6 +158,15 @@ message RegionLoad {
/** The current data locality for ssd for region in the regionserver */
optional float data_locality_for_ssd = 23;
/** The current blocks local weight for region in the regionserver */
optional uint64 blocks_local_weight = 24;
/** The current blocks local weight with ssd for region in the regionserver */
optional uint64 blocks_local_with_ssd_weight = 25;
/** The current blocks total weight for region in the regionserver */
optional uint64 blocks_total_weight = 26;
}
message UserLoad {

40
hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
View File

@ -51,7 +51,7 @@ public class HDFSBlocksDistribution {
*/
public static class HostAndWeight {
private String host;
private final String host;
private long weight;
private long weightForSsd;
@ -228,7 +228,7 @@ public class HDFSBlocksDistribution {
* Implementations 'visit' hostAndWeight.
*/
public interface Visitor {
float visit(final HostAndWeight hostAndWeight);
long visit(final HostAndWeight hostAndWeight);
}
/**
@ -236,8 +236,12 @@ public class HDFSBlocksDistribution {
* @return the locality index of the given host
*/
public float getBlockLocalityIndex(String host) {
return getBlockLocalityIndexInternal(host,
e -> (float) e.weight / (float) uniqueBlocksTotalWeight);
if (uniqueBlocksTotalWeight == 0) {
return 0.0f;
} else {
return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight)
/ (float) uniqueBlocksTotalWeight;
}
}
/**
@ -245,16 +249,36 @@ public class HDFSBlocksDistribution {
* @return the locality index with ssd of the given host
*/
public float getBlockLocalityIndexForSsd(String host) {
return getBlockLocalityIndexInternal(host,
e -> (float) e.weightForSsd / (float) uniqueBlocksTotalWeight);
if (uniqueBlocksTotalWeight == 0) {
return 0.0f;
} else {
return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd)
/ (float) uniqueBlocksTotalWeight;
}
}
/**
* @param host the host name
* @return the blocks local weight of the given host
*/
public long getBlocksLocalWeight(String host) {
return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight);
}
/**
* @param host the host name
* @return the blocks local with ssd weight of the given host
*/
public long getBlocksLocalWithSsdWeight(String host) {
return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd);
}
/**
* @param host the host name
* @return the locality index of the given host
*/
private float getBlockLocalityIndexInternal(String host, Visitor visitor) {
float localityIndex = 0;
private long getBlocksLocalityWeightInternal(String host, Visitor visitor) {
long localityIndex = 0;
HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) {
localityIndex = visitor.visit(hostAndWeight);

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -1696,6 +1696,9 @@ public class HRegionServer extends Thread implements
HDFSBlocksDistribution hdfsBd = r.getHDFSBlocksDistribution();
float dataLocality = hdfsBd.getBlockLocalityIndex(serverName.getHostname());
float dataLocalityForSsd = hdfsBd.getBlockLocalityIndexForSsd(serverName.getHostname());
long blocksTotalWeight = hdfsBd.getUniqueBlocksTotalWeight();
long blocksLocalWeight = hdfsBd.getBlocksLocalWeight(serverName.getHostname());
long blocksLocalWithSsdWeight = hdfsBd.getBlocksLocalWithSsdWeight(serverName.getHostname());
if (regionLoadBldr == null) {
regionLoadBldr = RegionLoad.newBuilder();
}
@ -1724,6 +1727,9 @@ public class HRegionServer extends Thread implements
.setCurrentCompactedKVs(currentCompactedKVs)
.setDataLocality(dataLocality)
.setDataLocalityForSsd(dataLocalityForSsd)
.setBlocksLocalWeight(blocksLocalWeight)
.setBlocksLocalWithSsdWeight(blocksLocalWithSsdWeight)
.setBlocksTotalWeight(blocksTotalWeight)
.setLastMajorCompactionTs(r.getOldestHfileTs(true));
r.setCompleteSequenceId(regionLoadBldr);

19
hbase-server/src/main/resources/hbase-webapps/master/table.jsp
View File

@ -808,9 +808,14 @@
long totalMemSize = 0;
long totalCompactingCells = 0;
long totalCompactedCells = 0;
long totalBlocksTotalWeight = 0;
long totalBlocksLocalWeight = 0;
long totalBlocksLocalWithSsdWeight = 0;
String totalCompactionProgress = "";
String totalMemSizeStr = ZEROMB;
String totalSizeStr = ZEROMB;
String totalLocality = "";
String totalLocalityForSsd = "";
Map<ServerName, Integer> regDistribution = new TreeMap<>();
Map<ServerName, Integer> primaryRegDistribution = new TreeMap<>();
List<HRegionLocation> regions = r.getAllRegionLocations();
@ -835,6 +840,9 @@
totalStoreFileSizeMB += regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE);
totalCompactingCells += regionMetrics.getCompactingCellCount();
totalCompactedCells += regionMetrics.getCompactedCellCount();
totalBlocksTotalWeight += regionMetrics.getBlocksTotalWeight();
totalBlocksLocalWeight += regionMetrics.getBlocksLocalWeight();
totalBlocksLocalWithSsdWeight += regionMetrics.getBlocksLocalWithSsdWeight();
} else {
RegionMetrics load0 = getEmptyRegionMetrics(regionInfo);
regionsToLoad.put(regionInfo, load0);
@ -858,7 +866,12 @@
totalCompactionProgress = String.format("%.2f", 100 *
((float) totalCompactedCells / totalCompactingCells)) + "%";
}
if (totalBlocksTotalWeight > 0) {
totalLocality = String.format("%.1f",
((float) totalBlocksLocalWeight / totalBlocksTotalWeight));
totalLocalityForSsd = String.format("%.1f",
((float) totalBlocksLocalWithSsdWeight / totalBlocksTotalWeight));
}
if(regions != null && regions.size() > 0) { %>
<h2>Table Regions</h2>
<div class="tabbable">
@ -963,8 +976,8 @@
<tr>
<th>Name(<%= String.format("%,1d", regions.size())%>)</th>
<th>Region Server</th>
<th>Locality</th>
<th>LocalityForSsd</th>
<th>Locality<br>(<%= totalLocality %>)</th>
<th>LocalityForSsd<br>(<%= totalLocalityForSsd %>)</th>
</tr>
</thead>
<tbody>

15
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
View File

@ -497,6 +497,21 @@ public class TestRegionsRecoveryChore {
public float getDataLocalityForSsd() {
return 0;
}
@Override
public long getBlocksLocalWeight() {
return 0;
}
@Override
public long getBlocksLocalWithSsdWeight() {
return 0;
}
@Override
public long getBlocksTotalWeight() {
return 0;
}
};
return regionMetrics;
}