Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26609 Round the size to MB or KB at the end of calculation in HRegionServer.createRegionLoad (#3967)

Browse Source 
Signed-off-by: Peter Somogyi <psomogyi@apache.org>
This commit is contained in:
Duo Zhang 2021-12-23 10:37:17 +08:00
parent a57f4976c1
commit 51d38b5e4f
1 changed files with 40 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -1733,6 +1733,20 @@ public class HRegionServer extends Thread implements
return regionServerAccounting; return regionServerAccounting;
} }
// Round the size with KB or MB.
// A trick here is that if the sizeInBytes is less than sizeUnit, we will round the size to 1
// instead of 0 if it is not 0, to avoid some schedulers think the region has no data. See
// HBASE-26340 for more details on why this is important.
private static int roundSize(long sizeInByte, int sizeUnit) {
if (sizeInByte == 0) {
return 0;
} else if (sizeInByte < sizeUnit) {
return 1;
} else {
return (int) Math.min(sizeInByte / sizeUnit, Integer.MAX_VALUE);
}
}
/** /**
* @param r Region to get RegionLoad for. * @param r Region to get RegionLoad for.
* @param regionLoadBldr the RegionLoad.Builder, can be null * @param regionLoadBldr the RegionLoad.Builder, can be null
@ -1746,16 +1760,14 @@ public class HRegionServer extends Thread implements
int storefiles = 0; int storefiles = 0;
int storeRefCount = 0; int storeRefCount = 0;
int maxCompactedStoreFileRefCount = 0; int maxCompactedStoreFileRefCount = 0;
int storeUncompressedSizeMB = 0; long storeUncompressedSize = 0L;
int storefileSizeMB = 0; long storefileSize = 0L;
long storefileSizeByte = 0L; long storefileIndexSize = 0L;
int memstoreSizeMB = (int) (r.getMemStoreDataSize() / 1024 / 1024); long rootLevelIndexSize = 0L;
long storefileIndexSizeKB = 0; long totalStaticIndexSize = 0L;
int rootLevelIndexSizeKB = 0; long totalStaticBloomSize = 0L;
int totalStaticIndexSizeKB = 0; long totalCompactingKVs = 0L;
int totalStaticBloomSizeKB = 0; long currentCompactedKVs = 0L;
long totalCompactingKVs = 0;
long currentCompactedKVs = 0;
List<HStore> storeList = r.getStores(); List<HStore> storeList = r.getStores();
stores += storeList.size(); stores += storeList.size();
for (HStore store : storeList) { for (HStore store : storeList) {
@ -1765,22 +1777,30 @@ public class HRegionServer extends Thread implements
int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount(); int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount();
maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount, maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount,
currentMaxCompactedStoreFileRefCount); currentMaxCompactedStoreFileRefCount);
storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024); storeUncompressedSize += store.getStoreSizeUncompressed();
storefileSizeByte += store.getStorefilesSize(); storefileSize += store.getStorefilesSize();
//TODO: storefileIndexSizeKB is same with rootLevelIndexSizeKB? //TODO: storefileIndexSizeKB is same with rootLevelIndexSizeKB?
storefileIndexSizeKB += store.getStorefilesRootLevelIndexSize() / 1024; storefileIndexSize += store.getStorefilesRootLevelIndexSize();
CompactionProgress progress = store.getCompactionProgress(); CompactionProgress progress = store.getCompactionProgress();
if (progress != null) { if (progress != null) {
totalCompactingKVs += progress.getTotalCompactingKVs(); totalCompactingKVs += progress.getTotalCompactingKVs();
currentCompactedKVs += progress.currentCompactedKVs; currentCompactedKVs += progress.currentCompactedKVs;
} }
rootLevelIndexSizeKB += (int) (store.getStorefilesRootLevelIndexSize() / 1024); rootLevelIndexSize += store.getStorefilesRootLevelIndexSize();
totalStaticIndexSizeKB += (int) (store.getTotalStaticIndexSize() / 1024); totalStaticIndexSize += store.getTotalStaticIndexSize();
totalStaticBloomSizeKB += (int) (store.getTotalStaticBloomSize() / 1024); totalStaticBloomSize += store.getTotalStaticBloomSize();
} }
//HBASE-26340 Fix false "0" size under 1MB
storefileSizeMB = storefileSizeByte > 0 && storefileSizeByte <= 1024 * 1024 int unitMB = 1024 * 1024;
? 1 : (int) storefileSizeByte / 1024 / 1024; int unitKB = 1024;
int memstoreSizeMB = roundSize(r.getMemStoreDataSize(), unitMB);
int storeUncompressedSizeMB = roundSize(storeUncompressedSize, unitMB);
int storefileSizeMB = roundSize(storefileSize, unitMB);
int storefileIndexSizeKB = roundSize(storefileIndexSize, unitKB);
int rootLevelIndexSizeKB = roundSize(rootLevelIndexSize, unitKB);
int totalStaticIndexSizeKB = roundSize(totalStaticIndexSize, unitKB);
int totalStaticBloomSizeKB = roundSize(totalStaticBloomSize, unitKB);
HDFSBlocksDistribution hdfsBd = r.getHDFSBlocksDistribution(); HDFSBlocksDistribution hdfsBd = r.getHDFSBlocksDistribution();
float dataLocality = hdfsBd.getBlockLocalityIndex(serverName.getHostname()); float dataLocality = hdfsBd.getBlockLocalityIndex(serverName.getHostname());
@ -1794,6 +1814,7 @@ public class HRegionServer extends Thread implements
if (regionSpecifier == null) { if (regionSpecifier == null) {
regionSpecifier = RegionSpecifier.newBuilder(); regionSpecifier = RegionSpecifier.newBuilder();
} }
regionSpecifier.setType(RegionSpecifierType.REGION_NAME); regionSpecifier.setType(RegionSpecifierType.REGION_NAME);
regionSpecifier.setValue(UnsafeByteOperations.unsafeWrap(name)); regionSpecifier.setValue(UnsafeByteOperations.unsafeWrap(name));
regionLoadBldr.setRegionSpecifier(regionSpecifier.build()) regionLoadBldr.setRegionSpecifier(regionSpecifier.build())