HBASE-14891 Add log for uncaught exception in RegionServerMetricsWrapperRunnable(Yu Li)
This commit is contained in:
chenheng
parent
71d41e0c9c
commit
999ae69155
|
|
@ -569,155 +569,160 @@ class MetricsRegionServerWrapperImpl
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
synchronized public void run() {
|
synchronized public void run() {
|
||||||
initBlockCache();
|
try {
|
||||||
initMobFileCache();
|
initBlockCache();
|
||||||
cacheStats = blockCache.getStats();
|
initMobFileCache();
|
||||||
|
cacheStats = blockCache.getStats();
|
||||||
|
|
||||||
HDFSBlocksDistribution hdfsBlocksDistribution =
|
HDFSBlocksDistribution hdfsBlocksDistribution =
|
||||||
new HDFSBlocksDistribution();
|
new HDFSBlocksDistribution();
|
||||||
HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions =
|
HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions =
|
||||||
new HDFSBlocksDistribution();
|
new HDFSBlocksDistribution();
|
||||||
|
|
||||||
long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0;
|
long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0;
|
||||||
long tempReadRequestsCount = 0, tempWriteRequestsCount = 0;
|
long tempReadRequestsCount = 0, tempWriteRequestsCount = 0;
|
||||||
long tempCheckAndMutateChecksFailed = 0;
|
long tempCheckAndMutateChecksFailed = 0;
|
||||||
long tempCheckAndMutateChecksPassed = 0;
|
long tempCheckAndMutateChecksPassed = 0;
|
||||||
long tempStorefileIndexSize = 0;
|
long tempStorefileIndexSize = 0;
|
||||||
long tempTotalStaticIndexSize = 0;
|
long tempTotalStaticIndexSize = 0;
|
||||||
long tempTotalStaticBloomSize = 0;
|
long tempTotalStaticBloomSize = 0;
|
||||||
long tempNumMutationsWithoutWAL = 0;
|
long tempNumMutationsWithoutWAL = 0;
|
||||||
long tempDataInMemoryWithoutWAL = 0;
|
long tempDataInMemoryWithoutWAL = 0;
|
||||||
double tempPercentFileLocal = 0;
|
double tempPercentFileLocal = 0;
|
||||||
double tempPercentFileLocalSecondaryRegions = 0;
|
double tempPercentFileLocalSecondaryRegions = 0;
|
||||||
long tempFlushedCellsCount = 0;
|
long tempFlushedCellsCount = 0;
|
||||||
long tempCompactedCellsCount = 0;
|
long tempCompactedCellsCount = 0;
|
||||||
long tempMajorCompactedCellsCount = 0;
|
long tempMajorCompactedCellsCount = 0;
|
||||||
long tempFlushedCellsSize = 0;
|
long tempFlushedCellsSize = 0;
|
||||||
long tempCompactedCellsSize = 0;
|
long tempCompactedCellsSize = 0;
|
||||||
long tempMajorCompactedCellsSize = 0;
|
long tempMajorCompactedCellsSize = 0;
|
||||||
long tempCellsCountCompactedToMob = 0;
|
long tempCellsCountCompactedToMob = 0;
|
||||||
long tempCellsCountCompactedFromMob = 0;
|
long tempCellsCountCompactedFromMob = 0;
|
||||||
long tempCellsSizeCompactedToMob = 0;
|
long tempCellsSizeCompactedToMob = 0;
|
||||||
long tempCellsSizeCompactedFromMob = 0;
|
long tempCellsSizeCompactedFromMob = 0;
|
||||||
long tempMobFlushCount = 0;
|
long tempMobFlushCount = 0;
|
||||||
long tempMobFlushedCellsCount = 0;
|
long tempMobFlushedCellsCount = 0;
|
||||||
long tempMobFlushedCellsSize = 0;
|
long tempMobFlushedCellsSize = 0;
|
||||||
long tempMobScanCellsCount = 0;
|
long tempMobScanCellsCount = 0;
|
||||||
long tempMobScanCellsSize = 0;
|
long tempMobScanCellsSize = 0;
|
||||||
long tempBlockedRequestsCount = 0;
|
long tempBlockedRequestsCount = 0;
|
||||||
|
|
||||||
for (Region r : regionServer.getOnlineRegionsLocalContext()) {
|
for (Region r : regionServer.getOnlineRegionsLocalContext()) {
|
||||||
tempNumMutationsWithoutWAL += r.getNumMutationsWithoutWAL();
|
tempNumMutationsWithoutWAL += r.getNumMutationsWithoutWAL();
|
||||||
tempDataInMemoryWithoutWAL += r.getDataInMemoryWithoutWAL();
|
tempDataInMemoryWithoutWAL += r.getDataInMemoryWithoutWAL();
|
||||||
tempReadRequestsCount += r.getReadRequestsCount();
|
tempReadRequestsCount += r.getReadRequestsCount();
|
||||||
tempWriteRequestsCount += r.getWriteRequestsCount();
|
tempWriteRequestsCount += r.getWriteRequestsCount();
|
||||||
tempCheckAndMutateChecksFailed += r.getCheckAndMutateChecksFailed();
|
tempCheckAndMutateChecksFailed += r.getCheckAndMutateChecksFailed();
|
||||||
tempCheckAndMutateChecksPassed += r.getCheckAndMutateChecksPassed();
|
tempCheckAndMutateChecksPassed += r.getCheckAndMutateChecksPassed();
|
||||||
tempBlockedRequestsCount += r.getBlockedRequestsCount();
|
tempBlockedRequestsCount += r.getBlockedRequestsCount();
|
||||||
List<Store> storeList = r.getStores();
|
List<Store> storeList = r.getStores();
|
||||||
tempNumStores += storeList.size();
|
tempNumStores += storeList.size();
|
||||||
for (Store store : storeList) {
|
for (Store store : storeList) {
|
||||||
tempNumStoreFiles += store.getStorefilesCount();
|
tempNumStoreFiles += store.getStorefilesCount();
|
||||||
tempMemstoreSize += store.getMemStoreSize();
|
tempMemstoreSize += store.getMemStoreSize();
|
||||||
tempStoreFileSize += store.getStorefilesSize();
|
tempStoreFileSize += store.getStorefilesSize();
|
||||||
tempStorefileIndexSize += store.getStorefilesIndexSize();
|
tempStorefileIndexSize += store.getStorefilesIndexSize();
|
||||||
tempTotalStaticBloomSize += store.getTotalStaticBloomSize();
|
tempTotalStaticBloomSize += store.getTotalStaticBloomSize();
|
||||||
tempTotalStaticIndexSize += store.getTotalStaticIndexSize();
|
tempTotalStaticIndexSize += store.getTotalStaticIndexSize();
|
||||||
tempFlushedCellsCount += store.getFlushedCellsCount();
|
tempFlushedCellsCount += store.getFlushedCellsCount();
|
||||||
tempCompactedCellsCount += store.getCompactedCellsCount();
|
tempCompactedCellsCount += store.getCompactedCellsCount();
|
||||||
tempMajorCompactedCellsCount += store.getMajorCompactedCellsCount();
|
tempMajorCompactedCellsCount += store.getMajorCompactedCellsCount();
|
||||||
tempFlushedCellsSize += store.getFlushedCellsSize();
|
tempFlushedCellsSize += store.getFlushedCellsSize();
|
||||||
tempCompactedCellsSize += store.getCompactedCellsSize();
|
tempCompactedCellsSize += store.getCompactedCellsSize();
|
||||||
tempMajorCompactedCellsSize += store.getMajorCompactedCellsSize();
|
tempMajorCompactedCellsSize += store.getMajorCompactedCellsSize();
|
||||||
if (store instanceof HMobStore) {
|
if (store instanceof HMobStore) {
|
||||||
HMobStore mobStore = (HMobStore) store;
|
HMobStore mobStore = (HMobStore) store;
|
||||||
tempCellsCountCompactedToMob += mobStore.getCellsCountCompactedToMob();
|
tempCellsCountCompactedToMob += mobStore.getCellsCountCompactedToMob();
|
||||||
tempCellsCountCompactedFromMob += mobStore.getCellsCountCompactedFromMob();
|
tempCellsCountCompactedFromMob += mobStore.getCellsCountCompactedFromMob();
|
||||||
tempCellsSizeCompactedToMob += mobStore.getCellsSizeCompactedToMob();
|
tempCellsSizeCompactedToMob += mobStore.getCellsSizeCompactedToMob();
|
||||||
tempCellsSizeCompactedFromMob += mobStore.getCellsSizeCompactedFromMob();
|
tempCellsSizeCompactedFromMob += mobStore.getCellsSizeCompactedFromMob();
|
||||||
tempMobFlushCount += mobStore.getMobFlushCount();
|
tempMobFlushCount += mobStore.getMobFlushCount();
|
||||||
tempMobFlushedCellsCount += mobStore.getMobFlushedCellsCount();
|
tempMobFlushedCellsCount += mobStore.getMobFlushedCellsCount();
|
||||||
tempMobFlushedCellsSize += mobStore.getMobFlushedCellsSize();
|
tempMobFlushedCellsSize += mobStore.getMobFlushedCellsSize();
|
||||||
tempMobScanCellsCount += mobStore.getMobScanCellsCount();
|
tempMobScanCellsCount += mobStore.getMobScanCellsCount();
|
||||||
tempMobScanCellsSize += mobStore.getMobScanCellsSize();
|
tempMobScanCellsSize += mobStore.getMobScanCellsSize();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution();
|
||||||
|
hdfsBlocksDistribution.add(distro);
|
||||||
|
if (r.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
|
||||||
|
hdfsBlocksDistributionSecondaryRegions.add(distro);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
float localityIndex = hdfsBlocksDistribution.getBlockLocalityIndex(
|
||||||
|
regionServer.getServerName().getHostname());
|
||||||
|
tempPercentFileLocal = Double.isNaN(tempBlockedRequestsCount) ? 0 : (localityIndex * 100);
|
||||||
|
|
||||||
HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution();
|
float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions
|
||||||
hdfsBlocksDistribution.add(distro);
|
.getBlockLocalityIndex(regionServer.getServerName().getHostname());
|
||||||
if (r.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
|
tempPercentFileLocalSecondaryRegions = Double.
|
||||||
hdfsBlocksDistributionSecondaryRegions.add(distro);
|
isNaN(localityIndexSecondaryRegions) ? 0 : (localityIndexSecondaryRegions * 100);
|
||||||
|
|
||||||
|
// Compute the number of requests per second
|
||||||
|
long currentTime = EnvironmentEdgeManager.currentTime();
|
||||||
|
|
||||||
|
// assume that it took PERIOD seconds to start the executor.
|
||||||
|
// this is a guess but it's a pretty good one.
|
||||||
|
if (lastRan == 0) {
|
||||||
|
lastRan = currentTime - period;
|
||||||
}
|
}
|
||||||
|
// If we've time traveled keep the last requests per second.
|
||||||
|
if ((currentTime - lastRan) > 0) {
|
||||||
|
long currentRequestCount = getTotalRequestCount();
|
||||||
|
requestsPerSecond = (currentRequestCount - lastRequestCount) /
|
||||||
|
((currentTime - lastRan) / 1000.0);
|
||||||
|
lastRequestCount = currentRequestCount;
|
||||||
|
}
|
||||||
|
lastRan = currentTime;
|
||||||
|
|
||||||
|
WALProvider provider = regionServer.walFactory.getWALProvider();
|
||||||
|
WALProvider metaProvider = regionServer.walFactory.getMetaWALProvider();
|
||||||
|
numWALFiles = (provider == null ? 0 : provider.getNumLogFiles()) +
|
||||||
|
(metaProvider == null ? 0 : metaProvider.getNumLogFiles());
|
||||||
|
walFileSize = (provider == null ? 0 : provider.getLogFileSize()) +
|
||||||
|
(provider == null ? 0 : provider.getLogFileSize());
|
||||||
|
// Copy over computed values so that no thread sees half computed values.
|
||||||
|
numStores = tempNumStores;
|
||||||
|
numStoreFiles = tempNumStoreFiles;
|
||||||
|
memstoreSize = tempMemstoreSize;
|
||||||
|
storeFileSize = tempStoreFileSize;
|
||||||
|
readRequestsCount = tempReadRequestsCount;
|
||||||
|
writeRequestsCount = tempWriteRequestsCount;
|
||||||
|
checkAndMutateChecksFailed = tempCheckAndMutateChecksFailed;
|
||||||
|
checkAndMutateChecksPassed = tempCheckAndMutateChecksPassed;
|
||||||
|
storefileIndexSize = tempStorefileIndexSize;
|
||||||
|
totalStaticIndexSize = tempTotalStaticIndexSize;
|
||||||
|
totalStaticBloomSize = tempTotalStaticBloomSize;
|
||||||
|
numMutationsWithoutWAL = tempNumMutationsWithoutWAL;
|
||||||
|
dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL;
|
||||||
|
percentFileLocal = tempPercentFileLocal;
|
||||||
|
percentFileLocalSecondaryRegions = tempPercentFileLocalSecondaryRegions;
|
||||||
|
flushedCellsCount = tempFlushedCellsCount;
|
||||||
|
compactedCellsCount = tempCompactedCellsCount;
|
||||||
|
majorCompactedCellsCount = tempMajorCompactedCellsCount;
|
||||||
|
flushedCellsSize = tempFlushedCellsSize;
|
||||||
|
compactedCellsSize = tempCompactedCellsSize;
|
||||||
|
majorCompactedCellsSize = tempMajorCompactedCellsSize;
|
||||||
|
cellsCountCompactedToMob = tempCellsCountCompactedToMob;
|
||||||
|
cellsCountCompactedFromMob = tempCellsCountCompactedFromMob;
|
||||||
|
cellsSizeCompactedToMob = tempCellsSizeCompactedToMob;
|
||||||
|
cellsSizeCompactedFromMob = tempCellsSizeCompactedFromMob;
|
||||||
|
mobFlushCount = tempMobFlushCount;
|
||||||
|
mobFlushedCellsCount = tempMobFlushedCellsCount;
|
||||||
|
mobFlushedCellsSize = tempMobFlushedCellsSize;
|
||||||
|
mobScanCellsCount = tempMobScanCellsCount;
|
||||||
|
mobScanCellsSize = tempMobScanCellsSize;
|
||||||
|
mobFileCacheAccessCount = mobFileCache.getAccessCount();
|
||||||
|
mobFileCacheMissCount = mobFileCache.getMissCount();
|
||||||
|
mobFileCacheHitRatio = Double.
|
||||||
|
isNaN(mobFileCache.getHitRatio())?0:mobFileCache.getHitRatio();
|
||||||
|
mobFileCacheEvictedCount = mobFileCache.getEvictedFileCount();
|
||||||
|
mobFileCacheCount = mobFileCache.getCacheSize();
|
||||||
|
blockedRequestsCount = tempBlockedRequestsCount;
|
||||||
|
} catch (Throwable e) {
|
||||||
|
LOG.warn("Caught exception! Will suppress and retry.", e);
|
||||||
}
|
}
|
||||||
float localityIndex = hdfsBlocksDistribution.getBlockLocalityIndex(
|
|
||||||
regionServer.getServerName().getHostname());
|
|
||||||
tempPercentFileLocal = Double.isNaN(tempBlockedRequestsCount) ? 0 : (localityIndex * 100);
|
|
||||||
|
|
||||||
float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions
|
|
||||||
.getBlockLocalityIndex(regionServer.getServerName().getHostname());
|
|
||||||
tempPercentFileLocalSecondaryRegions =
|
|
||||||
Double.isNaN(localityIndexSecondaryRegions) ? 0 : (localityIndexSecondaryRegions * 100);
|
|
||||||
|
|
||||||
// Compute the number of requests per second
|
|
||||||
long currentTime = EnvironmentEdgeManager.currentTime();
|
|
||||||
|
|
||||||
// assume that it took PERIOD seconds to start the executor.
|
|
||||||
// this is a guess but it's a pretty good one.
|
|
||||||
if (lastRan == 0) {
|
|
||||||
lastRan = currentTime - period;
|
|
||||||
}
|
|
||||||
// If we've time traveled keep the last requests per second.
|
|
||||||
if ((currentTime - lastRan) > 0) {
|
|
||||||
long currentRequestCount = getTotalRequestCount();
|
|
||||||
requestsPerSecond = (currentRequestCount - lastRequestCount) /
|
|
||||||
((currentTime - lastRan) / 1000.0);
|
|
||||||
lastRequestCount = currentRequestCount;
|
|
||||||
}
|
|
||||||
lastRan = currentTime;
|
|
||||||
|
|
||||||
WALProvider provider = regionServer.walFactory.getWALProvider();
|
|
||||||
WALProvider metaProvider = regionServer.walFactory.getMetaWALProvider();
|
|
||||||
numWALFiles = (provider == null ? 0 : provider.getNumLogFiles()) +
|
|
||||||
(metaProvider == null ? 0 : metaProvider.getNumLogFiles());
|
|
||||||
walFileSize = (provider == null ? 0 : provider.getLogFileSize()) +
|
|
||||||
(provider == null ? 0 : provider.getLogFileSize());
|
|
||||||
// Copy over computed values so that no thread sees half computed values.
|
|
||||||
numStores = tempNumStores;
|
|
||||||
numStoreFiles = tempNumStoreFiles;
|
|
||||||
memstoreSize = tempMemstoreSize;
|
|
||||||
storeFileSize = tempStoreFileSize;
|
|
||||||
readRequestsCount = tempReadRequestsCount;
|
|
||||||
writeRequestsCount = tempWriteRequestsCount;
|
|
||||||
checkAndMutateChecksFailed = tempCheckAndMutateChecksFailed;
|
|
||||||
checkAndMutateChecksPassed = tempCheckAndMutateChecksPassed;
|
|
||||||
storefileIndexSize = tempStorefileIndexSize;
|
|
||||||
totalStaticIndexSize = tempTotalStaticIndexSize;
|
|
||||||
totalStaticBloomSize = tempTotalStaticBloomSize;
|
|
||||||
numMutationsWithoutWAL = tempNumMutationsWithoutWAL;
|
|
||||||
dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL;
|
|
||||||
percentFileLocal = tempPercentFileLocal;
|
|
||||||
percentFileLocalSecondaryRegions = tempPercentFileLocalSecondaryRegions;
|
|
||||||
flushedCellsCount = tempFlushedCellsCount;
|
|
||||||
compactedCellsCount = tempCompactedCellsCount;
|
|
||||||
majorCompactedCellsCount = tempMajorCompactedCellsCount;
|
|
||||||
flushedCellsSize = tempFlushedCellsSize;
|
|
||||||
compactedCellsSize = tempCompactedCellsSize;
|
|
||||||
majorCompactedCellsSize = tempMajorCompactedCellsSize;
|
|
||||||
cellsCountCompactedToMob = tempCellsCountCompactedToMob;
|
|
||||||
cellsCountCompactedFromMob = tempCellsCountCompactedFromMob;
|
|
||||||
cellsSizeCompactedToMob = tempCellsSizeCompactedToMob;
|
|
||||||
cellsSizeCompactedFromMob = tempCellsSizeCompactedFromMob;
|
|
||||||
mobFlushCount = tempMobFlushCount;
|
|
||||||
mobFlushedCellsCount = tempMobFlushedCellsCount;
|
|
||||||
mobFlushedCellsSize = tempMobFlushedCellsSize;
|
|
||||||
mobScanCellsCount = tempMobScanCellsCount;
|
|
||||||
mobScanCellsSize = tempMobScanCellsSize;
|
|
||||||
mobFileCacheAccessCount = mobFileCache.getAccessCount();
|
|
||||||
mobFileCacheMissCount = mobFileCache.getMissCount();
|
|
||||||
mobFileCacheHitRatio = Double.isNaN(mobFileCache.getHitRatio())?0:mobFileCache.getHitRatio();
|
|
||||||
mobFileCacheEvictedCount = mobFileCache.getEvictedFileCount();
|
|
||||||
mobFileCacheCount = mobFileCache.getCacheSize();
|
|
||||||
blockedRequestsCount = tempBlockedRequestsCount;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue