HBASE-15102 Fix HeapMemoryTuner overtuning memstore
HeapMemoryTuner often over tunes memstore without looking at the lower limit of the previous memstore size and causing a situation in which memstore used size suddenly exceeds the total memstore size. Signed-off-by: Elliott Clark <eclark@apache.org>
This commit is contained in:
Ashu Pachauri
Elliott Clark
parent
7d7a2d8712
commit
a61f3ecc8b
|
|
@ -124,30 +124,113 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
|
|||
|
||||
@Override
|
||||
public TunerResult tune(TunerContext context) {
|
||||
long blockedFlushCount = context.getBlockedFlushCount();
|
||||
long unblockedFlushCount = context.getUnblockedFlushCount();
|
||||
long evictCount = context.getEvictCount();
|
||||
long cacheMissCount = context.getCacheMissCount();
|
||||
long totalFlushCount = blockedFlushCount+unblockedFlushCount;
|
||||
rollingStatsForCacheMisses.insertDataValue(cacheMissCount);
|
||||
rollingStatsForFlushes.insertDataValue(totalFlushCount);
|
||||
rollingStatsForEvictions.insertDataValue(evictCount);
|
||||
StepDirection newTuneDirection = StepDirection.NEUTRAL;
|
||||
float curMemstoreSize = context.getCurMemStoreSize();
|
||||
float curBlockCacheSize = context.getCurBlockCacheSize();
|
||||
addToRollingStats(context);
|
||||
|
||||
if (ignoreInitialPeriods < numPeriodsToIgnore) {
|
||||
// Ignoring the first few tuner periods
|
||||
ignoreInitialPeriods++;
|
||||
rollingStatsForTunerSteps.insertDataValue(0);
|
||||
return NO_OP_TUNER_RESULT;
|
||||
}
|
||||
String tunerLog = "";
|
||||
StepDirection newTuneDirection = getTuneDirection(context);
|
||||
|
||||
float newMemstoreSize;
|
||||
float newBlockCacheSize;
|
||||
|
||||
// Adjusting step size for tuning to get to steady state or restart from steady state.
|
||||
// Even if the step size was 4% and 32 GB memory size, we will be shifting 1 GB back and forth
|
||||
// per tuner operation and it can affect the performance of cluster so we keep on decreasing
|
||||
// step size until everything settles.
|
||||
if (prevTuneDirection == StepDirection.NEUTRAL
|
||||
&& newTuneDirection != StepDirection.NEUTRAL
|
||||
&& rollingStatsForTunerSteps.getDeviation() < TUNER_STEP_EPS) {
|
||||
// Restarting the tuning from steady state and setting step size to maximum.
|
||||
// The deviation cannot be that low if last period was neutral and some recent periods were
|
||||
// not neutral.
|
||||
step = maximumStepSize;
|
||||
} else if ((newTuneDirection == StepDirection.INCREASE_MEMSTORE_SIZE
|
||||
&& decayingTunerStepSizeSum < 0) ||
|
||||
(newTuneDirection == StepDirection.INCREASE_BLOCK_CACHE_SIZE
|
||||
&& decayingTunerStepSizeSum > 0)) {
|
||||
// Current step is opposite of past tuner actions so decrease the step size to reach steady
|
||||
// state.
|
||||
step = step/2.00f;
|
||||
}
|
||||
if (step < minimumStepSize) {
|
||||
// If step size is too small then we do nothing.
|
||||
LOG.debug("Tuner step size is too low; we will not perform any tuning this time.");
|
||||
step = 0.0f;
|
||||
newTuneDirection = StepDirection.NEUTRAL;
|
||||
}
|
||||
// Increase / decrease the memstore / block cahce sizes depending on new tuner step.
|
||||
float globalMemstoreLowerMark = HeapMemorySizeUtil.getGlobalMemStoreLowerMark(conf,
|
||||
curMemstoreSize);
|
||||
// We don't want to exert immediate pressure on memstore. So, we decrease its size gracefully;
|
||||
// we set a minimum bar in the middle of the total memstore size and the lower limit.
|
||||
float minMemstoreSize = ((globalMemstoreLowerMark + 1) * curMemstoreSize) / 2.00f;
|
||||
|
||||
switch (newTuneDirection) {
|
||||
case INCREASE_BLOCK_CACHE_SIZE:
|
||||
if (curMemstoreSize - step < minMemstoreSize) {
|
||||
step = curMemstoreSize - minMemstoreSize;
|
||||
}
|
||||
newMemstoreSize = curMemstoreSize - step;
|
||||
newBlockCacheSize = curBlockCacheSize + step;
|
||||
rollingStatsForTunerSteps.insertDataValue(-(int)(step*100000));
|
||||
decayingTunerStepSizeSum = (decayingTunerStepSizeSum - step)/2.00f;
|
||||
break;
|
||||
case INCREASE_MEMSTORE_SIZE:
|
||||
newBlockCacheSize = curBlockCacheSize - step;
|
||||
newMemstoreSize = curMemstoreSize + step;
|
||||
rollingStatsForTunerSteps.insertDataValue((int)(step*100000));
|
||||
decayingTunerStepSizeSum = (decayingTunerStepSizeSum + step)/2.00f;
|
||||
break;
|
||||
default:
|
||||
prevTuneDirection = StepDirection.NEUTRAL;
|
||||
rollingStatsForTunerSteps.insertDataValue(0);
|
||||
decayingTunerStepSizeSum = (decayingTunerStepSizeSum)/2.00f;
|
||||
return NO_OP_TUNER_RESULT;
|
||||
}
|
||||
// Check we are within max/min bounds.
|
||||
if (newMemstoreSize > globalMemStorePercentMaxRange) {
|
||||
newMemstoreSize = globalMemStorePercentMaxRange;
|
||||
} else if (newMemstoreSize < globalMemStorePercentMinRange) {
|
||||
newMemstoreSize = globalMemStorePercentMinRange;
|
||||
}
|
||||
if (newBlockCacheSize > blockCachePercentMaxRange) {
|
||||
newBlockCacheSize = blockCachePercentMaxRange;
|
||||
} else if (newBlockCacheSize < blockCachePercentMinRange) {
|
||||
newBlockCacheSize = blockCachePercentMinRange;
|
||||
}
|
||||
TUNER_RESULT.setBlockCacheSize(newBlockCacheSize);
|
||||
TUNER_RESULT.setMemstoreSize(newMemstoreSize);
|
||||
prevTuneDirection = newTuneDirection;
|
||||
return TUNER_RESULT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine best direction of tuning base on given context.
|
||||
* @param context The tuner context.
|
||||
* @return tuning direction.
|
||||
*/
|
||||
private StepDirection getTuneDirection(TunerContext context) {
|
||||
StepDirection newTuneDirection = StepDirection.NEUTRAL;
|
||||
long blockedFlushCount = context.getBlockedFlushCount();
|
||||
long unblockedFlushCount = context.getUnblockedFlushCount();
|
||||
long evictCount = context.getEvictCount();
|
||||
long cacheMissCount = context.getCacheMissCount();
|
||||
long totalFlushCount = blockedFlushCount+unblockedFlushCount;
|
||||
float curMemstoreSize = context.getCurMemStoreSize();
|
||||
float curBlockCacheSize = context.getCurBlockCacheSize();
|
||||
StringBuilder tunerLog = new StringBuilder();
|
||||
// We can consider memstore or block cache to be sufficient if
|
||||
// we are using only a minor fraction of what have been already provided to it.
|
||||
boolean earlyMemstoreSufficientCheck = totalFlushCount == 0
|
||||
|| context.getCurMemStoreUsed() < context.getCurMemStoreSize()*sufficientMemoryLevel;
|
||||
|| context.getCurMemStoreUsed() < curMemstoreSize * sufficientMemoryLevel;
|
||||
boolean earlyBlockCacheSufficientCheck = evictCount == 0 ||
|
||||
context.getCurBlockCacheUsed() < context.getCurBlockCacheSize()*sufficientMemoryLevel;
|
||||
float newMemstoreSize;
|
||||
float newBlockCacheSize;
|
||||
context.getCurBlockCacheUsed() < curBlockCacheSize * sufficientMemoryLevel;
|
||||
if (earlyMemstoreSufficientCheck && earlyBlockCacheSufficientCheck) {
|
||||
// Both memstore and block cache memory seems to be sufficient. No operation required.
|
||||
newTuneDirection = StepDirection.NEUTRAL;
|
||||
|
|
@ -172,11 +255,11 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
|
|||
// Reverting previous step as it was not useful.
|
||||
// Tuning failed to decrease evictions or tuning resulted in large number of flushes.
|
||||
newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
|
||||
tunerLog += "Reverting previous tuning.";
|
||||
tunerLog.append("We will revert previous tuning");
|
||||
if ((double)evictCount > rollingStatsForEvictions.getMean()) {
|
||||
tunerLog += " As could not decrease evctions sufficiently.";
|
||||
tunerLog.append(" because we could not decrease evictions sufficiently.");
|
||||
} else {
|
||||
tunerLog += " As number of flushes rose significantly.";
|
||||
tunerLog.append(" because the number of flushes rose significantly.");
|
||||
}
|
||||
isReverting = true;
|
||||
}
|
||||
|
|
@ -188,11 +271,11 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
|
|||
// Reverting previous step as it was not useful.
|
||||
// Tuning failed to decrease flushes or tuning resulted in large number of evictions.
|
||||
newTuneDirection = StepDirection.INCREASE_BLOCK_CACHE_SIZE;
|
||||
tunerLog += "Reverting previous tuning.";
|
||||
tunerLog.append("We will revert previous tuning");
|
||||
if ((double)totalFlushCount > rollingStatsForFlushes.getMean()) {
|
||||
tunerLog += " As could not decrease flushes sufficiently.";
|
||||
tunerLog.append(" because we could not decrease flushes sufficiently.");
|
||||
} else {
|
||||
tunerLog += " As number of evictions rose significantly.";
|
||||
tunerLog.append(" because number of evictions rose significantly.");
|
||||
}
|
||||
isReverting = true;
|
||||
}
|
||||
|
|
@ -224,88 +307,43 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
|
|||
rollingStatsForFlushes.getDeviation()*0.80) {
|
||||
// more misses , increasing cache size
|
||||
newTuneDirection = StepDirection.INCREASE_BLOCK_CACHE_SIZE;
|
||||
tunerLog +=
|
||||
"Increasing block cache size as observed increase in number of cache misses.";
|
||||
tunerLog.append(
|
||||
"Going to increase block cache size due to increase in number of cache misses.");
|
||||
} else if ((double)cacheMissCount < rollingStatsForCacheMisses.getMean() -
|
||||
rollingStatsForCacheMisses.getDeviation()*0.80 &&
|
||||
(double)totalFlushCount > rollingStatsForFlushes.getMean() +
|
||||
rollingStatsForFlushes.getDeviation()*0.80) {
|
||||
// more flushes , increasing memstore size
|
||||
newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
|
||||
tunerLog += "Increasing memstore size as observed increase in number of flushes.";
|
||||
tunerLog.append("Going to increase memstore size due to increase in number of flushes.");
|
||||
} else if (blockedFlushCount > 0 && prevTuneDirection == StepDirection.NEUTRAL) {
|
||||
// we do not want blocked flushes
|
||||
newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
|
||||
tunerLog += "Increasing memstore size as observed "
|
||||
+ blockedFlushCount + " blocked flushes.";
|
||||
tunerLog.append("Going to increase memstore size due to"
|
||||
+ blockedFlushCount + " blocked flushes.");
|
||||
} else {
|
||||
// Default. Not enough facts to do tuning.
|
||||
tunerLog.append("Going to do nothing because we "
|
||||
+ "could not determine best tuning direction");
|
||||
newTuneDirection = StepDirection.NEUTRAL;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Adjusting step size for tuning to get to steady state or restart from steady state.
|
||||
// Even if the step size was 4% and 32 GB memory size, we will be shifting 1 GB back and forth
|
||||
// per tuner operation and it can affect the performance of cluster so we keep on decreasing
|
||||
// step size until everything settles.
|
||||
if (prevTuneDirection == StepDirection.NEUTRAL
|
||||
&& newTuneDirection != StepDirection.NEUTRAL
|
||||
&& rollingStatsForTunerSteps.getDeviation() < TUNER_STEP_EPS) {
|
||||
// Restarting the tuning from steady state and setting step size to maximum.
|
||||
// The deviation cannot be that low if last period was neutral and some recent periods were
|
||||
// not neutral.
|
||||
step = maximumStepSize;
|
||||
} else if ((newTuneDirection == StepDirection.INCREASE_MEMSTORE_SIZE
|
||||
&& decayingTunerStepSizeSum < 0) ||
|
||||
(newTuneDirection == StepDirection.INCREASE_BLOCK_CACHE_SIZE
|
||||
&& decayingTunerStepSizeSum > 0)) {
|
||||
// Current step is opposite of past tuner actions so decrease the step size to reach steady
|
||||
// state.
|
||||
step = step/2.00f;
|
||||
}
|
||||
if (step < minimumStepSize) {
|
||||
// If step size is too small then we do nothing.
|
||||
step = 0.0f;
|
||||
newTuneDirection = StepDirection.NEUTRAL;
|
||||
}
|
||||
// Increase / decrease the memstore / block cahce sizes depending on new tuner step.
|
||||
switch (newTuneDirection) {
|
||||
case INCREASE_BLOCK_CACHE_SIZE:
|
||||
newBlockCacheSize = context.getCurBlockCacheSize() + step;
|
||||
newMemstoreSize = context.getCurMemStoreSize() - step;
|
||||
rollingStatsForTunerSteps.insertDataValue(-(int)(step*100000));
|
||||
decayingTunerStepSizeSum = (decayingTunerStepSizeSum - step)/2.00f;
|
||||
break;
|
||||
case INCREASE_MEMSTORE_SIZE:
|
||||
newBlockCacheSize = context.getCurBlockCacheSize() - step;
|
||||
newMemstoreSize = context.getCurMemStoreSize() + step;
|
||||
rollingStatsForTunerSteps.insertDataValue((int)(step*100000));
|
||||
decayingTunerStepSizeSum = (decayingTunerStepSizeSum + step)/2.00f;
|
||||
break;
|
||||
default:
|
||||
prevTuneDirection = StepDirection.NEUTRAL;
|
||||
rollingStatsForTunerSteps.insertDataValue(0);
|
||||
decayingTunerStepSizeSum = (decayingTunerStepSizeSum)/2.00f;
|
||||
return NO_OP_TUNER_RESULT;
|
||||
}
|
||||
// Check we are within max/min bounds.
|
||||
if (newMemstoreSize > globalMemStorePercentMaxRange) {
|
||||
newMemstoreSize = globalMemStorePercentMaxRange;
|
||||
} else if (newMemstoreSize < globalMemStorePercentMinRange) {
|
||||
newMemstoreSize = globalMemStorePercentMinRange;
|
||||
}
|
||||
if (newBlockCacheSize > blockCachePercentMaxRange) {
|
||||
newBlockCacheSize = blockCachePercentMaxRange;
|
||||
} else if (newBlockCacheSize < blockCachePercentMinRange) {
|
||||
newBlockCacheSize = blockCachePercentMinRange;
|
||||
}
|
||||
TUNER_RESULT.setBlockCacheSize(newBlockCacheSize);
|
||||
TUNER_RESULT.setMemstoreSize(newMemstoreSize);
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug(tunerLog);
|
||||
LOG.debug(tunerLog.toString());
|
||||
}
|
||||
prevTuneDirection = newTuneDirection;
|
||||
return TUNER_RESULT;
|
||||
return newTuneDirection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the given context to the rolling tuner stats.
|
||||
* @param context The tuner context.
|
||||
*/
|
||||
private void addToRollingStats(TunerContext context) {
|
||||
rollingStatsForCacheMisses.insertDataValue(context.getCacheMissCount());
|
||||
rollingStatsForFlushes.insertDataValue(context.getBlockedFlushCount() +
|
||||
context.getUnblockedFlushCount());
|
||||
rollingStatsForEvictions.insertDataValue(context.getEvictCount());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -227,6 +227,7 @@ public class TestHeapMemoryManager {
|
|||
blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
|
||||
regionServerAccounting.setTestMemstoreSize(0);
|
||||
Configuration conf = HBaseConfiguration.create();
|
||||
conf.setFloat(HeapMemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
|
||||
conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
|
||||
conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
|
||||
conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
|
||||
|
|
@ -238,6 +239,11 @@ public class TestHeapMemoryManager {
|
|||
new RegionServerStub(conf), new RegionServerAccountingStub());
|
||||
long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
|
||||
long oldBlockCacheSize = blockCache.maxSize;
|
||||
long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
|
||||
long maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
|
||||
float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
|
||||
maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
|
||||
DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
|
||||
final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
|
||||
heapMemoryManager.start(choreService);
|
||||
blockCache.evictBlock(null);
|
||||
|
|
@ -245,20 +251,21 @@ public class TestHeapMemoryManager {
|
|||
blockCache.evictBlock(null);
|
||||
// Allow the tuner to run once and do necessary memory up
|
||||
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
|
||||
assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldMemstoreHeapSize,
|
||||
memStoreFlusher.memstoreSize);
|
||||
assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldBlockCacheSize,
|
||||
blockCache.maxSize);
|
||||
assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
|
||||
assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
|
||||
oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
|
||||
oldBlockCacheSize = blockCache.maxSize;
|
||||
oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
|
||||
maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
|
||||
maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
|
||||
maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
|
||||
DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
|
||||
// Do some more evictions before the next run of HeapMemoryTuner
|
||||
blockCache.evictBlock(null);
|
||||
// Allow the tuner to run once and do necessary memory up
|
||||
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
|
||||
assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldMemstoreHeapSize,
|
||||
memStoreFlusher.memstoreSize);
|
||||
assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldBlockCacheSize,
|
||||
blockCache.maxSize);
|
||||
assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
|
||||
assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
Loading…
Reference in New Issue