Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14742 Fix TestHeapMemoryManager being flakey

This commit is contained in:
Elliott Clark 2015-11-02 09:50:48 -08:00
parent 1b8b612b8b
commit ca4d27c700
1 changed files with 45 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
View File

@ -30,9 +30,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
@ -53,6 +55,8 @@ import org.junit.experimental.categories.Category;
@Category({RegionServerTests.class, SmallTests.class})
public class TestHeapMemoryManager {
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
@Test
@ -129,7 +133,8 @@ public class TestHeapMemoryManager {
memStoreFlusher.requestFlush(null, false);
memStoreFlusher.flushType = FlushType.ABOVE_LOWER_MARK;
memStoreFlusher.requestFlush(null, false);
Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
// Allow the tuner to run once and do necessary memory up
Thread.sleep(1500);
// No changes should be made by tuner as we already have lot of empty space
assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
assertEquals(oldBlockCacheSize, blockCache.maxSize);
@ -160,7 +165,8 @@ public class TestHeapMemoryManager {
blockCache.evictBlock(null);
blockCache.evictBlock(null);
blockCache.evictBlock(null);
Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
// Allow the tuner to run once and do necessary memory up
Thread.sleep(1500);
// No changes should be made by tuner as we already have lot of empty space
assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
assertEquals(oldBlockCacheSize, blockCache.maxSize);
@ -193,7 +199,8 @@ public class TestHeapMemoryManager {
memStoreFlusher.requestFlush(null, false);
memStoreFlusher.requestFlush(null, false);
memStoreFlusher.requestFlush(null, false);
Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
// Allow the tuner to run once and do necessary memory up
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
memStoreFlusher.memstoreSize);
assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
@ -204,7 +211,8 @@ public class TestHeapMemoryManager {
memStoreFlusher.flushType = FlushType.ABOVE_LOWER_MARK;
memStoreFlusher.requestFlush(null, false);
memStoreFlusher.requestFlush(null, false);
Thread.sleep(1500);
// Allow the tuner to run once and do necessary memory up
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
memStoreFlusher.memstoreSize);
assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
@ -236,7 +244,8 @@ public class TestHeapMemoryManager {
blockCache.evictBlock(null);
blockCache.evictBlock(null);
blockCache.evictBlock(null);
Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
// Allow the tuner to run once and do necessary memory up
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldMemstoreHeapSize,
memStoreFlusher.memstoreSize);
assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldBlockCacheSize,
@ -245,7 +254,8 @@ public class TestHeapMemoryManager {
oldBlockCacheSize = blockCache.maxSize;
// Do some more evictions before the next run of HeapMemoryTuner
blockCache.evictBlock(null);
Thread.sleep(1500);
// Allow the tuner to run once and do necessary memory up
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldMemstoreHeapSize,
memStoreFlusher.memstoreSize);
assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldBlockCacheSize,
@ -280,7 +290,8 @@ public class TestHeapMemoryManager {
memStoreFlusher.requestFlush(null, false);
memStoreFlusher.requestFlush(null, false);
blockCache.evictBlock(null);
Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
// Allow the tuner to run once and do necessary memory up
Thread.sleep(1500);
// No changes should happen as there is undefined increase in flushes and evictions
assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
assertEquals(oldBlockCacheSize, blockCache.maxSize);
@ -289,7 +300,8 @@ public class TestHeapMemoryManager {
memStoreFlusher.requestFlush(null, false);
memStoreFlusher.requestFlush(null, false);
memStoreFlusher.requestFlush(null, false);
Thread.sleep(1500);
// Allow the tuner to run once and do necessary memory up
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
memStoreFlusher.memstoreSize);
assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
@ -325,7 +337,8 @@ public class TestHeapMemoryManager {
memStoreFlusher.requestFlush(null, false);
blockCache.evictBlock(null);
blockCache.evictBlock(null);
Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
// Allow the tuner to run once and do necessary memory up
Thread.sleep(1500);
// No changes should happen as there is undefined increase in flushes and evictions
assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
assertEquals(oldBlockCacheSize, blockCache.maxSize);
@ -336,6 +349,7 @@ public class TestHeapMemoryManager {
blockCache.evictBlock(null);
blockCache.evictBlock(null);
blockCache.evictBlock(null);
// Allow the tuner to run once and do necessary memory up
Thread.sleep(1500);
assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
memStoreFlusher.memstoreSize);
@ -364,13 +378,15 @@ public class TestHeapMemoryManager {
// Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner
CustomHeapMemoryTuner.memstoreSize = 0.78f;
CustomHeapMemoryTuner.blockCacheSize = 0.02f;
Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
// Allow the tuner to run once and do necessary memory up
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore
assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache
// Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner
CustomHeapMemoryTuner.blockCacheSize = 0.75f;
CustomHeapMemoryTuner.memstoreSize = 0.05f;
Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
// Allow the tuner to run once and do necessary memory up
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache
assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore
}
@ -422,6 +438,7 @@ public class TestHeapMemoryManager {
heapMemoryManager.start(choreService);
CustomHeapMemoryTuner.memstoreSize = 0.7f;
CustomHeapMemoryTuner.blockCacheSize = 0.3f;
// Allow the tuner to run once and do necessary memory up
Thread.sleep(1500);
assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
assertEquals(oldBlockCacheSize, blockCache.maxSize);
@ -465,14 +482,16 @@ public class TestHeapMemoryManager {
heapMemoryManager.start(choreService);
CustomHeapMemoryTuner.memstoreSize = 0.4f;
CustomHeapMemoryTuner.blockCacheSize = 0.4f;
Thread.sleep(1500);
// Allow the tuner to run once and do necessary memory up
Thread.sleep(1500);
// The size should not get changes as the collection of memstore size and L1 and L2 block cache
// size will cross the ax allowed 80% mark
assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
assertEquals(oldBlockCacheSize, blockCache.maxSize);
CustomHeapMemoryTuner.memstoreSize = 0.1f;
CustomHeapMemoryTuner.blockCacheSize = 0.5f;
Thread.sleep(1500);
// Allow the tuner to run once and do necessary memory up
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
assertHeapSpace(0.1f, memStoreFlusher.memstoreSize);
assertHeapSpace(0.5f, blockCache.maxSize);
}
@ -485,7 +504,7 @@ public class TestHeapMemoryManager {
private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace, long newHeapSpace) {
double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent);
// Tolerable error
double error = 0.999;
double error = 0.95;
if (expectedDeltaPercent > 0) {
assertTrue(expctedMinDelta*error <= (double)(newHeapSpace - oldHeapSpace));
assertTrue(expctedMinDelta/error >= (double)(newHeapSpace - oldHeapSpace));
@ -495,6 +514,18 @@ public class TestHeapMemoryManager {
}
}
private void waitForTune(final MemstoreFlusherStub memStoreFlusher,
final long oldMemstoreHeapSize) throws Exception {
// Allow the tuner to run once and do necessary memory up
UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
return oldMemstoreHeapSize != memStoreFlusher.memstoreSize;
}
});
}
private static class BlockCacheStub implements ResizableBlockCache {
CacheStats stats = new CacheStats("test");
long maxSize = 0;