From 920cf9d7d43e12f7ec3c636a531dbe6a9ecdc824 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Tue, 15 Mar 2022 20:06:05 +0800 Subject: [PATCH] HBASE-26835 Rewrite TestLruAdaptiveBlockCache to make it more stable (#4219) Signed-off-by: Xiaolin Ha (cherry picked from commit 5ad51e8b2b4b78bc7f92eea04465e24a7577d3ad) --- .../io/hfile/TestLruAdaptiveBlockCache.java | 86 +++++++++---------- 1 file changed, 40 insertions(+), 46 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruAdaptiveBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruAdaptiveBlockCache.java index a922b308c00..f600aaca21f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruAdaptiveBlockCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruAdaptiveBlockCache.java @@ -66,6 +66,8 @@ public class TestLruAdaptiveBlockCache { private static final Logger LOG = LoggerFactory.getLogger(TestLruAdaptiveBlockCache.class); + private static final Configuration CONF = HBaseConfiguration.create(); + @Test public void testCacheEvictionThreadSafe() throws Exception { long maxSize = 100000; @@ -74,13 +76,10 @@ public class TestLruAdaptiveBlockCache { final long blockSize = calculateBlockSizeDefault(maxSize, numBlocks); assertTrue("calculateBlockSize appears broken.", blockSize * numBlocks <= maxSize); - final Configuration conf = HBaseConfiguration.create(); final LruAdaptiveBlockCache cache = new LruAdaptiveBlockCache(maxSize, blockSize); EvictionThread evictionThread = cache.getEvictionThread(); assertNotNull(evictionThread); - while (!evictionThread.isEnteringRun()) { - Thread.sleep(1000); - } + Waiter.waitFor(CONF, 10000, 100, () -> evictionThread.isEnteringRun()); final String hfileName = "hfile"; int threads = 10; final int blocksPerThread = 5 * numBlocks; @@ -102,7 +101,7 @@ public class TestLruAdaptiveBlockCache { service.shutdown(); // The test may fail here if the evict thread frees the blocks too fast service.awaitTermination(10, TimeUnit.MINUTES); - Waiter.waitFor(conf, 10000, 100, new ExplainingPredicate() { + Waiter.waitFor(CONF, 10000, 100, new ExplainingPredicate() { @Override public boolean evaluate() throws Exception { return cache.getBlockCount() == 0; @@ -133,9 +132,7 @@ public class TestLruAdaptiveBlockCache { CachedItem[] blocks = generateFixedBlocks(numBlocks + 1, blockSize, "block"); // Make sure eviction thread has entered run method - while (!evictionThread.isEnteringRun()) { - Thread.sleep(1); - } + Waiter.waitFor(CONF, 10000, 10, () -> evictionThread.isEnteringRun()); // Add all the blocks for (CachedItem block : blocks) { @@ -143,11 +140,18 @@ public class TestLruAdaptiveBlockCache { } // wait until at least one eviction has run - int n = 0; - while(cache.getStats().getEvictionCount() == 0) { - Thread.sleep(200); - assertTrue("Eviction never happened.", n++ < 20); - } + Waiter.waitFor(CONF, 30000, 200, new ExplainingPredicate() { + + @Override + public boolean evaluate() throws Exception { + return cache.getStats().getEvictionCount() > 0; + } + + @Override + public String explainFailure() throws Exception { + return "Eviction never happened."; + } + }); // let cache stabilize // On some systems, the cache will run multiple evictions before it attains @@ -156,22 +160,20 @@ public class TestLruAdaptiveBlockCache { // evicts another. I think this is due to the delta between minSize and // acceptableSize, combined with variance between object overhead on // different environments. - n = 0; - for (long prevCnt = 0 /* < number of blocks added */, - curCnt = cache.getBlockCount(); - prevCnt != curCnt; prevCnt = curCnt, curCnt = cache.getBlockCount()) { + int n = 0; + for (long prevCnt = 0 /* < number of blocks added */, curCnt = + cache.getBlockCount(); prevCnt != curCnt; prevCnt = curCnt, curCnt = cache.getBlockCount()) { Thread.sleep(200); - assertTrue("Cache never stabilized.", n++ < 20); + assertTrue("Cache never stabilized.", n++ < 100); } long evictionCount = cache.getStats().getEvictionCount(); assertTrue(evictionCount >= 1); - System.out.println("Background Evictions run: " + evictionCount); + LOG.info("Background Evictions run: {}", evictionCount); } @Test public void testCacheSimple() throws Exception { - long maxSize = 1000000; long blockSize = calculateBlockSizeDefault(maxSize, 101); @@ -233,7 +235,6 @@ public class TestLruAdaptiveBlockCache { @Test public void testCacheEvictionSimple() throws Exception { - long maxSize = 100000; long blockSize = calculateBlockSizeDefault(maxSize, 10); @@ -275,14 +276,13 @@ public class TestLruAdaptiveBlockCache { @Test public void testCacheEvictionTwoPriorities() throws Exception { - long maxSize = 100000; long blockSize = calculateBlockSizeDefault(maxSize, 10); LruAdaptiveBlockCache cache = new LruAdaptiveBlockCache(maxSize,blockSize,false); - CachedItem [] singleBlocks = generateFixedBlocks(5, 10000, "single"); - CachedItem [] multiBlocks = generateFixedBlocks(5, 10000, "multi"); + CachedItem[] singleBlocks = generateFixedBlocks(5, 10000, "single"); + CachedItem[] multiBlocks = generateFixedBlocks(5, 10000, "multi"); long expectedCacheSize = cache.heapSize(); @@ -339,7 +339,6 @@ public class TestLruAdaptiveBlockCache { @Test public void testCacheEvictionThreePriorities() throws Exception { - long maxSize = 100000; long blockSize = calculateBlockSize(maxSize, 10); @@ -359,9 +358,9 @@ public class TestLruAdaptiveBlockCache { 500, 0.01f); - CachedItem [] singleBlocks = generateFixedBlocks(5, blockSize, "single"); - CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi"); - CachedItem [] memoryBlocks = generateFixedBlocks(5, blockSize, "memory"); + CachedItem[] singleBlocks = generateFixedBlocks(5, blockSize, "single"); + CachedItem[] multiBlocks = generateFixedBlocks(5, blockSize, "multi"); + CachedItem[] memoryBlocks = generateFixedBlocks(5, blockSize, "memory"); long expectedCacheSize = cache.heapSize(); @@ -617,8 +616,8 @@ public class TestLruAdaptiveBlockCache { 500, 0.01f); - CachedItem [] singleBlocks = generateFixedBlocks(20, blockSize, "single"); - CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi"); + CachedItem[] singleBlocks = generateFixedBlocks(20, blockSize, "single"); + CachedItem[] multiBlocks = generateFixedBlocks(5, blockSize, "multi"); // Add 5 multi blocks for (CachedItem block : multiBlocks) { @@ -627,7 +626,7 @@ public class TestLruAdaptiveBlockCache { } // Add 5 single blocks - for(int i=0;i<5;i++) { + for (int i = 0; i < 5; i++) { cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]); } @@ -654,7 +653,7 @@ public class TestLruAdaptiveBlockCache { // blocks evicted. Inserting 13 blocks should yield 3 more evictions and // 12 more evicted. - for(int i=5;i<18;i++) { + for (int i = 5; i < 18; i++) { cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]); } @@ -688,9 +687,8 @@ public class TestLruAdaptiveBlockCache { 500, 0.01f); - CachedItem [] tooLong = generateFixedBlocks(10, 1024+5, "long"); - CachedItem [] small = generateFixedBlocks(15, 600, "small"); - + CachedItem[] tooLong = generateFixedBlocks(10, 1024+5, "long"); + CachedItem[] small = generateFixedBlocks(15, 600, "small"); for (CachedItem i:tooLong) { cache.cacheBlock(i.cacheKey, i); @@ -712,7 +710,6 @@ public class TestLruAdaptiveBlockCache { // test setMaxSize @Test public void testResizeBlockCache() throws Exception { - long maxSize = 300000; long blockSize = calculateBlockSize(maxSize, 31); @@ -732,13 +729,12 @@ public class TestLruAdaptiveBlockCache { 500, 0.01f); - CachedItem [] singleBlocks = generateFixedBlocks(10, blockSize, "single"); - CachedItem [] multiBlocks = generateFixedBlocks(10, blockSize, "multi"); - CachedItem [] memoryBlocks = generateFixedBlocks(10, blockSize, "memory"); + CachedItem[] singleBlocks = generateFixedBlocks(10, blockSize, "single"); + CachedItem[] multiBlocks = generateFixedBlocks(10, blockSize, "multi"); + CachedItem[] memoryBlocks = generateFixedBlocks(10, blockSize, "memory"); // Add all blocks from all priorities - for(int i=0;i<10;i++) { - + for (int i = 0; i < 10; i++) { // Just add single blocks cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]); @@ -939,11 +935,11 @@ public class TestLruAdaptiveBlockCache { return blocks; } - private CachedItem [] generateFixedBlocks(int numBlocks, long size, String pfx) { + private CachedItem[] generateFixedBlocks(int numBlocks, long size, String pfx) { return generateFixedBlocks(numBlocks, (int)size, pfx); } - private CachedItem [] generateRandomBlocks(int numBlocks, long maxSize) { + private CachedItem[] generateRandomBlocks(int numBlocks, long maxSize) { CachedItem [] blocks = new CachedItem[numBlocks]; Random rand = ThreadLocalRandom.current(); for(int i=0;i evictionThread.isEnteringRun()); final String hfileName = "hfile"; for (int blockIndex = 0; blockIndex <= numBlocks * 3000; ++blockIndex) {