Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-27750: Update the list of prefetched Hfiles upon block eviction (#5140) 

Signed-off-by: Wellington Chevreuil <wchevreuil@apache.org>
This commit is contained in:
Kota-SH 2023-03-27 04:49:55 -05:00 committed by Wellington Ramos Chevreuil
parent 6d9ac13f80
commit 61e18a3d6c
3 changed files with 24 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
View File

@ -368,6 +368,7 @@ public class BucketCache implements BlockCache, HeapSize {
void startBucketCachePersisterThread() { void startBucketCachePersisterThread() {
BucketCachePersister cachePersister = BucketCachePersister cachePersister =
new BucketCachePersister(this, bucketcachePersistInterval); new BucketCachePersister(this, bucketcachePersistInterval);
cachePersister.setDaemon(true);
cachePersister.start(); cachePersister.start();
} }
@ -599,6 +600,9 @@ public class BucketCache implements BlockCache, HeapSize {
cacheStats.evicted(bucketEntry.getCachedTime(), cacheKey.isPrimary()); cacheStats.evicted(bucketEntry.getCachedTime(), cacheKey.isPrimary());
} }
if (ioEngine.isPersistent()) { if (ioEngine.isPersistent()) {
if (prefetchedFileListPath != null) {
PrefetchExecutor.removePrefetchedFileWhileEvict(cacheKey.getHfileName());
}
setCacheInconsistent(true); setCacheInconsistent(true);
} }
} }

2
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchRSClose.java
View File

@ -107,7 +107,7 @@ public class TestPrefetchRSClose {
table.put(put1); table.put(put1);
TEST_UTIL.flush(tableName); TEST_UTIL.flush(tableName);
} finally { } finally {
Thread.sleep(1500); Thread.sleep(2000);
} }
// Default interval for cache persistence is 1000ms. So after 1000ms, both the persistence files // Default interval for cache persistence is 1000ms. So after 1000ms, both the persistence files

22
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCachePersister.java
View File

@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileBlock; import org.apache.hadoop.hbase.io.hfile.HFileBlock;
import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.io.hfile.PrefetchExecutor;
import org.apache.hadoop.hbase.io.hfile.RandomKeyValueUtil; import org.apache.hadoop.hbase.io.hfile.RandomKeyValueUtil;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.IOTests;
@ -119,21 +120,36 @@ public class TestBucketCachePersister {
@Test @Test
public void testPrefetchPersistenceCrashNegative() throws Exception { public void testPrefetchPersistenceCrashNegative() throws Exception {
long bucketCachePersistInterval = 3000; long bucketCachePersistInterval = Long.MAX_VALUE;
Configuration conf = setupBucketCacheConfig(bucketCachePersistInterval); Configuration conf = setupBucketCacheConfig(bucketCachePersistInterval);
BucketCache bucketCache = setupBucketCache(conf); BucketCache bucketCache = setupBucketCache(conf);
CacheConfig cacheConf = new CacheConfig(conf, bucketCache); CacheConfig cacheConf = new CacheConfig(conf, bucketCache);
FileSystem fs = HFileSystem.get(conf); FileSystem fs = HFileSystem.get(conf);
// Load Cache // Load Cache
Path storeFile = writeStoreFile("TestPrefetch2", conf, cacheConf, fs); Path storeFile = writeStoreFile("TestPrefetch2", conf, cacheConf, fs);
Path storeFile2 = writeStoreFile("TestPrefetch3", conf, cacheConf, fs);
readStoreFile(storeFile, 0, fs, cacheConf, conf, bucketCache); readStoreFile(storeFile, 0, fs, cacheConf, conf, bucketCache);
readStoreFile(storeFile2, 0, fs, cacheConf, conf, bucketCache);
assertFalse(new File(testDir + "/prefetch.persistence").exists()); assertFalse(new File(testDir + "/prefetch.persistence").exists());
assertFalse(new File(testDir + "/bucket.persistence").exists()); assertFalse(new File(testDir + "/bucket.persistence").exists());
cleanupBucketCache(bucketCache); cleanupBucketCache(bucketCache);
} }
@Test
public void testPrefetchListUponBlockEviction() throws Exception {
Configuration conf = setupBucketCacheConfig(200);
BucketCache bucketCache1 = setupBucketCache(conf);
CacheConfig cacheConf = new CacheConfig(conf, bucketCache1);
FileSystem fs = HFileSystem.get(conf);
// Load Blocks in cache
Path storeFile = writeStoreFile("TestPrefetch3", conf, cacheConf, fs);
readStoreFile(storeFile, 0, fs, cacheConf, conf, bucketCache1);
Thread.sleep(500);
// Evict Blocks from cache
BlockCacheKey bucketCacheKey = bucketCache1.backingMap.entrySet().iterator().next().getKey();
assertTrue(PrefetchExecutor.isFilePrefetched(storeFile.getName()));
bucketCache1.evictBlock(bucketCacheKey);
assertFalse(PrefetchExecutor.isFilePrefetched(storeFile.getName()));
}
public void readStoreFile(Path storeFilePath, long offset, FileSystem fs, CacheConfig cacheConf, public void readStoreFile(Path storeFilePath, long offset, FileSystem fs, CacheConfig cacheConf,
Configuration conf, BucketCache bucketCache) throws Exception { Configuration conf, BucketCache bucketCache) throws Exception {
// Open the file // Open the file