HBASE-15338: Add a option to disable the data block cache for testing the performance of underlying file system
This commit is contained in:
Liu Shaohui
parent
ef712df944
commit
4483b3cbed
|
|
@ -44,6 +44,12 @@ import com.google.common.annotations.VisibleForTesting;
|
|||
public class CacheConfig {
|
||||
private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
|
||||
|
||||
/**
|
||||
* Configuration key to cache data blocks on read. Bloom blocks and index blocks are always be
|
||||
* cached if the block cache is enabled.
|
||||
*/
|
||||
public static final String CACHE_DATA_ON_READ_KEY = "hbase.block.data.cacheonread";
|
||||
|
||||
/**
|
||||
* Configuration key to cache data blocks on write. There are separate
|
||||
* switches for bloom blocks and non-root index blocks.
|
||||
|
|
@ -136,8 +142,9 @@ public class CacheConfig {
|
|||
private static final String EXTERNAL_BLOCKCACHE_KEY = "hbase.blockcache.use.external";
|
||||
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
|
||||
|
||||
private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY="hbase.blockcache.external.class";
|
||||
private static final String DROP_BEHIND_CACHE_COMPACTION_KEY="hbase.hfile.drop.behind.compaction";
|
||||
private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY = "hbase.blockcache.external.class";
|
||||
private static final String DROP_BEHIND_CACHE_COMPACTION_KEY =
|
||||
"hbase.hfile.drop.behind.compaction";
|
||||
private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true;
|
||||
|
||||
/**
|
||||
|
|
@ -219,7 +226,8 @@ public class CacheConfig {
|
|||
*/
|
||||
public CacheConfig(Configuration conf, HColumnDescriptor family) {
|
||||
this(CacheConfig.instantiateBlockCache(conf),
|
||||
family.isBlockCacheEnabled(),
|
||||
conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ)
|
||||
&& family.isBlockCacheEnabled(),
|
||||
family.isInMemory(),
|
||||
// For the following flags we enable them regardless of per-schema settings
|
||||
// if they are enabled in the global configuration.
|
||||
|
|
@ -236,7 +244,7 @@ public class CacheConfig {
|
|||
DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(),
|
||||
conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
|
||||
HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1(),
|
||||
conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
|
||||
conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -247,7 +255,7 @@ public class CacheConfig {
|
|||
*/
|
||||
public CacheConfig(Configuration conf) {
|
||||
this(CacheConfig.instantiateBlockCache(conf),
|
||||
DEFAULT_CACHE_DATA_ON_READ,
|
||||
conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ),
|
||||
DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set
|
||||
// strictly from conf
|
||||
conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
|
||||
|
|
@ -258,7 +266,7 @@ public class CacheConfig {
|
|||
conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
|
||||
conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
|
||||
HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1),
|
||||
conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
|
||||
conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -432,7 +440,7 @@ public class CacheConfig {
|
|||
* @return true if data blocks should be compressed in the cache, false if not
|
||||
*/
|
||||
public boolean shouldCacheDataCompressed() {
|
||||
return isBlockCacheEnabled() && this.cacheDataCompressed;
|
||||
return isBlockCacheEnabled() && this.cacheDataOnRead && this.cacheDataCompressed;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -442,7 +450,7 @@ public class CacheConfig {
|
|||
if (!isBlockCacheEnabled()) return false;
|
||||
switch (category) {
|
||||
case DATA:
|
||||
return this.cacheDataCompressed;
|
||||
return this.cacheDataOnRead && this.cacheDataCompressed;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1426,7 +1426,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
|
||||
this.isPrimaryReplicaReader());
|
||||
|
||||
cacheBlock &= cacheConf.shouldCacheDataOnRead();
|
||||
cacheBlock &= cacheConf.shouldCacheBlockOnRead(BlockType.META.getCategory());
|
||||
if (cacheConf.isBlockCacheEnabled()) {
|
||||
HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true,
|
||||
BlockType.META, null);
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
package org.apache.hadoop.hbase.io.hfile;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
|
|
@ -34,9 +35,11 @@ import org.apache.hadoop.fs.FileSystem;
|
|||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.testclassification.IOTests;
|
||||
import org.apache.hadoop.hbase.testclassification.LargeTests;
|
||||
import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
|
||||
import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType;
|
||||
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
|
||||
import org.apache.hadoop.hbase.nio.ByteBuff;
|
||||
|
|
@ -210,6 +213,72 @@ public class TestCacheConfig {
|
|||
return cc.getBlockCache().getBlockCount();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDisableCacheDataBlock() throws IOException {
|
||||
Configuration conf = HBaseConfiguration.create();
|
||||
CacheConfig cacheConfig = new CacheConfig(conf);
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA));
|
||||
assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA));
|
||||
assertFalse(cacheConfig.shouldCacheDataCompressed());
|
||||
assertFalse(cacheConfig.shouldCacheDataOnWrite());
|
||||
assertTrue(cacheConfig.shouldCacheDataOnRead());
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX));
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META));
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM));
|
||||
assertFalse(cacheConfig.shouldCacheBloomsOnWrite());
|
||||
assertFalse(cacheConfig.shouldCacheIndexesOnWrite());
|
||||
|
||||
conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true);
|
||||
conf.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, true);
|
||||
conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, true);
|
||||
conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, true);
|
||||
|
||||
cacheConfig = new CacheConfig(conf);
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA));
|
||||
assertTrue(cacheConfig.shouldCacheCompressed(BlockCategory.DATA));
|
||||
assertTrue(cacheConfig.shouldCacheDataCompressed());
|
||||
assertTrue(cacheConfig.shouldCacheDataOnWrite());
|
||||
assertTrue(cacheConfig.shouldCacheDataOnRead());
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX));
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META));
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM));
|
||||
assertTrue(cacheConfig.shouldCacheBloomsOnWrite());
|
||||
assertTrue(cacheConfig.shouldCacheIndexesOnWrite());
|
||||
|
||||
conf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, false);
|
||||
conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false);
|
||||
|
||||
cacheConfig = new CacheConfig(conf);
|
||||
assertFalse(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA));
|
||||
assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA));
|
||||
assertFalse(cacheConfig.shouldCacheDataCompressed());
|
||||
assertFalse(cacheConfig.shouldCacheDataOnWrite());
|
||||
assertFalse(cacheConfig.shouldCacheDataOnRead());
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX));
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META));
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM));
|
||||
assertTrue(cacheConfig.shouldCacheBloomsOnWrite());
|
||||
assertTrue(cacheConfig.shouldCacheIndexesOnWrite());
|
||||
|
||||
conf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, true);
|
||||
conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false);
|
||||
|
||||
HColumnDescriptor family = new HColumnDescriptor("testDisableCacheDataBlock");
|
||||
family.setBlockCacheEnabled(false);
|
||||
|
||||
cacheConfig = new CacheConfig(conf, family);
|
||||
assertFalse(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA));
|
||||
assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA));
|
||||
assertFalse(cacheConfig.shouldCacheDataCompressed());
|
||||
assertFalse(cacheConfig.shouldCacheDataOnWrite());
|
||||
assertFalse(cacheConfig.shouldCacheDataOnRead());
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX));
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META));
|
||||
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM));
|
||||
assertTrue(cacheConfig.shouldCacheBloomsOnWrite());
|
||||
assertTrue(cacheConfig.shouldCacheIndexesOnWrite());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCacheConfigDefaultLRUBlockCache() {
|
||||
CacheConfig cc = new CacheConfig(this.conf);
|
||||
|
|
@ -351,4 +420,4 @@ public class TestCacheConfig {
|
|||
blocks.get(BlockType.DATA) == null? 0:
|
||||
blocks.get(BlockType.DATA).intValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue