Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-15338: Add a option to disable the data block cache for testing the performance of underlying file system

This commit is contained in:
Liu Shaohui 2016-03-07 16:37:07 +08:00
parent ef712df944
commit 4483b3cbed
3 changed files with 87 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
View File

@ -44,6 +44,12 @@ import com.google.common.annotations.VisibleForTesting;
public class CacheConfig {
private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
/**
* Configuration key to cache data blocks on read. Bloom blocks and index blocks are always be
* cached if the block cache is enabled.
*/
public static final String CACHE_DATA_ON_READ_KEY = "hbase.block.data.cacheonread";
/**
* Configuration key to cache data blocks on write. There are separate
* switches for bloom blocks and non-root index blocks.
@ -136,8 +142,9 @@ public class CacheConfig {
private static final String EXTERNAL_BLOCKCACHE_KEY = "hbase.blockcache.use.external";
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY="hbase.blockcache.external.class";
private static final String DROP_BEHIND_CACHE_COMPACTION_KEY="hbase.hfile.drop.behind.compaction";
private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY = "hbase.blockcache.external.class";
private static final String DROP_BEHIND_CACHE_COMPACTION_KEY =
"hbase.hfile.drop.behind.compaction";
private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true;
/**
@ -219,7 +226,8 @@ public class CacheConfig {
*/
public CacheConfig(Configuration conf, HColumnDescriptor family) {
this(CacheConfig.instantiateBlockCache(conf),
family.isBlockCacheEnabled(),
conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ)
&& family.isBlockCacheEnabled(),
family.isInMemory(),
// For the following flags we enable them regardless of per-schema settings
// if they are enabled in the global configuration.
@ -236,7 +244,7 @@ public class CacheConfig {
DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(),
conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1(),
conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
);
}
@ -247,7 +255,7 @@ public class CacheConfig {
*/
public CacheConfig(Configuration conf) {
this(CacheConfig.instantiateBlockCache(conf),
DEFAULT_CACHE_DATA_ON_READ,
conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ),
DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set
// strictly from conf
conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
@ -258,7 +266,7 @@ public class CacheConfig {
conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1),
conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
);
}
@ -432,7 +440,7 @@ public class CacheConfig {
* @return true if data blocks should be compressed in the cache, false if not
*/
public boolean shouldCacheDataCompressed() {
return isBlockCacheEnabled() && this.cacheDataCompressed;
return isBlockCacheEnabled() && this.cacheDataOnRead && this.cacheDataCompressed;
}
/**
@ -442,7 +450,7 @@ public class CacheConfig {
if (!isBlockCacheEnabled()) return false;
switch (category) {
case DATA:
return this.cacheDataCompressed;
return this.cacheDataOnRead && this.cacheDataCompressed;
default:
return false;
}

2
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
View File

@ -1426,7 +1426,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
this.isPrimaryReplicaReader());
cacheBlock &= cacheConf.shouldCacheDataOnRead();
cacheBlock &= cacheConf.shouldCacheBlockOnRead(BlockType.META.getCategory());
if (cacheConf.isBlockCacheEnabled()) {
HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true,
BlockType.META, null);

69
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.io.hfile;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
@ -34,9 +35,11 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
import org.apache.hadoop.hbase.nio.ByteBuff;
@ -210,6 +213,72 @@ public class TestCacheConfig {
return cc.getBlockCache().getBlockCount();
}
@Test
public void testDisableCacheDataBlock() throws IOException {
Configuration conf = HBaseConfiguration.create();
CacheConfig cacheConfig = new CacheConfig(conf);
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA));
assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA));
assertFalse(cacheConfig.shouldCacheDataCompressed());
assertFalse(cacheConfig.shouldCacheDataOnWrite());
assertTrue(cacheConfig.shouldCacheDataOnRead());
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX));
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META));
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM));
assertFalse(cacheConfig.shouldCacheBloomsOnWrite());
assertFalse(cacheConfig.shouldCacheIndexesOnWrite());
conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true);
conf.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, true);
conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, true);
conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, true);
cacheConfig = new CacheConfig(conf);
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA));
assertTrue(cacheConfig.shouldCacheCompressed(BlockCategory.DATA));
assertTrue(cacheConfig.shouldCacheDataCompressed());
assertTrue(cacheConfig.shouldCacheDataOnWrite());
assertTrue(cacheConfig.shouldCacheDataOnRead());
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX));
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META));
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM));
assertTrue(cacheConfig.shouldCacheBloomsOnWrite());
assertTrue(cacheConfig.shouldCacheIndexesOnWrite());
conf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, false);
conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false);
cacheConfig = new CacheConfig(conf);
assertFalse(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA));
assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA));
assertFalse(cacheConfig.shouldCacheDataCompressed());
assertFalse(cacheConfig.shouldCacheDataOnWrite());
assertFalse(cacheConfig.shouldCacheDataOnRead());
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX));
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META));
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM));
assertTrue(cacheConfig.shouldCacheBloomsOnWrite());
assertTrue(cacheConfig.shouldCacheIndexesOnWrite());
conf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, true);
conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false);
HColumnDescriptor family = new HColumnDescriptor("testDisableCacheDataBlock");
family.setBlockCacheEnabled(false);
cacheConfig = new CacheConfig(conf, family);
assertFalse(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA));
assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA));
assertFalse(cacheConfig.shouldCacheDataCompressed());
assertFalse(cacheConfig.shouldCacheDataOnWrite());
assertFalse(cacheConfig.shouldCacheDataOnRead());
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX));
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META));
assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM));
assertTrue(cacheConfig.shouldCacheBloomsOnWrite());
assertTrue(cacheConfig.shouldCacheIndexesOnWrite());
}
@Test
public void testCacheConfigDefaultLRUBlockCache() {
CacheConfig cc = new CacheConfig(this.conf);