Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-5017 Bump the default hfile.block.cache.size because of HFileV2 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1213950 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jean-Daniel Cryans 2011-12-13 21:55:36 +00:00
parent 4de54f3552
commit 69b7530f60
8 changed files with 26 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -30,6 +30,7 @@ Release 0.92.0 - Unreleased
HBASE-4648 Bytes.toBigDecimal() doesn't use offset (Bryan Keller via Lars H)
HBASE-4715 Remove stale broke .rb scripts from bin dir
HBASE-3433 Remove the KV copy of every KV in Scan; introduced by HBASE-3232 (Lars H)
HBASE-5017 Bump the default hfile.block.cache.size because of HFileV2
BUG FIXES
HBASE-3280 YouAreDeadException being swallowed in HRS getMaster

4
src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
View File

@ -74,7 +74,9 @@ public class HBaseConfiguration extends Configuration {
private static void checkForClusterFreeMemoryLimit(Configuration conf) {
float globalMemstoreLimit = conf.getFloat("hbase.regionserver.global.memstore.upperLimit", 0.4f);
int gml = (int)(globalMemstoreLimit * CONVERT_TO_PERCENTAGE);
float blockCacheUpperLimit = conf.getFloat("hfile.block.cache.size", 0.2f);
float blockCacheUpperLimit =
conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE);
if (CONVERT_TO_PERCENTAGE - (gml + bcul)
< (int)(CONVERT_TO_PERCENTAGE *

8
src/main/java/org/apache/hadoop/hbase/HConstants.java
View File

@ -540,6 +540,14 @@ public final class HConstants {
public static final String HBASE_REGION_SPLIT_POLICY_KEY =
"hbase.regionserver.region.split.policy";
/**
* Configuration key for the size of the block cache
*/
public static final String HFILE_BLOCK_CACHE_SIZE_KEY =
"hfile.block.cache.size";
public static final float HFILE_BLOCK_CACHE_SIZE_DEFAULT = 0.25f;
/*
* Minimum percentage of free heap necessary for a successful cluster startup.
*/

12
src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
View File

@ -24,6 +24,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.util.DirectMemoryUtils;
import org.apache.hadoop.util.StringUtils;
@ -34,12 +35,6 @@ import org.apache.hadoop.util.StringUtils;
public class CacheConfig {
private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
/**
* Configuration key for the size of the block cache, in bytes.
*/
public static final String HFILE_BLOCK_CACHE_SIZE_KEY =
"hfile.block.cache.size";
/**
* Configuration key to cache data blocks on write. There are separate
* switches for bloom blocks and non-root index blocks.
@ -312,13 +307,14 @@ public class CacheConfig {
if (globalBlockCache != null) return globalBlockCache;
if (blockCacheDisabled) return null;
float cachePercentage = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, 0.2f);
float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
if (cachePercentage == 0L) {
blockCacheDisabled = true;
return null;
}
if (cachePercentage > 1.0) {
throw new IllegalArgumentException(HFILE_BLOCK_CACHE_SIZE_KEY +
throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
" must be between 0.0 and 1.0, not > 1.0");
}

3
src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
View File

@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KeyComparator;
import org.apache.hadoop.hbase.io.HbaseMapWritable;
@ -278,7 +279,7 @@ public class HFile {
public static final WriterFactory getWriterFactoryNoCache(Configuration
conf) {
Configuration tempConf = new Configuration(conf);
tempConf.setFloat(CacheConfig.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
return HFile.getWriterFactory(conf, new CacheConfig(tempConf));
}

3
src/main/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -57,7 +57,6 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.ipc.HBaseServer;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
@ -217,7 +216,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
throws IOException, KeeperException, InterruptedException {
this.conf = new Configuration(conf);
// Disable the block cache on the master
this.conf.setFloat(CacheConfig.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
// Set how many times to retry talking to another server over HConnection.
HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);
// Server to handle client requests.

2
src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
View File

@ -85,8 +85,6 @@ import com.google.common.collect.Ordering;
public class StoreFile {
static final Log LOG = LogFactory.getLog(StoreFile.class.getName());
static final String HFILE_BLOCK_CACHE_SIZE_KEY = "hfile.block.cache.size";
public static enum BloomType {
/**
* Bloomfilters disabled

6
src/main/resources/hbase-default.xml
View File

@ -446,11 +446,11 @@
</property>
<property>
<name>hfile.block.cache.size</name>
<value>0.2</value>
<value>0.25</value>
<description>
Percentage of maximum heap (-Xmx setting) to allocate to block cache
used by HFile/StoreFile. Default of 0.2 means allocate 20%.
Set to 0 to disable.
used by HFile/StoreFile. Default of 0.25 means allocate 25%.
Set to 0 to disable but it's not recommended.
</description>
</property>
<property>