From 69b7530f60058d990a60f6573570b16ed2c09f1f Mon Sep 17 00:00:00 2001 From: Jean-Daniel Cryans Date: Tue, 13 Dec 2011 21:55:36 +0000 Subject: [PATCH] HBASE-5017 Bump the default hfile.block.cache.size because of HFileV2 git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1213950 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + .../apache/hadoop/hbase/HBaseConfiguration.java | 4 +++- .../java/org/apache/hadoop/hbase/HConstants.java | 8 ++++++++ .../apache/hadoop/hbase/io/hfile/CacheConfig.java | 12 ++++-------- .../org/apache/hadoop/hbase/io/hfile/HFile.java | 3 ++- .../org/apache/hadoop/hbase/master/HMaster.java | 3 +-- .../hadoop/hbase/regionserver/StoreFile.java | 2 -- src/main/resources/hbase-default.xml | 14 +++++++------- 8 files changed, 26 insertions(+), 21 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index c9b9d2c40f8..16a233953e6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -30,6 +30,7 @@ Release 0.92.0 - Unreleased HBASE-4648 Bytes.toBigDecimal() doesn't use offset (Bryan Keller via Lars H) HBASE-4715 Remove stale broke .rb scripts from bin dir HBASE-3433 Remove the KV copy of every KV in Scan; introduced by HBASE-3232 (Lars H) + HBASE-5017 Bump the default hfile.block.cache.size because of HFileV2 BUG FIXES HBASE-3280 YouAreDeadException being swallowed in HRS getMaster diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java index 8d22fb9a926..df2d8445ec3 100644 --- a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java +++ b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java @@ -74,7 +74,9 @@ public class HBaseConfiguration extends Configuration { private static void checkForClusterFreeMemoryLimit(Configuration conf) { float globalMemstoreLimit = conf.getFloat("hbase.regionserver.global.memstore.upperLimit", 0.4f); int gml = (int)(globalMemstoreLimit * CONVERT_TO_PERCENTAGE); - float blockCacheUpperLimit = conf.getFloat("hfile.block.cache.size", 0.2f); + float blockCacheUpperLimit = + conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, + HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT); int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE); if (CONVERT_TO_PERCENTAGE - (gml + bcul) < (int)(CONVERT_TO_PERCENTAGE * diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java index 1cf58a924b5..3c8384684f7 100644 --- a/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -540,6 +540,14 @@ public final class HConstants { public static final String HBASE_REGION_SPLIT_POLICY_KEY = "hbase.regionserver.region.split.policy"; + /** + * Configuration key for the size of the block cache + */ + public static final String HFILE_BLOCK_CACHE_SIZE_KEY = + "hfile.block.cache.size"; + + public static final float HFILE_BLOCK_CACHE_SIZE_DEFAULT = 0.25f; + /* * Minimum percentage of free heap necessary for a successful cluster startup. */ diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 6e5c2899177..6461d53735c 100644 --- a/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.util.DirectMemoryUtils; import org.apache.hadoop.util.StringUtils; @@ -34,12 +35,6 @@ import org.apache.hadoop.util.StringUtils; public class CacheConfig { private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName()); - /** - * Configuration key for the size of the block cache, in bytes. - */ - public static final String HFILE_BLOCK_CACHE_SIZE_KEY = - "hfile.block.cache.size"; - /** * Configuration key to cache data blocks on write. There are separate * switches for bloom blocks and non-root index blocks. @@ -312,13 +307,14 @@ public class CacheConfig { if (globalBlockCache != null) return globalBlockCache; if (blockCacheDisabled) return null; - float cachePercentage = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, 0.2f); + float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, + HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT); if (cachePercentage == 0L) { blockCacheDisabled = true; return null; } if (cachePercentage > 1.0) { - throw new IllegalArgumentException(HFILE_BLOCK_CACHE_SIZE_KEY + + throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " must be between 0.0 and 1.0, not > 1.0"); } diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 764f9e04a49..397d3daadc7 100644 --- a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KeyComparator; import org.apache.hadoop.hbase.io.HbaseMapWritable; @@ -278,7 +279,7 @@ public class HFile { public static final WriterFactory getWriterFactoryNoCache(Configuration conf) { Configuration tempConf = new Configuration(conf); - tempConf.setFloat(CacheConfig.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); + tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); return HFile.getWriterFactory(conf, new CacheConfig(tempConf)); } diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 20d00cb4543..f92ba0d09ae 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -57,7 +57,6 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.executor.EventHandler; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType; -import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.ipc.HBaseRPC; import org.apache.hadoop.hbase.ipc.HBaseServer; import org.apache.hadoop.hbase.ipc.HMasterInterface; @@ -217,7 +216,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { throws IOException, KeeperException, InterruptedException { this.conf = new Configuration(conf); // Disable the block cache on the master - this.conf.setFloat(CacheConfig.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); + this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); // Set how many times to retry talking to another server over HConnection. HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG); // Server to handle client requests. diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java index 6c0dd69e3b7..ff6c17e4481 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java @@ -85,8 +85,6 @@ import com.google.common.collect.Ordering; public class StoreFile { static final Log LOG = LogFactory.getLog(StoreFile.class.getName()); - static final String HFILE_BLOCK_CACHE_SIZE_KEY = "hfile.block.cache.size"; - public static enum BloomType { /** * Bloomfilters disabled diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml index 9f8757fc811..99f31949ac3 100644 --- a/src/main/resources/hbase-default.xml +++ b/src/main/resources/hbase-default.xml @@ -445,13 +445,13 @@ - hfile.block.cache.size - 0.2 - - Percentage of maximum heap (-Xmx setting) to allocate to block cache - used by HFile/StoreFile. Default of 0.2 means allocate 20%. - Set to 0 to disable. - + hfile.block.cache.size + 0.25 + + Percentage of maximum heap (-Xmx setting) to allocate to block cache + used by HFile/StoreFile. Default of 0.25 means allocate 25%. + Set to 0 to disable but it's not recommended. + hbase.hash.type