Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-11527 Cluster free memory limit check should consider L2 block cache size also when L2 cache is onheap. (Anoop)

This commit is contained in:
anoopsjohn 2014-08-11 13:38:59 +05:30
parent b8f2f67d4c
commit 886ae4691f
12 changed files with 256 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
View File

@ -24,6 +24,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.util.VersionInfo;
/**
@ -35,9 +36,6 @@ public class HBaseConfiguration extends Configuration {
private static final Log LOG = LogFactory.getLog(HBaseConfiguration.class);
// a constant to convert a fraction to a percentage
private static final int CONVERT_TO_PERCENTAGE = 100;
/**
* Instantinating HBaseConfiguration() is deprecated. Please use
* HBaseConfiguration#create() to construct a plain Configuration
@ -73,39 +71,12 @@ public class HBaseConfiguration extends Configuration {
}
}
private static void checkForClusterFreeMemoryLimit(Configuration conf) {
if (conf.get("hbase.regionserver.global.memstore.upperLimit") != null) {
LOG.warn("hbase.regionserver.global.memstore.upperLimit is deprecated by "
+ "hbase.regionserver.global.memstore.size");
}
float globalMemstoreSize = conf.getFloat("hbase.regionserver.global.memstore.size",
conf.getFloat("hbase.regionserver.global.memstore.upperLimit", 0.4f));
int gml = (int)(globalMemstoreSize * CONVERT_TO_PERCENTAGE);
float blockCacheUpperLimit =
conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE);
if (CONVERT_TO_PERCENTAGE - (gml + bcul)
< (int)(CONVERT_TO_PERCENTAGE *
HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD)) {
throw new RuntimeException(
"Current heap configuration for MemStore and BlockCache exceeds " +
"the threshold required for successful cluster operation. " +
"The combined value cannot exceed 0.8. Please check " +
"the settings for hbase.regionserver.global.memstore.size and " +
"hfile.block.cache.size in your configuration. " +
"hbase.regionserver.global.memstore.size is " +
globalMemstoreSize +
" hfile.block.cache.size is " + blockCacheUpperLimit);
}
}
public static Configuration addHbaseResources(Configuration conf) {
conf.addResource("hbase-default.xml");
conf.addResource("hbase-site.xml");
checkDefaultsVersion(conf);
checkForClusterFreeMemoryLimit(conf);
HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf);
return conf;
}

22
hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
View File

@ -1017,6 +1017,28 @@ public final class HConstants {
public static final String HBASE_COORDINATED_STATE_MANAGER_CLASS =
"hbase.coordinated.state.manager.class";
/**
* Configuration keys for Bucket cache
*/
// TODO moving these bucket cache implementation specific configs to this level is violation of
// encapsulation. But as these has to be referred from hbase-common and bucket cache
// sits in hbase-server, there were no other go! Can we move the cache implementation to
// hbase-common?
/**
* Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path
* to the file that will host the file-based cache. See BucketCache#getIOEngineFromName() for
* list of supported ioengine options.
* <p>Set this option and a non-zero {@link #BUCKET_CACHE_SIZE_KEY} to enable bucket cache.
*/
public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
/**
* When using bucket cache, this is a float that EITHER represents a percentage of total heap
* memory size to give to the cache (if < 1.0) OR, it is the capacity in megabytes of the cache.
*/
public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
private HConstants() {
// Can't be instantiated with this ctor.
}

145
hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java Normal file
View File

@ -0,0 +1,145 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.io.util;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryUsage;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
@InterfaceAudience.Private
public class HeapMemorySizeUtil {
public static final String MEMSTORE_SIZE_KEY = "hbase.regionserver.global.memstore.size";
public static final String MEMSTORE_SIZE_OLD_KEY =
"hbase.regionserver.global.memstore.upperLimit";
public static final String MEMSTORE_SIZE_LOWER_LIMIT_KEY =
"hbase.regionserver.global.memstore.size.lower.limit";
public static final String MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY =
"hbase.regionserver.global.memstore.lowerLimit";
public static final float DEFAULT_MEMSTORE_SIZE = 0.4f;
// Default lower water mark limit is 95% size of memstore size.
public static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f;
private static final Log LOG = LogFactory.getLog(HeapMemorySizeUtil.class);
// a constant to convert a fraction to a percentage
private static final int CONVERT_TO_PERCENTAGE = 100;
/**
* Checks whether we have enough heap memory left out after portion for Memstore and Block cache.
* We need atleast 20% of heap left out for other RS functions.
* @param conf
*/
public static void checkForClusterFreeMemoryLimit(Configuration conf) {
if (conf.get(MEMSTORE_SIZE_OLD_KEY) != null) {
LOG.warn(MEMSTORE_SIZE_OLD_KEY + " is deprecated by " + MEMSTORE_SIZE_KEY);
}
float globalMemstoreSize = getGlobalMemStorePercent(conf, false);
int gml = (int)(globalMemstoreSize * CONVERT_TO_PERCENTAGE);
float blockCacheUpperLimit = getBlockCacheHeapPercent(conf);
int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE);
if (CONVERT_TO_PERCENTAGE - (gml + bcul)
< (int)(CONVERT_TO_PERCENTAGE *
HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD)) {
throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
+ "the threshold required for successful cluster operation. "
+ "The combined value cannot exceed 0.8. Please check "
+ "the settings for hbase.regionserver.global.memstore.size and "
+ "hfile.block.cache.size in your configuration. "
+ "hbase.regionserver.global.memstore.size is " + globalMemstoreSize
+ " hfile.block.cache.size is " + blockCacheUpperLimit);
}
}
/**
* Retrieve global memstore configured size as percentage of total heap.
* @param conf
* @param logInvalid
*/
public static float getGlobalMemStorePercent(final Configuration c, final boolean logInvalid) {
float limit = c.getFloat(MEMSTORE_SIZE_KEY,
c.getFloat(MEMSTORE_SIZE_OLD_KEY, DEFAULT_MEMSTORE_SIZE));
if (limit > 0.8f || limit < 0.05f) {
if (logInvalid) {
LOG.warn("Setting global memstore limit to default of " + DEFAULT_MEMSTORE_SIZE
+ " because supplied value outside allowed range of 0.05 -> 0.8");
}
limit = DEFAULT_MEMSTORE_SIZE;
}
return limit;
}
/**
* Retrieve configured size for global memstore lower water mark as percentage of total heap.
* @param conf
* @param globalMemStorePercent
*/
public static float getGlobalMemStoreLowerMark(final Configuration c, float globalMemStorePercent) {
String lowMarkPercentStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_KEY);
if (lowMarkPercentStr != null) {
return Float.parseFloat(lowMarkPercentStr);
}
String lowerWaterMarkOldValStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY);
if (lowerWaterMarkOldValStr != null) {
LOG.warn(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " is deprecated. Instead use "
+ MEMSTORE_SIZE_LOWER_LIMIT_KEY);
float lowerWaterMarkOldVal = Float.parseFloat(lowerWaterMarkOldValStr);
if (lowerWaterMarkOldVal > globalMemStorePercent) {
lowerWaterMarkOldVal = globalMemStorePercent;
LOG.info("Setting globalMemStoreLimitLowMark == globalMemStoreLimit " + "because supplied "
+ MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " was > " + MEMSTORE_SIZE_OLD_KEY);
}
return lowerWaterMarkOldVal / globalMemStorePercent;
}
return DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT;
}
/**
* Retrieve configured size for on heap block cache as percentage of total heap.
* @param conf
*/
public static float getBlockCacheHeapPercent(final Configuration conf) {
// L1 block cache is always on heap
float l1CachePercent = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
float l2CachePercent = getL2BlockCacheHeapPercent(conf);
return l1CachePercent + l2CachePercent;
}
/**
* @param conf
* @return The on heap size for L2 block cache.
*/
public static float getL2BlockCacheHeapPercent(Configuration conf) {
float l2CachePercent = 0.0F;
String bucketCacheIOEngineName = conf.get(HConstants.BUCKET_CACHE_IOENGINE_KEY, null);
// L2 block cache can be on heap when IOEngine is "heap"
if (bucketCacheIOEngineName != null && bucketCacheIOEngineName.startsWith("heap")) {
float bucketCachePercentage = conf.getFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0F);
MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
l2CachePercent = bucketCachePercentage < 1 ? bucketCachePercentage
: (bucketCachePercentage * 1024 * 1024) / mu.getMax();
}
return l2CachePercent;
}
}

19
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
View File

@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hbase.io.hfile;
import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY;
import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryUsage;
@ -78,21 +81,6 @@ public class CacheConfig {
* Configuration keys for Bucket cache
*/
/**
* Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path
* to the file that will host the file-based cache. See BucketCache#getIOEngineFromName() for
* list of supported ioengine options.
*
* <p>Set this option and a non-zero {@link #BUCKET_CACHE_SIZE_KEY} to enable bucket cache.
*/
public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
/**
* When using bucket cache, this is a float that EITHER represents a percentage of total heap
* memory size to give to the cache (if < 1.0) OR, it is the capacity in megabytes of the cache.
*/
public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
/**
* If the chosen ioengine can persist its state across restarts, the path to the file to
* persist to.
@ -123,7 +111,6 @@ public class CacheConfig {
public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
/**
* Configuration key to prefetch all blocks of a given file into the block cache

2
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
View File

@ -281,7 +281,7 @@ public class TableMapReduceUtil {
public static void resetCacheConfig(Configuration conf) {
conf.setFloat(
HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
conf.setFloat(CacheConfig.BUCKET_CACHE_SIZE_KEY, 0f);
conf.setFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0f);
conf.setFloat("hbase.bucketcache.size", 0f);
}

5
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
View File

@ -27,6 +27,7 @@ import static org.apache.hadoop.hbase.regionserver.HeapMemoryManager.MEMSTORE_SI
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
@ -109,8 +110,8 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
this.blockCachePercentMaxRange = conf.getFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY,
conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT));
this.globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY,
MemStoreFlusher.getGlobalMemStorePercent(conf));
HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false));
this.globalMemStorePercentMaxRange = conf.getFloat(MEMSTORE_SIZE_MAX_RANGE_KEY,
MemStoreFlusher.getGlobalMemStorePercent(conf));
HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false));
}
}

31
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
View File

@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.util.ReflectionUtils;
@ -67,6 +68,7 @@ public class HeapMemoryManager {
private float blockCachePercent;
private float blockCachePercentMinRange;
private float blockCachePercentMaxRange;
private float l2BlockCachePercent;
private final ResizableBlockCache blockCache;
private final FlushRequester memStoreFlusher;
@ -96,20 +98,10 @@ public class HeapMemoryManager {
}
private boolean doInit(Configuration conf) {
globalMemStorePercent = MemStoreFlusher.getGlobalMemStorePercent(conf);
globalMemStorePercent = HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false);
blockCachePercent = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
int gml = (int) (globalMemStorePercent * CONVERT_TO_PERCENTAGE);
int bcul = (int) (blockCachePercent * CONVERT_TO_PERCENTAGE);
if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
+ "the threshold required for successful cluster operation. "
+ "The combined value cannot exceed 0.8. Please check " + "the settings for "
+ MemStoreFlusher.MEMSTORE_SIZE_KEY + " and " + HFILE_BLOCK_CACHE_SIZE_KEY
+ " in your configuration. " + MemStoreFlusher.MEMSTORE_SIZE_KEY + " is "
+ globalMemStorePercent + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is "
+ blockCachePercent);
}
HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf);
// Initialize max and min range for memstore heap space
globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY,
globalMemStorePercent);
@ -117,14 +109,14 @@ public class HeapMemoryManager {
globalMemStorePercent);
if (globalMemStorePercent < globalMemStorePercentMinRange) {
LOG.warn("Setting " + MEMSTORE_SIZE_MIN_RANGE_KEY + " to " + globalMemStorePercent
+ ", same value as " + MemStoreFlusher.MEMSTORE_SIZE_KEY
+ ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
+ " because supplied value greater than initial memstore size value.");
globalMemStorePercentMinRange = globalMemStorePercent;
conf.setFloat(MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercentMinRange);
}
if (globalMemStorePercent > globalMemStorePercentMaxRange) {
LOG.warn("Setting " + MEMSTORE_SIZE_MAX_RANGE_KEY + " to " + globalMemStorePercent
+ ", same value as " + MemStoreFlusher.MEMSTORE_SIZE_KEY
+ ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
+ " because supplied value less than initial memstore size value.");
globalMemStorePercentMaxRange = globalMemStorePercent;
conf.setFloat(MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercentMaxRange);
@ -155,8 +147,9 @@ public class HeapMemoryManager {
return false;
}
gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE);
bcul = (int) (blockCachePercentMinRange * CONVERT_TO_PERCENTAGE);
int gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE);
this.l2BlockCachePercent = HeapMemorySizeUtil.getL2BlockCacheHeapPercent(conf);
int bcul = (int) ((blockCachePercentMinRange + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
+ "the threshold required for successful cluster operation. "
@ -167,7 +160,7 @@ public class HeapMemoryManager {
+ blockCachePercentMinRange);
}
gml = (int) (globalMemStorePercentMinRange * CONVERT_TO_PERCENTAGE);
bcul = (int) (blockCachePercentMaxRange * CONVERT_TO_PERCENTAGE);
bcul = (int) ((blockCachePercentMaxRange + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
+ "the threshold required for successful cluster operation. "
@ -258,11 +251,11 @@ public class HeapMemoryManager {
blockCacheSize = blockCachePercentMaxRange;
}
int gml = (int) (memstoreSize * CONVERT_TO_PERCENTAGE);
int bcul = (int) (blockCacheSize * CONVERT_TO_PERCENTAGE);
int bcul = (int) ((blockCacheSize + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
LOG.info("Current heap configuration from HeapMemoryTuner exceeds "
+ "the threshold required for successful cluster operation. "
+ "The combined value cannot exceed 0.8. " + MemStoreFlusher.MEMSTORE_SIZE_KEY
+ "The combined value cannot exceed 0.8. " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
+ " is " + memstoreSize + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is "
+ blockCacheSize);
// TODO can adjust the value so as not exceed 80%. Is that correct? may be.

4
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java
View File

@ -30,6 +30,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.regionserver.HeapMemStoreLAB.Chunk;
import org.apache.hadoop.util.StringUtils;
@ -192,7 +193,8 @@ public class MemStoreChunkPool {
throw new IllegalArgumentException(CHUNK_POOL_MAXSIZE_KEY + " must be between 0.0 and 1.0");
}
long heapMax = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
long globalMemStoreLimit = (long) (heapMax * MemStoreFlusher.getGlobalMemStorePercent(conf));
long globalMemStoreLimit = (long) (heapMax * HeapMemorySizeUtil.getGlobalMemStorePercent(conf,
false));
int chunkSize = conf.getInt(HeapMemStoreLAB.CHUNK_SIZE_KEY,
HeapMemStoreLAB.CHUNK_SIZE_DEFAULT);
int maxCount = (int) (globalMemStoreLimit * poolSizePercentage / chunkSize);

50
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
View File

@ -44,6 +44,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.DroppedSnapshotException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.HasThread;
@ -67,17 +68,6 @@ import com.google.common.base.Preconditions;
@InterfaceAudience.Private
class MemStoreFlusher implements FlushRequester {
static final Log LOG = LogFactory.getLog(MemStoreFlusher.class);
static final String MEMSTORE_SIZE_KEY = "hbase.regionserver.global.memstore.size";
private static final String MEMSTORE_SIZE_OLD_KEY =
"hbase.regionserver.global.memstore.upperLimit";
private static final String MEMSTORE_SIZE_LOWER_LIMIT_KEY =
"hbase.regionserver.global.memstore.size.lower.limit";
private static final String MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY =
"hbase.regionserver.global.memstore.lowerLimit";
private static final float DEFAULT_MEMSTORE_SIZE = 0.4f;
// Default lower water mark limit is 95% size of memstore size.
private static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f;
// These two data members go together. Any entry in the one must have
// a corresponding entry in the other.
@ -113,10 +103,10 @@ class MemStoreFlusher implements FlushRequester {
this.threadWakeFrequency =
conf.getLong(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
float globalMemStorePercent = getGlobalMemStorePercent(conf);
float globalMemStorePercent = HeapMemorySizeUtil.getGlobalMemStorePercent(conf, true);
this.globalMemStoreLimit = (long) (max * globalMemStorePercent);
this.globalMemStoreLimitLowMarkPercent =
getGlobalMemStoreLowerMark(conf, globalMemStorePercent);
HeapMemorySizeUtil.getGlobalMemStoreLowerMark(conf, globalMemStorePercent);
this.globalMemStoreLimitLowMark =
(long) (this.globalMemStoreLimit * this.globalMemStoreLimitLowMarkPercent);
@ -131,40 +121,6 @@ class MemStoreFlusher implements FlushRequester {
", maxHeap=" + StringUtils.humanReadableInt(max));
}
/**
* Retrieve global memstore configured size as percentage of total heap.
*/
static float getGlobalMemStorePercent(final Configuration c) {
float limit = c.getFloat(MEMSTORE_SIZE_KEY,
c.getFloat(MEMSTORE_SIZE_OLD_KEY, DEFAULT_MEMSTORE_SIZE));
if (limit > 0.8f || limit < 0.05f) {
LOG.warn("Setting global memstore limit to default of " + DEFAULT_MEMSTORE_SIZE
+ " because supplied value outside allowed range of 0.05 -> 0.8");
limit = DEFAULT_MEMSTORE_SIZE;
}
return limit;
}
private static float getGlobalMemStoreLowerMark(final Configuration c, float globalMemStorePercent) {
String lowMarkPercentStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_KEY);
if (lowMarkPercentStr != null) {
return Float.parseFloat(lowMarkPercentStr);
}
String lowerWaterMarkOldValStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY);
if (lowerWaterMarkOldValStr != null) {
LOG.warn(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " is deprecated. Instead use "
+ MEMSTORE_SIZE_LOWER_LIMIT_KEY);
float lowerWaterMarkOldVal = Float.parseFloat(lowerWaterMarkOldValStr);
if (lowerWaterMarkOldVal > globalMemStorePercent) {
lowerWaterMarkOldVal = globalMemStorePercent;
LOG.info("Setting globalMemStoreLimitLowMark == globalMemStoreLimit " + "because supplied "
+ MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " was > " + MEMSTORE_SIZE_OLD_KEY);
}
return lowerWaterMarkOldVal / globalMemStorePercent;
}
return DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT;
}
public Counter getUpdatesBlockedMsHighWater() {
return this.updatesBlockedMsHighWater;
}

5
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
View File

@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.io.hfile.TestCacheConfig.DataCacheEntry;
import org.apache.hadoop.hbase.io.hfile.TestCacheConfig.IndexCacheEntry;
@ -76,8 +77,8 @@ public class TestBlockCacheReporting {
@Test
public void testBucketCache() throws JsonGenerationException, JsonMappingException, IOException {
this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap");
this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, 100);
this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100);
CacheConfig cc = new CacheConfig(this.conf);
assertTrue(cc.getBlockCache() instanceof CombinedBlockCache);
logPerBlock(cc.getBlockCache());

16
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
View File

@ -214,13 +214,13 @@ public class TestCacheConfig {
*/
@Test
public void testOffHeapBucketCacheConfig() {
this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap");
this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
doBucketCacheConfigTest();
}
@Test
public void testOnHeapBucketCacheConfig() {
this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "heap");
this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "heap");
doBucketCacheConfigTest();
}
@ -231,7 +231,7 @@ public class TestCacheConfig {
Path p = new Path(htu.getDataTestDir(), "bc.txt");
FileSystem fs = FileSystem.get(this.conf);
fs.create(p).close();
this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "file:" + p);
this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "file:" + p);
doBucketCacheConfigTest();
} finally {
htu.cleanupTestDir();
@ -240,7 +240,7 @@ public class TestCacheConfig {
private void doBucketCacheConfigTest() {
final int bcSize = 100;
this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, bcSize);
this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, bcSize);
CacheConfig cc = new CacheConfig(this.conf);
basicBlockCacheOps(cc, false, false);
assertTrue(cc.getBlockCache() instanceof CombinedBlockCache);
@ -263,7 +263,7 @@ public class TestCacheConfig {
*/
@Test (timeout=10000)
public void testBucketCacheConfigL1L2Setup() {
this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap");
this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
// Make lru size is smaller than bcSize for sure. Need this to be true so when eviction
// from L1 happens, it does not fail because L2 can't take the eviction because block too big.
this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.001f);
@ -272,7 +272,7 @@ public class TestCacheConfig {
final int bcSize = 100;
long bcExpectedSize = 100 * 1024 * 1024; // MB.
assertTrue(lruExpectedSize < bcExpectedSize);
this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, bcSize);
this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, bcSize);
this.conf.setBoolean(CacheConfig.BUCKET_CACHE_COMBINED_KEY, false);
CacheConfig cc = new CacheConfig(this.conf);
basicBlockCacheOps(cc, false, false);
@ -317,8 +317,8 @@ public class TestCacheConfig {
*/
@Test
public void testCacheDataInL1() {
this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap");
this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, 100);
this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100);
CacheConfig cc = new CacheConfig(this.conf);
assertTrue(cc.getBlockCache() instanceof CombinedBlockCache);
CombinedBlockCache cbc = (CombinedBlockCache)cc.getBlockCache();

51
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
View File

@ -29,6 +29,7 @@ import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.SmallTests;
@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheStats;
import org.apache.hadoop.hbase.io.hfile.Cacheable;
import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
@ -246,6 +248,55 @@ public class TestHeapMemoryManager {
assertEquals(oldBlockCacheSize, blockCache.maxSize);
}
@Test
public void testWhenL2BlockCacheIsOnHeap() throws Exception {
HeapMemoryManager heapMemoryManager = null;
BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.3));
Configuration conf = HBaseConfiguration.create();
conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
conf.setFloat(HeapMemorySizeUtil.MEMSTORE_SIZE_KEY, 0.4F);
conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.3F);
conf.setFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0.1F);
conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "heap");
conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
HeapMemoryTuner.class);
try {
heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(
conf));
fail("Should have failed as the collective heap memory need is above 80%");
} catch (Exception e) {
}
// Change the max/min ranges for memstore and bock cache so as to pass the criteria check
conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.6f);
conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.6f);
heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(
conf));
long oldMemstoreSize = memStoreFlusher.memstoreSize;
long oldBlockCacheSize = blockCache.maxSize;
heapMemoryManager.start();
CustomHeapMemoryTuner.memstoreSize = 0.4f;
CustomHeapMemoryTuner.blockCacheSize = 0.4f;
Thread.sleep(1500);
// The size should not get changes as the collection of memstore size and L1 and L2 block cache
// size will cross the ax allowed 80% mark
assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
assertEquals(oldBlockCacheSize, blockCache.maxSize);
CustomHeapMemoryTuner.memstoreSize = 0.1f;
CustomHeapMemoryTuner.blockCacheSize = 0.5f;
Thread.sleep(1500);
assertHeapSpace(0.1f, memStoreFlusher.memstoreSize);
assertHeapSpace(0.5f, blockCache.maxSize);
}
private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) {
long expected = (long) (this.maxHeapSize * expectedHeapPercentage);
assertEquals(expected, currentHeapSpace);