HBASE-1027 Make global flusher check work with percentages rather than hard code memory sizes.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@723417 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -97,6 +97,8 @@ Release 0.19.0 - Unreleased
    HBASE-927   We don't recover if HRS hosting -ROOT-/.META. goes down -
                (fix bug in createTable which caused tests to fail)
    HBASE-1039  Compaction fails if bloomfilters are enabled
+   HBASE-1027  Make global flusher check work with percentages rather than
+               hard code memory sizes
 
   IMPROVEMENTS
    HBASE-901   Add a limit to key length, check key and value length on client side

conf/hbase-default.xml

@@ -191,19 +191,19 @@
     </description>
   </property>
   <property>
-    <name>hbase.regionserver.globalMemcacheLimit</name>
+    <name>hbase.regionserver.globalMemcache.upperLimit</name>
-    <value>536870912</value>
+    <value>0.4</value>
     <description>Maximum size of all memcaches in a region server before new 
-      updates are blocked and flushes are forced. Defaults to 512MB.
+      updates are blocked and flushes are forced. Defaults to 40% of heap.
     </description>
   </property>
   <property>
-    <name>hbase.regionserver.globalMemcacheLimitlowMark</name>
+    <name>hbase.regionserver.globalMemcache.lowerLimit</name>
-    <value>256435456</value>
+    <value>0.25</value>
     <description>When memcaches are being forced to flush to make room in
-      memory, keep flushing until we hit this mark. Defaults to 256MB. Setting
+      memory, keep flushing until we hit this mark. Defaults to 30% of heap. 
-      this value equal to hbase.regionserver.globalmemcachelimit causes the 
+      This value equal to hbase.regionserver.globalmemcache.upperLimit causes
-      minimum possible flushing to occur when updates are blocked due to 
+      the minimum possible flushing to occur when updates are blocked due to 
       memcache limiting.
     </description>
   </property>  

src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java

@@ -20,20 +20,22 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
-import java.util.concurrent.BlockingQueue;
+import java.lang.management.ManagementFactory;
-import java.util.concurrent.LinkedBlockingQueue;
+import java.util.ConcurrentModificationException;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.TimeUnit;
 import java.util.HashSet;
 import java.util.SortedMap;
-import java.util.ConcurrentModificationException;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.DroppedSnapshotException;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * Thread that flushes cache on request
@@ -58,6 +60,13 @@ class MemcacheFlusher extends Thread implements FlushRequester {
   protected final long globalMemcacheLimit;
   protected final long globalMemcacheLimitLowMark;
   
+  public static final float DEFAULT_UPPER = 0.4f;
+  public static final float DEFAULT_LOWER = 0.25f;
+  public static final String UPPER_KEY =
+    "hbase.regionserver.globalMemcache.upperLimit";
+  public static final String LOWER_KEY =
+    "hbase.regionserver.globalMemcache.lowerLimit";
+  
   /**
    * @param conf
    * @param server
@@ -66,16 +75,47 @@ class MemcacheFlusher extends Thread implements FlushRequester {
       final HRegionServer server) {
     super();
     this.server = server;
-    threadWakeFrequency = conf.getLong(
+    this.threadWakeFrequency =
-        HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
+      conf.getLong(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
+    long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
+    this.globalMemcacheLimit = globalMemcacheLimit(max, DEFAULT_UPPER,
+      UPPER_KEY, conf);
+    long lower = globalMemcacheLimit(max, DEFAULT_LOWER, LOWER_KEY, conf);
+    if (lower > this.globalMemcacheLimit) {
+      lower = this.globalMemcacheLimit;
+      LOG.info("Setting globalMemcacheLimitLowMark == globalMemcacheLimit " +
+        "because supplied " + LOWER_KEY + " was > " + UPPER_KEY);
+    }
+    this.globalMemcacheLimitLowMark = lower;
+    LOG.info("globalMemcacheLimit=" +
+      StringUtils.humanReadableInt(this.globalMemcacheLimit) +
+      ", globalMemcacheLimitLowMark=" +
+      StringUtils.humanReadableInt(this.globalMemcacheLimitLowMark) +
+      ", maxHeap=" + StringUtils.humanReadableInt(max));
+  }
 
-    // default memcache limit of 512MB
+  /**
-    globalMemcacheLimit = 
+   * Calculate size using passed <code>key</code> for configured
-      conf.getLong("hbase.regionserver.globalMemcacheLimit", 512 * 1024 * 1024);
+   * percentage of <code>max</code>.
-    // default memcache low mark limit of 256MB, which is half the upper limit
+   * @param max
-    globalMemcacheLimitLowMark = 
+   * @param defaultLimit
-      conf.getLong("hbase.regionserver.globalMemcacheLimitLowMark", 
+   * @param key
-        globalMemcacheLimit / 2);        
+   * @param c
+   * @return Limit.
+   */
+  static long globalMemcacheLimit(final long max,
+     final float defaultLimit, final String key, final HBaseConfiguration c) {
+    float limit = c.getFloat(key, defaultLimit);
+    return getMemcacheLimit(max, limit, defaultLimit);
+  }
+  
+  static long getMemcacheLimit(final long max, final float limit,
+      final float defaultLimit) {
+    if (limit >= 0.9f || limit < 0.1f) {
+      LOG.warn("Setting global memcache limit to default of " + defaultLimit +
+        " because supplied value outside allowed range of 0.1 -> 0.9");
+    }
+    return (long)(max * limit);
   }
   
   @Override
@@ -229,15 +269,18 @@ class MemcacheFlusher extends Thread implements FlushRequester {
       // flush the region with the biggest memcache
       if (m.size() <= 0) {
         LOG.info("No online regions to flush though we've been asked flush " +
-            "some; globalMemcacheSize=" + globalMemcacheSize +
+          "some; globalMemcacheSize=" +
-            ", globalMemcacheLimitLowMark=" + this.globalMemcacheLimitLowMark);
+          StringUtils.humanReadableInt(globalMemcacheSize) +
+          ", globalMemcacheLimitLowMark=" +
+          StringUtils.humanReadableInt(this.globalMemcacheLimitLowMark));
         break;
       }
       HRegion biggestMemcacheRegion = m.remove(m.firstKey());
       LOG.info("Forced flushing of " +  biggestMemcacheRegion.toString() +
         " because global memcache limit of " + this.globalMemcacheLimit +
-        " exceeded; currently " + globalMemcacheSize + " and flushing till " +
+        " exceeded; currently " +
-        this.globalMemcacheLimitLowMark);
+        StringUtils.humanReadableInt(globalMemcacheSize) + " and flushing till " +
+        StringUtils.humanReadableInt(this.globalMemcacheLimitLowMark));
       if (!flushRegion(biggestMemcacheRegion, true)) {
         LOG.warn("Flush failed");
         break;

src/test/org/apache/hadoop/hbase/TestGlobalMemcacheLimit.java

@@ -1,148 +0,0 @@
-/**
- * Copyright 2008 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.io.BatchUpdate;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Test setting the global memcache size for a region server. When it reaches 
- * this size, any puts should be blocked while one or more forced flushes occurs
- * to bring the memcache size back down. 
- */
-public class TestGlobalMemcacheLimit extends HBaseClusterTestCase {
-  final byte[] ONE_KB = new byte[1024];
-
-  HTable table1;
-  HTable table2;
-  HRegionServer server;
-  
-  long keySize =  COLFAMILY_NAME1.length + 9 + 8;
-  long rowSize = keySize + ONE_KB.length;
-  
-  /**
-   * Get our hands into the cluster configuration before the hbase cluster 
-   * starts up.
-   */
-  @Override
-  public void preHBaseClusterSetup() {
-    // we'll use a 2MB global memcache for testing's sake.
-    conf.setInt("hbase.regionserver.globalMemcacheLimit", 2 * 1024 * 1024);
-    // low memcache mark will be 1MB
-    conf.setInt("hbase.regionserver.globalMemcacheLimitLowMark", 
-      1 * 1024 * 1024);
-    // make sure we don't do any optional flushes and confuse my tests.
-    conf.setInt("hbase.regionserver.optionalcacheflushinterval", 120000);
-  }
-  
-  /**
-   * Create a table that we'll use to test.
-   */
-  @Override
-  public void postHBaseClusterSetup() throws IOException {
-    HTableDescriptor desc1 = createTableDescriptor("testTable1");
-    HTableDescriptor desc2 = createTableDescriptor("testTable2");
-    HBaseAdmin admin = new HBaseAdmin(conf);
-    admin.createTable(desc1);
-    admin.createTable(desc2);
-    table1 = new HTable(conf, "testTable1");
-    table2 = new HTable(conf, "testTable2");    
-    server = cluster.getRegionServer(0);    
-    
-    // there is a META region in play, and those are probably still in
-    // the memcache for ROOT. flush it out.
-    for (HRegion region : server.getOnlineRegions()) {
-      region.flushcache();
-    }
-    // We used to assert that the memsize here was zero but with the addition
-    // of region historian, its no longer true; an entry is added for the
-    // flushes run above.
-  }
-  
-  /**
-   * Make sure that region server thinks all the memcaches are as big as we were
-   * hoping they would be.
-   */
-  public void testMemcacheSizeAccounting() throws IOException {
-    // put some data in each of the two tables
-    long dataSize = populate(table1, 500, 0) + populate(table2, 500, 0);
-    
-    // make sure the region server says it is using as much memory as we think
-    // it is.
-    // Global cache size is now polluted by region historian data.  We used
-    // to be able to do direct compare of global memcache and the data added
-    // but not since HBASE-533 went in.  Compare has to be a bit sloppy.
-    assertTrue("Global memcache size",
-      dataSize <= server.getGlobalMemcacheSize());
-  }
-  
-  /**
-   * Test that a put gets blocked and a flush is forced as expected when we 
-   * reach the memcache size limit.
-   */
-  public void testBlocksAndForcesFlush() throws IOException {
-    // put some data in each of the two tables
-    long startingDataSize = populate(table1, 500, 0) + populate(table2, 500, 0);
-    
-    // at this point we have 1052000 bytes in memcache. now, we'll keep adding 
-    // data to one of the tables until just before the global memcache limit,
-    // noting that the globalMemcacheSize keeps growing as expected. then, we'll
-    // do another put, causing it to go over the limit. when we look at the
-    // globablMemcacheSize now, it should be <= the low limit. 
-    long dataNeeded = (2 * 1024 * 1024) - startingDataSize;
-    double numRows = (double)dataNeeded / (double)rowSize;
-    int preFlushRows = (int)Math.floor(numRows);
-  
-    long dataAdded = populate(table1, preFlushRows, 500);
-    // Global cache size is now polluted by region historian data.  We used
-    // to be able to do direct compare of global memcache and the data added
-    // but not since HBASE-533 went in.
-    long cacheSize = server.getGlobalMemcacheSize();
-    assertTrue("Expected memcache size", (dataAdded + startingDataSize) <= cacheSize);
-        
-    populate(table1, 2, preFlushRows + 500);
-    assertTrue("Post-flush memcache size", server.getGlobalMemcacheSize() <= 1024 * 1024);
-  }
-  
-  private long populate(HTable table, int numRows, int startKey)
-  throws IOException {
-    long total = 0;
-    BatchUpdate batchUpdate = null;
-    byte [] column = COLFAMILY_NAME1;
-    for (int i = startKey; i < startKey + numRows; i++) {
-      byte [] key = Bytes.toBytes("row_" + String.format("%1$5d", i));
-      total += key.length;
-      total += column.length;
-      total += 8;
-      total += ONE_KB.length;
-      batchUpdate = new BatchUpdate(key);
-      batchUpdate.put(column, ONE_KB);
-      table.commit(batchUpdate);
-    }
-    return total;
-  }
-}