Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-3421 Very wide rows -- 30M plus -- cause us OOME (Nate Putnam)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1173335 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Zhihong Yu 2011-09-20 20:21:27 +00:00
parent 1434e96117
commit bcaecda371
2 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -288,6 +288,7 @@ Release 0.91.0 - Unreleased
HBASE-4428 Two methods in CacheTestUtils don't call setDaemon() on the threads HBASE-4428 Two methods in CacheTestUtils don't call setDaemon() on the threads
HBASE-4400 .META. getting stuck if RS hosting it is dead and znode state is in HBASE-4400 .META. getting stuck if RS hosting it is dead and znode state is in
RS_ZK_REGION_OPENED (Ramkrishna) RS_ZK_REGION_OPENED (Ramkrishna)
HBASE-3421 Very wide rows -- 30M plus -- cause us OOME (Nate Putnam)
IMPROVEMENTS IMPROVEMENTS
HBASE-3290 Max Compaction Size (Nicolas Spiegelberg via Stack) HBASE-3290 Max Compaction Size (Nicolas Spiegelberg via Stack)

5
src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
View File

@ -118,6 +118,7 @@ public class Store implements HeapSize {
private final String storeNameStr; private final String storeNameStr;
private final boolean inMemory; private final boolean inMemory;
private CompactionProgress progress; private CompactionProgress progress;
private final int compactionKVMax;
/* /*
* List of store files inside this store. This is an immutable list that * List of store files inside this store. This is an immutable list that
@ -210,6 +211,7 @@ public class Store implements HeapSize {
this.maxCompactSize this.maxCompactSize
= conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE); = conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE);
this.compactRatio = conf.getFloat("hbase.hstore.compaction.ratio", 1.2F); this.compactRatio = conf.getFloat("hbase.hstore.compaction.ratio", 1.2F);
this.compactionKVMax = conf.getInt("hbase.hstore.compaction.kv.max", 10);
if (Store.closeCheckInterval == 0) { if (Store.closeCheckInterval == 0) {
Store.closeCheckInterval = conf.getInt( Store.closeCheckInterval = conf.getInt(
@ -1146,7 +1148,8 @@ public class Store implements HeapSize {
// since scanner.next() can return 'false' but still be delivering data, // since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop. // we have to use a do/while loop.
ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(); ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
while (scanner.next(kvs)) { // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
while (scanner.next(kvs,this.compactionKVMax)) {
if (writer == null && !kvs.isEmpty()) { if (writer == null && !kvs.isEmpty()) {
writer = createWriterInTmp(maxKeyCount, writer = createWriterInTmp(maxKeyCount,
this.compactionCompression); this.compactionCompression);