Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-15171 Avoid counting duplicate kv and generating lots of small hfiles in PutSortReducer (Yu Li)

This commit is contained in:
tedyu 2016-01-27 09:49:45 -08:00
parent 6d56ecfcf7
commit 630ad95c92
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
View File

@ -67,9 +67,11 @@ public class PutSortReducer extends
for (Cell cell: cells) { for (Cell cell: cells) {
KeyValue kv = KeyValueUtil.ensureKeyValueTypeForMR(cell); KeyValue kv = KeyValueUtil.ensureKeyValueTypeForMR(cell);
map.add(kv); map.add(kv);
curSize += kv.heapSize();
} }
} }
for(KeyValue kv: map){
curSize +=kv.heapSize();
}
} }
context.setStatus("Read " + map.size() + " entries of " + map.getClass() context.setStatus("Read " + map.size() + " entries of " + map.getClass()
+ "(" + StringUtils.humanReadableInt(curSize) + ")"); + "(" + StringUtils.humanReadableInt(curSize) + ")");