HBASE-15171 Avoid counting duplicate kv and generating lots of small hfiles in PutSortReducer (Yu Li)

This commit is contained in:
tedyu 2016-01-27 09:49:45 -08:00
parent 6d56ecfcf7
commit 630ad95c92
1 changed files with 3 additions and 1 deletions

View File

@ -67,9 +67,11 @@ public class PutSortReducer extends
for (Cell cell: cells) {
KeyValue kv = KeyValueUtil.ensureKeyValueTypeForMR(cell);
map.add(kv);
curSize += kv.heapSize();
}
}
for(KeyValue kv: map){
curSize +=kv.heapSize();
}
}
context.setStatus("Read " + map.size() + " entries of " + map.getClass()
+ "(" + StringUtils.humanReadableInt(curSize) + ")");