Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-6169. MergeQueue should release reference to the current item from key and value at the end of the iteration to save memory. (Zhihai Xu via kasha)

This commit is contained in:
Karthik Kambatla 2014-11-20 15:36:57 -08:00
parent eb4045e765
commit 90194ca1cb
3 changed files with 16 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/CHANGES.txt
View File

@ -237,6 +237,10 @@ Release 2.7.0 - UNRELEASED
OPTIMIZATIONS OPTIMIZATIONS
MAPREDUCE-6169. MergeQueue should release reference to the current item
from key and value at the end of the iteration to save memory.
(Zhihai Xu via kasha)
BUG FIXES BUG FIXES
MAPREDUCE-5918. LineRecordReader can return the same decompressor to MAPREDUCE-5918. LineRecordReader can return the same decompressor to

11
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java
View File

@ -528,9 +528,17 @@ public class Merger {
} }
} }
private void resetKeyValue() {
key = null;
value.reset(new byte[] {}, 0);
diskIFileValue.reset(new byte[] {}, 0);
}
public boolean next() throws IOException { public boolean next() throws IOException {
if (size() == 0) if (size() == 0) {
resetKeyValue();
return false; return false;
}
if (minSegment != null) { if (minSegment != null) {
//minSegment is non-null for all invocations of next except the first //minSegment is non-null for all invocations of next except the first
@ -539,6 +547,7 @@ public class Merger {
adjustPriorityQueue(minSegment); adjustPriorityQueue(minSegment);
if (size() == 0) { if (size() == 0) {
minSegment = null; minSegment = null;
resetKeyValue();
return false; return false;
} }
} }

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
View File

@ -294,6 +294,8 @@ public class TestMerger {
// Now there should be no more input // Now there should be no more input
Assert.assertFalse(mergeQueue.next()); Assert.assertFalse(mergeQueue.next());
Assert.assertEquals(1.0f, mergeQueue.getProgress().get(), epsilon); Assert.assertEquals(1.0f, mergeQueue.getProgress().get(), epsilon);
Assert.assertTrue(mergeQueue.getKey() == null);
Assert.assertEquals(0, mergeQueue.getValue().getData().length);
} }
private Progressable getReporter() { private Progressable getReporter() {