From 59587d9fad4319d10f9306a796fcd84d6b2f5adb Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Mon, 16 Sep 2013 14:18:42 +0000 Subject: [PATCH] MAPREDUCE-5493. Cleanup in-memory & on-disk segments to prevent leak on shuffle completion. Contributed by Jason Lowe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1523660 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapreduce/task/reduce/MergeManagerImpl.java | 3 +++ .../apache/hadoop/mapreduce/task/reduce/TestMerger.java | 7 ++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index fc26e8d4fca..58cf3c6ff2b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -267,6 +267,9 @@ Release 2.1.1-beta - UNRELEASED MAPREDUCE-5164. mapred job and queue commands omit HADOOP_CLIENT_OPTS (Nemon Lou via devaraj) + MAPREDUCE-5493. Cleanup in-memory & on-disk segments to prevent leak on + shuffle completion. (jlowe via acmurthy) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java index e52f4bf6593..ca3bed93998 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java @@ -355,8 +355,11 @@ public class MergeManagerImpl implements MergeManager { List> memory = new ArrayList>(inMemoryMergedMapOutputs); + inMemoryMergedMapOutputs.clear(); memory.addAll(inMemoryMapOutputs); + inMemoryMapOutputs.clear(); List disk = new ArrayList(onDiskMapOutputs); + onDiskMapOutputs.clear(); return finalMerge(jobConf, rfs, memory, disk); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java index 41a1848c35f..f1d54359b82 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java @@ -82,7 +82,7 @@ public class TestMerger { } @Test - public void testInMemoryMerger() throws IOException { + public void testInMemoryMerger() throws Throwable { JobID jobId = new JobID("a", 0); TaskAttemptID reduceId = new TaskAttemptID( new TaskID(jobId, TaskType.REDUCE, 0), 0); @@ -132,6 +132,11 @@ public class TestMerger { readOnDiskMapOutput(conf, fs, outPath, keys, values); Assert.assertEquals(keys, Arrays.asList("apple", "banana", "carrot")); Assert.assertEquals(values, Arrays.asList("disgusting", "pretty good", "delicious")); + + mergeManager.close(); + Assert.assertEquals(0, mergeManager.inMemoryMapOutputs.size()); + Assert.assertEquals(0, mergeManager.inMemoryMergedMapOutputs.size()); + Assert.assertEquals(0, mergeManager.onDiskMapOutputs.size()); } private byte[] writeMapOutput(Configuration conf, Map keysToValues)