diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ff929b0a339..11ddd455261 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -647,6 +647,9 @@ Release 2.1.0-beta - 2013-08-06 MAPREDUCE-5419. TestSlive is getting FileNotFound Exception (Robert Parker via jlowe) + MAPREDUCE-5399. Unnecessary Configuration instantiation in IFileInputStream + slows down merge. (Stanislav Barton via Sandy Ryza) + BREAKDOWN OF HADOOP-8562 SUBTASKS MAPREDUCE-4739. Some MapReduce tests fail to find winutils. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java index f409d2298eb..cfcf0f2c6c7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java @@ -81,6 +81,8 @@ public class BackupStore { private boolean inReset = false; private boolean clearMarkFlag = false; private boolean lastSegmentEOF = false; + + private Configuration conf; public BackupStore(Configuration conf, TaskAttemptID taskid) throws IOException { @@ -106,6 +108,8 @@ public class BackupStore { fileCache = new FileCache(conf); tid = taskid; + this.conf = conf; + LOG.info("Created a new BackupStore with a memory of " + maxSize); } @@ -500,7 +504,7 @@ public class BackupStore { Reader reader = new org.apache.hadoop.mapreduce.task.reduce.InMemoryReader(null, (org.apache.hadoop.mapred.TaskAttemptID) tid, - dataOut.getData(), 0, usedSize); + dataOut.getData(), 0, usedSize, conf); Segment segment = new Segment(reader, false); segmentList.add(segment); LOG.debug("Added Memory Segment to List. List Size is " + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java index 543ff3f9cc7..7b8f63a2f33 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.mapred.IFile.Reader; import org.apache.hadoop.mapreduce.TaskAttemptID; @@ -39,11 +40,11 @@ public class InMemoryReader extends Reader { DataInputBuffer memDataIn = new DataInputBuffer(); private int start; private int length; - + public InMemoryReader(MergeManagerImpl merger, TaskAttemptID taskAttemptId, - byte[] data, int start, int length) + byte[] data, int start, int length, Configuration conf) throws IOException { - super(null, null, length - start, null, null); + super(conf, null, length - start, null, null); this.merger = merger; this.taskAttemptId = taskAttemptId; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java index c6f9a36951f..e52f4bf6593 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java @@ -613,7 +613,7 @@ public class MergeManagerImpl implements MergeManager { fullSize -= size; Reader reader = new InMemoryReader(MergeManagerImpl.this, mo.getMapId(), - data, 0, (int)size); + data, 0, (int)size, jobConf); inMemorySegments.add(new Segment(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null)));