Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-6105. nconsistent configuration in property mapreduce.reduce.shuffle.merge.percent. Contributed by Ray Chiang. 

(cherry picked from commit 26a23b11598b3757ed4973781890e2cd2fc270df)
This commit is contained in:
Harsh J 2015-03-17 01:17:34 +05:30
parent 4568acaa62
commit 0e9f78dda2
3 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -8,6 +8,9 @@ Release 2.8.0 - UNRELEASED
IMPROVEMENTS IMPROVEMENTS
MAPREDUCE-6105. Inconsistent configuration in property
mapreduce.reduce.shuffle.merge.percent. (Ray Chiang via harsh)
MAPREDUCE-4414. Add main methods to JobConf and YarnConfiguration, MAPREDUCE-4414. Add main methods to JobConf and YarnConfiguration,
for debug purposes. (Plamen Jeliazkov via harsh) for debug purposes. (Plamen Jeliazkov via harsh)

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
View File

@ -300,6 +300,7 @@ public interface MRJobConfig {
= "mapreduce.reduce.shuffle.memory.limit.percent"; = "mapreduce.reduce.shuffle.memory.limit.percent";
public static final String SHUFFLE_MERGE_PERCENT = "mapreduce.reduce.shuffle.merge.percent"; public static final String SHUFFLE_MERGE_PERCENT = "mapreduce.reduce.shuffle.merge.percent";
public static final float DEFAULT_SHUFFLE_MERGE_PERCENT = 0.66f;
public static final String REDUCE_FAILURES_MAXPERCENT = "mapreduce.reduce.failures.maxpercent"; public static final String REDUCE_FAILURES_MAXPERCENT = "mapreduce.reduce.failures.maxpercent";

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
View File

@ -191,8 +191,9 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> {
this.memToMemMergeOutputsThreshold = this.memToMemMergeOutputsThreshold =
jobConf.getInt(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, ioSortFactor); jobConf.getInt(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, ioSortFactor);
this.mergeThreshold = (long)(this.memoryLimit * this.mergeThreshold = (long)(this.memoryLimit *
jobConf.getFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, jobConf.getFloat(
0.90f)); MRJobConfig.SHUFFLE_MERGE_PERCENT,
MRJobConfig.DEFAULT_SHUFFLE_MERGE_PERCENT));
LOG.info("MergerManager: memoryLimit=" + memoryLimit + ", " + LOG.info("MergerManager: memoryLimit=" + memoryLimit + ", " +
"maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " + "maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " +
"mergeThreshold=" + mergeThreshold + ", " + "mergeThreshold=" + mergeThreshold + ", " +