MAPREDUCE-6105. nconsistent configuration in property mapreduce.reduce.shuffle.merge.percent. Contributed by Ray Chiang.
(cherry picked from commit 26a23b11598b3757ed4973781890e2cd2fc270df)
This commit is contained in:
parent
4568acaa62
commit
0e9f78dda2
|
@ -8,6 +8,9 @@ Release 2.8.0 - UNRELEASED
|
|||
|
||||
IMPROVEMENTS
|
||||
|
||||
MAPREDUCE-6105. Inconsistent configuration in property
|
||||
mapreduce.reduce.shuffle.merge.percent. (Ray Chiang via harsh)
|
||||
|
||||
MAPREDUCE-4414. Add main methods to JobConf and YarnConfiguration,
|
||||
for debug purposes. (Plamen Jeliazkov via harsh)
|
||||
|
||||
|
|
|
@ -300,6 +300,7 @@ public interface MRJobConfig {
|
|||
= "mapreduce.reduce.shuffle.memory.limit.percent";
|
||||
|
||||
public static final String SHUFFLE_MERGE_PERCENT = "mapreduce.reduce.shuffle.merge.percent";
|
||||
public static final float DEFAULT_SHUFFLE_MERGE_PERCENT = 0.66f;
|
||||
|
||||
public static final String REDUCE_FAILURES_MAXPERCENT = "mapreduce.reduce.failures.maxpercent";
|
||||
|
||||
|
|
|
@ -191,8 +191,9 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> {
|
|||
this.memToMemMergeOutputsThreshold =
|
||||
jobConf.getInt(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, ioSortFactor);
|
||||
this.mergeThreshold = (long)(this.memoryLimit *
|
||||
jobConf.getFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT,
|
||||
0.90f));
|
||||
jobConf.getFloat(
|
||||
MRJobConfig.SHUFFLE_MERGE_PERCENT,
|
||||
MRJobConfig.DEFAULT_SHUFFLE_MERGE_PERCENT));
|
||||
LOG.info("MergerManager: memoryLimit=" + memoryLimit + ", " +
|
||||
"maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " +
|
||||
"mergeThreshold=" + mergeThreshold + ", " +
|
||||
|
|
Loading…
Reference in New Issue