MAPREDUCE-6105. nconsistent configuration in property mapreduce.reduce.shuffle.merge.percent. Contributed by Ray Chiang.
This commit is contained in:
parent
ce5de93a58
commit
685dbafbe2
|
@ -253,6 +253,9 @@ Release 2.8.0 - UNRELEASED
|
||||||
|
|
||||||
IMPROVEMENTS
|
IMPROVEMENTS
|
||||||
|
|
||||||
|
MAPREDUCE-6105. Inconsistent configuration in property
|
||||||
|
mapreduce.reduce.shuffle.merge.percent. (Ray Chiang via harsh)
|
||||||
|
|
||||||
MAPREDUCE-4414. Add main methods to JobConf and YarnConfiguration,
|
MAPREDUCE-4414. Add main methods to JobConf and YarnConfiguration,
|
||||||
for debug purposes. (Plamen Jeliazkov via harsh)
|
for debug purposes. (Plamen Jeliazkov via harsh)
|
||||||
|
|
||||||
|
|
|
@ -305,6 +305,7 @@ public interface MRJobConfig {
|
||||||
= "mapreduce.reduce.shuffle.memory.limit.percent";
|
= "mapreduce.reduce.shuffle.memory.limit.percent";
|
||||||
|
|
||||||
public static final String SHUFFLE_MERGE_PERCENT = "mapreduce.reduce.shuffle.merge.percent";
|
public static final String SHUFFLE_MERGE_PERCENT = "mapreduce.reduce.shuffle.merge.percent";
|
||||||
|
public static final float DEFAULT_SHUFFLE_MERGE_PERCENT = 0.66f;
|
||||||
|
|
||||||
public static final String REDUCE_FAILURES_MAXPERCENT = "mapreduce.reduce.failures.maxpercent";
|
public static final String REDUCE_FAILURES_MAXPERCENT = "mapreduce.reduce.failures.maxpercent";
|
||||||
|
|
||||||
|
|
|
@ -191,8 +191,9 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> {
|
||||||
this.memToMemMergeOutputsThreshold =
|
this.memToMemMergeOutputsThreshold =
|
||||||
jobConf.getInt(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, ioSortFactor);
|
jobConf.getInt(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, ioSortFactor);
|
||||||
this.mergeThreshold = (long)(this.memoryLimit *
|
this.mergeThreshold = (long)(this.memoryLimit *
|
||||||
jobConf.getFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT,
|
jobConf.getFloat(
|
||||||
0.90f));
|
MRJobConfig.SHUFFLE_MERGE_PERCENT,
|
||||||
|
MRJobConfig.DEFAULT_SHUFFLE_MERGE_PERCENT));
|
||||||
LOG.info("MergerManager: memoryLimit=" + memoryLimit + ", " +
|
LOG.info("MergerManager: memoryLimit=" + memoryLimit + ", " +
|
||||||
"maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " +
|
"maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " +
|
||||||
"mergeThreshold=" + mergeThreshold + ", " +
|
"mergeThreshold=" + mergeThreshold + ", " +
|
||||||
|
|
Loading…
Reference in New Issue