From 2d904b3fa3bb406415f6e7b9807fcd5afa564f04 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Thu, 14 Aug 2014 18:07:06 +0000 Subject: [PATCH] MAPREDUCE-5906. Inconsistent configuration in property "mapreduce.reduce.shuffle.input.buffer.percent" (Akira AJISAKA via aw) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618007 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 1 + .../apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a44f2e41ada..7dc4436a887 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -92,6 +92,9 @@ Trunk (Unreleased) MAPREDUCE-4791. Javadoc for KeyValueTextInputFormat should include default separator and how to change it (Akira AJISAKA via aw) + MAPREDUCE-5906. Inconsistent configuration in property + "mapreduce.reduce.shuffle.input.buffer.percent" (Akira AJISAKA via aw) + BUG FIXES MAPREDUCE-5714. Removed forceful JVM exit in shutDownJob. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 4795af78d2a..0a586967d01 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -265,6 +265,7 @@ public interface MRJobConfig { public static final String REDUCE_MEMORY_TOTAL_BYTES = "mapreduce.reduce.memory.totalbytes"; public static final String SHUFFLE_INPUT_BUFFER_PERCENT = "mapreduce.reduce.shuffle.input.buffer.percent"; + public static final float DEFAULT_SHUFFLE_INPUT_BUFFER_PERCENT = 0.70f; public static final String SHUFFLE_MEMORY_LIMIT_PERCENT = "mapreduce.reduce.shuffle.memory.limit.percent"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java index 4d83995dcfd..49aa8576d60 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java @@ -156,7 +156,8 @@ public class MergeManagerImpl implements MergeManager { this.rfs = ((LocalFileSystem)localFS).getRaw(); final float maxInMemCopyUse = - jobConf.getFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 0.90f); + jobConf.getFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, + MRJobConfig.DEFAULT_SHUFFLE_INPUT_BUFFER_PERCENT); if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0) { throw new IllegalArgumentException("Invalid value for " + MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT + ": " +