MAPREDUCE-6793. io.sort.factor code default and mapred-default.xml values inconsistent. Contributed by Gera Shegalov.
(cherry picked from commit 683e0c71fe
)
This commit is contained in:
parent
774b0cd845
commit
4b289d567e
|
@ -967,7 +967,8 @@ public class MapTask extends Task {
|
|||
//sanity checks
|
||||
final float spillper =
|
||||
job.getFloat(JobContext.MAP_SORT_SPILL_PERCENT, (float)0.8);
|
||||
final int sortmb = job.getInt(JobContext.IO_SORT_MB, 100);
|
||||
final int sortmb = job.getInt(MRJobConfig.IO_SORT_MB,
|
||||
MRJobConfig.DEFAULT_IO_SORT_MB);
|
||||
indexCacheMemoryLimit = job.getInt(JobContext.INDEX_CACHE_MEMORY_LIMIT,
|
||||
INDEX_CACHE_MEMORY_LIMIT_DEFAULT);
|
||||
if (spillper > (float)1.0 || spillper <= (float)0.0) {
|
||||
|
@ -1919,7 +1920,8 @@ public class MapTask extends Task {
|
|||
}
|
||||
}
|
||||
|
||||
int mergeFactor = job.getInt(JobContext.IO_SORT_FACTOR, 100);
|
||||
int mergeFactor = job.getInt(MRJobConfig.IO_SORT_FACTOR,
|
||||
MRJobConfig.DEFAULT_IO_SORT_FACTOR);
|
||||
// sort the segments only if there are intermediate merges
|
||||
boolean sortSegments = segmentList.size() > mergeFactor;
|
||||
//merge
|
||||
|
|
|
@ -212,8 +212,12 @@ public interface MRJobConfig {
|
|||
|
||||
public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor";
|
||||
|
||||
public static final int DEFAULT_IO_SORT_FACTOR = 10;
|
||||
|
||||
public static final String IO_SORT_MB = "mapreduce.task.io.sort.mb";
|
||||
|
||||
public static final int DEFAULT_IO_SORT_MB = 100;
|
||||
|
||||
public static final String INDEX_CACHE_MEMORY_LIMIT = "mapreduce.task.index.cache.limit.bytes";
|
||||
|
||||
public static final String PRESERVE_FAILED_TASK_FILES = "mapreduce.task.files.preserve.failedtasks";
|
||||
|
|
|
@ -175,7 +175,8 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> {
|
|||
MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES,
|
||||
Runtime.getRuntime().maxMemory()) * maxInMemCopyUse);
|
||||
|
||||
this.ioSortFactor = jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, 100);
|
||||
this.ioSortFactor = jobConf.getInt(MRJobConfig.IO_SORT_FACTOR,
|
||||
MRJobConfig.DEFAULT_IO_SORT_FACTOR);
|
||||
|
||||
final float singleShuffleMemoryLimitPercent =
|
||||
jobConf.getFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT,
|
||||
|
|
|
@ -207,6 +207,13 @@ public class TestMergeManager {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIoSortDefaults() {
|
||||
final JobConf jobConf = new JobConf();
|
||||
assertEquals(10, jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, 100));
|
||||
assertEquals(100, jobConf.getInt(MRJobConfig.IO_SORT_MB, 10));
|
||||
}
|
||||
|
||||
@SuppressWarnings({ "unchecked", "deprecation" })
|
||||
@Test(timeout=10000)
|
||||
public void testOnDiskMerger() throws IOException, URISyntaxException,
|
||||
|
|
Loading…
Reference in New Issue