From 1047c883ba01a252dbd4203e525fb1ff8ea313aa Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Thu, 26 Feb 2015 14:24:19 -0800 Subject: [PATCH] MAPREDUCE-6223. TestJobConf#testNegativeValueForTaskVmem failures. (Varun Saxena via kasha) --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/mapred/JobConf.java | 6 ++--- .../org/apache/hadoop/conf/TestJobConf.java | 23 ++++++++++--------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d2c4ab8ecce..6cb17547397 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -390,6 +390,9 @@ Release 2.7.0 - UNRELEASED MAPREDUCE-2815. JavaDoc does not generate correctly for MultithreadedMapRunner. (Chris Palmer via aajisaka) + MAPREDUCE-6223. TestJobConf#testNegativeValueForTaskVmem failures. + (Varun Saxena via kasha) + Release 2.6.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java index 98a643f112d..315c8299872 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -1826,8 +1826,7 @@ public class JobConf extends Configuration { public long getMemoryForMapTask() { long value = getDeprecatedMemoryValue(); if (value < 0) { - return getLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, - JobContext.DEFAULT_MAP_MEMORY_MB); + return getMemoryRequired(TaskType.MAP); } return value; } @@ -1853,8 +1852,7 @@ public class JobConf extends Configuration { public long getMemoryForReduceTask() { long value = getDeprecatedMemoryValue(); if (value < 0) { - return getLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, - JobContext.DEFAULT_REDUCE_MEMORY_MB); + return getMemoryRequired(TaskType.REDUCE); } return value; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java index e380d9200cf..f67ba1fb16b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java @@ -19,9 +19,7 @@ package org.apache.hadoop.conf; import org.junit.Assert; import org.junit.Test; - import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobContext; import org.apache.hadoop.mapreduce.MRJobConfig; public class TestJobConf { @@ -87,8 +85,10 @@ public class TestJobConf { configuration.set("mapred.task.maxvmem" , String.valueOf(-1)); configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1"); configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1"); - Assert.assertEquals(configuration.getMemoryForMapTask(),-1); - Assert.assertEquals(configuration.getMemoryForReduceTask(),-1); + Assert.assertEquals(configuration.getMemoryForMapTask(), + MRJobConfig.DEFAULT_MAP_MEMORY_MB); + Assert.assertEquals(configuration.getMemoryForReduceTask(), + MRJobConfig.DEFAULT_REDUCE_MEMORY_MB); configuration = new JobConf(); configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024)); @@ -96,7 +96,6 @@ public class TestJobConf { configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "3"); Assert.assertEquals(configuration.getMemoryForMapTask(),2); Assert.assertEquals(configuration.getMemoryForReduceTask(),2); - } /** @@ -106,7 +105,7 @@ public class TestJobConf { @Test public void testNegativeValueForTaskVmem() { JobConf configuration = new JobConf(); - + configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-3"); Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB, configuration.getMemoryForMapTask()); @@ -129,10 +128,12 @@ public class TestJobConf { configuration.set(MRJobConfig.MAP_MEMORY_MB, "-5"); configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "-6"); - Assert.assertEquals(-5, configuration.getMemoryForMapTask()); - Assert.assertEquals(-6, configuration.getMemoryForReduceTask()); + Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB, + configuration.getMemoryForMapTask()); + Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB, + configuration.getMemoryForReduceTask()); } - + /** * Test deprecated accessor and mutator method for mapred.task.maxvmem */ @@ -144,13 +145,13 @@ public class TestJobConf { configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300)); configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1)); Assert.assertEquals( - configuration.getMaxVirtualMemoryForTask(), 300 * 1024 * 1024); + configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024); configuration = new JobConf(); configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1)); configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(200)); Assert.assertEquals( - configuration.getMaxVirtualMemoryForTask(), 200 * 1024 * 1024); + configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024); configuration = new JobConf(); configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));