diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b6c42e1f03f..c35047306ac 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -164,6 +164,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4151. RM scheduler web page should filter apps to those that are relevant to scheduling (Jason Lowe via tgraves) + MAPREDUCE-4134. Remove references of mapred.child.ulimit etc. since they + are not being used any more (Ravi Prakash via bobby) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java index 4d6787918f9..fde88dbe09c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -187,9 +187,6 @@ public class JobConf extends Configuration { * /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: * -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc * - * The configuration variable {@link #MAPRED_TASK_ULIMIT} can be used to - * control the maximum virtual memory of the child processes. - * * The configuration variable {@link #MAPRED_TASK_ENV} can be used to pass * other environment variables to the child processes. * @@ -210,9 +207,6 @@ public class JobConf extends Configuration { * /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: * -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc * - * The configuration variable {@link #MAPRED_MAP_TASK_ULIMIT} can be used to - * control the maximum virtual memory of the map processes. - * * The configuration variable {@link #MAPRED_MAP_TASK_ENV} can be used to pass * other environment variables to the map processes. */ @@ -230,9 +224,6 @@ public class JobConf extends Configuration { * /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: * -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc * - * The configuration variable {@link #MAPRED_REDUCE_TASK_ULIMIT} can be used - * to control the maximum virtual memory of the reduce processes. - * * The configuration variable {@link #MAPRED_REDUCE_TASK_ENV} can be used to * pass process environment variables to the reduce processes. */ @@ -242,36 +233,32 @@ public class JobConf extends Configuration { public static final String DEFAULT_MAPRED_TASK_JAVA_OPTS = "-Xmx200m"; /** - * Configuration key to set the maximum virutal memory available to the child - * map and reduce tasks (in kilo-bytes). - * - * Note: This must be greater than or equal to the -Xmx passed to the JavaVM - * via {@link #MAPRED_TASK_JAVA_OPTS}, else the VM might not start. - * - * @deprecated Use {@link #MAPRED_MAP_TASK_ULIMIT} or - * {@link #MAPRED_REDUCE_TASK_ULIMIT} + * @deprecated + * Configuration key to set the maximum virtual memory available to the child + * map and reduce tasks (in kilo-bytes). This has been deprecated and will no + * longer have any effect. */ @Deprecated public static final String MAPRED_TASK_ULIMIT = "mapred.child.ulimit"; /** - * Configuration key to set the maximum virutal memory available to the - * map tasks (in kilo-bytes). - * - * Note: This must be greater than or equal to the -Xmx passed to the JavaVM - * via {@link #MAPRED_MAP_TASK_JAVA_OPTS}, else the VM might not start. + * @deprecated + * Configuration key to set the maximum virtual memory available to the + * map tasks (in kilo-bytes). This has been deprecated and will no + * longer have any effect. */ - public static final String MAPRED_MAP_TASK_ULIMIT = JobContext.MAP_ULIMIT; + @Deprecated + public static final String MAPRED_MAP_TASK_ULIMIT = "mapreduce.map.ulimit"; /** - * Configuration key to set the maximum virutal memory available to the - * reduce tasks (in kilo-bytes). - * - * Note: This must be greater than or equal to the -Xmx passed to the JavaVM - * via {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}, else the VM might not start. + * @deprecated + * Configuration key to set the maximum virtual memory available to the + * reduce tasks (in kilo-bytes). This has been deprecated and will no + * longer have any effect. */ - public static final String MAPRED_REDUCE_TASK_ULIMIT = - JobContext.REDUCE_ULIMIT; + @Deprecated + public static final String MAPRED_REDUCE_TASK_ULIMIT = + "mapreduce.reduce.ulimit"; /** @@ -1966,6 +1953,15 @@ public class JobConf extends Configuration { + " Instead use " + JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY + " and " + JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY); } + if(get(JobConf.MAPRED_TASK_ULIMIT) != null ) { + LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_TASK_ULIMIT)); + } + if(get(JobConf.MAPRED_MAP_TASK_ULIMIT) != null ) { + LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_MAP_TASK_ULIMIT)); + } + if(get(JobConf.MAPRED_REDUCE_TASK_ULIMIT) != null ) { + LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_REDUCE_TASK_ULIMIT)); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 339deef3506..50b6eed7ce3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -183,8 +183,6 @@ public interface MRJobConfig { public static final String MAP_JAVA_OPTS = "mapreduce.map.java.opts"; - public static final String MAP_ULIMIT = "mapreduce.map.ulimit"; - public static final String MAP_MAX_ATTEMPTS = "mapreduce.map.maxattempts"; public static final String MAP_DEBUG_SCRIPT = "mapreduce.map.debug.script"; @@ -243,8 +241,6 @@ public interface MRJobConfig { public static final String REDUCE_JAVA_OPTS = "mapreduce.reduce.java.opts"; - public static final String REDUCE_ULIMIT = "mapreduce.reduce.ulimit"; - public static final String MAPREDUCE_JOB_DIR = "mapreduce.job.dir"; public static final String REDUCE_MAX_ATTEMPTS = "mapreduce.reduce.maxattempts"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java index 438be6598fa..fde6aa4647f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java @@ -314,8 +314,6 @@ public class ConfigUtil { new String[] {MRJobConfig.MAP_ENV}); Configuration.addDeprecation("mapred.map.child.java.opts", new String[] {MRJobConfig.MAP_JAVA_OPTS}); - Configuration.addDeprecation("mapred.map.child.ulimit", - new String[] {MRJobConfig.MAP_ULIMIT}); Configuration.addDeprecation("mapred.map.max.attempts", new String[] {MRJobConfig.MAP_MAX_ATTEMPTS}); Configuration.addDeprecation("mapred.map.task.debug.script", @@ -362,8 +360,6 @@ public class ConfigUtil { new String[] {MRJobConfig.REDUCE_ENV}); Configuration.addDeprecation("mapred.reduce.child.java.opts", new String[] {MRJobConfig.REDUCE_JAVA_OPTS}); - Configuration.addDeprecation("mapred.reduce.child.ulimit", - new String[] {MRJobConfig.REDUCE_ULIMIT}); Configuration.addDeprecation("mapred.reduce.max.attempts", new String[] {MRJobConfig.REDUCE_MAX_ATTEMPTS}); Configuration.addDeprecation("mapred.reduce.parallel.copies", diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index a2bfa8080f8..5c533e6f26e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -411,9 +411,6 @@ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc - The configuration variable mapred.child.ulimit can be used to control the - maximum virtual memory of the child processes. - Usage of -Djava.library.path can cause programs to no longer function if hadoop native libraries are used. These values should instead be set as part of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and @@ -431,20 +428,6 @@ - - mapred.child.ulimit - - The maximum virtual memory, in KB, of a process launched by the - Map-Reduce framework. This can be used to control both the Mapper/Reducer - tasks and applications using Hadoop Pipes, Hadoop Streaming etc. - By default it is left unspecified to let cluster admins control it via - limits.conf and other such relevant mechanisms. - - Note: mapred.child.ulimit must be greater than or equal to the -Xmx passed to - JavaVM, else the VM might not start. - - - mapreduce.admin.user.env LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native diff --git a/hadoop-mapreduce-project/src/java/mapred-default.xml b/hadoop-mapreduce-project/src/java/mapred-default.xml index 0d0a91d7787..322e497ada4 100644 --- a/hadoop-mapreduce-project/src/java/mapred-default.xml +++ b/hadoop-mapreduce-project/src/java/mapred-default.xml @@ -403,9 +403,6 @@ For example, to enable verbose gc logging to a file named for the taskid in /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc - - The configuration variable mapred.child.ulimit can be used to control the - maximum virtual memory of the child processes. @@ -419,20 +416,6 @@ - - mapred.child.ulimit - - The maximum virtual memory, in KB, of a process launched by the - Map-Reduce framework. This can be used to control both the Mapper/Reducer - tasks and applications using Hadoop Pipes, Hadoop Streaming etc. - By default it is left unspecified to let cluster admins control it via - limits.conf and other such relevant mechanisms. - - Note: mapred.child.ulimit must be greater than or equal to the -Xmx passed to - JavaVM, else the VM might not start. - - - mapreduce.map.log.level INFO diff --git a/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java b/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java index 286961ef7e0..20f1f9484ab 100644 --- a/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java +++ b/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java @@ -33,12 +33,6 @@ class MapTaskRunner extends TaskRunner { super.getChildJavaOpts(jobConf, JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS)); } - - @Override - public int getChildUlimit(JobConf jobConf) { - return jobConf.getInt(JobConf.MAPRED_MAP_TASK_ULIMIT, - super.getChildUlimit(jobConf)); - } @Override public String getChildEnv(JobConf jobConf) { diff --git a/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java b/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java index 2a69714477d..9f6b21e0a97 100644 --- a/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java +++ b/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java @@ -41,12 +41,6 @@ class ReduceTaskRunner extends TaskRunner { super.getChildJavaOpts(jobConf, JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS)); } - - @Override - public int getChildUlimit(JobConf jobConf) { - return jobConf.getInt(JobConf.MAPRED_REDUCE_TASK_ULIMIT, - super.getChildUlimit(jobConf)); - } @Override public String getChildEnv(JobConf jobConf) { diff --git a/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java b/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java index 2a300eedf89..deffe71ea3a 100644 --- a/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java +++ b/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java @@ -110,20 +110,7 @@ abstract class TaskRunner extends Thread { public String getChildJavaOpts(JobConf jobConf, String defaultValue) { return jobConf.get(JobConf.MAPRED_TASK_JAVA_OPTS, defaultValue); } - - /** - * Get the maximum virtual memory of the child map/reduce tasks. - * @param jobConf job configuration - * @return the maximum virtual memory of the child task or -1 if - * none is specified - * @deprecated Use limits specific to the map or reduce tasks set via - * {@link JobConf#MAPRED_MAP_TASK_ULIMIT} or - * {@link JobConf#MAPRED_REDUCE_TASK_ULIMIT} - */ - @Deprecated - public int getChildUlimit(JobConf jobConf) { - return jobConf.getInt(JobConf.MAPRED_TASK_ULIMIT, -1); - } + /** * Get the environment variables for the child map/reduce tasks. @@ -188,8 +175,7 @@ abstract class TaskRunner extends Thread { tracker.addToMemoryManager(t.getTaskID(), t.isMapTask(), conf); - // set memory limit using ulimit if feasible and necessary ... - List setup = getVMSetupCmd(); + List setup = new ArrayList(); // Set up the redirection of the task's stdout and stderr streams File[] logFiles = prepareLogFiles(taskid, t.isTaskCleanupTask()); @@ -310,26 +296,6 @@ abstract class TaskRunner extends Thread { t.setJobFile(localTaskFile.toString()); } - /** - * @return - */ - private List getVMSetupCmd() { - - int ulimit = getChildUlimit(conf); - if (ulimit <= 0) { - return null; - } - List setup = null; - String[] ulimitCmd = Shell.getUlimitMemoryCommand(ulimit); - if (ulimitCmd != null) { - setup = new ArrayList(); - for (String arg : ulimitCmd) { - setup.add(arg); - } - } - return setup; - } - /** * Parse the given string and return an array of individual java opts. Split * on whitespace and replace the special string "@taskid@" with the task ID