Merge -c 1489236 from trunk to branch-2 to fix MAPREDUCE-5245. Added back constants to JobConf to fix incompatibilities. Contributed by Zhijie Shen.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1489237 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -353,6 +353,9 @@ Release 2.1.0-beta - UNRELEASED
     MAPREDUCE-5296. Fix backward incompatibility for JobControl by adding the
     omitted addJob. (Zhijie Shen via acmurthy) 
 
+    MAPREDUCE-5245. Added back constants to JobConf to fix incompatibilities.  
+    (Zhijie Shen via acmurthy) 
+
   BREAKDOWN OF HADOOP-8562 SUBTASKS
 
     MAPREDUCE-4739. Some MapReduce tests fail to find winutils.

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java

@@ -20,9 +20,6 @@ package org.apache.hadoop.mapred;
 
 
 import java.io.IOException;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.util.Enumeration;
 import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
@@ -122,8 +119,8 @@ public class JobConf extends Configuration {
   }
 
   /**
-   * @deprecated Use {@link #MAPRED_JOB_MAP_MEMORY_MB_PROPERTY} and
+   * @deprecated Use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY} and
-   * {@link #MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY}
+   * {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}
    */
   @Deprecated
   public static final String MAPRED_TASK_MAXVMEM_PROPERTY =
@@ -167,12 +164,28 @@ public class JobConf extends Configuration {
    */
   public static final String DEFAULT_QUEUE_NAME = "default";
 
-  static final String MAPRED_JOB_MAP_MEMORY_MB_PROPERTY = 
+  static final String MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY =
       JobContext.MAP_MEMORY_MB;
 
-  static final String MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY =
+  static final String MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY =
     JobContext.REDUCE_MEMORY_MB;
 
+  /**
+   * The variable is kept for M/R 1.x applications, while M/R 2.x applications
+   * should use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY}
+   */
+  @Deprecated
+  public static final String MAPRED_JOB_MAP_MEMORY_MB_PROPERTY =
+      "mapred.job.map.memory.mb";
+
+  /**
+   * The variable is kept for M/R 1.x applications, while M/R 2.x applications
+   * should use {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}
+   */
+  @Deprecated
+  public static final String MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY =
+      "mapred.job.reduce.memory.mb";
+
   /** Pattern for the default unpacking behavior for job jars */
   public static final Pattern UNPACK_JAR_PATTERN_DEFAULT =
     Pattern.compile("(?:classes/|lib/).*");
@@ -335,8 +348,67 @@ public class JobConf extends Configuration {
    * Default logging level for map/reduce tasks.
    */
   public static final Level DEFAULT_LOG_LEVEL = Level.INFO;
-  
+
-  
+  /**
+   * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+   * use {@link MRJobConfig#WORKFLOW_ID} instead
+   */
+  @Deprecated
+  public static final String WORKFLOW_ID = MRJobConfig.WORKFLOW_ID;
+
+  /**
+   * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+   * use {@link MRJobConfig#WORKFLOW_NAME} instead
+   */
+  @Deprecated
+  public static final String WORKFLOW_NAME = MRJobConfig.WORKFLOW_NAME;
+
+  /**
+   * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+   * use {@link MRJobConfig#WORKFLOW_NODE_NAME} instead
+   */
+  @Deprecated
+  public static final String WORKFLOW_NODE_NAME =
+      MRJobConfig.WORKFLOW_NODE_NAME;
+
+  /**
+   * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+   * use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_STRING} instead
+   */
+  @Deprecated
+  public static final String WORKFLOW_ADJACENCY_PREFIX_STRING =
+      MRJobConfig.WORKFLOW_ADJACENCY_PREFIX_STRING;
+
+  /**
+   * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+   * use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_PATTERN} instead
+   */
+  @Deprecated
+  public static final String WORKFLOW_ADJACENCY_PREFIX_PATTERN =
+      MRJobConfig.WORKFLOW_ADJACENCY_PREFIX_PATTERN;
+
+  /**
+   * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+   * use {@link MRJobConfig#WORKFLOW_TAGS} instead
+   */
+  @Deprecated
+  public static final String WORKFLOW_TAGS = MRJobConfig.WORKFLOW_TAGS;
+
+  /**
+   * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+   * not use it
+   */
+  @Deprecated
+  public static final String MAPREDUCE_RECOVER_JOB =
+      "mapreduce.job.restart.recover";
+
+  /**
+   * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+   * not use it
+   */
+  @Deprecated
+  public static final boolean DEFAULT_MAPREDUCE_RECOVER_JOB = true;
+
   /**
    * Construct a map/reduce job configuration.
    */
@@ -1695,6 +1767,12 @@ public class JobConf extends Configuration {
    */
   public long getMemoryForMapTask() {
     long value = getDeprecatedMemoryValue();
+    if (value == DISABLED_MEMORY_LIMIT) {
+      value = normalizeMemoryConfigValue(
+                getLong(JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY,
+                          DISABLED_MEMORY_LIMIT));
+    }
+    // In case that M/R 1.x applications use the old property name
     if (value == DISABLED_MEMORY_LIMIT) {
       value = normalizeMemoryConfigValue(
                 getLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY,
@@ -1704,6 +1782,8 @@ public class JobConf extends Configuration {
   }
 
   public void setMemoryForMapTask(long mem) {
+    setLong(JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, mem);
+    // In case that M/R 1.x applications use the old property name
     setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, mem);
   }
 
@@ -1722,6 +1802,12 @@ public class JobConf extends Configuration {
    */
   public long getMemoryForReduceTask() {
     long value = getDeprecatedMemoryValue();
+    if (value == DISABLED_MEMORY_LIMIT) {
+      value = normalizeMemoryConfigValue(
+                getLong(JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY,
+                        DISABLED_MEMORY_LIMIT));
+    }
+    // In case that M/R 1.x applications use the old property name
     if (value == DISABLED_MEMORY_LIMIT) {
       value = normalizeMemoryConfigValue(
                 getLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY,
@@ -1745,6 +1831,8 @@ public class JobConf extends Configuration {
   }
 
   public void setMemoryForReduceTask(long mem) {
+    setLong(JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, mem);
+    // In case that M/R 1.x applications use the old property name
     setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, mem);
   }
 
@@ -1927,8 +2015,8 @@ public class JobConf extends Configuration {
   private void checkAndWarnDeprecation() {
     if(get(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY) != null) {
       LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY)
-                + " Instead use " + JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY
+                + " Instead use " + JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY
-                + " and " + JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
+                + " and " + JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY);
     }
     if(get(JobConf.MAPRED_TASK_ULIMIT) != null ) {
       LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_TASK_ULIMIT));

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java

@@ -22,6 +22,7 @@ import java.util.regex.Pattern;
 import static org.junit.Assert.*;
 
 import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
 import org.junit.Test;
 
 /**
@@ -152,4 +153,36 @@ public class TestJobConf {
         JobConf.deprecatedString("key"));
 
   }
+
+  /**
+   * Ensure that M/R 1.x applications can get and set task virtual memory with
+   * old property names
+   */
+  @SuppressWarnings("deprecation")
+  @Test (timeout = 1000)
+  public void testDeprecatedPropertyNameForTaskVmem() {
+    JobConf configuration = new JobConf();
+
+    configuration.setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, 1024);
+    configuration.setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, 1024);
+    Assert.assertEquals(1024, configuration.getMemoryForMapTask());
+    Assert.assertEquals(1024, configuration.getMemoryForReduceTask());
+    // Make sure new property names aren't broken by the old ones
+    configuration.setLong(JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, 1025);
+    configuration.setLong(JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, 1025);
+    Assert.assertEquals(1025, configuration.getMemoryForMapTask());
+    Assert.assertEquals(1025, configuration.getMemoryForReduceTask());
+
+    configuration.setMemoryForMapTask(2048);
+    configuration.setMemoryForReduceTask(2048);
+    Assert.assertEquals(2048, configuration.getLong(
+        JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, -1));
+    Assert.assertEquals(2048, configuration.getLong(
+        JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, -1));
+    // Make sure new property names aren't broken by the old ones
+    Assert.assertEquals(2048, configuration.getLong(
+        JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, -1));
+    Assert.assertEquals(2048, configuration.getLong(
+        JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, -1));
+  }
 }