diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 1588c865550..129e63cd5ac 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -1364,6 +1364,8 @@ Release 0.23.0 - Unreleased MAPREDUCE-3066. Fixed default ResourceTracker address for the NodeManager. (Chris Riccomini via acmurthy) + MAPREDUCE-3044. Pipes jobs stuck without making progress. (mahadev) + Release 0.22.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java index fc25d06da24..4f2941a78cf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java @@ -31,6 +31,8 @@ import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; public class MapReduceChildJVM { @@ -164,8 +166,8 @@ private static String getChildJavaOpts(JobConf jobConf, boolean isMapTask) { private static void setupLog4jProperties(Vector vargs, long logSize) { vargs.add("-Dlog4j.configuration=container-log4j.properties"); - vargs.add("-Dhadoop.yarn.mr.containerLogDir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); - vargs.add("-Dhadoop.yarn.mr.totalLogFileSize=" + logSize); + vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); + vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize); } public static List getVMCommand( diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java index 7e978e9cf94..597b2edaa39 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java @@ -44,6 +44,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SecureIOUtils; import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.util.ProcessTree; import org.apache.hadoop.util.Shell; import org.apache.log4j.Appender; @@ -75,10 +76,18 @@ public class TaskLog { } } } - + + public static String getMRv2LogDir() { + return System.getProperty(MRJobConfig.TASK_LOG_DIR); + } + public static File getTaskLogFile(TaskAttemptID taskid, boolean isCleanup, LogName filter) { - return new File(getAttemptDir(taskid, isCleanup), filter.toString()); + if (getMRv2LogDir() != null) { + return new File(getMRv2LogDir(), filter.toString()); + } else { + return new File(getAttemptDir(taskid, isCleanup), filter.toString()); + } } static File getRealTaskLogFileLocation(TaskAttemptID taskid, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java index cb8b476ac75..0a108d73b63 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java @@ -18,6 +18,7 @@ package org.apache.hadoop.mapred.pipes; +import java.io.BufferedInputStream; import java.io.File; import java.io.IOException; import java.net.ServerSocket; @@ -26,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Random; import javax.crypto.SecretKey; @@ -111,7 +113,6 @@ class Application