From b6d9e6b5f7622dfde98158a0588c2550e4dbe3d1 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 7 Mar 2012 01:28:34 +0000 Subject: [PATCH] svn merge -c 1297825 from trunk to branch-0.23 FIXES: MAPREDUCE-3975. Default value not set for Configuration parameter mapreduce.job.local.dir (Eric Payne via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1297826 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/mapred/YarnChild.java | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 21d642d699f..4f7aa278fff 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -206,6 +206,9 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-3977. LogAggregationService leaks log aggregator objects (Jason Lowe via bobby) + MAPREDUCE-3975. Default value not set for Configuration parameter + mapreduce.job.local.dir (Eric Payne via bobby) + Release 0.23.1 - 2012-02-17 NEW FEATURES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java index 4eb12204f93..0687ab6cbd6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java @@ -35,6 +35,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSError; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; @@ -227,12 +228,21 @@ class YarnChild { /** * Configure mapred-local dirs. This config is used by the task for finding * out an output directory. + * @throws IOException */ - private static void configureLocalDirs(Task task, JobConf job) { + private static void configureLocalDirs(Task task, JobConf job) throws IOException { String[] localSysDirs = StringUtils.getTrimmedStrings( System.getenv(ApplicationConstants.LOCAL_DIR_ENV)); job.setStrings(MRConfig.LOCAL_DIR, localSysDirs); LOG.info(MRConfig.LOCAL_DIR + " for child: " + job.get(MRConfig.LOCAL_DIR)); + LocalDirAllocator lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR); + Path workDir = lDirAlloc.getLocalPathForWrite("work", job); + FileSystem lfs = FileSystem.getLocal(job).getRaw(); + if (!lfs.mkdirs(workDir)) { + throw new IOException("Mkdirs failed to create " + + workDir.toString()); + } + job.set(MRJobConfig.JOB_LOCAL_DIR,workDir.toString()); } private static JobConf configureTask(Task task, Credentials credentials,