From f047f9cac65e431fbccf1db9cc806565eb7fd7e6 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Tue, 2 Jul 2013 23:19:36 +0000 Subject: [PATCH] MAPREDUCE-5355. MiniMRYarnCluster with localFs does not work on Windows. Contributed by Chuan Liu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1499148 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 17ec69020ac..22c4fe34614 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -600,6 +600,9 @@ Release 2.1.0-beta - 2013-07-02 MAPREDUCE-5177. Use common utils FileUtil#setReadable/Writable/Executable & FileUtil#canRead/Write/Execute. (Ivan Mitic via suresh) + MAPREDUCE-5355. MiniMRYarnCluster with localFs does not work on Windows. + (Chuan Liu via cnauroth) + MAPREDUCE-5291. Change MR App to use updated property names in container-log4j.properties. (Zhijie Shen via sseth) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java index 93a91243736..6eb5789d89e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.LocalContainerLauncher; import org.apache.hadoop.mapred.ShuffleHandler; @@ -87,6 +88,21 @@ public class MiniMRYarnCluster extends MiniYARNCluster { try { Path stagingPath = FileContext.getFileContext(conf).makeQualified( new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR))); + /* + * Re-configure the staging path on Windows if the file system is localFs. + * We need to use a absolute path that contains the drive letter. The unit + * test could run on a different drive than the AM. We can run into the + * issue that job files are localized to the drive where the test runs on, + * while the AM starts on a different drive and fails to find the job + * metafiles. Using absolute path can avoid this ambiguity. + */ + if (Path.WINDOWS) { + if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) { + conf.set(MRJobConfig.MR_AM_STAGING_DIR, + new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR)) + .getAbsolutePath()); + } + } FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf); if (fc.util().exists(stagingPath)) { LOG.info(stagingPath + " exists! deleting...");