From 1d915238a6a06d09e1789532994f00f496bd969c Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 30 Jul 2013 20:04:37 +0000 Subject: [PATCH] MAPREDUCE-5385. Fixed a bug with JobContext getCacheFiles API. Contributed by Omkar Vinit Joshi. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1508595 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../mapred/TestMRWithDistributedCache.java | 16 ++++++++++------ .../hadoop/mapreduce/lib/map/WrappedMapper.java | 2 +- .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 12 ++++++++---- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b90dbb62302..a174eb366cf 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -197,6 +197,9 @@ Release 2.1.1-beta - UNRELEASED BUG FIXES + MAPREDUCE-5385. Fixed a bug with JobContext getCacheFiles API. (Omkar Vinit + Joshi via vinodkv) + Release 2.1.0-beta - 2013-08-06 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java index ce40bd1fc1c..3bd7f052fc1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java @@ -89,22 +89,26 @@ public class TestMRWithDistributedCache extends TestCase { @Override public void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); - Path[] files = context.getLocalCacheFiles(); - Path[] archives = context.getLocalCacheArchives(); + Path[] localFiles = context.getLocalCacheFiles(); + URI[] files = context.getCacheFiles(); + Path[] localArchives = context.getLocalCacheArchives(); + URI[] archives = context.getCacheArchives(); FileSystem fs = LocalFileSystem.get(conf); // Check that 2 files and 2 archives are present + TestCase.assertEquals(2, localFiles.length); + TestCase.assertEquals(2, localArchives.length); TestCase.assertEquals(2, files.length); TestCase.assertEquals(2, archives.length); // Check lengths of the files - TestCase.assertEquals(1, fs.getFileStatus(files[0]).getLen()); - TestCase.assertTrue(fs.getFileStatus(files[1]).getLen() > 1); + TestCase.assertEquals(1, fs.getFileStatus(localFiles[0]).getLen()); + TestCase.assertTrue(fs.getFileStatus(localFiles[1]).getLen() > 1); // Check extraction of the archive - TestCase.assertTrue(fs.exists(new Path(archives[0], + TestCase.assertTrue(fs.exists(new Path(localArchives[0], "distributed.jar.inside3"))); - TestCase.assertTrue(fs.exists(new Path(archives[1], + TestCase.assertTrue(fs.exists(new Path(localArchives[1], "distributed.jar.inside4"))); // Check the class loaders diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java index 10761c18d26..95c4b90c0f9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java @@ -144,7 +144,7 @@ public class WrappedMapper @Override public URI[] getCacheFiles() throws IOException { - return mapContext.getCacheArchives(); + return mapContext.getCacheFiles(); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java index cb84d30510e..1ca1fd400b4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java @@ -422,16 +422,20 @@ public class TestMRJobs { @Override public void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); - Path[] files = context.getLocalCacheFiles(); - Path[] archives = context.getLocalCacheArchives(); + Path[] localFiles = context.getLocalCacheFiles(); + URI[] files = context.getCacheFiles(); + Path[] localArchives = context.getLocalCacheArchives(); + URI[] archives = context.getCacheArchives(); // Check that 4 (2 + appjar + DistrubutedCacheChecker jar) files // and 2 archives are present + Assert.assertEquals(4, localFiles.length); Assert.assertEquals(4, files.length); + Assert.assertEquals(2, localArchives.length); Assert.assertEquals(2, archives.length); // Check lengths of the files - Map filesMap = pathsToMap(files); + Map filesMap = pathsToMap(localFiles); Assert.assertTrue(filesMap.containsKey("distributed.first.symlink")); Assert.assertEquals(1, localFs.getFileStatus( filesMap.get("distributed.first.symlink")).getLen()); @@ -440,7 +444,7 @@ public class TestMRJobs { filesMap.get("distributed.second.jar")).getLen() > 1); // Check extraction of the archive - Map archivesMap = pathsToMap(archives); + Map archivesMap = pathsToMap(localArchives); Assert.assertTrue(archivesMap.containsKey("distributed.third.jar")); Assert.assertTrue(localFs.exists(new Path( archivesMap.get("distributed.third.jar"), "distributed.jar.inside3")));