Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-5385. Fixed a bug with JobContext getCacheFiles API. Contributed by Omkar Vinit Joshi. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1508595 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-07-30 20:04:37 +00:00
parent 52694344fd
commit 1d915238a6
4 changed files with 22 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -197,6 +197,9 @@ Release 2.1.1-beta - UNRELEASED
BUG FIXES BUG FIXES
MAPREDUCE-5385. Fixed a bug with JobContext getCacheFiles API. (Omkar Vinit
Joshi via vinodkv)
Release 2.1.0-beta - 2013-08-06 Release 2.1.0-beta - 2013-08-06
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

16
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
View File

@ -89,22 +89,26 @@ public static class DistributedCacheChecker extends
@Override @Override
public void setup(Context context) throws IOException { public void setup(Context context) throws IOException {
Configuration conf = context.getConfiguration(); Configuration conf = context.getConfiguration();
Path[] files = context.getLocalCacheFiles(); Path[] localFiles = context.getLocalCacheFiles();
Path[] archives = context.getLocalCacheArchives(); URI[] files = context.getCacheFiles();
Path[] localArchives = context.getLocalCacheArchives();
URI[] archives = context.getCacheArchives();
FileSystem fs = LocalFileSystem.get(conf); FileSystem fs = LocalFileSystem.get(conf);
// Check that 2 files and 2 archives are present // Check that 2 files and 2 archives are present
TestCase.assertEquals(2, localFiles.length);
TestCase.assertEquals(2, localArchives.length);
TestCase.assertEquals(2, files.length); TestCase.assertEquals(2, files.length);
TestCase.assertEquals(2, archives.length); TestCase.assertEquals(2, archives.length);
// Check lengths of the files // Check lengths of the files
TestCase.assertEquals(1, fs.getFileStatus(files[0]).getLen()); TestCase.assertEquals(1, fs.getFileStatus(localFiles[0]).getLen());
TestCase.assertTrue(fs.getFileStatus(files[1]).getLen() > 1); TestCase.assertTrue(fs.getFileStatus(localFiles[1]).getLen() > 1);
// Check extraction of the archive // Check extraction of the archive
TestCase.assertTrue(fs.exists(new Path(archives[0], TestCase.assertTrue(fs.exists(new Path(localArchives[0],
"distributed.jar.inside3"))); "distributed.jar.inside3")));
TestCase.assertTrue(fs.exists(new Path(archives[1], TestCase.assertTrue(fs.exists(new Path(localArchives[1],
"distributed.jar.inside4"))); "distributed.jar.inside4")));
// Check the class loaders // Check the class loaders

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java
View File

@ -144,7 +144,7 @@ public URI[] getCacheArchives() throws IOException {
@Override @Override
public URI[] getCacheFiles() throws IOException { public URI[] getCacheFiles() throws IOException {
return mapContext.getCacheArchives(); return mapContext.getCacheFiles();
} }
@Override @Override

12
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
View File

@ -422,16 +422,20 @@ public static class DistributedCacheChecker extends
@Override @Override
public void setup(Context context) throws IOException { public void setup(Context context) throws IOException {
Configuration conf = context.getConfiguration(); Configuration conf = context.getConfiguration();
Path[] files = context.getLocalCacheFiles(); Path[] localFiles = context.getLocalCacheFiles();
Path[] archives = context.getLocalCacheArchives(); URI[] files = context.getCacheFiles();
Path[] localArchives = context.getLocalCacheArchives();
URI[] archives = context.getCacheArchives();
// Check that 4 (2 + appjar + DistrubutedCacheChecker jar) files // Check that 4 (2 + appjar + DistrubutedCacheChecker jar) files
// and 2 archives are present // and 2 archives are present
Assert.assertEquals(4, localFiles.length);
Assert.assertEquals(4, files.length); Assert.assertEquals(4, files.length);
Assert.assertEquals(2, localArchives.length);
Assert.assertEquals(2, archives.length); Assert.assertEquals(2, archives.length);
// Check lengths of the files // Check lengths of the files
Map<String, Path> filesMap = pathsToMap(files); Map<String, Path> filesMap = pathsToMap(localFiles);
Assert.assertTrue(filesMap.containsKey("distributed.first.symlink")); Assert.assertTrue(filesMap.containsKey("distributed.first.symlink"));
Assert.assertEquals(1, localFs.getFileStatus( Assert.assertEquals(1, localFs.getFileStatus(
filesMap.get("distributed.first.symlink")).getLen()); filesMap.get("distributed.first.symlink")).getLen());
@ -440,7 +444,7 @@ public void setup(Context context) throws IOException {
filesMap.get("distributed.second.jar")).getLen() > 1); filesMap.get("distributed.second.jar")).getLen() > 1);
// Check extraction of the archive // Check extraction of the archive
Map<String, Path> archivesMap = pathsToMap(archives); Map<String, Path> archivesMap = pathsToMap(localArchives);
Assert.assertTrue(archivesMap.containsKey("distributed.third.jar")); Assert.assertTrue(archivesMap.containsKey("distributed.third.jar"));
Assert.assertTrue(localFs.exists(new Path( Assert.assertTrue(localFs.exists(new Path(
archivesMap.get("distributed.third.jar"), "distributed.jar.inside3"))); archivesMap.get("distributed.third.jar"), "distributed.jar.inside3")));