Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge -r 1227237:1227238 from trunk to branch. FIXES: MAPREDUCE-1744 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1227789 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2012-01-05 19:50:43 +00:00
parent 3af44a310d
commit a7f8694d59
4 changed files with 36 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -357,6 +357,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3615. Fix some ant test failures. (Thomas Graves via sseth) MAPREDUCE-3615. Fix some ant test failures. (Thomas Graves via sseth)
MAPREDUCE-1744. DistributedCache creates its own FileSytem instance when
adding a file/archive to the path. (Dick King via tucu)
Release 0.23.0 - 2011-11-01 Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
View File

@ -1030,7 +1030,7 @@ public class Job extends JobContextImpl implements JobContext {
public void addFileToClassPath(Path file) public void addFileToClassPath(Path file)
throws IOException { throws IOException {
ensureState(JobState.DEFINE); ensureState(JobState.DEFINE);
DistributedCache.addFileToClassPath(file, conf); DistributedCache.addFileToClassPath(file, conf, file.getFileSystem(conf));
} }
/** /**
@ -1045,7 +1045,7 @@ public class Job extends JobContextImpl implements JobContext {
public void addArchiveToClassPath(Path archive) public void addArchiveToClassPath(Path archive)
throws IOException { throws IOException {
ensureState(JobState.DEFINE); ensureState(JobState.DEFINE);
DistributedCache.addArchiveToClassPath(archive, conf); DistributedCache.addArchiveToClassPath(archive, conf, archive.getFileSystem(conf));
} }
/** /**

32
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java
View File

@ -276,13 +276,26 @@ public class DistributedCache {
*/ */
@Deprecated @Deprecated
public static void addFileToClassPath(Path file, Configuration conf) public static void addFileToClassPath(Path file, Configuration conf)
throws IOException {
addFileToClassPath(file, conf, file.getFileSystem(conf));
}
/**
* Add a file path to the current set of classpath entries. It adds the file
* to cache as well. Intended to be used by user code.
*
* @param file Path of the file to be added
* @param conf Configuration that contains the classpath setting
* @param fs FileSystem with respect to which {@code archivefile} should
* be interpreted.
*/
public static void addFileToClassPath
(Path file, Configuration conf, FileSystem fs)
throws IOException { throws IOException {
String classpath = conf.get(MRJobConfig.CLASSPATH_FILES); String classpath = conf.get(MRJobConfig.CLASSPATH_FILES);
conf.set(MRJobConfig.CLASSPATH_FILES, classpath == null ? file.toString() conf.set(MRJobConfig.CLASSPATH_FILES, classpath == null ? file.toString()
: classpath + "," + file.toString()); : classpath + "," + file.toString());
FileSystem fs = FileSystem.get(conf);
URI uri = fs.makeQualified(file).toUri(); URI uri = fs.makeQualified(file).toUri();
addCacheFile(uri, conf); addCacheFile(uri, conf);
} }
@ -318,10 +331,23 @@ public class DistributedCache {
@Deprecated @Deprecated
public static void addArchiveToClassPath(Path archive, Configuration conf) public static void addArchiveToClassPath(Path archive, Configuration conf)
throws IOException { throws IOException {
addArchiveToClassPath(archive, conf, archive.getFileSystem(conf));
}
/**
* Add an archive path to the current set of classpath entries. It adds the
* archive to cache as well. Intended to be used by user code.
*
* @param archive Path of the archive to be added
* @param conf Configuration that contains the classpath setting
* @param fs FileSystem with respect to which {@code archive} should be interpreted.
*/
public static void addArchiveToClassPath
(Path archive, Configuration conf, FileSystem fs)
throws IOException {
String classpath = conf.get(MRJobConfig.CLASSPATH_ARCHIVES); String classpath = conf.get(MRJobConfig.CLASSPATH_ARCHIVES);
conf.set(MRJobConfig.CLASSPATH_ARCHIVES, classpath == null ? archive conf.set(MRJobConfig.CLASSPATH_ARCHIVES, classpath == null ? archive
.toString() : classpath + "," + archive.toString()); .toString() : classpath + "," + archive.toString());
FileSystem fs = FileSystem.get(conf);
URI uri = fs.makeQualified(archive).toUri(); URI uri = fs.makeQualified(archive).toUri();
addCacheArchive(uri, conf); addCacheArchive(uri, conf);

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java
View File

@ -196,7 +196,7 @@ public class TestMROldApiJobs {
file.close(); file.close();
} }
DistributedCache.addFileToClassPath(TestMRJobs.APP_JAR, conf); DistributedCache.addFileToClassPath(TestMRJobs.APP_JAR, conf, fs);
conf.setOutputCommitter(CustomOutputCommitter.class); conf.setOutputCommitter(CustomOutputCommitter.class);
conf.setInputFormat(TextInputFormat.class); conf.setInputFormat(TextInputFormat.class);
conf.setOutputKeyClass(LongWritable.class); conf.setOutputKeyClass(LongWritable.class);