MAPREDUCE-1744. DistributedCache creates its own FileSytem instance when adding a file/archive to the path. (Dick King via tucu)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1227238 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
55e94dc5ef
commit
c27de4bd0b
|
@ -392,6 +392,9 @@ Release 0.23.1 - Unreleased
|
|||
MAPREDUCE-3490. Fixed MapReduce AM to count failed maps also towards Reduce
|
||||
ramp up. (Sharad Agarwal and Arun C Murthy via vinodkv)
|
||||
|
||||
MAPREDUCE-1744. DistributedCache creates its own FileSytem instance when
|
||||
adding a file/archive to the path. (Dick King via tucu)
|
||||
|
||||
Release 0.23.0 - 2011-11-01
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
@ -1030,7 +1030,7 @@ public class Job extends JobContextImpl implements JobContext {
|
|||
public void addFileToClassPath(Path file)
|
||||
throws IOException {
|
||||
ensureState(JobState.DEFINE);
|
||||
DistributedCache.addFileToClassPath(file, conf);
|
||||
DistributedCache.addFileToClassPath(file, conf, file.getFileSystem(conf));
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -1045,7 +1045,7 @@ public class Job extends JobContextImpl implements JobContext {
|
|||
public void addArchiveToClassPath(Path archive)
|
||||
throws IOException {
|
||||
ensureState(JobState.DEFINE);
|
||||
DistributedCache.addArchiveToClassPath(archive, conf);
|
||||
DistributedCache.addArchiveToClassPath(archive, conf, archive.getFileSystem(conf));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -269,7 +269,7 @@ public class DistributedCache {
|
|||
/**
|
||||
* Add an file path to the current set of classpath entries It adds the file
|
||||
* to cache as well. Intended to be used by user code.
|
||||
*
|
||||
*
|
||||
* @param file Path of the file to be added
|
||||
* @param conf Configuration that contains the classpath setting
|
||||
* @deprecated Use {@link Job#addFileToClassPath(Path)} instead
|
||||
|
@ -277,12 +277,25 @@ public class DistributedCache {
|
|||
@Deprecated
|
||||
public static void addFileToClassPath(Path file, Configuration conf)
|
||||
throws IOException {
|
||||
addFileToClassPath(file, conf, file.getFileSystem(conf));
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a file path to the current set of classpath entries. It adds the file
|
||||
* to cache as well. Intended to be used by user code.
|
||||
*
|
||||
* @param file Path of the file to be added
|
||||
* @param conf Configuration that contains the classpath setting
|
||||
* @param fs FileSystem with respect to which {@code archivefile} should
|
||||
* be interpreted.
|
||||
*/
|
||||
public static void addFileToClassPath
|
||||
(Path file, Configuration conf, FileSystem fs)
|
||||
throws IOException {
|
||||
String classpath = conf.get(MRJobConfig.CLASSPATH_FILES);
|
||||
conf.set(MRJobConfig.CLASSPATH_FILES, classpath == null ? file.toString()
|
||||
: classpath + "," + file.toString());
|
||||
FileSystem fs = FileSystem.get(conf);
|
||||
URI uri = fs.makeQualified(file).toUri();
|
||||
|
||||
addCacheFile(uri, conf);
|
||||
}
|
||||
|
||||
|
@ -318,10 +331,23 @@ public class DistributedCache {
|
|||
@Deprecated
|
||||
public static void addArchiveToClassPath(Path archive, Configuration conf)
|
||||
throws IOException {
|
||||
addArchiveToClassPath(archive, conf, archive.getFileSystem(conf));
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an archive path to the current set of classpath entries. It adds the
|
||||
* archive to cache as well. Intended to be used by user code.
|
||||
*
|
||||
* @param archive Path of the archive to be added
|
||||
* @param conf Configuration that contains the classpath setting
|
||||
* @param fs FileSystem with respect to which {@code archive} should be interpreted.
|
||||
*/
|
||||
public static void addArchiveToClassPath
|
||||
(Path archive, Configuration conf, FileSystem fs)
|
||||
throws IOException {
|
||||
String classpath = conf.get(MRJobConfig.CLASSPATH_ARCHIVES);
|
||||
conf.set(MRJobConfig.CLASSPATH_ARCHIVES, classpath == null ? archive
|
||||
.toString() : classpath + "," + archive.toString());
|
||||
FileSystem fs = FileSystem.get(conf);
|
||||
URI uri = fs.makeQualified(archive).toUri();
|
||||
|
||||
addCacheArchive(uri, conf);
|
||||
|
|
|
@ -196,7 +196,7 @@ public class TestMROldApiJobs {
|
|||
file.close();
|
||||
}
|
||||
|
||||
DistributedCache.addFileToClassPath(TestMRJobs.APP_JAR, conf);
|
||||
DistributedCache.addFileToClassPath(TestMRJobs.APP_JAR, conf, fs);
|
||||
conf.setOutputCommitter(CustomOutputCommitter.class);
|
||||
conf.setInputFormat(TextInputFormat.class);
|
||||
conf.setOutputKeyClass(LongWritable.class);
|
||||
|
|
Loading…
Reference in New Issue