HADOOP-10392. Use FileSystem#makeQualified(Path) instead of Path#makeQualified(FileSystem) (ajisakaa via aw)

This commit is contained in:
Allen Wittenauer 2017-08-11 09:25:56 -07:00
parent d964062f66
commit 4222c97108
No known key found for this signature in database
GPG Key ID: E01B34FBE846DF38
30 changed files with 78 additions and 75 deletions

View File

@ -295,8 +295,8 @@ public class FileUtil {
Path dst) Path dst)
throws IOException { throws IOException {
if (srcFS == dstFS) { if (srcFS == dstFS) {
String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR; String srcq = srcFS.makeQualified(src).toString() + Path.SEPARATOR;
String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR; String dstq = dstFS.makeQualified(dst).toString() + Path.SEPARATOR;
if (dstq.startsWith(srcq)) { if (dstq.startsWith(srcq)) {
if (srcq.length() == dstq.length()) { if (srcq.length() == dstq.length()) {
throw new IOException("Cannot copy " + src + " to itself."); throw new IOException("Cannot copy " + src + " to itself.");

View File

@ -505,7 +505,7 @@ public class FTPFileSystem extends FileSystem {
long modTime = -1; // Modification time of root dir not known. long modTime = -1; // Modification time of root dir not known.
Path root = new Path("/"); Path root = new Path("/");
return new FileStatus(length, isDir, blockReplication, blockSize, return new FileStatus(length, isDir, blockReplication, blockSize,
modTime, root.makeQualified(this)); modTime, this.makeQualified(root));
} }
String pathName = parentPath.toUri().getPath(); String pathName = parentPath.toUri().getPath();
FTPFile[] ftpFiles = client.listFiles(pathName); FTPFile[] ftpFiles = client.listFiles(pathName);
@ -546,7 +546,7 @@ public class FTPFileSystem extends FileSystem {
String group = ftpFile.getGroup(); String group = ftpFile.getGroup();
Path filePath = new Path(parentPath, ftpFile.getName()); Path filePath = new Path(parentPath, ftpFile.getName());
return new FileStatus(length, isDir, blockReplication, blockSize, modTime, return new FileStatus(length, isDir, blockReplication, blockSize, modTime,
accessTime, permission, user, group, filePath.makeQualified(this)); accessTime, permission, user, group, this.makeQualified(filePath));
} }
@Override @Override

View File

@ -1883,7 +1883,7 @@ public class SequenceFile {
@Deprecated @Deprecated
public Reader(FileSystem fs, Path file, public Reader(FileSystem fs, Path file,
Configuration conf) throws IOException { Configuration conf) throws IOException {
this(conf, file(file.makeQualified(fs))); this(conf, file(fs.makeQualified(file)));
} }
/** /**

View File

@ -218,8 +218,8 @@ public class TestLocalFileSystem {
@Test @Test
public void testHomeDirectory() throws IOException { public void testHomeDirectory() throws IOException {
Path home = new Path(System.getProperty("user.home")) Path home = fileSys.makeQualified(
.makeQualified(fileSys); new Path(System.getProperty("user.home")));
Path fsHome = fileSys.getHomeDirectory(); Path fsHome = fileSys.getHomeDirectory();
assertEquals(home, fsHome); assertEquals(home, fsHome);
} }
@ -229,7 +229,7 @@ public class TestLocalFileSystem {
Path path = new Path(TEST_ROOT_DIR, "foo%bar"); Path path = new Path(TEST_ROOT_DIR, "foo%bar");
writeFile(fileSys, path, 1); writeFile(fileSys, path, 1);
FileStatus status = fileSys.getFileStatus(path); FileStatus status = fileSys.getFileStatus(path);
assertEquals(path.makeQualified(fileSys), status.getPath()); assertEquals(fileSys.makeQualified(path), status.getPath());
cleanupFile(fileSys, path); cleanupFile(fileSys, path);
} }

View File

@ -170,7 +170,7 @@ public class FileBench extends Configured implements Tool {
for(int i = 0; i < argv.length; ++i) { for(int i = 0; i < argv.length; ++i) {
try { try {
if ("-dir".equals(argv[i])) { if ("-dir".equals(argv[i])) {
root = new Path(argv[++i]).makeQualified(fs); root = fs.makeQualified(new Path(argv[++i]));
System.out.println("DIR: " + root.toString()); System.out.println("DIR: " + root.toString());
} else if ("-seed".equals(argv[i])) { } else if ("-seed".equals(argv[i])) {
job.setLong("filebench.seed", Long.valueOf(argv[++i])); job.setLong("filebench.seed", Long.valueOf(argv[++i]));

View File

@ -50,8 +50,8 @@ public class MiniMRClientClusterFactory {
FileSystem fs = FileSystem.get(conf); FileSystem fs = FileSystem.get(conf);
Path testRootDir = new Path("target", identifier + "-tmpDir") Path testRootDir = fs.makeQualified(
.makeQualified(fs); new Path("target", identifier + "-tmpDir"));
Path appJar = new Path(testRootDir, "MRAppJar.jar"); Path appJar = new Path(testRootDir, "MRAppJar.jar");
// Copy MRAppJar and make it private. // Copy MRAppJar and make it private.

View File

@ -47,9 +47,9 @@ public class TestCombineFileInputFormat {
throw new RuntimeException("init failure", e); throw new RuntimeException("init failure", e);
} }
} }
private static Path workDir = private static Path workDir = localFs.makeQualified(new Path(
new Path(new Path(System.getProperty("test.build.data", "/tmp")), System.getProperty("test.build.data", "/tmp"),
"TestCombineFileInputFormat").makeQualified(localFs); "TestCombineFileInputFormat"));
private static void writeFile(FileSystem fs, Path name, private static void writeFile(FileSystem fs, Path name,
String contents) throws IOException { String contents) throws IOException {

View File

@ -53,10 +53,9 @@ public class TestCombineSequenceFileInputFormat {
} }
} }
@SuppressWarnings("deprecation") private static Path workDir = localFs.makeQualified(new Path(
private static Path workDir = System.getProperty("test.build.data", "/tmp"),
new Path(new Path(System.getProperty("test.build.data", "/tmp")), "TestCombineSequenceFileInputFormat"));
"TestCombineSequenceFileInputFormat").makeQualified(localFs);
@Test(timeout=10000) @Test(timeout=10000)
public void testFormat() throws Exception { public void testFormat() throws Exception {

View File

@ -60,10 +60,9 @@ public class TestCombineTextInputFormat {
} }
} }
@SuppressWarnings("deprecation") private static Path workDir = localFs.makeQualified(new Path(
private static Path workDir = System.getProperty("test.build.data", "/tmp"),
new Path(new Path(System.getProperty("test.build.data", "/tmp")), "TestCombineTextInputFormat"));
"TestCombineTextInputFormat").makeQualified(localFs);
// A reporter that does nothing // A reporter that does nothing
private static final Reporter voidReporter = Reporter.NULL; private static final Reporter voidReporter = Reporter.NULL;

View File

@ -84,9 +84,9 @@ public class TestConcatenatedCompressedInput {
public void after() { public void after() {
ZlibFactory.loadNativeZLib(); ZlibFactory.loadNativeZLib();
} }
private static Path workDir = private static Path workDir = localFs.makeQualified(new Path(
new Path(new Path(System.getProperty("test.build.data", "/tmp")), System.getProperty("test.build.data", "/tmp"),
"TestConcatenatedCompressedInput").makeQualified(localFs); "TestConcatenatedCompressedInput"));
private static LineReader makeStream(String str) throws IOException { private static LineReader makeStream(String str) throws IOException {
return new LineReader(new ByteArrayInputStream(str.getBytes("UTF-8")), return new LineReader(new ByteArrayInputStream(str.getBytes("UTF-8")),

View File

@ -342,8 +342,8 @@ public class TestMapRed extends Configured implements Tool {
values.add(m); values.add(m);
m = m.replace((char)('A' + i - 1), (char)('A' + i)); m = m.replace((char)('A' + i - 1), (char)('A' + i));
} }
Path testdir = new Path( Path testdir = fs.makeQualified(new Path(
System.getProperty("test.build.data","/tmp")).makeQualified(fs); System.getProperty("test.build.data","/tmp")));
fs.delete(testdir, true); fs.delete(testdir, true);
Path inFile = new Path(testdir, "nullin/blah"); Path inFile = new Path(testdir, "nullin/blah");
SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, inFile, SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, inFile,

View File

@ -75,8 +75,8 @@ public class TestMiniMRChildTask {
} }
} }
private static Path TEST_ROOT_DIR = new Path("target", private static Path TEST_ROOT_DIR = localFs.makeQualified(
TestMiniMRChildTask.class.getName() + "-tmpDir").makeQualified(localFs); new Path("target", TestMiniMRChildTask.class.getName() + "-tmpDir"));
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar"); static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
/** /**

View File

@ -61,10 +61,10 @@ public class TestTextInputFormat {
throw new RuntimeException("init failure", e); throw new RuntimeException("init failure", e);
} }
} }
@SuppressWarnings("deprecation")
private static Path workDir = private static Path workDir = localFs.makeQualified(new Path(
new Path(new Path(System.getProperty("test.build.data", "/tmp")), System.getProperty("test.build.data", "/tmp"),
"TestTextInputFormat").makeQualified(localFs); "TestTextInputFormat"));
@Test (timeout=500000) @Test (timeout=500000)
public void testFormat() throws Exception { public void testFormat() throws Exception {

View File

@ -50,8 +50,8 @@ public class TestWrappedRecordReaderClassloader {
assertTrue(job.getClassLoader() instanceof Fake_ClassLoader); assertTrue(job.getClassLoader() instanceof Fake_ClassLoader);
FileSystem fs = FileSystem.get(job); FileSystem fs = FileSystem.get(job);
Path testdir = new Path(System.getProperty("test.build.data", "/tmp")) Path testdir = fs.makeQualified(new Path(
.makeQualified(fs); System.getProperty("test.build.data", "/tmp")));
Path base = new Path(testdir, "/empty"); Path base = new Path(testdir, "/empty");
Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") }; Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") };

View File

@ -50,8 +50,8 @@ public class TestWrappedRRClassloader {
assertTrue(conf.getClassLoader() instanceof Fake_ClassLoader); assertTrue(conf.getClassLoader() instanceof Fake_ClassLoader);
FileSystem fs = FileSystem.get(conf); FileSystem fs = FileSystem.get(conf);
Path testdir = new Path(System.getProperty("test.build.data", "/tmp")) Path testdir = fs.makeQualified(new Path(
.makeQualified(fs); System.getProperty("test.build.data", "/tmp")));
Path base = new Path(testdir, "/empty"); Path base = new Path(testdir, "/empty");
Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") }; Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") };

View File

@ -330,7 +330,7 @@ public class MRAsyncDiskService {
* Returns the normalized path of a path. * Returns the normalized path of a path.
*/ */
private String normalizePath(String path) { private String normalizePath(String path) {
return (new Path(path)).makeQualified(this.localFileSystem) return this.localFileSystem.makeQualified(new Path(path))
.toUri().getPath(); .toUri().getPath();
} }

View File

@ -128,8 +128,8 @@ public class TestMRJobs {
} }
} }
private static Path TEST_ROOT_DIR = new Path("target", private static Path TEST_ROOT_DIR = localFs.makeQualified(
TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs); new Path("target", TestMRJobs.class.getName() + "-tmpDir"));
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar"); static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
private static final String OUTPUT_ROOT_DIR = "/tmp/" + private static final String OUTPUT_ROOT_DIR = "/tmp/" +
TestMRJobs.class.getSimpleName(); TestMRJobs.class.getSimpleName();

View File

@ -73,8 +73,8 @@ public class TestMRJobsWithHistoryService {
} }
} }
private static Path TEST_ROOT_DIR = new Path("target", private static Path TEST_ROOT_DIR = localFs.makeQualified(
TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs); new Path("target", TestMRJobs.class.getName() + "-tmpDir"));
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar"); static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
@Before @Before

View File

@ -473,7 +473,7 @@ public class HadoopArchives implements Tool {
conf.setLong(HAR_BLOCKSIZE_LABEL, blockSize); conf.setLong(HAR_BLOCKSIZE_LABEL, blockSize);
conf.setLong(HAR_PARTSIZE_LABEL, partSize); conf.setLong(HAR_PARTSIZE_LABEL, partSize);
conf.set(DST_HAR_LABEL, archiveName); conf.set(DST_HAR_LABEL, archiveName);
conf.set(SRC_PARENT_LABEL, parentPath.makeQualified(fs).toString()); conf.set(SRC_PARENT_LABEL, fs.makeQualified(parentPath).toString());
conf.setInt(HAR_REPLICATION_LABEL, repl); conf.setInt(HAR_REPLICATION_LABEL, repl);
Path outputPath = new Path(dest, archiveName); Path outputPath = new Path(dest, archiveName);
FileOutputFormat.setOutputPath(conf, outputPath); FileOutputFormat.setOutputPath(conf, outputPath);

View File

@ -447,7 +447,7 @@ public class Gridmix extends Configured implements Tool {
// Create <ioPath> with 777 permissions // Create <ioPath> with 777 permissions
final FileSystem inputFs = ioPath.getFileSystem(conf); final FileSystem inputFs = ioPath.getFileSystem(conf);
ioPath = ioPath.makeQualified(inputFs); ioPath = inputFs.makeQualified(ioPath);
boolean succeeded = false; boolean succeeded = false;
try { try {
succeeded = FileSystem.mkdirs(inputFs, ioPath, succeeded = FileSystem.mkdirs(inputFs, ioPath,

View File

@ -116,7 +116,7 @@ class PseudoLocalFs extends FileSystem {
* @throws FileNotFoundException * @throws FileNotFoundException
*/ */
long validateFileNameFormat(Path path) throws FileNotFoundException { long validateFileNameFormat(Path path) throws FileNotFoundException {
path = path.makeQualified(this); path = this.makeQualified(path);
boolean valid = true; boolean valid = true;
long fileSize = 0; long fileSize = 0;
if (!path.toUri().getScheme().equals(getUri().getScheme())) { if (!path.toUri().getScheme().equals(getUri().getScheme())) {
@ -329,4 +329,10 @@ class PseudoLocalFs extends FileSystem {
throw new UnsupportedOperationException("SetWorkingDirectory " throw new UnsupportedOperationException("SetWorkingDirectory "
+ "is not supported in pseudo local file system."); + "is not supported in pseudo local file system.");
} }
@Override
public Path makeQualified(Path path) {
// skip FileSystem#checkPath() to validate some other Filesystems
return path.makeQualified(this.getUri(), this.getWorkingDirectory());
}
} }

View File

@ -48,8 +48,8 @@ public class TestFilePool {
try { try {
final Configuration conf = new Configuration(); final Configuration conf = new Configuration();
final FileSystem fs = FileSystem.getLocal(conf).getRaw(); final FileSystem fs = FileSystem.getLocal(conf).getRaw();
return new Path(System.getProperty("test.build.data", "/tmp"), return fs.makeQualified(new Path(
"testFilePool").makeQualified(fs); System.getProperty("test.build.data", "/tmp"), "testFilePool"));
} catch (IOException e) { } catch (IOException e) {
fail(); fail();
} }

View File

@ -48,8 +48,8 @@ public class TestFileQueue {
public static void setup() throws IOException { public static void setup() throws IOException {
final Configuration conf = new Configuration(); final Configuration conf = new Configuration();
final FileSystem fs = FileSystem.getLocal(conf).getRaw(); final FileSystem fs = FileSystem.getLocal(conf).getRaw();
final Path p = new Path(System.getProperty("test.build.data", "/tmp"), final Path p = fs.makeQualified(new Path(
"testFileQueue").makeQualified(fs); System.getProperty("test.build.data", "/tmp"), "testFileQueue"));
fs.delete(p, true); fs.delete(p, true);
final byte[] b = new byte[BLOCK]; final byte[] b = new byte[BLOCK];
for (int i = 0; i < NFILES; ++i) { for (int i = 0; i < NFILES; ++i) {
@ -71,8 +71,8 @@ public class TestFileQueue {
public static void cleanup() throws IOException { public static void cleanup() throws IOException {
final Configuration conf = new Configuration(); final Configuration conf = new Configuration();
final FileSystem fs = FileSystem.getLocal(conf).getRaw(); final FileSystem fs = FileSystem.getLocal(conf).getRaw();
final Path p = new Path(System.getProperty("test.build.data", "/tmp"), final Path p = fs.makeQualified(new Path(
"testFileQueue").makeQualified(fs); System.getProperty("test.build.data", "/tmp"), "testFileQueue"));
fs.delete(p, true); fs.delete(p, true);
} }

View File

@ -224,7 +224,7 @@ public class TestPseudoLocalFs {
// Validate operations on valid qualified path // Validate operations on valid qualified path
path = new Path("myPsedoFile.1237"); path = new Path("myPsedoFile.1237");
path = path.makeQualified(pfs); path = pfs.makeQualified(path);
validateGetFileStatus(pfs, path, true); validateGetFileStatus(pfs, path, true);
validateCreate(pfs, path, true); validateCreate(pfs, path, true);
validateOpen(pfs, path, true); validateOpen(pfs, path, true);

View File

@ -40,8 +40,8 @@ public class TestUserResolve {
public static void createRootDir() throws IOException { public static void createRootDir() throws IOException {
conf = new Configuration(); conf = new Configuration();
fs = FileSystem.getLocal(conf); fs = FileSystem.getLocal(conf);
rootDir = new Path(new Path(System.getProperty("test.build.data", "/tmp")) rootDir = new Path(fs.makeQualified(new Path(
.makeQualified(fs), "gridmixUserResolve"); System.getProperty("test.build.data", "/tmp"))), "gridmixUserResolve");
} }
/** /**

View File

@ -278,7 +278,7 @@ public class SwiftTestUtils extends org.junit.Assert {
noteAction(action); noteAction(action);
try { try {
if (fileSystem != null) { if (fileSystem != null) {
fileSystem.delete(new Path(cleanupPath).makeQualified(fileSystem), fileSystem.delete(fileSystem.makeQualified(new Path(cleanupPath)),
true); true);
} }
} catch (Exception e) { } catch (Exception e) {

View File

@ -159,7 +159,7 @@ public class SwiftFileSystemBaseTest extends Assert implements
* @return a qualified path instance * @return a qualified path instance
*/ */
protected Path path(String pathString) { protected Path path(String pathString) {
return new Path(pathString).makeQualified(fs); return fs.makeQualified(new Path(pathString));
} }
/** /**

View File

@ -126,7 +126,7 @@ public class TestSwiftFileSystemPartitionedUploads extends
SwiftTestUtils.compareByteArrays(src, dest, len); SwiftTestUtils.compareByteArrays(src, dest, len);
FileStatus status; FileStatus status;
final Path qualifiedPath = path.makeQualified(fs); final Path qualifiedPath = fs.makeQualified(path);
status = fs.getFileStatus(qualifiedPath); status = fs.getFileStatus(qualifiedPath);
//now see what block location info comes back. //now see what block location info comes back.
//This will vary depending on the Swift version, so the results //This will vary depending on the Swift version, so the results
@ -216,7 +216,7 @@ public class TestSwiftFileSystemPartitionedUploads extends
private FileStatus validatePathLen(Path path, int len) throws IOException { private FileStatus validatePathLen(Path path, int len) throws IOException {
//verify that the length is what was written in a direct status check //verify that the length is what was written in a direct status check
final Path qualifiedPath = path.makeQualified(fs); final Path qualifiedPath = fs.makeQualified(path);
FileStatus[] parentDirListing = fs.listStatus(qualifiedPath.getParent()); FileStatus[] parentDirListing = fs.listStatus(qualifiedPath.getParent());
StringBuilder listing = lsToString(parentDirListing); StringBuilder listing = lsToString(parentDirListing);
String parentDirLS = listing.toString(); String parentDirLS = listing.toString();

View File

@ -57,8 +57,8 @@ public class TestHistograms {
public void testHistograms() throws IOException { public void testHistograms() throws IOException {
final Configuration conf = new Configuration(); final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf); final FileSystem lfs = FileSystem.getLocal(conf);
final Path rootInputDir = new Path( final Path rootInputDir = lfs.makeQualified(new Path(
System.getProperty("test.tools.input.dir", "")).makeQualified(lfs); System.getProperty("test.tools.input.dir", "target/input")));
final Path rootInputFile = new Path(rootInputDir, "rumen/histogram-tests"); final Path rootInputFile = new Path(rootInputDir, "rumen/histogram-tests");
@ -132,7 +132,7 @@ public class TestHistograms {
final FileSystem lfs = FileSystem.getLocal(conf); final FileSystem lfs = FileSystem.getLocal(conf);
for (String arg : args) { for (String arg : args) {
Path filePath = new Path(arg).makeQualified(lfs); Path filePath = lfs.makeQualified(new Path(arg));
String fileName = filePath.getName(); String fileName = filePath.getName();
if (fileName.startsWith("input")) { if (fileName.startsWith("input")) {
LoggedDiscreteCDF newResult = histogramFileToCDF(filePath, lfs); LoggedDiscreteCDF newResult = histogramFileToCDF(filePath, lfs);

View File

@ -22,13 +22,11 @@ import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.regex.Pattern;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.TreeSet; import java.util.TreeSet;
@ -41,12 +39,12 @@ import org.apache.commons.cli.Options;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileInputFormat;
@ -56,7 +54,6 @@ import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.KeyValueTextInputFormat; import org.apache.hadoop.mapred.KeyValueTextInputFormat;
import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.SequenceFileAsTextInputFormat; import org.apache.hadoop.mapred.SequenceFileAsTextInputFormat;
import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileInputFormat;
@ -65,6 +62,7 @@ import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.lib.LazyOutputFormat; import org.apache.hadoop.mapred.lib.LazyOutputFormat;
import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner; import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner;
import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer; import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.streaming.io.IdentifierResolver; import org.apache.hadoop.streaming.io.IdentifierResolver;
import org.apache.hadoop.streaming.io.InputWriter; import org.apache.hadoop.streaming.io.InputWriter;
import org.apache.hadoop.streaming.io.OutputReader; import org.apache.hadoop.streaming.io.OutputReader;
@ -297,7 +295,10 @@ public class StreamJob implements Tool {
try { try {
Path path = new Path(file); Path path = new Path(file);
FileSystem localFs = FileSystem.getLocal(config_); FileSystem localFs = FileSystem.getLocal(config_);
String finalPath = path.makeQualified(localFs).toString(); Path qualifiedPath = path.makeQualified(
localFs.getUri(), localFs.getWorkingDirectory());
validate(qualifiedPath);
String finalPath = qualifiedPath.toString();
if(fileList.length() > 0) { if(fileList.length() > 0) {
fileList.append(','); fileList.append(',');
} }
@ -313,7 +314,6 @@ public class StreamJob implements Tool {
tmpFiles = tmpFiles + "," + fileList; tmpFiles = tmpFiles + "," + fileList;
} }
config_.set("tmpfiles", tmpFiles); config_.set("tmpfiles", tmpFiles);
validate(packageFiles_);
} }
String fsName = cmdLine.getOptionValue("dfs"); String fsName = cmdLine.getOptionValue("dfs");
@ -391,14 +391,13 @@ public class StreamJob implements Tool {
return OptionBuilder.withDescription(desc).create(name); return OptionBuilder.withDescription(desc).create(name);
} }
private void validate(final List<String> values) private void validate(final Path path) throws IOException {
throws IllegalArgumentException { try {
for (String file : values) { path.getFileSystem(config_).access(path, FsAction.READ);
File f = new File(file); } catch (FileNotFoundException e) {
if (!FileUtil.canRead(f)) { fail("File: " + path + " does not exist.");
fail("File: " + f.getAbsolutePath() } catch (AccessControlException e) {
+ " does not exist, or is not readable."); fail("File: " + path + " is not readable.");
}
} }
} }