Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

svn merge -c 1296386 trunk to branch-0.23 FIXES HADOOP-8131. FsShell put doesn't correctly handle a non-existent dir (Daryn Sharp via bobby) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1296389 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-03-02 19:10:15 +00:00
parent 2da46a4e7a
commit 17e840e585
5 changed files with 165 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -166,6 +166,9 @@ Release 0.23.2 - UNRELEASED
HADOOP-8050. Deadlock in metrics. (Kihwal Lee via mattf)
HADOOP-8131. FsShell put doesn't correctly handle a non-existent dir
(Daryn Sharp via bobby)
Release 0.23.1 - 2012-02-17
INCOMPATIBLE CHANGES

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
View File

@ -112,10 +112,12 @@ abstract class CommandWithDestination extends FsCommand {
if (!dst.stat.isDirectory()) {
throw new PathIsNotDirectoryException(dst.toString());
}
} else {
if (dst.exists && !dst.stat.isDirectory() && !overwrite) {
} else if (dst.exists) {
if (!dst.stat.isDirectory() && !overwrite) {
throw new PathExistsException(dst.toString());
}
} else if (!dst.parentExists()) {
throw new PathNotFoundException(dst.toString());
}
super.processArguments(args);
}

14
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
View File

@ -181,6 +181,20 @@ public class PathData {
return tmpFile;
}
/**
* Test if the parent directory exists
* @return boolean indicating parent exists
* @throws IOException upon unexpected error
*/
public boolean parentExists() throws IOException {
String uriPath = uri.getPath();
String name = uriPath.substring(uriPath.lastIndexOf("/")+1);
// Path will munch off the chars that indicate a dir, so there's no way
// to perform this test except by examining the raw basename we maintain
return (name.isEmpty() || name.equals(".") || name.equals(".."))
? fs.exists(path) : fs.exists(path.getParent());
}
/**
* Returns a list of PathData objects of the items contained in the given
* directory.

140
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java
View File

@ -38,8 +38,10 @@ public class TestFsShellCopy {
conf = new Configuration();
shell = new FsShell(conf);
lfs = FileSystem.getLocal(conf);
testRootDir = new Path(
System.getProperty("test.build.data","test/build/data"), "testShellCopy");
testRootDir = lfs.makeQualified(new Path(
System.getProperty("test.build.data","test/build/data"),
"testShellCopy"));
lfs.mkdirs(testRootDir);
srcPath = new Path(testRootDir, "srcFile");
dstPath = new Path(testRootDir, "dstFile");
@ -94,4 +96,138 @@ public class TestFsShellCopy {
private void shellRun(int n, String ... args) throws Exception {
assertEquals(n, shell.run(args));
}
@Test
public void testCopyFileFromLocal() throws Exception {
Path testRoot = new Path(testRootDir, "testPutFile");
lfs.delete(testRoot, true);
lfs.mkdirs(testRoot);
Path targetDir = new Path(testRoot, "target");
Path filePath = new Path(testRoot, new Path("srcFile"));
lfs.create(filePath).close();
checkPut(filePath, targetDir);
}
@Test
public void testCopyDirFromLocal() throws Exception {
Path testRoot = new Path(testRootDir, "testPutDir");
lfs.delete(testRoot, true);
lfs.mkdirs(testRoot);
Path targetDir = new Path(testRoot, "target");
Path dirPath = new Path(testRoot, new Path("srcDir"));
lfs.mkdirs(dirPath);
lfs.create(new Path(dirPath, "srcFile")).close();
checkPut(dirPath, targetDir);
}
private void checkPut(Path srcPath, Path targetDir)
throws Exception {
lfs.delete(targetDir, true);
lfs.mkdirs(targetDir);
lfs.setWorkingDirectory(targetDir);
final Path dstPath = new Path("path");
final Path childPath = new Path(dstPath, "childPath");
lfs.setWorkingDirectory(targetDir);
// copy to new file, then again
prepPut(dstPath, false, false);
checkPut(0, srcPath, dstPath);
if (lfs.isFile(srcPath)) {
checkPut(1, srcPath, dstPath);
} else { // directory works because it copies into the dir
// clear contents so the check won't think there are extra paths
prepPut(dstPath, true, true);
checkPut(0, srcPath, dstPath);
}
// copy to non-existent subdir
prepPut(childPath, false, false);
checkPut(1, srcPath, dstPath);
// copy into dir, then with another name
prepPut(dstPath, true, true);
checkPut(0, srcPath, dstPath);
prepPut(childPath, true, true);
checkPut(0, srcPath, childPath);
// try to put to pwd with existing dir
prepPut(targetDir, true, true);
checkPut(0, srcPath, null);
prepPut(targetDir, true, true);
checkPut(0, srcPath, new Path("."));
// try to put to pwd with non-existent cwd
prepPut(dstPath, false, true);
lfs.setWorkingDirectory(dstPath);
checkPut(1, srcPath, null);
prepPut(dstPath, false, true);
checkPut(1, srcPath, new Path("."));
}
private void prepPut(Path dst, boolean create,
boolean isDir) throws IOException {
lfs.delete(dst, true);
assertFalse(lfs.exists(dst));
if (create) {
if (isDir) {
lfs.mkdirs(dst);
assertTrue(lfs.isDirectory(dst));
} else {
lfs.mkdirs(new Path(dst.getName()));
lfs.create(dst).close();
assertTrue(lfs.isFile(dst));
}
}
}
private void checkPut(int exitCode, Path src, Path dest) throws Exception {
String argv[] = null;
if (dest != null) {
argv = new String[]{ "-put", src.toString(), pathAsString(dest) };
} else {
argv = new String[]{ "-put", src.toString() };
dest = new Path(Path.CUR_DIR);
}
Path target;
if (lfs.exists(dest)) {
if (lfs.isDirectory(dest)) {
target = new Path(pathAsString(dest), src.getName());
} else {
target = dest;
}
} else {
target = new Path(lfs.getWorkingDirectory(), dest);
}
boolean targetExists = lfs.exists(target);
Path parent = lfs.makeQualified(target).getParent();
System.out.println("COPY src["+src.getName()+"] -> ["+dest+"] as ["+target+"]");
String lsArgv[] = new String[]{ "-ls", "-R", pathAsString(parent) };
shell.run(lsArgv);
int gotExit = shell.run(argv);
System.out.println("copy exit:"+gotExit);
lsArgv = new String[]{ "-ls", "-R", pathAsString(parent) };
shell.run(lsArgv);
if (exitCode == 0) {
assertTrue(lfs.exists(target));
assertTrue(lfs.isFile(src) == lfs.isFile(target));
assertEquals(1, lfs.listStatus(lfs.makeQualified(target).getParent()).length);
} else {
assertEquals(targetExists, lfs.exists(target));
}
assertEquals(exitCode, gotExit);
}
// path handles "." rather oddly
private String pathAsString(Path p) {
String s = (p == null) ? Path.CUR_DIR : p.toString();
return s.isEmpty() ? Path.CUR_DIR : s;
}
}

8
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
View File

@ -280,11 +280,15 @@ public class TestFsShellReturnCode {
System.setErr(out);
final String results;
try {
Path tdir = new Path(TEST_ROOT_DIR, "notNullCopy");
fileSys.delete(tdir, true);
fileSys.mkdirs(tdir);
String[] args = new String[3];
args[0] = "-get";
args[1] = "/invalidPath";
args[2] = "/test/tmp";
args[1] = tdir+"/invalidSrc";
args[2] = tdir+"/invalidDst";
assertTrue("file exists", !fileSys.exists(new Path(args[1])));
assertTrue("file exists", !fileSys.exists(new Path(args[2])));
int run = shell.run(args);
results = bytes.toString();
assertEquals("Return code should be 1", 1, run);