From 8c293a7ff128f36c52fde1007716ea7ebe2b55e1 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Thu, 20 Dec 2012 16:30:20 +0000 Subject: [PATCH] svn merge -c 1424566 FIXES: HADOOP-9105. FsShell -moveFromLocal erroneously fails (daryn via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1424574 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../org/apache/hadoop/fs/shell/Command.java | 10 ++++ .../apache/hadoop/fs/shell/MoveCommands.java | 19 +++++- .../org/apache/hadoop/fs/TestFsShellCopy.java | 60 +++++++++++++++++++ 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c4de56081d9..f75186ff8a7 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -919,6 +919,8 @@ Release 0.23.6 - UNRELEASED HADOOP-9038. unit-tests for AllocatorPerContext.PathIterator (Ivan A. Veselovsky via bobby) + HADOOP-9105. FsShell -moveFromLocal erroneously fails (daryn via bobby) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java index a5ce158cddd..47a1dc22774 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java @@ -307,6 +307,7 @@ abstract public class Command extends Configured { if (recursive && item.stat.isDirectory()) { recursePath(item); } + postProcessPath(item); } catch (IOException e) { displayError(e); } @@ -325,6 +326,15 @@ abstract public class Command extends Configured { throw new RuntimeException("processPath() is not implemented"); } + /** + * Hook for commands to implement an operation to be applied on each + * path for the command after being processed successfully + * @param item a {@link PathData} object + * @throws IOException if anything goes wrong... + */ + protected void postProcessPath(PathData item) throws IOException { + } + /** * Gets the directory listing for a path and invokes * {@link #processPaths(PathData, PathData...)} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java index e1ad1a2ef6d..ffb3483ef21 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java @@ -24,6 +24,7 @@ import java.util.LinkedList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.PathExistsException; import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal; /** Various commands for moving files */ @@ -49,7 +50,21 @@ class MoveCommands { @Override protected void processPath(PathData src, PathData target) throws IOException { - target.fs.moveFromLocalFile(src.path, target.path); + // unlike copy, don't merge existing dirs during move + if (target.exists && target.stat.isDirectory()) { + throw new PathExistsException(target.toString()); + } + super.processPath(src, target); + } + + @Override + protected void postProcessPath(PathData src) throws IOException { + if (!src.fs.delete(src.path, false)) { + // we have no way to know the actual error... + PathIOException e = new PathIOException(src.toString()); + e.setOperation("remove"); + throw e; + } } } @@ -95,4 +110,4 @@ class MoveCommands { } } } -} \ No newline at end of file +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java index 5b4f585c19d..72df11e6df6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java @@ -357,6 +357,66 @@ public class TestFsShellCopy { assertEquals(0, exit); assertEquals("f1\ndf1\ndf2\ndf3\nf2\n", readFile("out")); } + + + @Test + public void testMoveFileFromLocal() throws Exception { + Path testRoot = new Path(testRootDir, "testPutFile"); + lfs.delete(testRoot, true); + lfs.mkdirs(testRoot); + + Path target = new Path(testRoot, "target"); + Path srcFile = new Path(testRoot, new Path("srcFile")); + lfs.createNewFile(srcFile); + + int exit = shell.run(new String[]{ + "-moveFromLocal", srcFile.toString(), target.toString() }); + assertEquals(0, exit); + assertFalse(lfs.exists(srcFile)); + assertTrue(lfs.exists(target)); + assertTrue(lfs.isFile(target)); + } + + @Test + public void testMoveDirFromLocal() throws Exception { + Path testRoot = new Path(testRootDir, "testPutDir"); + lfs.delete(testRoot, true); + lfs.mkdirs(testRoot); + + Path srcDir = new Path(testRoot, "srcDir"); + lfs.mkdirs(srcDir); + Path targetDir = new Path(testRoot, "target"); + + int exit = shell.run(new String[]{ + "-moveFromLocal", srcDir.toString(), targetDir.toString() }); + assertEquals(0, exit); + assertFalse(lfs.exists(srcDir)); + assertTrue(lfs.exists(targetDir)); + } + + @Test + public void testMoveDirFromLocalDestExists() throws Exception { + Path testRoot = new Path(testRootDir, "testPutDir"); + lfs.delete(testRoot, true); + lfs.mkdirs(testRoot); + + Path srcDir = new Path(testRoot, "srcDir"); + lfs.mkdirs(srcDir); + Path targetDir = new Path(testRoot, "target"); + lfs.mkdirs(targetDir); + + int exit = shell.run(new String[]{ + "-moveFromLocal", srcDir.toString(), targetDir.toString() }); + assertEquals(0, exit); + assertFalse(lfs.exists(srcDir)); + assertTrue(lfs.exists(new Path(targetDir, srcDir.getName()))); + + lfs.mkdirs(srcDir); + exit = shell.run(new String[]{ + "-moveFromLocal", srcDir.toString(), targetDir.toString() }); + assertEquals(1, exit); + assertTrue(lfs.exists(srcDir)); + } private void createFile(Path ... paths) throws IOException { for (Path path : paths) {