From b7e885435ada55cc5b6b879b9763c9cfefceb591 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Tue, 8 Oct 2013 23:41:53 +0000 Subject: [PATCH] HADOOP-10030. FsShell -put/copyFromLocal should support Windows local path. Contributed by Chuan Liu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1530462 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/fs/shell/CopyCommands.java | 11 ++- .../org/apache/hadoop/fs/TestFsShellCopy.java | 71 ++++++++++++++----- 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 45bfaa7c0ff..c079b778e8e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -407,6 +407,9 @@ Release 2.2.1 - UNRELEASED HADOOP-10028. Malformed ssl-server.xml.example. (Haohui Mai via jing9) + HADOOP-10030. FsShell -put/copyFromLocal should support Windows local path. + (Chuan Liu via cnauroth) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java index db15d467ec3..21f9d2ca9e1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java @@ -204,13 +204,18 @@ protected void processOptions(LinkedList args) throws IOException { // commands operating on local paths have no need for glob expansion @Override protected List expandArgument(String arg) throws IOException { + List items = new LinkedList(); try { - List items = new LinkedList(); items.add(new PathData(new URI(arg), getConf())); - return items; } catch (URISyntaxException e) { - throw new IOException("unexpected URISyntaxException", e); + if (Path.WINDOWS) { + // Unlike URI, PathData knows how to parse Windows drive-letter paths. + items.add(new PathData(arg, getConf())); + } else { + throw new IOException("unexpected URISyntaxException", e); + } } + return items; } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java index 72df11e6df6..dbdb7da101b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java @@ -19,7 +19,9 @@ package org.apache.hadoop.fs; import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; +import java.io.File; import java.io.IOException; import org.apache.hadoop.conf.Configuration; @@ -106,7 +108,7 @@ public void testCopyFileFromLocal() throws Exception { Path targetDir = new Path(testRoot, "target"); Path filePath = new Path(testRoot, new Path("srcFile")); lfs.create(filePath).close(); - checkPut(filePath, targetDir); + checkPut(filePath, targetDir, false); } @Test @@ -119,10 +121,42 @@ public void testCopyDirFromLocal() throws Exception { Path dirPath = new Path(testRoot, new Path("srcDir")); lfs.mkdirs(dirPath); lfs.create(new Path(dirPath, "srcFile")).close(); - checkPut(dirPath, targetDir); + checkPut(dirPath, targetDir, false); } + + @Test + public void testCopyFileFromWindowsLocalPath() throws Exception { + assumeTrue(Path.WINDOWS); + String windowsTestRootPath = (new File(testRootDir.toUri().getPath() + .toString())).getAbsolutePath(); + Path testRoot = new Path(windowsTestRootPath, "testPutFile"); + lfs.delete(testRoot, true); + lfs.mkdirs(testRoot); + + Path targetDir = new Path(testRoot, "target"); + Path filePath = new Path(testRoot, new Path("srcFile")); + lfs.create(filePath).close(); + checkPut(filePath, targetDir, true); + } + + @Test + public void testCopyDirFromWindowsLocalPath() throws Exception { + assumeTrue(Path.WINDOWS); + String windowsTestRootPath = (new File(testRootDir.toUri().getPath() + .toString())).getAbsolutePath(); + Path testRoot = new Path(windowsTestRootPath, "testPutDir"); + lfs.delete(testRoot, true); + lfs.mkdirs(testRoot); + + Path targetDir = new Path(testRoot, "target"); + Path dirPath = new Path(testRoot, new Path("srcDir")); + lfs.mkdirs(dirPath); + lfs.create(new Path(dirPath, "srcFile")).close(); + checkPut(dirPath, targetDir, true); + } + - private void checkPut(Path srcPath, Path targetDir) + private void checkPut(Path srcPath, Path targetDir, boolean useWindowsPath) throws Exception { lfs.delete(targetDir, true); lfs.mkdirs(targetDir); @@ -134,37 +168,37 @@ private void checkPut(Path srcPath, Path targetDir) // copy to new file, then again prepPut(dstPath, false, false); - checkPut(0, srcPath, dstPath); + checkPut(0, srcPath, dstPath, useWindowsPath); if (lfs.isFile(srcPath)) { - checkPut(1, srcPath, dstPath); + checkPut(1, srcPath, dstPath, useWindowsPath); } else { // directory works because it copies into the dir // clear contents so the check won't think there are extra paths prepPut(dstPath, true, true); - checkPut(0, srcPath, dstPath); + checkPut(0, srcPath, dstPath, useWindowsPath); } // copy to non-existent subdir prepPut(childPath, false, false); - checkPut(1, srcPath, dstPath); + checkPut(1, srcPath, dstPath, useWindowsPath); // copy into dir, then with another name prepPut(dstPath, true, true); - checkPut(0, srcPath, dstPath); + checkPut(0, srcPath, dstPath, useWindowsPath); prepPut(childPath, true, true); - checkPut(0, srcPath, childPath); + checkPut(0, srcPath, childPath, useWindowsPath); // try to put to pwd with existing dir prepPut(targetDir, true, true); - checkPut(0, srcPath, null); + checkPut(0, srcPath, null, useWindowsPath); prepPut(targetDir, true, true); - checkPut(0, srcPath, new Path(".")); + checkPut(0, srcPath, new Path("."), useWindowsPath); // try to put to pwd with non-existent cwd prepPut(dstPath, false, true); lfs.setWorkingDirectory(dstPath); - checkPut(1, srcPath, null); + checkPut(1, srcPath, null, useWindowsPath); prepPut(dstPath, false, true); - checkPut(1, srcPath, new Path(".")); + checkPut(1, srcPath, new Path("."), useWindowsPath); } private void prepPut(Path dst, boolean create, @@ -183,12 +217,17 @@ private void prepPut(Path dst, boolean create, } } - private void checkPut(int exitCode, Path src, Path dest) throws Exception { + private void checkPut(int exitCode, Path src, Path dest, + boolean useWindowsPath) throws Exception { String argv[] = null; + String srcPath = src.toString(); + if (useWindowsPath) { + srcPath = (new File(srcPath)).getAbsolutePath(); + } if (dest != null) { - argv = new String[]{ "-put", src.toString(), pathAsString(dest) }; + argv = new String[]{ "-put", srcPath, pathAsString(dest) }; } else { - argv = new String[]{ "-put", src.toString() }; + argv = new String[]{ "-put", srcPath }; dest = new Path(Path.CUR_DIR); }