HADOOP-9527. Add symlink support to LocalFileSystem on Windows. Contributed by Arpit Agarwal.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1511118 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth 2013-08-06 20:35:52 +00:00
parent 3e0203a3a4
commit f2942687d4
10 changed files with 199 additions and 83 deletions

View File

@ -347,6 +347,9 @@ Release 2.1.1-beta - UNRELEASED
HADOOP-9806 PortmapInterface should check if the procedure is out-of-range
(brandonli)
HADOOP-9527. Add symlink support to LocalFileSystem on Windows.
(Arpit Agarwal via cnauroth)
Release 2.1.0-beta - 2013-08-06
INCOMPATIBLE CHANGES

View File

@ -142,7 +142,28 @@ public class FileUtil {
}
return deleteImpl(dir, true);
}
/**
* Returns the target of the given symlink. Returns the empty string if
* the given path does not refer to a symlink or there is an error
* accessing the symlink.
* @param f File representing the symbolic link.
* @return The target of the symbolic link, empty string on error or if not
* a symlink.
*/
public static String readLink(File f) {
/* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
* use getCanonicalPath in File to get the target of the symlink but that
* does not indicate if the given path refers to a symlink.
*/
try {
return Shell.execCommand(
Shell.getReadlinkCommand(f.toString())).trim();
} catch (IOException x) {
return "";
}
}
/*
* Pure-Java implementation of "chmod +rwx f".
*/
@ -737,15 +758,18 @@ public class FileUtil {
* On Windows, when symlink creation fails due to security
* setting, we will log a warning. The return code in this
* case is 2.
*
* @param target the target for symlink
* @param linkname the symlink
* @return value returned by the command
* @return 0 on success
*/
public static int symLink(String target, String linkname) throws IOException{
// Run the input paths through Java's File so that they are converted to the
// native OS form
File targetFile = new File(target);
File linkFile = new File(linkname);
File targetFile = new File(
Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString());
File linkFile = new File(
Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString());
// If not on Java7+, copy a file instead of creating a symlink since
// Java6 has close to no support for symlinks on Windows. Specifically
@ -757,9 +781,16 @@ public class FileUtil {
// is symlinked under userlogs and userlogs are generated afterwards).
if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
try {
LOG.info("FileUtil#symlink: On Java6, copying file instead "
+ linkname + " -> " + target);
org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
LOG.warn("FileUtil#symlink: On Windows+Java6, copying file instead " +
"of creating a symlink. Copying " + target + " -> " + linkname);
if (!linkFile.getParentFile().exists()) {
LOG.warn("Parent directory " + linkFile.getParent() +
" does not exist.");
return 1;
} else {
org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
}
} catch (IOException ex) {
LOG.warn("FileUtil#symlink failed to copy the file with error: "
+ ex.getMessage());
@ -769,10 +800,23 @@ public class FileUtil {
return 0;
}
String[] cmd = Shell.getSymlinkCommand(targetFile.getPath(),
linkFile.getPath());
ShellCommandExecutor shExec = new ShellCommandExecutor(cmd);
String[] cmd = Shell.getSymlinkCommand(
targetFile.toString(),
linkFile.toString());
ShellCommandExecutor shExec;
try {
if (Shell.WINDOWS &&
linkFile.getParentFile() != null &&
!new Path(target).isAbsolute()) {
// Relative links on Windows must be resolvable at the time of
// creation. To ensure this we run the shell command in the directory
// of the link.
//
shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile());
} else {
shExec = new ShellCommandExecutor(cmd);
}
shExec.execute();
} catch (Shell.ExitCodeException ec) {
int returnVal = ec.getExitCode();
@ -795,7 +839,7 @@ public class FileUtil {
}
return shExec.getExitCode();
}
/**
* Change the permissions on a filename.
* @param filename the name of the file to change

View File

@ -682,31 +682,13 @@ public class RawLocalFileSystem extends FileSystem {
if (createParent) {
mkdirs(link.getParent());
}
// NB: Use createSymbolicLink in java.nio.file.Path once available
try {
Shell.execCommand(Shell.getSymlinkCommand(
Path.getPathWithoutSchemeAndAuthority(target).toString(),
Path.getPathWithoutSchemeAndAuthority(makeAbsolute(link)).toString()));
} catch (IOException x) {
throw new IOException("Unable to create symlink: "+x.getMessage());
}
}
/**
* Returns the target of the given symlink. Returns the empty string if
* the given path does not refer to a symlink or there is an error
* accessing the symlink.
*/
private String readLink(Path p) {
/* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
* use getCanonicalPath in File to get the target of the symlink but that
* does not indicate if the given path refers to a symlink.
*/
try {
final String path = p.toUri().getPath();
return Shell.execCommand(Shell.READ_LINK_COMMAND, path).trim();
} catch (IOException x) {
return "";
// NB: Use createSymbolicLink in java.nio.file.Path once available
int result = FileUtil.symLink(target.toString(),
makeAbsolute(link).toString());
if (result != 0) {
throw new IOException("Error " + result + " creating symlink " +
link + " to " + target);
}
}
@ -729,7 +711,7 @@ public class RawLocalFileSystem extends FileSystem {
}
private FileStatus getFileLinkStatusInternal(final Path f) throws IOException {
String target = readLink(f);
String target = FileUtil.readLink(new File(f.toString()));
try {
FileStatus fs = getFileStatus(f);

View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs.local;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
@ -28,12 +29,12 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.DelegateToFileSystem;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Shell;
/**
* The RawLocalFs implementation of AbstractFileSystem.
@ -75,47 +76,29 @@ public class RawLocalFs extends DelegateToFileSystem {
@Override
public boolean supportsSymlinks() {
return true;
}
}
@Override
public void createSymlink(Path target, Path link, boolean createParent)
public void createSymlink(Path target, Path link, boolean createParent)
throws IOException {
final String targetScheme = target.toUri().getScheme();
if (targetScheme != null && !"file".equals(targetScheme)) {
throw new IOException("Unable to create symlink to non-local file "+
"system: "+target.toString());
"system: "+target.toString());
}
if (createParent) {
mkdir(link.getParent(), FsPermission.getDirDefault(), true);
}
// NB: Use createSymbolicLink in java.nio.file.Path once available
try {
Shell.execCommand(Shell.getSymlinkCommand(
Path.getPathWithoutSchemeAndAuthority(target).toString(),
Path.getPathWithoutSchemeAndAuthority(link).toString()));
} catch (IOException x) {
throw new IOException("Unable to create symlink: "+x.getMessage());
int result = FileUtil.symLink(target.toString(), link.toString());
if (result != 0) {
throw new IOException("Error " + result + " creating symlink " +
link + " to " + target);
}
}
/**
* Returns the target of the given symlink. Returns the empty string if
* the given path does not refer to a symlink or there is an error
* acessing the symlink.
*/
private String readLink(Path p) {
/* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
* use getCanonicalPath in File to get the target of the symlink but that
* does not indicate if the given path refers to a symlink.
*/
try {
final String path = p.toUri().getPath();
return Shell.execCommand(Shell.READ_LINK_COMMAND, path).trim();
} catch (IOException x) {
return "";
}
}
/**
* Return a FileStatus representing the given path. If the path refers
* to a symlink return a FileStatus representing the link rather than
@ -123,7 +106,7 @@ public class RawLocalFs extends DelegateToFileSystem {
*/
@Override
public FileStatus getFileLinkStatus(final Path f) throws IOException {
String target = readLink(f);
String target = FileUtil.readLink(new File(f.toString()));
try {
FileStatus fs = getFileStatus(f);
// If f refers to a regular file or directory

View File

@ -123,6 +123,12 @@ abstract public class Shell {
: new String[] { "ln", "-s", target, link };
}
/** Return a command to read the target of the a symbolic link*/
public static String[] getReadlinkCommand(String link) {
return WINDOWS ? new String[] { WINUTILS, "readlink", link }
: new String[] { "readlink", link };
}
/** Return a command for determining if process with specified pid is alive. */
public static String[] getCheckProcessIsAliveCommand(String pid) {
return Shell.WINDOWS ?

View File

@ -71,7 +71,8 @@ public abstract class FSTestWrapper implements FSWrapper {
public String getAbsoluteTestRootDir() throws IOException {
if (absTestRootDir == null) {
if (testRootDir.startsWith("/")) {
Path testRootPath = new Path(testRootDir);
if (testRootPath.isAbsolute()) {
absTestRootDir = testRootDir;
} else {
absTestRootDir = getWorkingDirectory().toString() + "/"

View File

@ -20,13 +20,10 @@ package org.apache.hadoop.fs;
import java.io.*;
import java.net.URI;
import java.util.EnumSet;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.test.GenericTestUtils;
import static org.junit.Assert.*;
@ -51,6 +48,13 @@ public abstract class SymlinkBaseTest {
abstract protected String testBaseDir2() throws IOException;
abstract protected URI testURI();
// Returns true if the filesystem is emulating symlink support. Certain
// checks will be bypassed if that is the case.
//
protected boolean emulatingSymlinksOnWindows() {
return false;
}
protected IOException unwrapException(IOException e) {
return e;
}
@ -156,8 +160,11 @@ public abstract class SymlinkBaseTest {
@Test(timeout=10000)
/** Try to create a directory given a path that refers to a symlink */
public void testMkdirExistingLink() throws IOException {
Path file = new Path(testBaseDir1() + "/targetFile");
createAndWriteFile(file);
Path dir = new Path(testBaseDir1()+"/link");
wrapper.createSymlink(new Path("/doesNotExist"), dir, false);
wrapper.createSymlink(file, dir, false);
try {
wrapper.mkdir(dir, FileContext.DEFAULT_PERM, false);
fail("Created a dir where a symlink exists");
@ -224,6 +231,7 @@ public abstract class SymlinkBaseTest {
@Test(timeout=10000)
/** Stat a link to a file */
public void testStatLinkToFile() throws IOException {
assumeTrue(!emulatingSymlinksOnWindows());
Path file = new Path(testBaseDir1()+"/file");
Path linkToFile = new Path(testBaseDir1()+"/linkToFile");
createAndWriteFile(file);
@ -232,8 +240,7 @@ public abstract class SymlinkBaseTest {
assertTrue(wrapper.isSymlink(linkToFile));
assertTrue(wrapper.isFile(linkToFile));
assertFalse(wrapper.isDir(linkToFile));
assertEquals(file.toUri().getPath(),
wrapper.getLinkTarget(linkToFile).toString());
assertEquals(file, wrapper.getLinkTarget(linkToFile));
// The local file system does not fully resolve the link
// when obtaining the file status
if (!"file".equals(getScheme())) {
@ -277,8 +284,7 @@ public abstract class SymlinkBaseTest {
assertFalse(wrapper.isFile(linkToDir));
assertTrue(wrapper.isDir(linkToDir));
assertEquals(dir.toUri().getPath(),
wrapper.getLinkTarget(linkToDir).toString());
assertEquals(dir, wrapper.getLinkTarget(linkToDir));
}
@Test(timeout=10000)
@ -351,6 +357,12 @@ public abstract class SymlinkBaseTest {
/* Assert that the given link to a file behaves as expected. */
private void checkLink(Path linkAbs, Path expectedTarget, Path targetQual)
throws IOException {
// If we are emulating symlinks then many of these checks will fail
// so we skip them.
//
assumeTrue(!emulatingSymlinksOnWindows());
Path dir = new Path(testBaseDir1());
// isFile/Directory
assertTrue(wrapper.isFile(linkAbs));
@ -400,7 +412,7 @@ public abstract class SymlinkBaseTest {
failureExpected = false;
}
try {
readFile(new Path(getScheme()+"://"+testBaseDir1()+"/linkToFile"));
readFile(new Path(getScheme()+":///"+testBaseDir1()+"/linkToFile"));
assertFalse(failureExpected);
} catch (Exception e) {
if (!failureExpected) {
@ -646,6 +658,7 @@ public abstract class SymlinkBaseTest {
@Test(timeout=10000)
/** Create symlink through a symlink */
public void testCreateLinkViaLink() throws IOException {
assumeTrue(!emulatingSymlinksOnWindows());
Path dir1 = new Path(testBaseDir1());
Path file = new Path(testBaseDir1(), "file");
Path linkToDir = new Path(testBaseDir2(), "linkToDir");
@ -688,6 +701,7 @@ public abstract class SymlinkBaseTest {
@Test(timeout=10000)
/** Test create symlink using the same path */
public void testCreateLinkTwice() throws IOException {
assumeTrue(!emulatingSymlinksOnWindows());
Path file = new Path(testBaseDir1(), "file");
Path link = new Path(testBaseDir1(), "linkToFile");
createAndWriteFile(file);
@ -783,7 +797,7 @@ public abstract class SymlinkBaseTest {
Path linkToDir = new Path(testBaseDir2(), "linkToDir");
Path fileViaLink = new Path(linkToDir, "test/file");
// Symlink to .. is not a problem since the .. is squashed early
assertEquals(testBaseDir1(), dotDot.toString());
assertEquals(new Path(testBaseDir1()), dotDot);
createAndWriteFile(file);
wrapper.createSymlink(dotDot, linkToDir, false);
readFile(fileViaLink);
@ -876,7 +890,8 @@ public abstract class SymlinkBaseTest {
assertFalse(wrapper.exists(linkViaLink));
// Check that we didn't rename the link target
assertTrue(wrapper.exists(file));
assertTrue(wrapper.getFileLinkStatus(linkNewViaLink).isSymlink());
assertTrue(wrapper.getFileLinkStatus(linkNewViaLink).isSymlink() ||
emulatingSymlinksOnWindows());
readFile(linkNewViaLink);
}
@ -1014,7 +1029,8 @@ public abstract class SymlinkBaseTest {
createAndWriteFile(file);
wrapper.createSymlink(file, link1, false);
wrapper.rename(link1, link2);
assertTrue(wrapper.getFileLinkStatus(link2).isSymlink());
assertTrue(wrapper.getFileLinkStatus(link2).isSymlink() ||
emulatingSymlinksOnWindows());
readFile(link2);
readFile(file);
assertFalse(wrapper.exists(link1));
@ -1038,8 +1054,11 @@ public abstract class SymlinkBaseTest {
}
wrapper.rename(link, file1, Rename.OVERWRITE);
assertFalse(wrapper.exists(link));
assertTrue(wrapper.getFileLinkStatus(file1).isSymlink());
assertEquals(file2, wrapper.getLinkTarget(file1));
if (!emulatingSymlinksOnWindows()) {
assertTrue(wrapper.getFileLinkStatus(file1).isSymlink());
assertEquals(file2, wrapper.getLinkTarget(file1));
}
}
@Test(timeout=10000)
@ -1078,16 +1097,21 @@ public abstract class SymlinkBaseTest {
@Test(timeout=10000)
/** Rename a symlink to itself */
public void testRenameSymlinkToItself() throws IOException {
Path file = new Path(testBaseDir1(), "file");
createAndWriteFile(file);
Path link = new Path(testBaseDir1(), "linkToFile1");
wrapper.createSymlink(new Path("/doestNotExist"), link, false);
wrapper.createSymlink(file, link, false);
try {
wrapper.rename(link, link);
fail("Failed to get expected IOException");
} catch (IOException e) {
assertTrue(unwrapException(e) instanceof FileAlreadyExistsException);
}
// Fails with overwrite as well
try {
wrapper.rename(link, link, Rename.OVERWRITE);
fail("Failed to get expected IOException");
} catch (IOException e) {
assertTrue(unwrapException(e) instanceof FileAlreadyExistsException);
}
@ -1096,6 +1120,7 @@ public abstract class SymlinkBaseTest {
@Test(timeout=10000)
/** Rename a symlink */
public void testRenameSymlink() throws IOException {
assumeTrue(!emulatingSymlinksOnWindows());
Path file = new Path(testBaseDir1(), "file");
Path link1 = new Path(testBaseDir1(), "linkToFile1");
Path link2 = new Path(testBaseDir1(), "linkToFile2");
@ -1193,6 +1218,7 @@ public abstract class SymlinkBaseTest {
@Test(timeout=10000)
/** Test rename the symlink's target */
public void testRenameLinkTarget() throws IOException {
assumeTrue(!emulatingSymlinksOnWindows());
Path file = new Path(testBaseDir1(), "file");
Path fileNew = new Path(testBaseDir1(), "fileNew");
Path link = new Path(testBaseDir1(), "linkToFile");

View File

@ -30,6 +30,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Shell;
import org.junit.Test;
/**
@ -61,6 +62,16 @@ abstract public class TestSymlinkLocalFS extends SymlinkBaseTest {
}
}
@Override
protected boolean emulatingSymlinksOnWindows() {
// Java 6 on Windows has very poor symlink support. Specifically
// Specifically File#length and File#renameTo do not work as expected.
// (see HADOOP-9061 for additional details)
// Hence some symlink tests will be skipped.
//
return (Shell.WINDOWS && !Shell.isJava7OrAbove());
}
@Override
public void testCreateDanglingLink() throws IOException {
// Dangling symlinks are not supported on Windows local file system.
@ -171,6 +182,7 @@ abstract public class TestSymlinkLocalFS extends SymlinkBaseTest {
* file scheme (eg file://host/tmp/test).
*/
public void testGetLinkStatusPartQualTarget() throws IOException {
assumeTrue(!emulatingSymlinksOnWindows());
Path fileAbs = new Path(testBaseDir1()+"/file");
Path fileQual = new Path(testURI().toString(), fileAbs);
Path dir = new Path(testBaseDir1());
@ -205,4 +217,14 @@ abstract public class TestSymlinkLocalFS extends SymlinkBaseTest {
// Excpected.
}
}
/** Test create symlink to . */
@Override
public void testCreateLinkToDot() throws IOException {
try {
super.testCreateLinkToDot();
} catch (IllegalArgumentException iae) {
// Expected.
}
}
}

View File

@ -17,8 +17,13 @@
*/
package org.apache.hadoop.fs;
import org.apache.hadoop.util.Shell;
import org.junit.BeforeClass;
import java.io.IOException;
import static org.junit.Assume.assumeTrue;
public class TestSymlinkLocalFSFileContext extends TestSymlinkLocalFS {
@BeforeClass
@ -27,4 +32,9 @@ public class TestSymlinkLocalFSFileContext extends TestSymlinkLocalFS {
wrapper = new FileContextTestWrapper(context);
}
@Override
public void testRenameFileWithDestParentSymlink() throws IOException {
assumeTrue(!Shell.WINDOWS);
super.testRenameFileWithDestParentSymlink();
}
}

View File

@ -17,13 +17,20 @@
*/
package org.apache.hadoop.fs;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.util.Shell;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
public class TestSymlinkLocalFSFileSystem extends TestSymlinkLocalFS {
@BeforeClass
@ -54,4 +61,36 @@ public class TestSymlinkLocalFSFileSystem extends TestSymlinkLocalFS {
@Override
@Test(timeout=1000)
public void testAccessFileViaInterSymlinkAbsTarget() throws IOException {}
@Override
public void testRenameFileWithDestParentSymlink() throws IOException {
assumeTrue(!Shell.WINDOWS);
super.testRenameFileWithDestParentSymlink();
}
@Override
@Test(timeout=10000)
/** Rename a symlink to itself */
public void testRenameSymlinkToItself() throws IOException {
Path file = new Path(testBaseDir1(), "file");
createAndWriteFile(file);
Path link = new Path(testBaseDir1(), "linkToFile1");
wrapper.createSymlink(file, link, false);
try {
wrapper.rename(link, link);
fail("Failed to get expected IOException");
} catch (IOException e) {
assertTrue(unwrapException(e) instanceof FileAlreadyExistsException);
}
// Fails with overwrite as well
try {
wrapper.rename(link, link, Rename.OVERWRITE);
fail("Failed to get expected IOException");
} catch (IOException e) {
// Todo: Fix this test when HADOOP-9819 is fixed.
assertTrue(unwrapException(e) instanceof FileAlreadyExistsException ||
unwrapException(e) instanceof FileNotFoundException);
}
}
}