diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c7bc33dfbe1..cdf7cfebf93 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -98,6 +98,9 @@ Release 2.0.5-beta - UNRELEASED HADOOP-9125. LdapGroupsMapping threw CommunicationException after some idle time. (Kai Zheng via atm) + HADOOP-9357. Fallback to default authority if not specified in FileContext. + (Andrew Wang via eli) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index dfbf84ad5c1..336dda44426 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -254,17 +254,33 @@ public final class FileContext { } /* - * Remove relative part - return "absolute": - * If input is relative path ("foo/bar") add wd: ie "//foo/bar" - * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path + * Resolve a relative path passed from the user. + * + * Relative paths are resolved against the current working directory + * (e.g. "foo/bar" becomes "//foo/bar"). + * Fully-qualified URIs (e.g. "hdfs://nn:p/foo/bar") and slash-relative paths * ("/foo/bar") are returned unchanged. * + * Additionally, we fix malformed URIs that specify a scheme but not an + * authority (e.g. "hdfs:///foo/bar"). Per RFC 2395, we remove the scheme + * if it matches the default FS, and let the default FS add in the default + * scheme and authority later (see {@link #AbstractFileSystem#checkPath}). + * * Applications that use FileContext should use #makeQualified() since - * they really want a fully qualified URI. + * they really want a fully-qualified URI. * Hence this method is not called makeAbsolute() and * has been deliberately declared private. */ private Path fixRelativePart(Path p) { + // Per RFC 2396 5.2, drop schema if there is a scheme but no authority. + if (p.hasSchemeAndNoAuthority()) { + String scheme = p.toUri().getScheme(); + if (scheme.equalsIgnoreCase(defaultFS.getUri().getScheme())) { + p = new Path(p.toUri().getSchemeSpecificPart()); + } + } + // Absolute paths are unchanged. Relative paths are resolved against the + // current working directory. if (p.isUriPathAbsolute()) { return p; } else { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java index c0ebebfe67e..2d548812e52 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java @@ -202,6 +202,10 @@ public class Path implements Comparable { return (isUriPathAbsolute() && uri.getScheme() == null && uri.getAuthority() == null); } + + public boolean hasSchemeAndNoAuthority() { + return uri.getScheme() != null && uri.getAuthority() == null; + } /** * True if the path component (i.e. directory) of this URI is absolute. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java index d6efc52c42a..df2f7eb9597 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java @@ -21,6 +21,8 @@ package org.apache.hadoop.fs; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.EnumSet; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -1164,6 +1166,40 @@ public abstract class FileContextMainOperationsBaseTest { Assert.assertEquals(fc.getFileStatus(file), fc.getFileLinkStatus(file)); } } + + /** + * Test that URIs with a scheme, no authority, and absolute path component + * resolve with the authority of the default FS. + */ + @Test(timeout=30000) + public void testAbsolutePathSchemeNoAuthority() throws IOException, + URISyntaxException { + Path file = getTestRootPath(fc, "test/file"); + createFile(file); + URI uri = file.toUri(); + URI noAuthorityUri = new URI(uri.getScheme(), null, uri.getPath(), + uri.getQuery(), uri.getFragment()); + Path noAuthority = new Path(noAuthorityUri); + Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority)); + } + + /** + * Test that URIs with a scheme, no authority, and relative path component + * resolve with the authority of the default FS. + */ + @Test(timeout=30000) + public void testRelativePathSchemeNoAuthority() throws IOException, + URISyntaxException { + Path workDir = new Path(getAbsoluteTestRootPath(fc), new Path("test")); + fc.setWorkingDirectory(workDir); + Path file = new Path(workDir, "file"); + createFile(file); + URI uri = file.toUri(); + URI noAuthorityUri = new URI(uri.getScheme() + ":file"); + System.out.println(noAuthorityUri); + Path noAuthority = new Path(noAuthorityUri); + Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority)); + } protected void createFile(Path path) throws IOException { FSDataOutputStream out = fc.create(path, EnumSet.of(CREATE),