From 53fed66d918e31a9c3d8bf0d4e6d3861c9b058b3 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Wed, 12 Feb 2014 02:12:11 +0000 Subject: [PATCH] HADOOP-10338. Cannot get the FileStatus of the root inode from the new Globber (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1567497 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/fs/Globber.java | 28 ++++++++++++++---- .../org/apache/hadoop/fs/TestGlobPaths.java | 29 +++++++++++++++++++ 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 49a32c72342..7bfad349293 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -334,6 +334,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10326. M/R jobs can not access S3 if Kerberos is enabled. (bc Wong via atm) + HADOOP-10338. Cannot get the FileStatus of the root inode from the new + Globber (cmccabe) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java index d00c387f9b3..5eee5e4fb3d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java @@ -114,7 +114,8 @@ class Globber { if (fs != null) { scheme = fs.getUri().getScheme(); } else { - scheme = fc.getDefaultFileSystem().getUri().getScheme(); + scheme = fc.getFSofPath(fc.fixRelativePart(path)). + getUri().getScheme(); } } return scheme; @@ -126,7 +127,8 @@ class Globber { if (fs != null) { authority = fs.getUri().getAuthority(); } else { - authority = fc.getDefaultFileSystem().getUri().getAuthority(); + authority = fc.getFSofPath(fc.fixRelativePart(path)). + getUri().getAuthority(); } } return authority ; @@ -162,18 +164,26 @@ class Globber { // Starting out at the root of the filesystem, we try to match // filesystem entries against pattern components. ArrayList candidates = new ArrayList(1); + // To get the "real" FileStatus of root, we'd have to do an expensive + // RPC to the NameNode. So we create a placeholder FileStatus which has + // the correct path, but defaults for the rest of the information. + // Later, if it turns out we actually want the FileStatus of root, we'll + // replace the placeholder with a real FileStatus obtained from the + // NameNode. + FileStatus rootPlaceholder; if (Path.WINDOWS && !components.isEmpty() && Path.isWindowsAbsolutePath(absPattern.toUri().getPath(), true)) { // On Windows the path could begin with a drive letter, e.g. /E:/foo. // We will skip matching the drive letter and start from listing the // root of the filesystem on that drive. String driveLetter = components.remove(0); - candidates.add(new FileStatus(0, true, 0, 0, 0, new Path(scheme, - authority, Path.SEPARATOR + driveLetter + Path.SEPARATOR))); + rootPlaceholder = new FileStatus(0, true, 0, 0, 0, new Path(scheme, + authority, Path.SEPARATOR + driveLetter + Path.SEPARATOR)); } else { - candidates.add(new FileStatus(0, true, 0, 0, 0, - new Path(scheme, authority, Path.SEPARATOR))); + rootPlaceholder = new FileStatus(0, true, 0, 0, 0, + new Path(scheme, authority, Path.SEPARATOR)); } + candidates.add(rootPlaceholder); for (int componentIdx = 0; componentIdx < components.size(); componentIdx++) { @@ -245,6 +255,12 @@ class Globber { candidates = newCandidates; } for (FileStatus status : candidates) { + // Use object equality to see if this status is the root placeholder. + // See the explanation for rootPlaceholder above for more information. + if (status == rootPlaceholder) { + status = getFileStatus(rootPlaceholder.getPath()); + if (status == null) continue; + } // HADOOP-3497 semantics: the user-defined filter is applied at the // end, once the full path is built up. if (filter.accept(status.getPath())) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java index 8eb9847ebb5..5ba797c8b01 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java @@ -21,6 +21,7 @@ import static org.junit.Assert.*; import java.io.IOException; import java.security.PrivilegedExceptionAction; +import java.util.UUID; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; @@ -1175,4 +1176,32 @@ public class TestGlobPaths { public void testReservedHdfsPathsOnFC() throws Exception { testOnFileContext(new TestReservedHdfsPaths()); } + + /** + * Test trying to glob the root. Regression test for HDFS-5888. + **/ + private static class TestGlobRoot implements FSTestWrapperGlobTest { + public void run(FSTestWrapper wrap, FSTestWrapper unprivilegedWrap, + FileSystem fs, FileContext fc) throws Exception { + final Path rootPath = new Path("/"); + FileStatus oldRootStatus = wrap.getFileStatus(rootPath); + String newOwner = UUID.randomUUID().toString(); + wrap.setOwner(new Path("/"), newOwner, null); + FileStatus[] status = + wrap.globStatus(rootPath, new AcceptAllPathFilter()); + Assert.assertEquals(1, status.length); + Assert.assertEquals(newOwner, status[0].getOwner()); + wrap.setOwner(new Path("/"), oldRootStatus.getOwner(), null); + } + } + + @Test + public void testGlobRootOnFS() throws Exception { + testOnFileSystem(new TestGlobRoot()); + } + + @Test + public void testGlobRootOnFC() throws Exception { + testOnFileContext(new TestGlobRoot()); + } }