diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e32580fe7a3..02ff55b5ab3 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -349,6 +349,8 @@ Release 2.3.0 - UNRELEASED HADOOP-9865. FileContext#globStatus has a regression with respect to relative path. (Chuan Lin via Colin Patrick McCabe) + HADOOP-9877. Fix listing of snapshot directories in globStatus. + (Binglin Chang via Andrew Wang) Release 2.1.1-beta - UNRELEASED diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java index 378311a71a2..57ad45e81d4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java @@ -62,6 +62,18 @@ private FileStatus getFileStatus(Path path) { } } + private FileStatus getFileLinkStatus(Path path) { + try { + if (fs != null) { + return fs.getFileLinkStatus(path); + } else { + return fc.getFileLinkStatus(path); + } + } catch (IOException e) { + return null; + } + } + private FileStatus[] listStatus(Path path) { try { if (fs != null) { @@ -122,6 +134,18 @@ private String authorityFromPath(Path path) throws IOException { return authority ; } + /** + * The glob filter builds a regexp per path component. If the component + * does not contain a shell metachar, then it falls back to appending the + * raw string to the list of built up paths. This raw path needs to have + * the quoting removed. Ie. convert all occurrences of "\X" to "X" + * @param name of the path component + * @return the unquoted path component + */ + private static String unquotePathComponent(String name) { + return name.replaceAll("\\\\(.)", "$1"); + } + public FileStatus[] glob() throws IOException { // First we get the scheme and authority of the pattern that was passed // in. @@ -176,14 +200,30 @@ public FileStatus[] glob() throws IOException { resolvedCandidate.isDirectory() == false) { continue; } - FileStatus[] children = listStatus(candidate.getPath()); - for (FileStatus child : children) { - // Set the child path based on the parent path. - // This keeps the symlinks in our path. - child.setPath(new Path(candidate.getPath(), - child.getPath().getName())); - if (globFilter.accept(child.getPath())) { - newCandidates.add(child); + // For components without pattern, we get its FileStatus directly + // using getFileLinkStatus for two reasons: + // 1. It should be faster to only get FileStatus needed rather than + // get all children. + // 2. Some special filesystem directories (e.g. HDFS snapshot + // directories) are not returned by listStatus, but do exist if + // checked explicitly via getFileLinkStatus. + if (globFilter.hasPattern()) { + FileStatus[] children = listStatus(candidate.getPath()); + for (FileStatus child : children) { + // Set the child path based on the parent path. + // This keeps the symlinks in our path. + child.setPath(new Path(candidate.getPath(), + child.getPath().getName())); + if (globFilter.accept(child.getPath())) { + newCandidates.add(child); + } + } + } else { + Path p = new Path(candidate.getPath(), unquotePathComponent(component)); + FileStatus s = getFileLinkStatus(p); + if (s != null) { + s.setPath(p); + newCandidates.add(s); } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java index 354f7aabfd6..877a491bf9b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.junit.After; import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; @@ -632,6 +633,20 @@ public void testGlobStatusFilterWithMultiplePathWildcardsAndNonTrivialFilter() filteredPaths)); } + protected Path getHiddenPathForTest() { + return null; + } + + @Test + public void testGlobStatusFilterWithHiddenPathTrivialFilter() + throws Exception { + Path hidden = getHiddenPathForTest(); + Assume.assumeNotNull(hidden); + FileStatus[] filteredPaths = fc.util().globStatus(hidden, DEFAULT_FILTER); + Assert.assertNotNull(filteredPaths); + Assert.assertEquals(1, filteredPaths.length); + } + @Test public void testWriteReadAndDeleteEmptyFile() throws Exception { writeReadAndDelete(0); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java index dcc19df3d4e..2fff29e38d4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java @@ -517,6 +517,26 @@ public FileStatus getFileStatus(Path p) throws IOException { } return stat; } + + @Override + public FileStatus getFileLinkStatus(Path p) throws IOException { + String f = makeQualified(p).toString(); + FileStatus stat = super.getFileLinkStatus(p); + + stat.getPermission(); + if (owners.containsKey(f)) { + stat.setOwner("STUB-"+owners.get(f)); + } else { + stat.setOwner("REAL-"+stat.getOwner()); + } + if (groups.containsKey(f)) { + stat.setGroup("STUB-"+groups.get(f)); + } else { + stat.setGroup("REAL-"+stat.getGroup()); + } + return stat; + } + } static class MyFsShell extends FsShell { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java index 6388bdd9e7a..8f5f14db614 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java @@ -59,6 +59,9 @@ public static void clusterSetupAtBegining() throws IOException, defaultWorkingDirectory = fc.makeQualified( new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true); + // Make defaultWorkingDirectory snapshottable to enable + // testGlobStatusFilterWithHiddenPathTrivialFilter + cluster.getFileSystem().allowSnapshot(defaultWorkingDirectory); } private static void restartCluster() throws IOException, LoginException { @@ -73,6 +76,9 @@ private static void restartCluster() throws IOException, LoginException { defaultWorkingDirectory = fc.makeQualified( new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true); + // Make defaultWorkingDirectory snapshottable to enable + // testGlobStatusFilterWithHiddenPathTrivialFilter + cluster.getFileSystem().allowSnapshot(defaultWorkingDirectory); } @AfterClass @@ -92,6 +98,11 @@ public void tearDown() throws Exception { super.tearDown(); } + @Override + protected Path getHiddenPathForTest() { + return new Path(defaultWorkingDirectory, ".snapshot"); + } + @Override protected Path getDefaultWorkingDirectory() { return defaultWorkingDirectory;