Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-5093. Merging change r1514366 from trunk to branch-2. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1514369 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth 2013-08-15 16:56:09 +00:00
parent d6049bd288
commit ed114b4b13
2 changed files with 116 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -104,6 +104,9 @@ Release 2.1.1-beta - UNRELEASED
HDFS-4632. globStatus using backslash for escaping does not work on Windows. HDFS-4632. globStatus using backslash for escaping does not work on Windows.
(Chuan Liu via cnauroth) (Chuan Liu via cnauroth)
HDFS-5093. TestGlobPaths should re-use the MiniDFSCluster to avoid failure
on Windows. (Chuan Liu via cnauroth)
Release 2.1.0-beta - 2013-08-06 Release 2.1.0-beta - 2013-08-06
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

218
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
View File

@ -20,7 +20,6 @@ package org.apache.hadoop.fs;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@ -30,8 +29,6 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.*; import org.junit.*;
import com.google.common.base.Joiner;
public class TestGlobPaths { public class TestGlobPaths {
static class RegexPathFilter implements PathFilter { static class RegexPathFilter implements PathFilter {
@ -50,6 +47,7 @@ public class TestGlobPaths {
static private MiniDFSCluster dfsCluster; static private MiniDFSCluster dfsCluster;
static private FileSystem fs; static private FileSystem fs;
static private FileContext fc;
static final private int NUM_OF_PATHS = 4; static final private int NUM_OF_PATHS = 4;
static private String USER_DIR; static private String USER_DIR;
private Path[] path = new Path[NUM_OF_PATHS]; private Path[] path = new Path[NUM_OF_PATHS];
@ -59,6 +57,7 @@ public class TestGlobPaths {
Configuration conf = new HdfsConfiguration(); Configuration conf = new HdfsConfiguration();
dfsCluster = new MiniDFSCluster.Builder(conf).build(); dfsCluster = new MiniDFSCluster.Builder(conf).build();
fs = FileSystem.get(conf); fs = FileSystem.get(conf);
fc = FileContext.getFileContext(conf);
USER_DIR = fs.getHomeDirectory().toUri().getPath().toString(); USER_DIR = fs.getHomeDirectory().toUri().getPath().toString();
} }
@ -803,28 +802,24 @@ public class TestGlobPaths {
/** /**
* Run a glob test on FileSystem. * Run a glob test on FileSystem.
*/ */
private static void testOnFileSystem(FSTestWrapperGlobTest test) throws Exception { private void testOnFileSystem(FSTestWrapperGlobTest test) throws Exception {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
try { try {
FileSystem fs = FileSystem.get(conf); fc.mkdir(new Path(USER_DIR), FsPermission.getDefault(), true);
test.run(new FileSystemTestWrapper(fs), fs, null); test.run(new FileSystemTestWrapper(fs), fs, null);
} finally { } finally {
cluster.shutdown(); fc.delete(new Path(USER_DIR), true);
} }
} }
/** /**
* Run a glob test on FileContext. * Run a glob test on FileContext.
*/ */
private static void testOnFileContext(FSTestWrapperGlobTest test) throws Exception { private void testOnFileContext(FSTestWrapperGlobTest test) throws Exception {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
try { try {
FileContext fc = FileContext.getFileContext(conf); fs.mkdirs(new Path(USER_DIR));
test.run(new FileContextTestWrapper(fc), null, fc); test.run(new FileContextTestWrapper(fc), null, fc);
} finally { } finally {
cluster.shutdown(); cleanupDFS();
} }
} }
@ -857,32 +852,33 @@ public class TestGlobPaths {
throws Exception { throws Exception {
// Test that globbing through a symlink to a directory yields a path // Test that globbing through a symlink to a directory yields a path
// containing that symlink. // containing that symlink.
wrap.mkdir(new Path("/alpha"), wrap.mkdir(new Path(USER_DIR + "/alpha"), FsPermission.getDirDefault(),
FsPermission.getDirDefault(), false); false);
wrap.createSymlink(new Path("/alpha"), new Path("/alphaLink"), false); wrap.createSymlink(new Path(USER_DIR + "/alpha"), new Path(USER_DIR
wrap.mkdir(new Path("/alphaLink/beta"), + "/alphaLink"), false);
wrap.mkdir(new Path(USER_DIR + "/alphaLink/beta"),
FsPermission.getDirDefault(), false); FsPermission.getDirDefault(), false);
// Test simple glob // Test simple glob
FileStatus[] statuses = FileStatus[] statuses = wrap.globStatus(new Path(USER_DIR + "/alpha/*"),
wrap.globStatus(new Path("/alpha/*"), new AcceptAllPathFilter());
Assert.assertEquals(1, statuses.length);
Assert.assertEquals("/alpha/beta",
statuses[0].getPath().toUri().getPath());
// Test glob through symlink
statuses =
wrap.globStatus(new Path("/alphaLink/*"), new AcceptAllPathFilter());
Assert.assertEquals(1, statuses.length);
Assert.assertEquals("/alphaLink/beta",
statuses[0].getPath().toUri().getPath());
// If the terminal path component in a globbed path is a symlink,
// we don't dereference that link.
wrap.createSymlink(new Path("beta"), new Path("/alphaLink/betaLink"),
false);
statuses = wrap.globStatus(new Path("/alpha/betaLi*"),
new AcceptAllPathFilter()); new AcceptAllPathFilter());
Assert.assertEquals(1, statuses.length); Assert.assertEquals(1, statuses.length);
Assert.assertEquals("/alpha/betaLink", Assert.assertEquals(USER_DIR + "/alpha/beta", statuses[0].getPath()
statuses[0].getPath().toUri().getPath()); .toUri().getPath());
// Test glob through symlink
statuses = wrap.globStatus(new Path(USER_DIR + "/alphaLink/*"),
new AcceptAllPathFilter());
Assert.assertEquals(1, statuses.length);
Assert.assertEquals(USER_DIR + "/alphaLink/beta", statuses[0].getPath()
.toUri().getPath());
// If the terminal path component in a globbed path is a symlink,
// we don't dereference that link.
wrap.createSymlink(new Path("beta"), new Path(USER_DIR
+ "/alphaLink/betaLink"), false);
statuses = wrap.globStatus(new Path(USER_DIR + "/alpha/betaLi*"),
new AcceptAllPathFilter());
Assert.assertEquals(1, statuses.length);
Assert.assertEquals(USER_DIR + "/alpha/betaLink", statuses[0].getPath()
.toUri().getPath());
// todo: test symlink-to-symlink-to-dir, etc. // todo: test symlink-to-symlink-to-dir, etc.
} }
} }
@ -902,58 +898,64 @@ public class TestGlobPaths {
* *
* Also test globbing dangling symlinks. It should NOT throw any exceptions! * Also test globbing dangling symlinks. It should NOT throw any exceptions!
*/ */
private static class TestGlobWithSymlinksToSymlinks private static class TestGlobWithSymlinksToSymlinks implements
implements FSTestWrapperGlobTest { FSTestWrapperGlobTest {
public void run(FSTestWrapper wrap, FileSystem fs, FileContext fc) public void run(FSTestWrapper wrap, FileSystem fs, FileContext fc)
throws Exception { throws Exception {
// Test that globbing through a symlink to a symlink to a directory // Test that globbing through a symlink to a symlink to a directory
// fully resolves // fully resolves
wrap.mkdir(new Path("/alpha"), FsPermission.getDirDefault(), false); wrap.mkdir(new Path(USER_DIR + "/alpha"), FsPermission.getDirDefault(),
wrap.createSymlink(new Path("/alpha"), new Path("/alphaLink"), false); false);
wrap.createSymlink(new Path("/alphaLink"), wrap.createSymlink(new Path(USER_DIR + "/alpha"), new Path(USER_DIR
new Path("/alphaLinkLink"), false); + "/alphaLink"), false);
wrap.mkdir(new Path("/alpha/beta"), FsPermission.getDirDefault(), false); wrap.createSymlink(new Path(USER_DIR + "/alphaLink"), new Path(USER_DIR
+ "/alphaLinkLink"), false);
wrap.mkdir(new Path(USER_DIR + "/alpha/beta"),
FsPermission.getDirDefault(), false);
// Test glob through symlink to a symlink to a directory // Test glob through symlink to a symlink to a directory
FileStatus statuses[] = FileStatus statuses[] = wrap.globStatus(new Path(USER_DIR
wrap.globStatus(new Path("/alphaLinkLink"), new AcceptAllPathFilter()); + "/alphaLinkLink"), new AcceptAllPathFilter());
Assert.assertEquals(1, statuses.length); Assert.assertEquals(1, statuses.length);
Assert.assertEquals("/alphaLinkLink", Assert.assertEquals(USER_DIR + "/alphaLinkLink", statuses[0].getPath()
statuses[0].getPath().toUri().getPath()); .toUri().getPath());
statuses = statuses = wrap.globStatus(new Path(USER_DIR + "/alphaLinkLink/*"),
wrap.globStatus(new Path("/alphaLinkLink/*"), new AcceptAllPathFilter()); new AcceptAllPathFilter());
Assert.assertEquals(1, statuses.length); Assert.assertEquals(1, statuses.length);
Assert.assertEquals("/alphaLinkLink/beta", Assert.assertEquals(USER_DIR + "/alphaLinkLink/beta", statuses[0]
statuses[0].getPath().toUri().getPath()); .getPath().toUri().getPath());
// Test glob of dangling symlink (theta does not actually exist) // Test glob of dangling symlink (theta does not actually exist)
wrap.createSymlink(new Path("theta"), new Path("/alpha/kappa"), false); wrap.createSymlink(new Path(USER_DIR + "theta"), new Path(USER_DIR
statuses = wrap.globStatus(new Path("/alpha/kappa/kappa"), + "/alpha/kappa"), false);
new AcceptAllPathFilter()); statuses = wrap.globStatus(new Path(USER_DIR + "/alpha/kappa/kappa"),
new AcceptAllPathFilter());
Assert.assertNull(statuses); Assert.assertNull(statuses);
// Test glob of symlinks // Test glob of symlinks
wrap.createFile("/alpha/beta/gamma"); wrap.createFile(USER_DIR + "/alpha/beta/gamma");
wrap.createSymlink(new Path("gamma"), wrap.createSymlink(new Path(USER_DIR + "gamma"), new Path(USER_DIR
new Path("/alpha/beta/gammaLink"), false); + "/alpha/beta/gammaLink"), false);
wrap.createSymlink(new Path("gammaLink"), wrap.createSymlink(new Path(USER_DIR + "gammaLink"), new Path(USER_DIR
new Path("/alpha/beta/gammaLinkLink"), false); + "/alpha/beta/gammaLinkLink"), false);
wrap.createSymlink(new Path("gammaLinkLink"), wrap.createSymlink(new Path(USER_DIR + "gammaLinkLink"), new Path(
new Path("/alpha/beta/gammaLinkLinkLink"), false); USER_DIR + "/alpha/beta/gammaLinkLinkLink"), false);
statuses = wrap.globStatus(new Path("/alpha/*/gammaLinkLinkLink"), statuses = wrap.globStatus(new Path(USER_DIR
new AcceptAllPathFilter()); + "/alpha/*/gammaLinkLinkLink"), new AcceptAllPathFilter());
Assert.assertEquals(1, statuses.length); Assert.assertEquals(1, statuses.length);
Assert.assertEquals("/alpha/beta/gammaLinkLinkLink", Assert.assertEquals(USER_DIR + "/alpha/beta/gammaLinkLinkLink",
statuses[0].getPath().toUri().getPath()); statuses[0].getPath().toUri().getPath());
statuses = wrap.globStatus(new Path("/alpha/beta/*"), statuses = wrap.globStatus(new Path(USER_DIR + "/alpha/beta/*"),
new AcceptAllPathFilter()); new AcceptAllPathFilter());
Assert.assertEquals("/alpha/beta/gamma;/alpha/beta/gammaLink;" + Assert.assertEquals(USER_DIR + "/alpha/beta/gamma;" + USER_DIR
"/alpha/beta/gammaLinkLink;/alpha/beta/gammaLinkLinkLink", + "/alpha/beta/gammaLink;" + USER_DIR + "/alpha/beta/gammaLinkLink;"
+ USER_DIR + "/alpha/beta/gammaLinkLinkLink",
TestPath.mergeStatuses(statuses)); TestPath.mergeStatuses(statuses));
// Let's create two symlinks that point to each other, and glob on them. // Let's create two symlinks that point to each other, and glob on them.
wrap.createSymlink(new Path("tweedledee"), wrap.createSymlink(new Path(USER_DIR + "tweedledee"), new Path(USER_DIR
new Path("/tweedledum"), false); + "/tweedledum"), false);
wrap.createSymlink(new Path("tweedledum"), wrap.createSymlink(new Path(USER_DIR + "tweedledum"), new Path(USER_DIR
new Path("/tweedledee"), false); + "/tweedledee"), false);
statuses = wrap.globStatus(new Path("/tweedledee/unobtainium"), statuses = wrap.globStatus(
new AcceptAllPathFilter()); new Path(USER_DIR + "/tweedledee/unobtainium"),
new AcceptAllPathFilter());
Assert.assertNull(statuses); Assert.assertNull(statuses);
} }
} }
@ -971,34 +973,39 @@ public class TestGlobPaths {
/** /**
* Test globbing symlinks with a custom PathFilter * Test globbing symlinks with a custom PathFilter
*/ */
private static class TestGlobSymlinksWithCustomPathFilter private static class TestGlobSymlinksWithCustomPathFilter implements
implements FSTestWrapperGlobTest { FSTestWrapperGlobTest {
public void run(FSTestWrapper wrap, FileSystem fs, FileContext fc) public void run(FSTestWrapper wrap, FileSystem fs, FileContext fc)
throws Exception { throws Exception {
// Test that globbing through a symlink to a symlink to a directory // Test that globbing through a symlink to a symlink to a directory
// fully resolves // fully resolves
wrap.mkdir(new Path("/alpha"), FsPermission.getDirDefault(), false); wrap.mkdir(new Path(USER_DIR + "/alpha"), FsPermission.getDirDefault(),
wrap.createSymlink(new Path("/alpha"), new Path("/alphaLinkz"), false); false);
wrap.mkdir(new Path("/alpha/beta"), FsPermission.getDirDefault(), false); wrap.createSymlink(new Path(USER_DIR + "/alpha"), new Path(USER_DIR
wrap.mkdir(new Path("/alpha/betaz"), FsPermission.getDirDefault(), false); + "/alphaLinkz"), false);
// Test glob through symlink to a symlink to a directory, with a PathFilter wrap.mkdir(new Path(USER_DIR + "/alpha/beta"),
FileStatus statuses[] = FsPermission.getDirDefault(), false);
wrap.globStatus(new Path("/alpha/beta"), new AcceptPathsEndingInZ()); wrap.mkdir(new Path(USER_DIR + "/alpha/betaz"),
FsPermission.getDirDefault(), false);
// Test glob through symlink to a symlink to a directory, with a
// PathFilter
FileStatus statuses[] = wrap.globStatus(
new Path(USER_DIR + "/alpha/beta"), new AcceptPathsEndingInZ());
Assert.assertNull(statuses); Assert.assertNull(statuses);
statuses = statuses = wrap.globStatus(new Path(USER_DIR + "/alphaLinkz/betaz"),
wrap.globStatus(new Path("/alphaLinkz/betaz"), new AcceptPathsEndingInZ()); new AcceptPathsEndingInZ());
Assert.assertEquals(1, statuses.length); Assert.assertEquals(1, statuses.length);
Assert.assertEquals("/alphaLinkz/betaz", Assert.assertEquals(USER_DIR + "/alphaLinkz/betaz", statuses[0].getPath()
statuses[0].getPath().toUri().getPath()); .toUri().getPath());
statuses = statuses = wrap.globStatus(new Path(USER_DIR + "/*/*"),
wrap.globStatus(new Path("/*/*"), new AcceptPathsEndingInZ()); new AcceptPathsEndingInZ());
Assert.assertEquals("/alpha/betaz;/alphaLinkz/betaz", Assert.assertEquals(USER_DIR + "/alpha/betaz;" + USER_DIR
TestPath.mergeStatuses(statuses)); + "/alphaLinkz/betaz", TestPath.mergeStatuses(statuses));
statuses = statuses = wrap.globStatus(new Path(USER_DIR + "/*/*"),
wrap.globStatus(new Path("/*/*"), new AcceptAllPathFilter()); new AcceptAllPathFilter());
Assert.assertEquals("/alpha/beta;/alpha/betaz;" + Assert.assertEquals(USER_DIR + "/alpha/beta;" + USER_DIR
"/alphaLinkz/beta;/alphaLinkz/betaz", + "/alpha/betaz;" + USER_DIR + "/alphaLinkz/beta;" + USER_DIR
TestPath.mergeStatuses(statuses)); + "/alphaLinkz/betaz", TestPath.mergeStatuses(statuses));
} }
} }
@ -1015,24 +1022,25 @@ public class TestGlobPaths {
/** /**
* Test that globStatus fills in the scheme even when it is not provided. * Test that globStatus fills in the scheme even when it is not provided.
*/ */
private static class TestGlobFillsInScheme private static class TestGlobFillsInScheme implements FSTestWrapperGlobTest {
implements FSTestWrapperGlobTest { public void run(FSTestWrapper wrap, FileSystem fs, FileContext fc)
public void run(FSTestWrapper wrap, FileSystem fs, FileContext fc)
throws Exception { throws Exception {
// Verify that the default scheme is hdfs, when we don't supply one. // Verify that the default scheme is hdfs, when we don't supply one.
wrap.mkdir(new Path("/alpha"), FsPermission.getDirDefault(), false); wrap.mkdir(new Path(USER_DIR + "/alpha"), FsPermission.getDirDefault(),
wrap.createSymlink(new Path("/alpha"), new Path("/alphaLink"), false); false);
FileStatus statuses[] = wrap.createSymlink(new Path(USER_DIR + "/alpha"), new Path(USER_DIR
wrap.globStatus(new Path("/alphaLink"), new AcceptAllPathFilter()); + "/alphaLink"), false);
FileStatus statuses[] = wrap.globStatus(
new Path(USER_DIR + "/alphaLink"), new AcceptAllPathFilter());
Assert.assertEquals(1, statuses.length); Assert.assertEquals(1, statuses.length);
Path path = statuses[0].getPath(); Path path = statuses[0].getPath();
Assert.assertEquals("/alphaLink", path.toUri().getPath()); Assert.assertEquals(USER_DIR + "/alphaLink", path.toUri().getPath());
Assert.assertEquals("hdfs", path.toUri().getScheme()); Assert.assertEquals("hdfs", path.toUri().getScheme());
if (fc != null) { if (fc != null) {
// If we're using FileContext, then we can list a file:/// URI. // If we're using FileContext, then we can list a file:/// URI.
// Since everyone should have the root directory, we list that. // Since everyone should have the root directory, we list that.
statuses = statuses = wrap.globStatus(new Path("file:///"),
wrap.globStatus(new Path("file:///"), new AcceptAllPathFilter()); new AcceptAllPathFilter());
Assert.assertEquals(1, statuses.length); Assert.assertEquals(1, statuses.length);
Path filePath = statuses[0].getPath(); Path filePath = statuses[0].getPath();
Assert.assertEquals("file", filePath.toUri().getScheme()); Assert.assertEquals("file", filePath.toUri().getScheme());