HADOOP-8589 ViewFs tests fail when tests and home dirs are nested (sanjay Radia)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406939 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sanjay Radia
parent
1594dd6965
commit
76d43a08ed
|
|
@ -272,10 +272,13 @@ Trunk (Unreleased)
|
|||
HADOOP-8918. test-patch.sh is parsing modified files wrong.
|
||||
(Raja Aluri via suresh)
|
||||
|
||||
HADOOP-8589 ViewFs tests fail when tests and home dirs are nested (sanjay Radia)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
HADOOP-7761. Improve the performance of raw comparisons. (todd)
|
||||
|
||||
HADOOP-8589 ViewFs tests fail when tests and home dirs are nested (sanjay Radia)
|
||||
Release 2.0.3-alpha - Unreleased
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
|||
|
|
@ -126,6 +126,11 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem {
|
|||
return fsImpl.getServerDefaults();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getHomeDirectory() {
|
||||
return fsImpl.getHomeDirectory();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getUriDefaultPort() {
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -154,12 +154,6 @@ class ChRootedFileSystem extends FilterFileSystem {
|
|||
new Path(chRootPathPartString + f.toUri().toString()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getHomeDirectory() {
|
||||
return new Path("/user/"+System.getProperty("user.name")).makeQualified(
|
||||
getUri(), null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getWorkingDirectory() {
|
||||
return workingDir;
|
||||
|
|
|
|||
|
|
@ -256,8 +256,9 @@ public class ViewFileSystem extends FileSystem {
|
|||
if (base == null) {
|
||||
base = "/user";
|
||||
}
|
||||
homeDir =
|
||||
this.makeQualified(new Path(base + "/" + ugi.getShortUserName()));
|
||||
homeDir = (base.equals("/") ?
|
||||
this.makeQualified(new Path(base + ugi.getShortUserName())):
|
||||
this.makeQualified(new Path(base + "/" + ugi.getShortUserName())));
|
||||
}
|
||||
return homeDir;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -248,8 +248,9 @@ public class ViewFs extends AbstractFileSystem {
|
|||
if (base == null) {
|
||||
base = "/user";
|
||||
}
|
||||
homeDir =
|
||||
this.makeQualified(new Path(base + "/" + ugi.getShortUserName()));
|
||||
homeDir = (base.equals("/") ?
|
||||
this.makeQualified(new Path(base + ugi.getShortUserName())):
|
||||
this.makeQualified(new Path(base + "/" + ugi.getShortUserName())));
|
||||
}
|
||||
return homeDir;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -61,19 +61,28 @@ public final class FileSystemTestHelper {
|
|||
return data;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* get testRootPath qualified for fSys
|
||||
*/
|
||||
public static Path getTestRootPath(FileSystem fSys) {
|
||||
return fSys.makeQualified(new Path(TEST_ROOT_DIR));
|
||||
}
|
||||
|
||||
/*
|
||||
* get testRootPath + pathString qualified for fSys
|
||||
*/
|
||||
public static Path getTestRootPath(FileSystem fSys, String pathString) {
|
||||
return fSys.makeQualified(new Path(TEST_ROOT_DIR, pathString));
|
||||
}
|
||||
|
||||
|
||||
// the getAbsolutexxx method is needed because the root test dir
|
||||
// can be messed up by changing the working dir.
|
||||
// can be messed up by changing the working dir since the TEST_ROOT_PATH
|
||||
// is often relative to the working directory of process
|
||||
// running the unit tests.
|
||||
|
||||
public static String getAbsoluteTestRootDir(FileSystem fSys)
|
||||
static String getAbsoluteTestRootDir(FileSystem fSys)
|
||||
throws IOException {
|
||||
// NOTE: can't cache because of different filesystems!
|
||||
//if (absTestRootDir == null)
|
||||
|
|
|
|||
|
|
@ -73,10 +73,10 @@ public class TestChRootedFileSystem {
|
|||
URI uri = fSys.getUri();
|
||||
Assert.assertEquals(chrootedTo.toUri(), uri);
|
||||
Assert.assertEquals(fSys.makeQualified(
|
||||
new Path("/user/" + System.getProperty("user.name"))),
|
||||
new Path(System.getProperty("user.home"))),
|
||||
fSys.getWorkingDirectory());
|
||||
Assert.assertEquals(fSys.makeQualified(
|
||||
new Path("/user/" + System.getProperty("user.name"))),
|
||||
new Path(System.getProperty("user.home"))),
|
||||
fSys.getHomeDirectory());
|
||||
/*
|
||||
* ChRootedFs as its uri like file:///chrootRoot.
|
||||
|
|
|
|||
|
|
@ -70,10 +70,10 @@ public class TestChRootedFs {
|
|||
URI uri = fc.getDefaultFileSystem().getUri();
|
||||
Assert.assertEquals(chrootedTo.toUri(), uri);
|
||||
Assert.assertEquals(fc.makeQualified(
|
||||
new Path("/user/" + System.getProperty("user.name"))),
|
||||
new Path(System.getProperty("user.home"))),
|
||||
fc.getWorkingDirectory());
|
||||
Assert.assertEquals(fc.makeQualified(
|
||||
new Path("/user/" + System.getProperty("user.name"))),
|
||||
new Path(System.getProperty("user.home"))),
|
||||
fc.getHomeDirectory());
|
||||
/*
|
||||
* ChRootedFs as its uri like file:///chrootRoot.
|
||||
|
|
|
|||
|
|
@ -39,44 +39,7 @@ public class TestFcMainOperationsLocalFs extends
|
|||
@Override
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
/**
|
||||
* create the test root on local_fs - the mount table will point here
|
||||
*/
|
||||
fclocal = FileContext.getLocalFSFileContext();
|
||||
targetOfTests = FileContextTestHelper.getTestRootPath(fclocal);
|
||||
// In case previous test was killed before cleanup
|
||||
fclocal.delete(targetOfTests, true);
|
||||
|
||||
fclocal.mkdir(targetOfTests, FileContext.DEFAULT_PERM, true);
|
||||
|
||||
|
||||
|
||||
|
||||
// We create mount table so that the test root on the viewFs points to
|
||||
// to the test root on the target.
|
||||
// DOing this helps verify the FileStatus.path.
|
||||
//
|
||||
// The test root by default when running eclipse
|
||||
// is a test dir below the working directory.
|
||||
// (see FileContextTestHelper).
|
||||
// Since viewFs has no built-in wd, its wd is /user/<username>.
|
||||
// If this test launched via ant (build.xml) the test root is absolute path
|
||||
|
||||
String srcTestRoot;
|
||||
if (FileContextTestHelper.TEST_ROOT_DIR.startsWith("/")) {
|
||||
srcTestRoot = FileContextTestHelper.TEST_ROOT_DIR;
|
||||
} else {
|
||||
srcTestRoot = "/user/" + System.getProperty("user.name") + "/" +
|
||||
FileContextTestHelper.TEST_ROOT_DIR;
|
||||
}
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
ConfigUtil.addLink(conf, srcTestRoot,
|
||||
targetOfTests.toUri());
|
||||
|
||||
fc = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf);
|
||||
//System.out.println("SRCOfTests = "+ FileContextTestHelper.getTestRootPath(fc, "test"));
|
||||
//System.out.println("TargetOfTests = "+ targetOfTests.toUri());
|
||||
fc = ViewFsTestSetup.setupForViewFsLocalFs();
|
||||
super.setUp();
|
||||
}
|
||||
|
||||
|
|
@ -84,6 +47,6 @@ public class TestFcMainOperationsLocalFs extends
|
|||
@After
|
||||
public void tearDown() throws Exception {
|
||||
super.tearDown();
|
||||
fclocal.delete(targetOfTests, true);
|
||||
ViewFsTestSetup.tearDownForViewFsLocalFs();
|
||||
}
|
||||
}
|
||||
|
|
@ -17,7 +17,10 @@
|
|||
*/
|
||||
package org.apache.hadoop.fs.viewfs;
|
||||
|
||||
import java.net.URI;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileContext;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.FileSystemTestHelper;
|
||||
import org.apache.hadoop.fs.FsConstants;
|
||||
|
|
@ -32,15 +35,20 @@ import org.mortbay.log.Log;
|
|||
*
|
||||
* If tests launched via ant (build.xml) the test root is absolute path
|
||||
* If tests launched via eclipse, the test root is
|
||||
* is a test dir below the working directory. (see FileSystemTestHelper).
|
||||
* Since viewFs has no built-in wd, its wd is /user/<username>
|
||||
* (or /User/<username> on mac)
|
||||
* is a test dir below the working directory. (see FileContextTestHelper)
|
||||
*
|
||||
* We set a viewFileSystems with mount point for
|
||||
* /<firstComponent>" pointing to the target fs's testdir
|
||||
* We set a viewFileSystems with 3 mount points:
|
||||
* 1) /<firstComponent>" of testdir pointing to same in target fs
|
||||
* 2) /<firstComponent>" of home pointing to same in target fs
|
||||
* 3) /<firstComponent>" of wd pointing to same in target fs
|
||||
* (note in many cases the link may be the same - viewFileSytem handles this)
|
||||
*
|
||||
* We also set the view file system's wd to point to the wd.
|
||||
*/
|
||||
public class ViewFileSystemTestSetup {
|
||||
|
||||
static public String ViewFSTestDir = "/testDir";
|
||||
|
||||
/**
|
||||
*
|
||||
* @param fsTarget - the target fs of the view fs.
|
||||
|
|
@ -56,24 +64,26 @@ public class ViewFileSystemTestSetup {
|
|||
fsTarget.delete(targetOfTests, true);
|
||||
fsTarget.mkdirs(targetOfTests);
|
||||
|
||||
// Setup a link from viewfs to targetfs for the first component of
|
||||
// path of testdir.
|
||||
|
||||
// Set up viewfs link for test dir as described above
|
||||
String testDir = FileSystemTestHelper.getTestRootPath(fsTarget).toUri()
|
||||
.getPath();
|
||||
int indexOf2ndSlash = testDir.indexOf('/', 1);
|
||||
String testDirFirstComponent = testDir.substring(0, indexOf2ndSlash);
|
||||
ConfigUtil.addLink(conf, testDirFirstComponent, fsTarget.makeQualified(
|
||||
new Path(testDirFirstComponent)).toUri());
|
||||
linkUpFirstComponents(conf, testDir, fsTarget, "test dir");
|
||||
|
||||
|
||||
// Set up viewfs link for home dir as described above
|
||||
setUpHomeDir(conf, fsTarget);
|
||||
|
||||
|
||||
// the test path may be relative to working dir - we need to make that work:
|
||||
// Set up viewfs link for wd as described above
|
||||
String wdDir = fsTarget.getWorkingDirectory().toUri().getPath();
|
||||
linkUpFirstComponents(conf, wdDir, fsTarget, "working dir");
|
||||
|
||||
// viewFs://home => fsTarget://home
|
||||
String homeDirRoot = fsTarget.getHomeDirectory()
|
||||
.getParent().toUri().getPath();
|
||||
ConfigUtil.addLink(conf, homeDirRoot,
|
||||
fsTarget.makeQualified(new Path(homeDirRoot)).toUri());
|
||||
ConfigUtil.setHomeDirConf(conf, homeDirRoot);
|
||||
Log.info("Home dir base " + homeDirRoot);
|
||||
|
||||
FileSystem fsView = FileSystem.get(FsConstants.VIEWFS_URI, conf);
|
||||
fsView.setWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd.
|
||||
Log.info("Working dir is: " + fsView.getWorkingDirectory());
|
||||
return fsView;
|
||||
}
|
||||
|
||||
|
|
@ -91,4 +101,33 @@ public class ViewFileSystemTestSetup {
|
|||
conf.set("fs.viewfs.impl", ViewFileSystem.class.getName());
|
||||
return conf;
|
||||
}
|
||||
|
||||
static void setUpHomeDir(Configuration conf, FileSystem fsTarget) {
|
||||
String homeDir = fsTarget.getHomeDirectory().toUri().getPath();
|
||||
int indexOf2ndSlash = homeDir.indexOf('/', 1);
|
||||
if (indexOf2ndSlash >0) {
|
||||
linkUpFirstComponents(conf, homeDir, fsTarget, "home dir");
|
||||
} else { // home dir is at root. Just link the home dir itse
|
||||
URI linkTarget = fsTarget.makeQualified(new Path(homeDir)).toUri();
|
||||
ConfigUtil.addLink(conf, homeDir, linkTarget);
|
||||
Log.info("Added link for home dir " + homeDir + "->" + linkTarget);
|
||||
}
|
||||
// Now set the root of the home dir for viewfs
|
||||
String homeDirRoot = fsTarget.getHomeDirectory().getParent().toUri().getPath();
|
||||
ConfigUtil.setHomeDirConf(conf, homeDirRoot);
|
||||
Log.info("Home dir base for viewfs" + homeDirRoot);
|
||||
}
|
||||
|
||||
/*
|
||||
* Set up link in config for first component of path to the same
|
||||
* in the target file system.
|
||||
*/
|
||||
static void linkUpFirstComponents(Configuration conf, String path, FileSystem fsTarget, String info) {
|
||||
int indexOf2ndSlash = path.indexOf('/', 1);
|
||||
String firstComponent = path.substring(0, indexOf2ndSlash);
|
||||
URI linkTarget = fsTarget.makeQualified(new Path(firstComponent)).toUri();
|
||||
ConfigUtil.addLink(conf, firstComponent, linkTarget);
|
||||
Log.info("Added link for " + info + " "
|
||||
+ firstComponent + "->" + linkTarget);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,12 +17,15 @@
|
|||
*/
|
||||
package org.apache.hadoop.fs.viewfs;
|
||||
|
||||
import java.net.URI;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileContext;
|
||||
import org.apache.hadoop.fs.FileContextTestHelper;
|
||||
import org.apache.hadoop.fs.FsConstants;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.viewfs.ConfigUtil;
|
||||
import org.mortbay.log.Log;
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -31,14 +34,21 @@ import org.apache.hadoop.fs.viewfs.ConfigUtil;
|
|||
*
|
||||
* If tests launched via ant (build.xml) the test root is absolute path
|
||||
* If tests launched via eclipse, the test root is
|
||||
* is a test dir below the working directory. (see FileContextTestHelper).
|
||||
* Since viewFs has no built-in wd, its wd is /user/<username>.
|
||||
* is a test dir below the working directory. (see FileContextTestHelper)
|
||||
*
|
||||
* We set up fc to be the viewFs with mount point for
|
||||
* /<firstComponent>" pointing to the local file system's testdir
|
||||
* We set a viewfs with 3 mount points:
|
||||
* 1) /<firstComponent>" of testdir pointing to same in target fs
|
||||
* 2) /<firstComponent>" of home pointing to same in target fs
|
||||
* 3) /<firstComponent>" of wd pointing to same in target fs
|
||||
* (note in many cases the link may be the same - viewfs handles this)
|
||||
*
|
||||
* We also set the view file system's wd to point to the wd.
|
||||
*/
|
||||
|
||||
public class ViewFsTestSetup {
|
||||
|
||||
static public String ViewFSTestDir = "/testDir";
|
||||
|
||||
|
||||
/*
|
||||
* return the ViewFS File context to be used for tests
|
||||
|
|
@ -47,30 +57,31 @@ public class ViewFsTestSetup {
|
|||
/**
|
||||
* create the test root on local_fs - the mount table will point here
|
||||
*/
|
||||
FileContext fclocal = FileContext.getLocalFSFileContext();
|
||||
Path targetOfTests = FileContextTestHelper.getTestRootPath(fclocal);
|
||||
FileContext fsTarget = FileContext.getLocalFSFileContext();
|
||||
Path targetOfTests = FileContextTestHelper.getTestRootPath(fsTarget);
|
||||
// In case previous test was killed before cleanup
|
||||
fclocal.delete(targetOfTests, true);
|
||||
fsTarget.delete(targetOfTests, true);
|
||||
|
||||
fclocal.mkdir(targetOfTests, FileContext.DEFAULT_PERM, true);
|
||||
|
||||
String srcTestFirstDir;
|
||||
if (FileContextTestHelper.TEST_ROOT_DIR.startsWith("/")) {
|
||||
int indexOf2ndSlash = FileContextTestHelper.TEST_ROOT_DIR.indexOf('/', 1);
|
||||
srcTestFirstDir = FileContextTestHelper.TEST_ROOT_DIR.substring(0, indexOf2ndSlash);
|
||||
} else {
|
||||
srcTestFirstDir = "/user";
|
||||
|
||||
}
|
||||
//System.out.println("srcTestFirstDir=" + srcTestFirstDir);
|
||||
|
||||
// Set up the defaultMT in the config with mount point links
|
||||
// The test dir is root is below /user/<userid>
|
||||
fsTarget.mkdir(targetOfTests, FileContext.DEFAULT_PERM, true);
|
||||
Configuration conf = new Configuration();
|
||||
ConfigUtil.addLink(conf, srcTestFirstDir,
|
||||
targetOfTests.toUri());
|
||||
|
||||
// Set up viewfs link for test dir as described above
|
||||
String testDir = FileContextTestHelper.getTestRootPath(fsTarget).toUri()
|
||||
.getPath();
|
||||
linkUpFirstComponents(conf, testDir, fsTarget, "test dir");
|
||||
|
||||
|
||||
// Set up viewfs link for home dir as described above
|
||||
setUpHomeDir(conf, fsTarget);
|
||||
|
||||
// the test path may be relative to working dir - we need to make that work:
|
||||
// Set up viewfs link for wd as described above
|
||||
String wdDir = fsTarget.getWorkingDirectory().toUri().getPath();
|
||||
linkUpFirstComponents(conf, wdDir, fsTarget, "working dir");
|
||||
|
||||
FileContext fc = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf);
|
||||
fc.setWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd.
|
||||
Log.info("Working dir is: " + fc.getWorkingDirectory());
|
||||
//System.out.println("SRCOfTests = "+ getTestRootPath(fc, "test"));
|
||||
//System.out.println("TargetOfTests = "+ targetOfTests.toUri());
|
||||
return fc;
|
||||
|
|
@ -86,4 +97,35 @@ public class ViewFsTestSetup {
|
|||
fclocal.delete(targetOfTests, true);
|
||||
}
|
||||
|
||||
|
||||
static void setUpHomeDir(Configuration conf, FileContext fsTarget) {
|
||||
String homeDir = fsTarget.getHomeDirectory().toUri().getPath();
|
||||
int indexOf2ndSlash = homeDir.indexOf('/', 1);
|
||||
if (indexOf2ndSlash >0) {
|
||||
linkUpFirstComponents(conf, homeDir, fsTarget, "home dir");
|
||||
} else { // home dir is at root. Just link the home dir itse
|
||||
URI linkTarget = fsTarget.makeQualified(new Path(homeDir)).toUri();
|
||||
ConfigUtil.addLink(conf, homeDir, linkTarget);
|
||||
Log.info("Added link for home dir " + homeDir + "->" + linkTarget);
|
||||
}
|
||||
// Now set the root of the home dir for viewfs
|
||||
String homeDirRoot = fsTarget.getHomeDirectory().getParent().toUri().getPath();
|
||||
ConfigUtil.setHomeDirConf(conf, homeDirRoot);
|
||||
Log.info("Home dir base for viewfs" + homeDirRoot);
|
||||
}
|
||||
|
||||
/*
|
||||
* Set up link in config for first component of path to the same
|
||||
* in the target file system.
|
||||
*/
|
||||
static void linkUpFirstComponents(Configuration conf, String path,
|
||||
FileContext fsTarget, String info) {
|
||||
int indexOf2ndSlash = path.indexOf('/', 1);
|
||||
String firstComponent = path.substring(0, indexOf2ndSlash);
|
||||
URI linkTarget = fsTarget.makeQualified(new Path(firstComponent)).toUri();
|
||||
ConfigUtil.addLink(conf, firstComponent, linkTarget);
|
||||
Log.info("Added link for " + info + " "
|
||||
+ firstComponent + "->" + linkTarget);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue