Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-7284 Trash and shell's rm does not work for viewfs (Sanjay Radia) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1127642 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sanjay Radia 2011-05-25 19:23:18 +00:00
parent 77b4fd6572
commit 1596a106e4
11 changed files with 246 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -247,6 +247,8 @@ Trunk (unreleased changes)
HADOOP-7282. ipc.Server.getRemoteIp() may return null. (John George
via szetszwo)
HADOOP-7284 Trash and shell's rm does not work for viewfs (Sanjay Radia)
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES

29
src/java/org/apache/hadoop/fs/Trash.java
View File

@ -17,6 +17,11 @@
*/
package org.apache.hadoop.fs;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.text.DateFormat;
@ -30,7 +35,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.StringUtils;
@ -86,7 +90,26 @@ public class Trash extends Configured {
FS_TRASH_INTERVAL_DEFAULT) *
MSECS_PER_MINUTE);
}
/**
* In case of the symlinks or mount points, one has to move the appropriate
* trashbin in the actual volume of the path p being deleted.
*
* Hence we get the file system of the fully-qualified resolved-path and
* then move the path p to the trashbin in that volume,
* @param fs - the filesystem of path p
* @param p - the path being deleted - to be moved to trasg
* @param conf - configuration
* @return false if the item is already in the trash or trash is disabled
* @throws IOException on error
*/
public static boolean moveToAppropriateTrash(FileSystem fs, Path p,
Configuration conf) throws IOException {
Path fullyResolvedPath = fs.resolvePath(p);
Trash trash = new Trash(FileSystem.get(fullyResolvedPath.toUri(), conf), conf);
return trash.moveToTrash(fullyResolvedPath);
}
private Trash(Path home, Configuration conf) throws IOException {
super(conf);
this.fs = home.getFileSystem(conf);
@ -122,7 +145,7 @@ public class Trash extends Configured {
if (!fs.exists(path)) // check that path exists
throw new FileNotFoundException(path.toString());
String qpath = path.makeQualified(fs).toString();
String qpath = fs.makeQualified(path).toString();
if (qpath.startsWith(trash.toString())) {
return false; // already in trash

3
src/java/org/apache/hadoop/fs/shell/Delete.java
View File

@ -86,8 +86,7 @@ class Delete extends FsCommand {
private boolean moveToTrash(PathData item) throws IOException {
boolean success = false;
if (!skipTrash) {
Trash trash = new Trash(item.fs, getConf());
success = (trash.isEnabled() && trash.moveToTrash(item.path));
success = Trash.moveToAppropriateTrash(item.fs, item.path, getConf());
}
return success;
}

3
src/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
View File

@ -116,8 +116,7 @@ class ChRootedFileSystem extends FileSystem {
chRootPathPart.toString().substring(1));
workingDir = getHomeDirectory();
// We don't use the wd of the myFs, (lets set it to root anyway)
myFs.setWorkingDirectory(chRootPathPart);
// We don't use the wd of the myFs
}
/**

25
src/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -157,8 +157,6 @@ public class ViewFileSystem extends FileSystem {
final String authority = theUri.getAuthority();
try {
myUri = new URI(FsConstants.VIEWFS_SCHEME, authority, "/", null, null);
workingDir =
this.makeQualified(new Path("/user/" + ugi.getShortUserName()));
fsState = new InodeTree<FileSystem>(conf, authority) {
@Override
@ -184,6 +182,7 @@ public class ViewFileSystem extends FileSystem {
// return MergeFs.createMergeFs(mergeFsURIList, config);
}
};
workingDir = this.getHomeDirectory();
} catch (URISyntaxException e) {
throw new IOException("URISyntax exception: " + theUri);
}
@ -234,6 +233,28 @@ public class ViewFileSystem extends FileSystem {
return res.targetFileSystem.resolvePath(res.remainingPath);
}
Path homeDir = null;
@Override
public Path getHomeDirectory() {
String base = "/user/";;
if (homeDir == null) {
// We deal with whether or not home dir should be /user (hdfs) or /Users (if run on local file system on mac
try {
if (this.exists(new Path("/user/"))) {
base = "/user/";
} else if (this.exists(new Path("/Users"))) {
base = "/Users/";
} else if (this.exists(new Path("/homes"))) {
base = "/homes/";
}
} catch (IOException e) {
// ignore - - the default of /user is good enough
}
homeDir = this.makeQualified(new Path(base + ugi.getShortUserName()));
}
return homeDir;
}
@Override
public Path getWorkingDirectory() {
return workingDir;

60
src/test/core/org/apache/hadoop/fs/TestTrash.java
View File

@ -24,7 +24,6 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
@ -57,10 +56,10 @@ public class TestTrash extends TestCase {
}
// check that the specified file is in Trash
protected static void checkTrash(FileSystem fs, Path trashRoot,
protected static void checkTrash(FileSystem trashFs, Path trashRoot,
Path path) throws IOException {
Path p = new Path(trashRoot+"/"+ path.toUri().getPath());
assertTrue(fs.exists(p));
assertTrue("Could not find file in trash: "+ p , trashFs.exists(p));
}
// counts how many instances of the file are in the Trash
@ -90,10 +89,19 @@ public class TestTrash extends TestCase {
assertTrue(!fs.exists(p));
}
protected static void trashShell(final FileSystem fs, final Path base)
/**
*
* Test trash for the shell's delete command for the default file system
* specified in the paramter conf
* @param conf
* @param base - the base path where files are created
* @param trashRoot - the expected place where the trashbin resides
* @throws IOException
*/
public static void trashShell(final Configuration conf, final Path base,
FileSystem trashRootFs, Path trashRoot)
throws IOException {
Configuration conf = new Configuration();
conf.set("fs.default.name", fs.getUri().toString());
FileSystem fs = FileSystem.get(conf);
conf.set(FS_TRASH_INTERVAL_KEY, "0"); // disabled
assertFalse(new Trash(conf).isEnabled());
@ -103,7 +111,12 @@ public class TestTrash extends TestCase {
FsShell shell = new FsShell();
shell.setConf(conf);
Path trashRoot = null;
if (trashRoot == null) {
trashRoot = shell.getCurrentTrashDir();
}
if (trashRootFs == null) {
trashRootFs = fs;
}
// First create a new directory with mkdirs
Path myPath = new Path(base, "test/mkdirs");
@ -143,8 +156,8 @@ public class TestTrash extends TestCase {
}
assertTrue(val == 0);
trashRoot = shell.getCurrentTrashDir();
checkTrash(fs, trashRoot, myFile);
checkTrash(trashRootFs, trashRoot, fs.makeQualified(myFile));
}
// Verify that we can recreate the file
@ -206,7 +219,7 @@ public class TestTrash extends TestCase {
{
Path toErase = new Path(trashRoot, "toErase");
int retVal = -1;
writeFile(fs, toErase);
writeFile(trashRootFs, toErase);
try {
retVal = shell.run(new String[] {"-rm", toErase.toString()});
} catch (Exception e) {
@ -214,8 +227,8 @@ public class TestTrash extends TestCase {
e.getLocalizedMessage());
}
assertTrue(retVal == 0);
checkNotInTrash (fs, trashRoot, toErase.toString());
checkNotInTrash (fs, trashRoot, toErase.toString()+".1");
checkNotInTrash (trashRootFs, trashRoot, toErase.toString());
checkNotInTrash (trashRootFs, trashRoot, toErase.toString()+".1");
}
// simulate Trash removal
@ -233,7 +246,7 @@ public class TestTrash extends TestCase {
}
// verify that after expunging the Trash, it really goes away
checkNotInTrash(fs, trashRoot, new Path(base, "test/mkdirs/myFile").toString());
checkNotInTrash(trashRootFs, trashRoot, new Path(base, "test/mkdirs/myFile").toString());
// recreate directory and file
mkdir(fs, myPath);
@ -252,7 +265,7 @@ public class TestTrash extends TestCase {
e.getLocalizedMessage());
}
assertTrue(val == 0);
checkTrash(fs, trashRoot, myFile);
checkTrash(trashRootFs, trashRoot, myFile);
args = new String[2];
args[0] = "-rmr";
@ -265,7 +278,7 @@ public class TestTrash extends TestCase {
e.getLocalizedMessage());
}
assertTrue(val == 0);
checkTrash(fs, trashRoot, myPath);
checkTrash(trashRootFs, trashRoot, myPath);
}
// attempt to remove parent of trash
@ -281,7 +294,7 @@ public class TestTrash extends TestCase {
e.getLocalizedMessage());
}
assertEquals("exit code", 1, val);
assertTrue(fs.exists(trashRoot));
assertTrue(trashRootFs.exists(trashRoot));
}
// Verify skip trash option really works
@ -307,7 +320,10 @@ public class TestTrash extends TestCase {
System.err.println("Exception raised from Trash.run " +
e.getLocalizedMessage());
}
assertFalse(fs.exists(trashRoot)); // No new Current should be created
assertFalse("Expected TrashRoot (" + trashRoot +
") to exist in file system:"
+ trashRootFs.getUri(),
trashRootFs.exists(trashRoot)); // No new Current should be created
assertFalse(fs.exists(myFile));
assertTrue(val == 0);
}
@ -335,7 +351,7 @@ public class TestTrash extends TestCase {
e.getLocalizedMessage());
}
assertFalse(fs.exists(trashRoot)); // No new Current should be created
assertFalse(trashRootFs.exists(trashRoot)); // No new Current should be created
assertFalse(fs.exists(myPath));
assertFalse(fs.exists(myFile));
assertTrue(val == 0);
@ -414,13 +430,13 @@ public class TestTrash extends TestCase {
public void testTrash() throws IOException {
Configuration conf = new Configuration();
conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class);
trashShell(FileSystem.getLocal(conf), TEST_DIR);
trashShell(conf, TEST_DIR, null, null);
}
public void testNonDefaultFS() throws IOException {
Configuration conf = new Configuration();
conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class);
conf.set("fs.default.name", "invalid://host/bar/foo");
conf.set("fs.defaultFS", "invalid://host/bar/foo");
trashNonDefaultFS(conf);
}
@ -438,7 +454,7 @@ public class TestTrash extends TestCase {
emptierThread.start();
FileSystem fs = FileSystem.getLocal(conf);
conf.set("fs.default.name", fs.getUri().toString());
conf.set("fs.defaultFS", fs.getUri().toString());
FsShell shell = new FsShell();
shell.setConf(conf);
shell.init();
@ -521,7 +537,7 @@ public class TestTrash extends TestCase {
conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class);
FileSystem fs = FileSystem.getLocal(conf);
conf.set("fs.default.name", fs.getUri().toString());
conf.set("fs.defaultFS", fs.getUri().toString());
conf.set(FS_TRASH_INTERVAL_KEY, "10"); //minutes..
FsShell shell = new FsShell();
shell.setConf(conf);

19
src/test/core/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs.viewfs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
@ -287,5 +288,21 @@ public class TestChRootedFileSystem {
Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory());
}
/*
* Test resolvePath(p)
*/
@Test
public void testResolvePath() throws IOException {
Assert.assertEquals(chrootedTo, fSys.resolvePath(new Path("/")));
FileSystemTestHelper.createFile(fSys, "/foo");
Assert.assertEquals(new Path(chrootedTo, "foo"),
fSys.resolvePath(new Path("/foo")));
}
@Test(expected=FileNotFoundException.class)
public void testResolvePathNonExisting() throws IOException {
fSys.resolvePath(new Path("/nonExisting"));
}
}

18
src/test/core/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs.viewfs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.EnumSet;
@ -288,5 +289,22 @@ public class TestChRootedFs {
Assert.assertEquals(absoluteDir, fc.getWorkingDirectory());
}
/*
* Test resolvePath(p)
*/
@Test
public void testResolvePath() throws IOException {
Assert.assertEquals(chrootedTo, fc.getDefaultFileSystem().resolvePath(new Path("/")));
FileContextTestHelper.createFile(fc, "/foo");
Assert.assertEquals(new Path(chrootedTo, "foo"),
fc.getDefaultFileSystem().resolvePath(new Path("/foo")));
}
@Test(expected=FileNotFoundException.class)
public void testResolvePathNonExisting() throws IOException {
fc.getDefaultFileSystem().resolvePath(new Path("/nonExisting"));
}
}

3
src/test/core/org/apache/hadoop/fs/viewfs/TestFSMainOperationsLocalFileSystem.java
View File

@ -37,7 +37,8 @@ public class TestFSMainOperationsLocalFileSystem extends FSMainOperationsBaseTes
public void setUp() throws Exception {
Configuration conf = new Configuration();
fcTarget = FileSystem.getLocal(conf);
fSys = ViewFileSystemTestSetup.setupForViewFs(fcTarget);
fSys = ViewFileSystemTestSetup.setupForViewFs(
ViewFileSystemTestSetup.configWithViewfsScheme(), fcTarget);
super.setUp();
}

103
src/test/core/org/apache/hadoop/fs/viewfs/TestViewFsTrash.java Normal file
View File

@ -0,0 +1,103 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.viewfs;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.TestTrash;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class TestViewFsTrash {
FileSystem fsTarget; // the target file system - the mount will point here
FileSystem fsView;
Path targetTestRoot;
Configuration conf;
static class TestLFS extends LocalFileSystem {
Path home;
TestLFS() throws IOException {
this(new Path(FileSystemTestHelper.TEST_ROOT_DIR));
}
TestLFS(Path home) throws IOException {
super();
this.home = home;
}
public Path getHomeDirectory() {
return home;
}
}
@Before
public void setUp() throws Exception {
fsTarget = FileSystem.getLocal(new Configuration());
targetTestRoot = FileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
// In case previous test was killed before cleanup
fsTarget.delete(targetTestRoot, true);
fsTarget.mkdirs(targetTestRoot);
fsTarget.mkdirs(new Path(targetTestRoot,"dir1"));
// Now we use the mount fs to set links to user and dir
// in the test root
// Set up the defaultMT in the config with our mount point links
conf = ViewFileSystemTestSetup.configWithViewfsScheme();
// create a link for home directory so that trash path works
// set up viewfs's home dir root to point to home dir root on target
// But home dir is differnt on linux, mac etc.
// Figure it out by calling home dir on target
String homeDir = fsTarget.getHomeDirectory().toUri().getPath();
int indexOf2ndSlash = homeDir.indexOf('/', 1);
String homeDirRoot = homeDir.substring(0, indexOf2ndSlash);
ConfigUtil.addLink(conf, homeDirRoot,
fsTarget.makeQualified(new Path(homeDirRoot)).toUri());
fsView = ViewFileSystemTestSetup.setupForViewFs(conf, fsTarget);
// set working dir so that relative paths
//fsView.setWorkingDirectory(new Path(fsTarget.getWorkingDirectory().toUri().getPath()));
conf.set("fs.defaultFS", FsConstants.VIEWFS_URI.toString());
}
@After
public void tearDown() throws Exception {
}
@Test
public void testTrash() throws IOException {
TestTrash.trashShell(conf, FileSystemTestHelper.getTestRootPath(fsView),
fsTarget, new Path(fsTarget.getHomeDirectory(), ".Trash/Current"));
}
}

31
src/test/core/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java
View File

@ -32,7 +32,8 @@ import org.apache.hadoop.fs.viewfs.ConfigUtil;
* If tests launched via ant (build.xml) the test root is absolute path
* If tests launched via eclipse, the test root is
* is a test dir below the working directory. (see FileSystemTestHelper).
* Since viewFs has no built-in wd, its wd is /user/<username>.
* Since viewFs has no built-in wd, its wd is /user/<username>
* (or /User/<username> on mac
*
* We set a viewFileSystems with mount point for
* /<firstComponent>" pointing to the target fs's testdir
@ -45,32 +46,28 @@ public class ViewFileSystemTestSetup {
* @return return the ViewFS File context to be used for tests
* @throws Exception
*/
static public FileSystem setupForViewFs(FileSystem fsTarget) throws Exception {
static public FileSystem setupForViewFs(Configuration conf, FileSystem fsTarget) throws Exception {
/**
* create the test root on local_fs - the mount table will point here
*/
Configuration conf = configWithViewfsScheme();
Path targetOfTests = FileSystemTestHelper.getTestRootPath(fsTarget);
// In case previous test was killed before cleanup
fsTarget.delete(targetOfTests, true);
fsTarget.mkdirs(targetOfTests);
String srcTestFirstDir;
if (FileSystemTestHelper.TEST_ROOT_DIR.startsWith("/")) {
int indexOf2ndSlash = FileSystemTestHelper.TEST_ROOT_DIR.indexOf('/', 1);
srcTestFirstDir = FileSystemTestHelper.TEST_ROOT_DIR.substring(0, indexOf2ndSlash);
} else {
srcTestFirstDir = "/user";
}
//System.out.println("srcTestFirstDir=" + srcTestFirstDir);
// Set up the defaultMT in the config with mount point links
// The test dir is root is below /user/<userid>
ConfigUtil.addLink(conf, srcTestFirstDir,
targetOfTests.toUri());
// Now set up a link from viewfs to targetfs for the first component of
// path of testdir. For example, if testdir is /user/<userid>/xx then
// a link from /user to tragetfs://user.
String testDir = FileSystemTestHelper.getTestRootPath(fsTarget).toUri().getPath();
int indexOf2ndSlash = testDir.indexOf('/', 1);
String testDirFirstComponent = testDir.substring(0, indexOf2ndSlash);
ConfigUtil.addLink(conf, testDirFirstComponent,
fsTarget.makeQualified(new Path(testDirFirstComponent)).toUri());
FileSystem fsView = FileSystem.get(FsConstants.VIEWFS_URI, conf);
//System.out.println("SRCOfTests = "+ getTestRootPath(fs, "test"));