Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-8957. Merge r1420965 from trunk 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1490115 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Suresh Srinivas 2013-06-06 03:29:43 +00:00
parent 0cb5e152d4
commit a35a2ca335
11 changed files with 84 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -239,6 +239,9 @@ Release 2.1.0-beta - UNRELEASED
HADOOP-9593. stack trace printed at ERROR for all yarn clients without
hadoop.home set (stevel)
HADOOP-8957 AbstractFileSystem#IsValidName should be overridden for
embedded file systems like ViewFs (Chris Nauroth via Sanjay Radia)
BREAKDOWN OF HADOOP-8562 SUBTASKS AND RELATED JIRAS
HADOOP-8924. Hadoop Common creating package-info.java must not depend on

14
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
View File

@ -85,14 +85,20 @@ public Statistics getStatistics() {
}
/**
* Prohibits names which contain a ".", "..", ":" or "/"
* Returns true if the specified string is considered valid in the path part
* of a URI by this file system. The default implementation enforces the rules
* of HDFS, but subclasses may override this method to implement specific
* validation rules for specific file systems.
*
* @param src String source filename to check, path part of the URI
* @return boolean true if the specified string is considered valid
*/
private static boolean isValidName(String src) {
// Check for ".." "." ":" "/"
public boolean isValidName(String src) {
// Prohibit ".." "." and anything containing ":"
StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR);
while(tokens.hasMoreTokens()) {
String element = tokens.nextToken();
if (element.equals("target/generated-sources") ||
if (element.equals("..") ||
element.equals(".") ||
(element.indexOf(":") >= 0)) {
return false;

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
View File

@ -278,4 +278,9 @@ public String getCanonicalServiceName() {
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
return myFs.getDelegationTokens(renewer);
}
@Override
public boolean isValidName(String src) {
return myFs.isValidName(src);
}
}

8
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
View File

@ -158,6 +158,14 @@ public FileStatus getFileLinkStatus(final Path f) throws IOException {
}
}
@Override
public boolean isValidName(String src) {
// Different local file systems have different validation rules. Skip
// validation here and just let the OS handle it. This is consistent with
// RawLocalFileSystem.
return true;
}
@Override
public Path getLinkTarget(Path f) throws IOException {
/* We should never get here. Valid local links are resolved transparently

9
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
View File

@ -83,7 +83,12 @@ protected Path fullPath(final Path path) {
return new Path((chRootPathPart.isRoot() ? "" : chRootPathPartString)
+ path.toUri().getPath());
}
@Override
public boolean isValidName(String src) {
return myFs.isValidName(fullPath(new Path(src)).toUri().toString());
}
public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
throws URISyntaxException {
super(fs.getUri(), fs.getUri().getScheme(),
@ -103,7 +108,7 @@ public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
// scheme:/// and scheme://authority/
myUri = new URI(myFs.getUri().toString() +
(myFs.getUri().getAuthority() == null ? "" : Path.SEPARATOR) +
chRootPathPart.toString().substring(1));
chRootPathPart.toUri().getPath().substring(1));
super.checkPath(theRoot);
}

26
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -64,6 +64,9 @@
@InterfaceAudience.Public
@InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */
public class ViewFileSystem extends FileSystem {
private static final Path ROOT_PATH = new Path(Path.SEPARATOR);
static AccessControlException readOnlyMountTable(final String operation,
final String p) {
return new AccessControlException(
@ -98,23 +101,6 @@ URI[] getTargets() {
InodeTree<FileSystem> fsState; // the fs state; ie the mount table
Path homeDir = null;
/**
* Prohibits names which contain a ".", "..", ":" or "/"
*/
private static boolean isValidName(final String src) {
// Check for ".." "." ":" "/"
final StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR);
while(tokens.hasMoreTokens()) {
String element = tokens.nextToken();
if (element.equals("..") ||
element.equals(".") ||
(element.indexOf(":") >= 0)) {
return false;
}
}
return true;
}
/**
* Make the path Absolute and get the path-part of a pathname.
* Checks that URI matches this file system
@ -126,10 +112,6 @@ private static boolean isValidName(final String src) {
private String getUriPath(final Path p) {
checkPath(p);
String s = makeAbsolute(p).toUri().getPath();
if (!isValidName(s)) {
throw new InvalidPathException("Path part " + s + " from URI" + p
+ " is not a valid filename.");
}
return s;
}
@ -689,7 +671,7 @@ public FileStatus getFileStatus(Path f) throws IOException {
PERMISSION_RRR, ugi.getUserName(), ugi.getGroupNames()[0],
new Path(theInternalDir.fullPath).makeQualified(
myUri, null));
myUri, ROOT_PATH));
}

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
View File

@ -597,6 +597,12 @@ public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
return result;
}
@Override
public boolean isValidName(String src) {
// Prefix validated at mount time and rest of path validated by mount target.
return true;
}
/*

19
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
View File

@ -25,6 +25,7 @@
import static org.apache.hadoop.fs.FileContextTestHelper.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContextTestHelper;
@ -36,6 +37,7 @@
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
public class TestChRootedFs {
FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
@ -308,4 +310,21 @@ public void testResolvePathNonExisting() throws IOException {
fc.getDefaultFileSystem().resolvePath(new Path("/nonExisting"));
}
@Test
public void testIsValidNameValidInBaseFs() throws Exception {
AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
ChRootedFs chRootedFs = new ChRootedFs(baseFs, new Path("/chroot"));
Mockito.doReturn(true).when(baseFs).isValidName(Mockito.anyString());
Assert.assertTrue(chRootedFs.isValidName("/test"));
Mockito.verify(baseFs).isValidName("/chroot/test");
}
@Test
public void testIsValidNameInvalidInBaseFs() throws Exception {
AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
ChRootedFs chRootedFs = new ChRootedFs(baseFs, new Path("/chroot"));
Mockito.doReturn(false).when(baseFs).isValidName(Mockito.anyString());
Assert.assertFalse(chRootedFs.isValidName("/test"));
Mockito.verify(baseFs).isValidName("/chroot/test");
}
}

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -345,6 +345,9 @@ Release 2.1.0-beta - UNRELEASED
HDFS-4815. TestRBWBlockInvalidation: Double call countReplicas() to fetch
corruptReplicas and liveReplicas is not needed. (Tian Hong Wang via atm)
HADOOP-8957 HDFS tests for AbstractFileSystem#IsValidName should be overridden for
embedded file systems like ViewFs (Chris Nauroth via Sanjay Radia)
BREAKDOWN OF HDFS-347 SUBTASKS AND RELATED JIRAS
HDFS-4353. Encapsulate connections to peers in Peer and PeerServer classes.

17
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
View File

@ -257,7 +257,22 @@ public void testEditsLogRename() throws Exception {
Assert.assertFalse(fs.exists(src1)); // ensure src1 is already renamed
Assert.assertTrue(fs.exists(dst1)); // ensure rename dst exists
}
@Test
public void testIsValidNameInvalidNames() {
String[] invalidNames = {
"/foo/../bar",
"/foo/./bar",
"/foo/:/bar",
"/foo:bar"
};
for (String invalidName: invalidNames) {
Assert.assertFalse(invalidName + " is not valid",
fc.getDefaultFileSystem().isValidName(invalidName));
}
}
private void oldRename(Path src, Path dst, boolean renameSucceeds,
boolean exception) throws Exception {
DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
View File

@ -640,9 +640,12 @@ public void testLocalhostReverseLookup() {
@Test (timeout=15000)
public void testIsValidName() {
assertFalse(DFSUtil.isValidName("/foo/../bar"));
assertFalse(DFSUtil.isValidName("/foo/./bar"));
assertFalse(DFSUtil.isValidName("/foo//bar"));
assertTrue(DFSUtil.isValidName("/"));
assertTrue(DFSUtil.isValidName("/bar/"));
assertFalse(DFSUtil.isValidName("/foo/:/bar"));
assertFalse(DFSUtil.isValidName("/foo:bar"));
}
@Test(timeout=5000)