Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13722. Code cleanup -- ViewFileSystem and InodeTree. Contributed by Manoj Govindassamy.

This commit is contained in:
Andrew Wang 2016-10-17 13:15:11 -07:00
parent 412c4c9a34
commit 0f4afc8100
3 changed files with 158 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
View File

@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -36,7 +36,6 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
/**
* InodeTree implements a mount-table as a tree of inodes.
* It is used to implement ViewFs and ViewFileSystem.
@ -48,8 +47,7 @@ import org.apache.hadoop.util.StringUtils;
*
* @param <T> is AbstractFileSystem or FileSystem
*
* The three main methods are
* {@link #InodeTreel(Configuration)} // constructor
* The two main methods are
* {@link #InodeTree(Configuration, String)} // constructor
* {@link #resolve(String, boolean)}
*/
@ -57,24 +55,24 @@ import org.apache.hadoop.util.StringUtils;
@InterfaceAudience.Private
@InterfaceStability.Unstable
abstract class InodeTree<T> {
static enum ResultKind {isInternalDir, isExternalDir;};
enum ResultKind {
INTERNAL_DIR,
EXTERNAL_DIR
}
static final Path SlashPath = new Path("/");
final INodeDir<T> root; // the root of the mount table
final String homedirPrefix; // the homedir config value for this mount table
List<MountPoint<T>> mountPoints = new ArrayList<MountPoint<T>>();
private final INodeDir<T> root; // the root of the mount table
private final String homedirPrefix; // the homedir for this mount table
private List<MountPoint<T>> mountPoints = new ArrayList<MountPoint<T>>();
static class MountPoint<T> {
String src;
INodeLink<T> target;
MountPoint(String srcPath, INodeLink<T> mountLink) {
src = srcPath;
target = mountLink;
}
}
/**
@ -92,10 +90,11 @@ abstract class InodeTree<T> {
*/
abstract static class INode<T> {
final String fullPath; // the full path to the root
public INode(String pathToNode, UserGroupInformation aUgi) {
fullPath = pathToNode;
}
};
}
/**
* Internal class to represent an internal dir of the mount table
@ -110,26 +109,17 @@ abstract class InodeTree<T> {
super(pathToNode, aUgi);
}
INode<T> resolve(final String pathComponent) throws FileNotFoundException {
final INode<T> result = resolveInternal(pathComponent);
if (result == null) {
throw new FileNotFoundException();
}
return result;
}
INode<T> resolveInternal(final String pathComponent) {
return children.get(pathComponent);
}
INodeDir<T> addDir(final String pathComponent,
final UserGroupInformation aUgi)
throws FileAlreadyExistsException {
final UserGroupInformation aUgi) throws FileAlreadyExistsException {
if (children.containsKey(pathComponent)) {
throw new FileAlreadyExistsException();
}
final INodeDir<T> newDir = new INodeDir<T>(fullPath+ (isRoot ? "" : "/") +
pathComponent, aUgi);
final INodeDir<T> newDir = new INodeDir<T>(fullPath +
(isRoot ? "" : "/") + pathComponent, aUgi);
children.put(pathComponent, newDir);
return newDir;
}
@ -144,11 +134,11 @@ abstract class InodeTree<T> {
}
/**
* In internal class to represent a mount link
* An internal class to represent a mount link.
* A mount link can be single dir link or a merge dir link.
* A merge dir link is a merge (junction) of links to dirs:
* example : <merge of 2 dirs
* example : merge of 2 dirs
* /users -> hdfs:nn1//users
* /users -> hdfs:nn2//users
*
@ -161,7 +151,7 @@ abstract class InodeTree<T> {
final T targetFileSystem; // file system object created from the link.
/**
* Construct a mergeLink
* Construct a mergeLink.
*/
INodeLink(final String pathToNode, final UserGroupInformation aUgi,
final T targetMergeFs, final URI[] aTargetDirLinkList) {
@ -172,7 +162,7 @@ abstract class InodeTree<T> {
}
/**
* Construct a simple link (i.e. not a mergeLink)
* Construct a simple link (i.e. not a mergeLink).
*/
INodeLink(final String pathToNode, final UserGroupInformation aUgi,
final T targetFs, final URI aTargetDirLink) {
@ -184,21 +174,19 @@ abstract class InodeTree<T> {
}
/**
* Get the target of the link
* If a merge link then it returned as "," separated URI list.
* Get the target of the link. If a merge link then it returned
* as "," separated URI list.
*/
Path getTargetLink() {
// is merge link - use "," as separator between the merged URIs
//String result = targetDirLinkList[0].toString();
StringBuilder result = new StringBuilder(targetDirLinkList[0].toString());
for (int i=1; i < targetDirLinkList.length; ++i) {
// If merge link, use "," as separator between the merged URIs
for (int i = 1; i < targetDirLinkList.length; ++i) {
result.append(',').append(targetDirLinkList[i].toString());
}
return new Path(result.toString());
}
}
private void createLink(final String src, final String target,
final boolean isLinkMerge, final UserGroupInformation aUgi)
throws URISyntaxException, IOException,
@ -206,14 +194,14 @@ abstract class InodeTree<T> {
// Validate that src is valid absolute path
final Path srcPath = new Path(src);
if (!srcPath.isAbsoluteAndSchemeAuthorityNull()) {
throw new IOException("ViewFs:Non absolute mount name in config:" + src);
throw new IOException("ViewFs: Non absolute mount name in config:" + src);
}
final String[] srcPaths = breakIntoPathComponents(src);
INodeDir<T> curInode = root;
int i;
// Ignore first initial slash, process all except last component
for (i = 1; i < srcPaths.length-1; i++) {
for (i = 1; i < srcPaths.length - 1; i++) {
final String iPath = srcPaths[i];
INode<T> nextInode = curInode.resolveInternal(iPath);
if (nextInode == null) {
@ -226,7 +214,7 @@ abstract class InodeTree<T> {
throw new FileAlreadyExistsException("Path " + nextInode.fullPath +
" already exists as link");
} else {
assert(nextInode instanceof INodeDir);
assert (nextInode instanceof INodeDir);
curInode = (INodeDir<T>) nextInode;
}
}
@ -264,10 +252,6 @@ abstract class InodeTree<T> {
mountPoints.add(new MountPoint<T>(src, newLink));
}
/**
* Below the "public" methods of InodeTree
*/
/**
* The user of this class must subclass and implement the following
* 3 abstract methods.
@ -325,9 +309,8 @@ abstract class InodeTree<T> {
// ignore - we set home dir from config
continue;
} else {
throw new IOException(
"ViewFs: Cannot initialize: Invalid entry in Mount table in config: "+
src);
throw new IOException("ViewFs: Cannot initialize: Invalid entry in " +
"Mount table in config: " + src);
}
final String target = si.getValue(); // link or merge link
createLink(src, target, isMergeLink, ugi);
@ -365,9 +348,9 @@ abstract class InodeTree<T> {
remainingPath = remainingP;
}
// isInternalDir of path resolution completed within the mount table
// Internal dir path resolution completed within the mount table
boolean isInternalDir() {
return (kind == ResultKind.isInternalDir);
return (kind == ResultKind.INTERNAL_DIR);
}
}
@ -380,11 +363,10 @@ abstract class InodeTree<T> {
*/
ResolveResult<T> resolve(final String p, final boolean resolveLastComponent)
throws FileNotFoundException {
// TO DO: - more efficient to not split the path, but simply compare
String[] path = breakIntoPathComponents(p);
if (path.length <= 1) { // special case for when path is "/"
ResolveResult<T> res =
new ResolveResult<T>(ResultKind.isInternalDir,
new ResolveResult<T>(ResultKind.INTERNAL_DIR,
root.InodeDirFs, root.fullPath, SlashPath);
return res;
}
@ -396,7 +378,7 @@ abstract class InodeTree<T> {
INode<T> nextInode = curInode.resolveInternal(path[i]);
if (nextInode == null) {
StringBuilder failedAt = new StringBuilder(path[0]);
for ( int j = 1; j <=i; ++j) {
for (int j = 1; j <= i; ++j) {
failedAt.append('/').append(path[j]);
}
throw (new FileNotFoundException(failedAt.toString()));
@ -405,17 +387,17 @@ abstract class InodeTree<T> {
if (nextInode instanceof INodeLink) {
final INodeLink<T> link = (INodeLink<T>) nextInode;
final Path remainingPath;
if (i >= path.length-1) {
if (i >= path.length - 1) {
remainingPath = SlashPath;
} else {
StringBuilder remainingPathStr = new StringBuilder("/" + path[i+1]);
for (int j = i+2; j< path.length; ++j) {
StringBuilder remainingPathStr = new StringBuilder("/" + path[i + 1]);
for (int j = i + 2; j < path.length; ++j) {
remainingPathStr.append('/').append(path[j]);
}
remainingPath = new Path(remainingPathStr.toString());
}
final ResolveResult<T> res =
new ResolveResult<T>(ResultKind.isExternalDir,
new ResolveResult<T>(ResultKind.EXTERNAL_DIR,
link.targetFileSystem, nextInode.fullPath, remainingPath);
return res;
} else if (nextInode instanceof INodeDir) {
@ -433,13 +415,13 @@ abstract class InodeTree<T> {
// that follows will do a children.get(remaningPath) and will have to
// strip-out the initial /
StringBuilder remainingPathStr = new StringBuilder("/" + path[i]);
for (int j = i+1; j< path.length; ++j) {
for (int j = i + 1; j < path.length; ++j) {
remainingPathStr.append('/').append(path[j]);
}
remainingPath = new Path(remainingPathStr.toString());
}
final ResolveResult<T> res =
new ResolveResult<T>(ResultKind.isInternalDir,
new ResolveResult<T>(ResultKind.INTERNAL_DIR,
curInode.InodeDirFs, curInode.fullPath, remainingPath);
return res;
}

73
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -89,8 +89,17 @@ public class ViewFileSystem extends FileSystem {
}
static public class MountPoint {
private Path src; // the src of the mount
private URI[] targets; // target of the mount; Multiple targets imply mergeMount
/**
* The source of the mount.
*/
private Path src;
/**
* One or more targets of the mount.
* Multiple targets imply MergeMount.
*/
private URI[] targets;
MountPoint(Path srcPath, URI[] targetURIs) {
src = srcPath;
targets = targetURIs;
@ -142,13 +151,12 @@ public class ViewFileSystem extends FileSystem {
/**
* Return the protocol scheme for the FileSystem.
* <p/>
*
* @return <code>viewfs</code>
*/
@Override
public String getScheme() {
return "viewfs";
return FsConstants.VIEWFS_SCHEME;
}
/**
@ -198,7 +206,6 @@ public class ViewFileSystem extends FileSystem {
}
/**
* Convenience Constructor for apps to call directly
* @param theUri which must be that of ViewFileSystem
@ -226,8 +233,7 @@ public class ViewFileSystem extends FileSystem {
}
@Override
public Path resolvePath(final Path f)
throws IOException {
public Path resolvePath(final Path f) throws IOException {
final InodeTree.ResolveResult<FileSystem> res;
res = fsState.resolve(getUriPath(f), true);
if (res.isInternalDir()) {
@ -271,8 +277,8 @@ public class ViewFileSystem extends FileSystem {
@Override
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
EnumSet<CreateFlag> flags, int bufferSize, short replication,
long blockSize, Progressable progress) throws IOException {
InodeTree.ResolveResult<FileSystem> res;
try {
res = fsState.resolve(getUriPath(f), false);
@ -280,8 +286,8 @@ public class ViewFileSystem extends FileSystem {
throw readOnlyMountTable("create", f);
}
assert(res.remainingPath != null);
return res.targetFileSystem.createNonRecursive(res.remainingPath, permission,
flags, bufferSize, replication, blockSize, progress);
return res.targetFileSystem.createNonRecursive(res.remainingPath,
permission, flags, bufferSize, replication, blockSize, progress);
}
@Override
@ -302,8 +308,7 @@ public class ViewFileSystem extends FileSystem {
@Override
public boolean delete(final Path f, final boolean recursive)
throws AccessControlException, FileNotFoundException,
IOException {
throws AccessControlException, FileNotFoundException, IOException {
InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true);
// If internal dir or target is a mount link (ie remainingPath is Slash)
@ -316,8 +321,7 @@ public class ViewFileSystem extends FileSystem {
@Override
@SuppressWarnings("deprecation")
public boolean delete(final Path f)
throws AccessControlException, FileNotFoundException,
IOException {
throws AccessControlException, FileNotFoundException, IOException {
return delete(f, true);
}
@ -339,7 +343,6 @@ public class ViewFileSystem extends FileSystem {
return res.targetFileSystem.getFileChecksum(res.remainingPath);
}
private static FileStatus fixFileStatus(FileStatus orig,
Path qualified) throws IOException {
// FileStatus#getPath is a fully qualified path relative to the root of
@ -367,7 +370,6 @@ public class ViewFileSystem extends FileSystem {
: new ViewFsFileStatus(orig, qualified);
}
@Override
public FileStatus getFileStatus(final Path f) throws AccessControlException,
FileNotFoundException, IOException {
@ -407,10 +409,10 @@ public class ViewFileSystem extends FileSystem {
@Override
public RemoteIterator<LocatedFileStatus>listLocatedStatus(final Path f,
final PathFilter filter) throws FileNotFoundException, IOException {
final InodeTree.ResolveResult<FileSystem> res = fsState
.resolve(getUriPath(f), true);
final RemoteIterator<LocatedFileStatus> statusIter = res.targetFileSystem
.listLocatedStatus(res.remainingPath);
final InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true);
final RemoteIterator<LocatedFileStatus> statusIter =
res.targetFileSystem.listLocatedStatus(res.remainingPath);
if (res.isInternalDir()) {
return statusIter;
@ -449,8 +451,7 @@ public class ViewFileSystem extends FileSystem {
@Override
public FSDataInputStream open(final Path f, final int bufferSize)
throws AccessControlException, FileNotFoundException,
IOException {
throws AccessControlException, FileNotFoundException, IOException {
InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true);
return res.targetFileSystem.open(res.remainingPath, bufferSize);
@ -507,8 +508,7 @@ public class ViewFileSystem extends FileSystem {
@Override
public void setOwner(final Path f, final String username,
final String groupname) throws AccessControlException,
FileNotFoundException,
IOException {
FileNotFoundException, IOException {
InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true);
res.targetFileSystem.setOwner(res.remainingPath, username, groupname);
@ -516,8 +516,7 @@ public class ViewFileSystem extends FileSystem {
@Override
public void setPermission(final Path f, final FsPermission permission)
throws AccessControlException, FileNotFoundException,
IOException {
throws AccessControlException, FileNotFoundException, IOException {
InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true);
res.targetFileSystem.setPermission(res.remainingPath, permission);
@ -525,8 +524,7 @@ public class ViewFileSystem extends FileSystem {
@Override
public boolean setReplication(final Path f, final short replication)
throws AccessControlException, FileNotFoundException,
IOException {
throws AccessControlException, FileNotFoundException, IOException {
InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true);
return res.targetFileSystem.setReplication(res.remainingPath, replication);
@ -534,8 +532,7 @@ public class ViewFileSystem extends FileSystem {
@Override
public void setTimes(final Path f, final long mtime, final long atime)
throws AccessControlException, FileNotFoundException,
IOException {
throws AccessControlException, FileNotFoundException, IOException {
InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true);
res.targetFileSystem.setTimes(res.remainingPath, mtime, atime);
@ -793,7 +790,7 @@ public class ViewFileSystem extends FileSystem {
return allPolicies;
}
/*
/**
* An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table.
* It is a read only mount tables and create, mkdir or delete operations
@ -826,8 +823,8 @@ public class ViewFileSystem extends FileSystem {
static private void checkPathIsSlash(final Path f) throws IOException {
if (f != InodeTree.SlashPath) {
throw new IOException (
"Internal implementation error: expected file name to be /" );
throw new IOException(
"Internal implementation error: expected file name to be /");
}
}
@ -838,14 +835,14 @@ public class ViewFileSystem extends FileSystem {
@Override
public Path getWorkingDirectory() {
throw new RuntimeException (
"Internal impl error: getWorkingDir should not have been called" );
throw new RuntimeException(
"Internal impl error: getWorkingDir should not have been called");
}
@Override
public void setWorkingDirectory(final Path new_dir) {
throw new RuntimeException (
"Internal impl error: getWorkingDir should not have been called" );
throw new RuntimeException(
"Internal impl error: getWorkingDir should not have been called");
}
@Override

26
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java
View File

@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -22,19 +22,14 @@ import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.viewfs.ConfigUtil;
import org.apache.hadoop.fs.viewfs.InodeTree;
import org.junit.Test;
public class TestViewFsConfig {
@Test(expected=FileAlreadyExistsException.class)
@Test(expected = FileAlreadyExistsException.class)
public void testInvalidConfig() throws IOException, URISyntaxException {
Configuration conf = new Configuration();
ConfigUtil.addLink(conf, "/internalDir/linkToDir2",
@ -42,29 +37,26 @@ public class TestViewFsConfig {
ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3",
new Path("file:///dir3").toUri());
class Foo { };
class Foo {
}
new InodeTree<Foo>(conf, null) {
@Override
protected
Foo getTargetFileSystem(final URI uri)
protected Foo getTargetFileSystem(final URI uri)
throws URISyntaxException, UnsupportedFileSystemException {
return null;
}
@Override
protected
Foo getTargetFileSystem(
org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo>
dir)
protected Foo getTargetFileSystem(
org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo> dir)
throws URISyntaxException {
return null;
}
@Override
protected
Foo getTargetFileSystem(URI[] mergeFsURIList)
protected Foo getTargetFileSystem(URI[] mergeFsURIList)
throws URISyntaxException, UnsupportedFileSystemException {
return null;
}