Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13722. Code cleanup -- ViewFileSystem and InodeTree. Contributed by Manoj Govindassamy.

This commit is contained in:
Andrew Wang 2016-10-17 13:15:11 -07:00
parent 412c4c9a34
commit 0f4afc8100
3 changed files with 158 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

206
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
View File

@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the * to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance * "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at * with the License. You may obtain a copy of the License at
* * <p>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* * <p>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -36,47 +36,45 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
/** /**
* InodeTree implements a mount-table as a tree of inodes. * InodeTree implements a mount-table as a tree of inodes.
* It is used to implement ViewFs and ViewFileSystem. * It is used to implement ViewFs and ViewFileSystem.
* In order to use it the caller must subclass it and implement * In order to use it the caller must subclass it and implement
* the abstract methods {@link #getTargetFileSystem(INodeDir)}, etc. * the abstract methods {@link #getTargetFileSystem(INodeDir)}, etc.
* *
* The mountable is initialized from the config variables as * The mountable is initialized from the config variables as
* specified in {@link ViewFs} * specified in {@link ViewFs}
* *
* @param <T> is AbstractFileSystem or FileSystem * @param <T> is AbstractFileSystem or FileSystem
* *
* The three main methods are * The two main methods are
* {@link #InodeTreel(Configuration)} // constructor
* {@link #InodeTree(Configuration, String)} // constructor * {@link #InodeTree(Configuration, String)} // constructor
* {@link #resolve(String, boolean)} * {@link #resolve(String, boolean)}
*/ */
@InterfaceAudience.Private @InterfaceAudience.Private
@InterfaceStability.Unstable @InterfaceStability.Unstable
abstract class InodeTree<T> { abstract class InodeTree<T> {
static enum ResultKind {isInternalDir, isExternalDir;}; enum ResultKind {
INTERNAL_DIR,
EXTERNAL_DIR
}
static final Path SlashPath = new Path("/"); static final Path SlashPath = new Path("/");
private final INodeDir<T> root; // the root of the mount table
final INodeDir<T> root; // the root of the mount table private final String homedirPrefix; // the homedir for this mount table
private List<MountPoint<T>> mountPoints = new ArrayList<MountPoint<T>>();
final String homedirPrefix; // the homedir config value for this mount table
List<MountPoint<T>> mountPoints = new ArrayList<MountPoint<T>>();
static class MountPoint<T> { static class MountPoint<T> {
String src; String src;
INodeLink<T> target; INodeLink<T> target;
MountPoint(String srcPath, INodeLink<T> mountLink) { MountPoint(String srcPath, INodeLink<T> mountLink) {
src = srcPath; src = srcPath;
target = mountLink; target = mountLink;
} }
} }
/** /**
* Breaks file path into component names. * Breaks file path into component names.
* @param path * @param path
@ -84,18 +82,19 @@ abstract class InodeTree<T> {
*/ */
static String[] breakIntoPathComponents(final String path) { static String[] breakIntoPathComponents(final String path) {
return path == null ? null : path.split(Path.SEPARATOR); return path == null ? null : path.split(Path.SEPARATOR);
} }
/** /**
* Internal class for inode tree * Internal class for inode tree
* @param <T> * @param <T>
*/ */
abstract static class INode<T> { abstract static class INode<T> {
final String fullPath; // the full path to the root final String fullPath; // the full path to the root
public INode(String pathToNode, UserGroupInformation aUgi) { public INode(String pathToNode, UserGroupInformation aUgi) {
fullPath = pathToNode; fullPath = pathToNode;
} }
}; }
/** /**
* Internal class to represent an internal dir of the mount table * Internal class to represent an internal dir of the mount table
@ -105,37 +104,28 @@ abstract class InodeTree<T> {
final Map<String,INode<T>> children = new HashMap<String,INode<T>>(); final Map<String,INode<T>> children = new HashMap<String,INode<T>>();
T InodeDirFs = null; // file system of this internal directory of mountT T InodeDirFs = null; // file system of this internal directory of mountT
boolean isRoot = false; boolean isRoot = false;
INodeDir(final String pathToNode, final UserGroupInformation aUgi) { INodeDir(final String pathToNode, final UserGroupInformation aUgi) {
super(pathToNode, aUgi); super(pathToNode, aUgi);
} }
INode<T> resolve(final String pathComponent) throws FileNotFoundException {
final INode<T> result = resolveInternal(pathComponent);
if (result == null) {
throw new FileNotFoundException();
}
return result;
}
INode<T> resolveInternal(final String pathComponent) { INode<T> resolveInternal(final String pathComponent) {
return children.get(pathComponent); return children.get(pathComponent);
} }
INodeDir<T> addDir(final String pathComponent, INodeDir<T> addDir(final String pathComponent,
final UserGroupInformation aUgi) final UserGroupInformation aUgi) throws FileAlreadyExistsException {
throws FileAlreadyExistsException {
if (children.containsKey(pathComponent)) { if (children.containsKey(pathComponent)) {
throw new FileAlreadyExistsException(); throw new FileAlreadyExistsException();
} }
final INodeDir<T> newDir = new INodeDir<T>(fullPath+ (isRoot ? "" : "/") + final INodeDir<T> newDir = new INodeDir<T>(fullPath +
pathComponent, aUgi); (isRoot ? "" : "/") + pathComponent, aUgi);
children.put(pathComponent, newDir); children.put(pathComponent, newDir);
return newDir; return newDir;
} }
void addLink(final String pathComponent, final INodeLink<T> link) void addLink(final String pathComponent, final INodeLink<T> link)
throws FileAlreadyExistsException { throws FileAlreadyExistsException {
if (children.containsKey(pathComponent)) { if (children.containsKey(pathComponent)) {
throw new FileAlreadyExistsException(); throw new FileAlreadyExistsException();
} }
@ -144,14 +134,14 @@ abstract class InodeTree<T> {
} }
/** /**
* In internal class to represent a mount link * An internal class to represent a mount link.
* A mount link can be single dir link or a merge dir link. * A mount link can be single dir link or a merge dir link.
* A merge dir link is a merge (junction) of links to dirs: * A merge dir link is a merge (junction) of links to dirs:
* example : <merge of 2 dirs * example : merge of 2 dirs
* /users -> hdfs:nn1//users * /users -> hdfs:nn1//users
* /users -> hdfs:nn2//users * /users -> hdfs:nn2//users
* *
* For a merge, each target is checked to be dir when created but if target * For a merge, each target is checked to be dir when created but if target
* is changed later it is then ignored (a dir with null entries) * is changed later it is then ignored (a dir with null entries)
*/ */
@ -159,9 +149,9 @@ abstract class InodeTree<T> {
final boolean isMergeLink; // true if MergeLink final boolean isMergeLink; // true if MergeLink
final URI[] targetDirLinkList; final URI[] targetDirLinkList;
final T targetFileSystem; // file system object created from the link. final T targetFileSystem; // file system object created from the link.
/** /**
* Construct a mergeLink * Construct a mergeLink.
*/ */
INodeLink(final String pathToNode, final UserGroupInformation aUgi, INodeLink(final String pathToNode, final UserGroupInformation aUgi,
final T targetMergeFs, final URI[] aTargetDirLinkList) { final T targetMergeFs, final URI[] aTargetDirLinkList) {
@ -170,9 +160,9 @@ abstract class InodeTree<T> {
targetDirLinkList = aTargetDirLinkList; targetDirLinkList = aTargetDirLinkList;
isMergeLink = true; isMergeLink = true;
} }
/** /**
* Construct a simple link (i.e. not a mergeLink) * Construct a simple link (i.e. not a mergeLink).
*/ */
INodeLink(final String pathToNode, final UserGroupInformation aUgi, INodeLink(final String pathToNode, final UserGroupInformation aUgi,
final T targetFs, final URI aTargetDirLink) { final T targetFs, final URI aTargetDirLink) {
@ -182,38 +172,36 @@ abstract class InodeTree<T> {
targetDirLinkList[0] = aTargetDirLink; targetDirLinkList[0] = aTargetDirLink;
isMergeLink = false; isMergeLink = false;
} }
/** /**
* Get the target of the link * Get the target of the link. If a merge link then it returned
* If a merge link then it returned as "," separated URI list. * as "," separated URI list.
*/ */
Path getTargetLink() { Path getTargetLink() {
// is merge link - use "," as separator between the merged URIs
//String result = targetDirLinkList[0].toString();
StringBuilder result = new StringBuilder(targetDirLinkList[0].toString()); StringBuilder result = new StringBuilder(targetDirLinkList[0].toString());
for (int i=1; i < targetDirLinkList.length; ++i) { // If merge link, use "," as separator between the merged URIs
for (int i = 1; i < targetDirLinkList.length; ++i) {
result.append(',').append(targetDirLinkList[i].toString()); result.append(',').append(targetDirLinkList[i].toString());
} }
return new Path(result.toString()); return new Path(result.toString());
} }
} }
private void createLink(final String src, final String target, private void createLink(final String src, final String target,
final boolean isLinkMerge, final UserGroupInformation aUgi) final boolean isLinkMerge, final UserGroupInformation aUgi)
throws URISyntaxException, IOException, throws URISyntaxException, IOException,
FileAlreadyExistsException, UnsupportedFileSystemException { FileAlreadyExistsException, UnsupportedFileSystemException {
// Validate that src is valid absolute path // Validate that src is valid absolute path
final Path srcPath = new Path(src); final Path srcPath = new Path(src);
if (!srcPath.isAbsoluteAndSchemeAuthorityNull()) { if (!srcPath.isAbsoluteAndSchemeAuthorityNull()) {
throw new IOException("ViewFs:Non absolute mount name in config:" + src); throw new IOException("ViewFs: Non absolute mount name in config:" + src);
} }
final String[] srcPaths = breakIntoPathComponents(src); final String[] srcPaths = breakIntoPathComponents(src);
INodeDir<T> curInode = root; INodeDir<T> curInode = root;
int i; int i;
// Ignore first initial slash, process all except last component // Ignore first initial slash, process all except last component
for (i = 1; i < srcPaths.length-1; i++) { for (i = 1; i < srcPaths.length - 1; i++) {
final String iPath = srcPaths[i]; final String iPath = srcPaths[i];
INode<T> nextInode = curInode.resolveInternal(iPath); INode<T> nextInode = curInode.resolveInternal(iPath);
if (nextInode == null) { if (nextInode == null) {
@ -226,11 +214,11 @@ abstract class InodeTree<T> {
throw new FileAlreadyExistsException("Path " + nextInode.fullPath + throw new FileAlreadyExistsException("Path " + nextInode.fullPath +
" already exists as link"); " already exists as link");
} else { } else {
assert(nextInode instanceof INodeDir); assert (nextInode instanceof INodeDir);
curInode = (INodeDir<T>) nextInode; curInode = (INodeDir<T>) nextInode;
} }
} }
// Now process the last component // Now process the last component
// Add the link in 2 cases: does not exist or a link exists // Add the link in 2 cases: does not exist or a link exists
String iPath = srcPaths[i];// last component String iPath = srcPaths[i];// last component
@ -241,9 +229,9 @@ abstract class InodeTree<T> {
strB.append('/').append(srcPaths[j]); strB.append('/').append(srcPaths[j]);
} }
throw new FileAlreadyExistsException("Path " + strB + throw new FileAlreadyExistsException("Path " + strB +
" already exists as dir; cannot create link here"); " already exists as dir; cannot create link here");
} }
final INodeLink<T> newLink; final INodeLink<T> newLink;
final String fullPath = curInode.fullPath + (curInode == root ? "" : "/") final String fullPath = curInode.fullPath + (curInode == root ? "" : "/")
+ iPath; + iPath;
@ -263,25 +251,21 @@ abstract class InodeTree<T> {
curInode.addLink(iPath, newLink); curInode.addLink(iPath, newLink);
mountPoints.add(new MountPoint<T>(src, newLink)); mountPoints.add(new MountPoint<T>(src, newLink));
} }
/**
* Below the "public" methods of InodeTree
*/
/** /**
* The user of this class must subclass and implement the following * The user of this class must subclass and implement the following
* 3 abstract methods. * 3 abstract methods.
* @throws IOException * @throws IOException
*/ */
protected abstract T getTargetFileSystem(final URI uri) protected abstract T getTargetFileSystem(final URI uri)
throws UnsupportedFileSystemException, URISyntaxException, IOException; throws UnsupportedFileSystemException, URISyntaxException, IOException;
protected abstract T getTargetFileSystem(final INodeDir<T> dir) protected abstract T getTargetFileSystem(final INodeDir<T> dir)
throws URISyntaxException; throws URISyntaxException;
protected abstract T getTargetFileSystem(final URI[] mergeFsURIList) protected abstract T getTargetFileSystem(final URI[] mergeFsURIList)
throws UnsupportedFileSystemException, URISyntaxException; throws UnsupportedFileSystemException, URISyntaxException;
/** /**
* Create Inode Tree from the specified mount-table specified in Config * Create Inode Tree from the specified mount-table specified in Config
* @param config - the mount table keys are prefixed with * @param config - the mount table keys are prefixed with
@ -294,7 +278,7 @@ abstract class InodeTree<T> {
*/ */
protected InodeTree(final Configuration config, final String viewName) protected InodeTree(final Configuration config, final String viewName)
throws UnsupportedFileSystemException, URISyntaxException, throws UnsupportedFileSystemException, URISyntaxException,
FileAlreadyExistsException, IOException { FileAlreadyExistsException, IOException {
String vName = viewName; String vName = viewName;
if (vName == null) { if (vName == null) {
vName = Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE; vName = Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE;
@ -303,9 +287,9 @@ abstract class InodeTree<T> {
root = new INodeDir<T>("/", UserGroupInformation.getCurrentUser()); root = new INodeDir<T>("/", UserGroupInformation.getCurrentUser());
root.InodeDirFs = getTargetFileSystem(root); root.InodeDirFs = getTargetFileSystem(root);
root.isRoot = true; root.isRoot = true;
final String mtPrefix = Constants.CONFIG_VIEWFS_PREFIX + "." + final String mtPrefix = Constants.CONFIG_VIEWFS_PREFIX + "." +
vName + "."; vName + ".";
final String linkPrefix = Constants.CONFIG_VIEWFS_LINK + "."; final String linkPrefix = Constants.CONFIG_VIEWFS_LINK + ".";
final String linkMergePrefix = Constants.CONFIG_VIEWFS_LINK_MERGE + "."; final String linkMergePrefix = Constants.CONFIG_VIEWFS_LINK_MERGE + ".";
boolean gotMountTableEntry = false; boolean gotMountTableEntry = false;
@ -325,18 +309,17 @@ abstract class InodeTree<T> {
// ignore - we set home dir from config // ignore - we set home dir from config
continue; continue;
} else { } else {
throw new IOException( throw new IOException("ViewFs: Cannot initialize: Invalid entry in " +
"ViewFs: Cannot initialize: Invalid entry in Mount table in config: "+ "Mount table in config: " + src);
src);
} }
final String target = si.getValue(); // link or merge link final String target = si.getValue(); // link or merge link
createLink(src, target, isMergeLink, ugi); createLink(src, target, isMergeLink, ugi);
} }
} }
if (!gotMountTableEntry) { if (!gotMountTableEntry) {
throw new IOException( throw new IOException(
"ViewFs: Cannot initialize: Empty Mount table in config for " + "ViewFs: Cannot initialize: Empty Mount table in config for " +
"viewfs://" + vName + "/"); "viewfs://" + vName + "/");
} }
} }
@ -344,7 +327,7 @@ abstract class InodeTree<T> {
* Resolve returns ResolveResult. * Resolve returns ResolveResult.
* The caller can continue the resolution of the remainingPath * The caller can continue the resolution of the remainingPath
* in the targetFileSystem. * in the targetFileSystem.
* *
* If the input pathname leads to link to another file system then * If the input pathname leads to link to another file system then
* the targetFileSystem is the one denoted by the link (except it is * the targetFileSystem is the one denoted by the link (except it is
* file system chrooted to link target. * file system chrooted to link target.
@ -356,7 +339,7 @@ abstract class InodeTree<T> {
final T targetFileSystem; final T targetFileSystem;
final String resolvedPath; final String resolvedPath;
final Path remainingPath; // to resolve in the target FileSystem final Path remainingPath; // to resolve in the target FileSystem
ResolveResult(final ResultKind k, final T targetFs, final String resolveP, ResolveResult(final ResultKind k, final T targetFs, final String resolveP,
final Path remainingP) { final Path remainingP) {
kind = k; kind = k;
@ -364,31 +347,30 @@ abstract class InodeTree<T> {
resolvedPath = resolveP; resolvedPath = resolveP;
remainingPath = remainingP; remainingPath = remainingP;
} }
// isInternalDir of path resolution completed within the mount table // Internal dir path resolution completed within the mount table
boolean isInternalDir() { boolean isInternalDir() {
return (kind == ResultKind.isInternalDir); return (kind == ResultKind.INTERNAL_DIR);
} }
} }
/** /**
* Resolve the pathname p relative to root InodeDir * Resolve the pathname p relative to root InodeDir
* @param p - inout path * @param p - inout path
* @param resolveLastComponent * @param resolveLastComponent
* @return ResolveResult which allows further resolution of the remaining path * @return ResolveResult which allows further resolution of the remaining path
* @throws FileNotFoundException * @throws FileNotFoundException
*/ */
ResolveResult<T> resolve(final String p, final boolean resolveLastComponent) ResolveResult<T> resolve(final String p, final boolean resolveLastComponent)
throws FileNotFoundException { throws FileNotFoundException {
// TO DO: - more efficient to not split the path, but simply compare String[] path = breakIntoPathComponents(p);
String[] path = breakIntoPathComponents(p);
if (path.length <= 1) { // special case for when path is "/" if (path.length <= 1) { // special case for when path is "/"
ResolveResult<T> res = ResolveResult<T> res =
new ResolveResult<T>(ResultKind.isInternalDir, new ResolveResult<T>(ResultKind.INTERNAL_DIR,
root.InodeDirFs, root.fullPath, SlashPath); root.InodeDirFs, root.fullPath, SlashPath);
return res; return res;
} }
INodeDir<T> curInode = root; INodeDir<T> curInode = root;
int i; int i;
// ignore first slash // ignore first slash
@ -396,27 +378,27 @@ abstract class InodeTree<T> {
INode<T> nextInode = curInode.resolveInternal(path[i]); INode<T> nextInode = curInode.resolveInternal(path[i]);
if (nextInode == null) { if (nextInode == null) {
StringBuilder failedAt = new StringBuilder(path[0]); StringBuilder failedAt = new StringBuilder(path[0]);
for ( int j = 1; j <=i; ++j) { for (int j = 1; j <= i; ++j) {
failedAt.append('/').append(path[j]); failedAt.append('/').append(path[j]);
} }
throw (new FileNotFoundException(failedAt.toString())); throw (new FileNotFoundException(failedAt.toString()));
} }
if (nextInode instanceof INodeLink) { if (nextInode instanceof INodeLink) {
final INodeLink<T> link = (INodeLink<T>) nextInode; final INodeLink<T> link = (INodeLink<T>) nextInode;
final Path remainingPath; final Path remainingPath;
if (i >= path.length-1) { if (i >= path.length - 1) {
remainingPath = SlashPath; remainingPath = SlashPath;
} else { } else {
StringBuilder remainingPathStr = new StringBuilder("/" + path[i+1]); StringBuilder remainingPathStr = new StringBuilder("/" + path[i + 1]);
for (int j = i+2; j< path.length; ++j) { for (int j = i + 2; j < path.length; ++j) {
remainingPathStr.append('/').append(path[j]); remainingPathStr.append('/').append(path[j]);
} }
remainingPath = new Path(remainingPathStr.toString()); remainingPath = new Path(remainingPathStr.toString());
} }
final ResolveResult<T> res = final ResolveResult<T> res =
new ResolveResult<T>(ResultKind.isExternalDir, new ResolveResult<T>(ResultKind.EXTERNAL_DIR,
link.targetFileSystem, nextInode.fullPath, remainingPath); link.targetFileSystem, nextInode.fullPath, remainingPath);
return res; return res;
} else if (nextInode instanceof INodeDir) { } else if (nextInode instanceof INodeDir) {
curInode = (INodeDir<T>) nextInode; curInode = (INodeDir<T>) nextInode;
@ -433,23 +415,23 @@ abstract class InodeTree<T> {
// that follows will do a children.get(remaningPath) and will have to // that follows will do a children.get(remaningPath) and will have to
// strip-out the initial / // strip-out the initial /
StringBuilder remainingPathStr = new StringBuilder("/" + path[i]); StringBuilder remainingPathStr = new StringBuilder("/" + path[i]);
for (int j = i+1; j< path.length; ++j) { for (int j = i + 1; j < path.length; ++j) {
remainingPathStr.append('/').append(path[j]); remainingPathStr.append('/').append(path[j]);
} }
remainingPath = new Path(remainingPathStr.toString()); remainingPath = new Path(remainingPathStr.toString());
} }
final ResolveResult<T> res = final ResolveResult<T> res =
new ResolveResult<T>(ResultKind.isInternalDir, new ResolveResult<T>(ResultKind.INTERNAL_DIR,
curInode.InodeDirFs, curInode.fullPath, remainingPath); curInode.InodeDirFs, curInode.fullPath, remainingPath);
return res; return res;
} }
List<MountPoint<T>> getMountPoints() { List<MountPoint<T>> getMountPoints() {
return mountPoints; return mountPoints;
} }
/** /**
* *
* @return home dir value from mount table; null if no config value * @return home dir value from mount table; null if no config value
* was found. * was found.
*/ */

91
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -87,10 +87,19 @@ public class ViewFileSystem extends FileSystem {
final Path p) { final Path p) {
return readOnlyMountTable(operation, p.toString()); return readOnlyMountTable(operation, p.toString());
} }
static public class MountPoint { static public class MountPoint {
private Path src; // the src of the mount /**
private URI[] targets; // target of the mount; Multiple targets imply mergeMount * The source of the mount.
*/
private Path src;
/**
* One or more targets of the mount.
* Multiple targets imply MergeMount.
*/
private URI[] targets;
MountPoint(Path srcPath, URI[] targetURIs) { MountPoint(Path srcPath, URI[] targetURIs) {
src = srcPath; src = srcPath;
targets = targetURIs; targets = targetURIs;
@ -142,19 +151,18 @@ public class ViewFileSystem extends FileSystem {
/** /**
* Return the protocol scheme for the FileSystem. * Return the protocol scheme for the FileSystem.
* <p/>
* *
* @return <code>viewfs</code> * @return <code>viewfs</code>
*/ */
@Override @Override
public String getScheme() { public String getScheme() {
return "viewfs"; return FsConstants.VIEWFS_SCHEME;
} }
/** /**
* Called after a new FileSystem instance is constructed. * Called after a new FileSystem instance is constructed.
* @param theUri a uri whose authority section names the host, port, etc. for * @param theUri a uri whose authority section names the host, port, etc. for
* this FileSystem * this FileSystem
* @param conf the configuration * @param conf the configuration
*/ */
@Override @Override
@ -197,8 +205,7 @@ public class ViewFileSystem extends FileSystem {
} }
} }
/** /**
* Convenience Constructor for apps to call directly * Convenience Constructor for apps to call directly
* @param theUri which must be that of ViewFileSystem * @param theUri which must be that of ViewFileSystem
@ -206,7 +213,7 @@ public class ViewFileSystem extends FileSystem {
* @throws IOException * @throws IOException
*/ */
ViewFileSystem(final URI theUri, final Configuration conf) ViewFileSystem(final URI theUri, final Configuration conf)
throws IOException { throws IOException {
this(); this();
initialize(theUri, conf); initialize(theUri, conf);
} }
@ -226,8 +233,7 @@ public class ViewFileSystem extends FileSystem {
} }
@Override @Override
public Path resolvePath(final Path f) public Path resolvePath(final Path f) throws IOException {
throws IOException {
final InodeTree.ResolveResult<FileSystem> res; final InodeTree.ResolveResult<FileSystem> res;
res = fsState.resolve(getUriPath(f), true); res = fsState.resolve(getUriPath(f), true);
if (res.isInternalDir()) { if (res.isInternalDir()) {
@ -271,8 +277,8 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, EnumSet<CreateFlag> flags, int bufferSize, short replication,
Progressable progress) throws IOException { long blockSize, Progressable progress) throws IOException {
InodeTree.ResolveResult<FileSystem> res; InodeTree.ResolveResult<FileSystem> res;
try { try {
res = fsState.resolve(getUriPath(f), false); res = fsState.resolve(getUriPath(f), false);
@ -280,8 +286,8 @@ public class ViewFileSystem extends FileSystem {
throw readOnlyMountTable("create", f); throw readOnlyMountTable("create", f);
} }
assert(res.remainingPath != null); assert(res.remainingPath != null);
return res.targetFileSystem.createNonRecursive(res.remainingPath, permission, return res.targetFileSystem.createNonRecursive(res.remainingPath,
flags, bufferSize, replication, blockSize, progress); permission, flags, bufferSize, replication, blockSize, progress);
} }
@Override @Override
@ -302,10 +308,9 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public boolean delete(final Path f, final boolean recursive) public boolean delete(final Path f, final boolean recursive)
throws AccessControlException, FileNotFoundException, throws AccessControlException, FileNotFoundException, IOException {
IOException {
InodeTree.ResolveResult<FileSystem> res = InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true); fsState.resolve(getUriPath(f), true);
// If internal dir or target is a mount link (ie remainingPath is Slash) // If internal dir or target is a mount link (ie remainingPath is Slash)
if (res.isInternalDir() || res.remainingPath == InodeTree.SlashPath) { if (res.isInternalDir() || res.remainingPath == InodeTree.SlashPath) {
throw readOnlyMountTable("delete", f); throw readOnlyMountTable("delete", f);
@ -316,9 +321,8 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
public boolean delete(final Path f) public boolean delete(final Path f)
throws AccessControlException, FileNotFoundException, throws AccessControlException, FileNotFoundException, IOException {
IOException { return delete(f, true);
return delete(f, true);
} }
@Override @Override
@ -339,7 +343,6 @@ public class ViewFileSystem extends FileSystem {
return res.targetFileSystem.getFileChecksum(res.remainingPath); return res.targetFileSystem.getFileChecksum(res.remainingPath);
} }
private static FileStatus fixFileStatus(FileStatus orig, private static FileStatus fixFileStatus(FileStatus orig,
Path qualified) throws IOException { Path qualified) throws IOException {
// FileStatus#getPath is a fully qualified path relative to the root of // FileStatus#getPath is a fully qualified path relative to the root of
@ -367,7 +370,6 @@ public class ViewFileSystem extends FileSystem {
: new ViewFsFileStatus(orig, qualified); : new ViewFsFileStatus(orig, qualified);
} }
@Override @Override
public FileStatus getFileStatus(final Path f) throws AccessControlException, public FileStatus getFileStatus(final Path f) throws AccessControlException,
FileNotFoundException, IOException { FileNotFoundException, IOException {
@ -407,10 +409,10 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public RemoteIterator<LocatedFileStatus>listLocatedStatus(final Path f, public RemoteIterator<LocatedFileStatus>listLocatedStatus(final Path f,
final PathFilter filter) throws FileNotFoundException, IOException { final PathFilter filter) throws FileNotFoundException, IOException {
final InodeTree.ResolveResult<FileSystem> res = fsState final InodeTree.ResolveResult<FileSystem> res =
.resolve(getUriPath(f), true); fsState.resolve(getUriPath(f), true);
final RemoteIterator<LocatedFileStatus> statusIter = res.targetFileSystem final RemoteIterator<LocatedFileStatus> statusIter =
.listLocatedStatus(res.remainingPath); res.targetFileSystem.listLocatedStatus(res.remainingPath);
if (res.isInternalDir()) { if (res.isInternalDir()) {
return statusIter; return statusIter;
@ -449,8 +451,7 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public FSDataInputStream open(final Path f, final int bufferSize) public FSDataInputStream open(final Path f, final int bufferSize)
throws AccessControlException, FileNotFoundException, throws AccessControlException, FileNotFoundException, IOException {
IOException {
InodeTree.ResolveResult<FileSystem> res = InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true); fsState.resolve(getUriPath(f), true);
return res.targetFileSystem.open(res.remainingPath, bufferSize); return res.targetFileSystem.open(res.remainingPath, bufferSize);
@ -507,8 +508,7 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public void setOwner(final Path f, final String username, public void setOwner(final Path f, final String username,
final String groupname) throws AccessControlException, final String groupname) throws AccessControlException,
FileNotFoundException, FileNotFoundException, IOException {
IOException {
InodeTree.ResolveResult<FileSystem> res = InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true); fsState.resolve(getUriPath(f), true);
res.targetFileSystem.setOwner(res.remainingPath, username, groupname); res.targetFileSystem.setOwner(res.remainingPath, username, groupname);
@ -516,8 +516,7 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public void setPermission(final Path f, final FsPermission permission) public void setPermission(final Path f, final FsPermission permission)
throws AccessControlException, FileNotFoundException, throws AccessControlException, FileNotFoundException, IOException {
IOException {
InodeTree.ResolveResult<FileSystem> res = InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true); fsState.resolve(getUriPath(f), true);
res.targetFileSystem.setPermission(res.remainingPath, permission); res.targetFileSystem.setPermission(res.remainingPath, permission);
@ -525,8 +524,7 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public boolean setReplication(final Path f, final short replication) public boolean setReplication(final Path f, final short replication)
throws AccessControlException, FileNotFoundException, throws AccessControlException, FileNotFoundException, IOException {
IOException {
InodeTree.ResolveResult<FileSystem> res = InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true); fsState.resolve(getUriPath(f), true);
return res.targetFileSystem.setReplication(res.remainingPath, replication); return res.targetFileSystem.setReplication(res.remainingPath, replication);
@ -534,8 +532,7 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public void setTimes(final Path f, final long mtime, final long atime) public void setTimes(final Path f, final long mtime, final long atime)
throws AccessControlException, FileNotFoundException, throws AccessControlException, FileNotFoundException, IOException {
IOException {
InodeTree.ResolveResult<FileSystem> res = InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true); fsState.resolve(getUriPath(f), true);
res.targetFileSystem.setTimes(res.remainingPath, mtime, atime); res.targetFileSystem.setTimes(res.remainingPath, mtime, atime);
@ -793,8 +790,8 @@ public class ViewFileSystem extends FileSystem {
return allPolicies; return allPolicies;
} }
/* /**
* An instance of this class represents an internal dir of the viewFs * An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table. * that is internal dir of the mount table.
* It is a read only mount tables and create, mkdir or delete operations * It is a read only mount tables and create, mkdir or delete operations
* are not allowed. * are not allowed.
@ -826,8 +823,8 @@ public class ViewFileSystem extends FileSystem {
static private void checkPathIsSlash(final Path f) throws IOException { static private void checkPathIsSlash(final Path f) throws IOException {
if (f != InodeTree.SlashPath) { if (f != InodeTree.SlashPath) {
throw new IOException ( throw new IOException(
"Internal implementation error: expected file name to be /" ); "Internal implementation error: expected file name to be /");
} }
} }
@ -838,14 +835,14 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public Path getWorkingDirectory() { public Path getWorkingDirectory() {
throw new RuntimeException ( throw new RuntimeException(
"Internal impl error: getWorkingDir should not have been called" ); "Internal impl error: getWorkingDir should not have been called");
} }
@Override @Override
public void setWorkingDirectory(final Path new_dir) { public void setWorkingDirectory(final Path new_dir) {
throw new RuntimeException ( throw new RuntimeException(
"Internal impl error: getWorkingDir should not have been called" ); "Internal impl error: getWorkingDir should not have been called");
} }
@Override @Override
@ -878,7 +875,7 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public BlockLocation[] getFileBlockLocations(final FileStatus fs, public BlockLocation[] getFileBlockLocations(final FileStatus fs,
final long start, final long len) throws final long start, final long len) throws
FileNotFoundException, IOException { FileNotFoundException, IOException {
checkPathIsSlash(fs.getPath()); checkPathIsSlash(fs.getPath());
throw new FileNotFoundException("Path points to dir not a file"); throw new FileNotFoundException("Path points to dir not a file");
@ -1055,7 +1052,7 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public void setXAttr(Path path, String name, byte[] value, public void setXAttr(Path path, String name, byte[] value,
EnumSet<XAttrSetFlag> flag) throws IOException { EnumSet<XAttrSetFlag> flag) throws IOException {
checkPathIsSlash(path); checkPathIsSlash(path);
throw readOnlyMountTable("setXAttr", path); throw readOnlyMountTable("setXAttr", path);
} }

48
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java
View File

@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the * to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance * "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at * with the License. You may obtain a copy of the License at
* * <p>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* * <p>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -22,49 +22,41 @@ import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.viewfs.ConfigUtil;
import org.apache.hadoop.fs.viewfs.InodeTree;
import org.junit.Test; import org.junit.Test;
public class TestViewFsConfig { public class TestViewFsConfig {
@Test(expected = FileAlreadyExistsException.class)
@Test(expected=FileAlreadyExistsException.class)
public void testInvalidConfig() throws IOException, URISyntaxException { public void testInvalidConfig() throws IOException, URISyntaxException {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
ConfigUtil.addLink(conf, "/internalDir/linkToDir2", ConfigUtil.addLink(conf, "/internalDir/linkToDir2",
new Path("file:///dir2").toUri()); new Path("file:///dir2").toUri());
ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3", ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3",
new Path("file:///dir3").toUri()); new Path("file:///dir3").toUri());
class Foo { }; class Foo {
}
new InodeTree<Foo>(conf, null) {
new InodeTree<Foo>(conf, null) {
@Override @Override
protected protected Foo getTargetFileSystem(final URI uri)
Foo getTargetFileSystem(final URI uri) throws URISyntaxException, UnsupportedFileSystemException {
throws URISyntaxException, UnsupportedFileSystemException {
return null;
}
@Override
protected
Foo getTargetFileSystem(
org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo>
dir)
throws URISyntaxException {
return null; return null;
} }
@Override @Override
protected protected Foo getTargetFileSystem(
Foo getTargetFileSystem(URI[] mergeFsURIList) org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo> dir)
throws URISyntaxException {
return null;
}
@Override
protected Foo getTargetFileSystem(URI[] mergeFsURIList)
throws URISyntaxException, UnsupportedFileSystemException { throws URISyntaxException, UnsupportedFileSystemException {
return null; return null;
} }