Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-10744. Internally optimize path component resolution. Contributed by Daryn Sharp. 

(cherry picked from commit 1ef8d7a638)
(cherry picked from commit 3dac73080a)
This commit is contained in:
Kihwal Lee 2016-09-02 11:31:48 -05:00 committed by Zhe Zhang
parent 9afc6615b0
commit 199ccd4915
11 changed files with 110 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
View File

@ -39,8 +39,7 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
@ -65,8 +64,7 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
@ -90,8 +88,7 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
@ -115,8 +112,7 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
@ -135,9 +131,8 @@ class FSDirAclOp {
throws IOException {
String src = srcArg;
checkAclsConfigFlag(fsd);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = fsd.getPermissionChecker();
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
@ -155,8 +150,7 @@ class FSDirAclOp {
FSDirectory fsd, String src) throws IOException {
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
String srcs = FSDirectory.normalizePath(src);
fsd.readLock();
try {

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
View File

@ -52,11 +52,10 @@ public class FSDirAttrOp {
throws IOException {
String src = srcArg;
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
INodesInPath iip;
fsd.writeLock();
try {
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
iip = fsd.getINodesInPath4Write(src);
fsd.checkOwner(pc, iip);
unprotectedSetPermission(fsd, src, permission);
@ -71,11 +70,10 @@ public class FSDirAttrOp {
FSDirectory fsd, String src, String username, String group)
throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
INodesInPath iip;
fsd.writeLock();
try {
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
iip = fsd.getINodesInPath4Write(src);
fsd.checkOwner(pc, iip);
if (!pc.isSuperUser()) {
@ -105,12 +103,11 @@ public class FSDirAttrOp {
}
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
INodesInPath iip;
fsd.writeLock();
try {
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
iip = fsd.getINodesInPath4Write(src);
// Write access is required to set access and modification times
if (fsd.isPermissionEnabled()) {
@ -138,10 +135,9 @@ public class FSDirAttrOp {
bm.verifyReplication(src, replication, null);
final boolean isFile;
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
fsd.writeLock();
try {
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath4Write(src);
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
@ -170,11 +166,10 @@ public class FSDirAttrOp {
+ DFS_STORAGE_POLICY_ENABLED_KEY + " is set to false.");
}
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
INodesInPath iip;
fsd.writeLock();
try {
src = FSDirectory.resolvePath(src, pathComponents, fsd);
src = FSDirectory.resolvePath(src, fsd);
iip = fsd.getINodesInPath4Write(src);
if (fsd.isPermissionEnabled()) {
@ -203,10 +198,9 @@ public class FSDirAttrOp {
static long getPreferredBlockSize(FSDirectory fsd, String src)
throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
fsd.readLock();
try {
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath(src, false);
if (fsd.isPermissionEnabled()) {
fsd.checkTraverse(pc, iip);

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
View File

@ -77,9 +77,8 @@ class FSDirDeleteOp {
throws IOException {
FSDirectory fsd = fsn.getFSDirectory();
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath4Write(src, false);
if (!recursive && fsd.isNonEmptyDirectory(iip)) {
throw new PathIsNotEmptyDirectoryException(src + " is non empty");

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
View File

@ -50,10 +50,9 @@ class FSDirMkdirOp {
throw new InvalidPathException(src);
}
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
fsd.writeLock();
try {
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
INodesInPath iip = fsd.getINodesInPath4Write(src);
if (fsd.isPermissionEnabled()) {
fsd.checkTraverse(pc, iip);

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
View File

@ -64,11 +64,9 @@ class FSDirRenameOp {
}
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] srcComponents = FSDirectory.getPathComponentsForReservedPath(src);
byte[][] dstComponents = FSDirectory.getPathComponentsForReservedPath(dst);
HdfsFileStatus resultingStat = null;
src = fsd.resolvePath(pc, src, srcComponents);
dst = fsd.resolvePath(pc, dst, dstComponents);
src = fsd.resolvePath(pc, src);
dst = fsd.resolvePath(pc, dst);
@SuppressWarnings("deprecation")
final boolean status = renameTo(fsd, pc, src, dst, logRetryCache);
if (status) {
@ -239,11 +237,9 @@ class FSDirRenameOp {
}
final FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] srcComponents = FSDirectory.getPathComponentsForReservedPath(src);
byte[][] dstComponents = FSDirectory.getPathComponentsForReservedPath(dst);
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
src = fsd.resolvePath(pc, src, srcComponents);
dst = fsd.resolvePath(pc, dst, dstComponents);
src = fsd.resolvePath(pc, src);
dst = fsd.resolvePath(pc, dst);
renameTo(fsd, pc, src, dst, collectedBlocks, logRetryCache, options);
INodesInPath dstIIP = fsd.getINodesInPath(dst, false);
HdfsFileStatus resultingStat = fsd.getAuditFileInfo(dstIIP);

17
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
View File

@ -48,18 +48,14 @@ class FSDirStatAndListingOp {
static DirectoryListing getListingInt(FSDirectory fsd, final String srcArg,
byte[] startAfter, boolean needLocation) throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory
.getPathComponentsForReservedPath(srcArg);
final String startAfterString = DFSUtil.bytes2String(startAfter);
final String src = fsd.resolvePath(pc, srcArg, pathComponents);
final String src = fsd.resolvePath(pc, srcArg);
final INodesInPath iip = fsd.getINodesInPath(src, true);
// Get file name when startAfter is an INodePath
if (FSDirectory.isReservedName(startAfterString)) {
byte[][] startAfterComponents = FSDirectory
.getPathComponentsForReservedPath(startAfterString);
try {
String tmp = FSDirectory.resolvePath(src, startAfterComponents, fsd);
String tmp = FSDirectory.resolvePath(startAfterString, fsd);
byte[][] regularPath = INode.getPathComponents(tmp);
startAfter = regularPath[regularPath.length - 1];
} catch (IOException e) {
@ -99,8 +95,7 @@ class FSDirStatAndListingOp {
throw new InvalidPathException("Invalid file name: " + src);
}
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, srcArg);
final INodesInPath iip = fsd.getINodesInPath(src, resolveLink);
boolean isSuperUser = true;
if (fsd.isPermissionEnabled()) {
@ -116,8 +111,7 @@ class FSDirStatAndListingOp {
*/
static boolean isFileClosed(FSDirectory fsd, String src) throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath(src, true);
if (fsd.isPermissionEnabled()) {
fsd.checkTraverse(pc, iip);
@ -127,9 +121,8 @@ class FSDirStatAndListingOp {
static ContentSummary getContentSummary(
FSDirectory fsd, String src) throws IOException {
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = fsd.getPermissionChecker();
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath(src, false);
if (fsd.isPermissionEnabled()) {
fsd.checkPermission(pc, iip, false, null, null, null,

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
View File

@ -52,11 +52,10 @@ class FSDirSymlinkOp {
}
FSPermissionChecker pc = fsn.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(link);
INodesInPath iip;
fsd.writeLock();
try {
link = fsd.resolvePath(pc, link, pathComponents);
link = fsd.resolvePath(pc, link);
iip = fsd.getINodesInPath4Write(link, false);
if (!createParent) {
fsd.verifyParentDir(iip, link);

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
View File

@ -67,8 +67,7 @@ class FSDirXAttrOp {
FSPermissionChecker pc = fsd.getPermissionChecker();
XAttrPermissionFilter.checkPermissionForApi(
pc, xAttr, FSDirectory.isReservedRawName(src));
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(xAttr);
INodesInPath iip;
@ -95,8 +94,7 @@ class FSDirXAttrOp {
if (!getAll) {
XAttrPermissionFilter.checkPermissionForApi(pc, xAttrs, isRawPath);
}
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath(src, true);
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.READ);
@ -135,8 +133,7 @@ class FSDirXAttrOp {
FSDirXAttrOp.checkXAttrsConfigFlag(fsd);
final FSPermissionChecker pc = fsd.getPermissionChecker();
final boolean isRawPath = FSDirectory.isReservedRawName(src);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath(src, true);
if (fsd.isPermissionEnabled()) {
/* To access xattr names, you need EXECUTE in the owning directory. */
@ -163,15 +160,13 @@ class FSDirXAttrOp {
FSPermissionChecker pc = fsd.getPermissionChecker();
XAttrPermissionFilter.checkPermissionForApi(
pc, xAttr, FSDirectory.isReservedRawName(src));
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(
src);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(xAttr);
INodesInPath iip;
fsd.writeLock();
try {
src = fsd.resolvePath(pc, src, pathComponents);
src = fsd.resolvePath(pc, src);
iip = fsd.getINodesInPath4Write(src);
checkXAttrChangeAccess(fsd, iip, xAttr, pc);

99
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
View File

@ -127,6 +127,8 @@ public class FSDirectory implements Closeable {
public final static String DOT_INODES_STRING = ".inodes";
public final static byte[] DOT_INODES =
DFSUtil.string2Bytes(DOT_INODES_STRING);
private final static byte[] DOT_DOT =
DFSUtil.string2Bytes("..");
INodeDirectory rootDir;
private final FSNamesystem namesystem;
@ -556,7 +558,6 @@ public class FSDirectory implements Closeable {
*
* @param pc The permission checker used when resolving path.
* @param path The path to resolve.
* @param pathComponents path components corresponding to the path
* @return if the path indicates an inode, return path after replacing up to
* <inodeid> with the corresponding path of the inode, else the path
* in {@code src} as is. If the path refers to a path in the "raw"
@ -564,12 +565,12 @@ public class FSDirectory implements Closeable {
* @throws FileNotFoundException
* @throws AccessControlException
*/
String resolvePath(FSPermissionChecker pc, String path, byte[][] pathComponents)
String resolvePath(FSPermissionChecker pc, String path)
throws FileNotFoundException, AccessControlException {
if (isReservedRawName(path) && isPermissionEnabled) {
pc.checkSuperuserPrivilege();
}
return resolvePath(path, pathComponents, this);
return resolvePath(path, this);
}
/**
@ -1389,13 +1390,6 @@ public class FSDirectory implements Closeable {
return components.toArray(new byte[components.size()][]);
}
/**
* @return path components for reserved path, else null.
*/
static byte[][] getPathComponentsForReservedPath(String src) {
return !isReservedName(src) ? null : INode.getPathComponents(src);
}
/** Check if a given inode name is reserved */
public static boolean isReservedName(INode inode) {
return CHECK_RESERVED_FILE_NAMES
@ -1412,6 +1406,12 @@ public class FSDirectory implements Closeable {
Path.SEPARATOR + RAW_STRING);
}
static boolean isReservedName(byte[][] components) {
return (components.length > 2) &&
Arrays.equals(INodeDirectory.ROOT_NAME, components[0]) &&
Arrays.equals(DOT_RESERVED, components[1]);
}
/**
* Resolve a /.reserved/... path to a non-reserved path.
* <p/>
@ -1431,7 +1431,6 @@ public class FSDirectory implements Closeable {
* unencrypted file).
*
* @param src path that is being processed
* @param pathComponents path components corresponding to the path
* @param fsd FSDirectory
* @return if the path indicates an inode, return path after replacing up to
* <inodeid> with the corresponding path of the inode, else the path
@ -1439,39 +1438,32 @@ public class FSDirectory implements Closeable {
* directory, return the non-raw pathname.
* @throws FileNotFoundException if inodeid is invalid
*/
static String resolvePath(String src, byte[][] pathComponents,
static String resolvePath(String src,
FSDirectory fsd) throws FileNotFoundException {
final int nComponents = (pathComponents == null) ?
0 : pathComponents.length;
if (nComponents <= 2) {
return src;
}
if (!Arrays.equals(DOT_RESERVED, pathComponents[1])) {
byte[][] pathComponents = INode.getPathComponents(src);
final int nComponents = pathComponents.length;
if (!isReservedName(pathComponents)) {
/* This is not a /.reserved/ path so do nothing. */
return src;
}
if (Arrays.equals(DOT_INODES, pathComponents[2])) {
} else if (Arrays.equals(DOT_INODES, pathComponents[2])) {
/* It's a /.reserved/.inodes path. */
if (nComponents > 3) {
return resolveDotInodesPath(src, pathComponents, fsd);
} else {
return src;
pathComponents = resolveDotInodesPath(pathComponents, fsd);
}
} else if (Arrays.equals(RAW, pathComponents[2])) {
/* It's /.reserved/raw so strip off the /.reserved/raw prefix. */
if (nComponents == 3) {
return Path.SEPARATOR;
pathComponents = new byte[][]{INodeDirectory.ROOT_NAME};
} else {
return constructRemainingPath("", pathComponents, 3);
pathComponents = constructRemainingPath(
new byte[][]{INodeDirectory.ROOT_NAME}, pathComponents, 3);
}
} else {
/* It's some sort of /.reserved/<unknown> path. Ignore it. */
return src;
}
// this double conversion will be unnecessary when resolving returns
// INodesInPath (needs components byte[][])
return DFSUtil.byteArray2PathString(pathComponents);
}
private static String resolveDotInodesPath(String src,
private static byte[][] resolveDotInodesPath(
byte[][] pathComponents, FSDirectory fsd)
throws FileNotFoundException {
final String inodeId = DFSUtil.bytes2String(pathComponents[3]);
@ -1479,48 +1471,47 @@ public class FSDirectory implements Closeable {
try {
id = Long.parseLong(inodeId);
} catch (NumberFormatException e) {
throw new FileNotFoundException("Invalid inode path: " + src);
throw new FileNotFoundException("Invalid inode path: " +
DFSUtil.byteArray2PathString(pathComponents));
}
if (id == INodeId.ROOT_INODE_ID && pathComponents.length == 4) {
return Path.SEPARATOR;
return new byte[][]{INodeDirectory.ROOT_NAME};
}
INode inode = fsd.getInode(id);
if (inode == null) {
throw new FileNotFoundException(
"File for given inode path does not exist: " + src);
"File for given inode path does not exist: " +
DFSUtil.byteArray2PathString(pathComponents));
}
// Handle single ".." for NFS lookup support.
if ((pathComponents.length > 4)
&& DFSUtil.bytes2String(pathComponents[4]).equals("..")) {
&& Arrays.equals(pathComponents[4], DOT_DOT)) {
INode parent = inode.getParent();
if (parent == null || parent.getId() == INodeId.ROOT_INODE_ID) {
// inode is root, or its parent is root.
return Path.SEPARATOR;
} else {
return parent.getFullPathName();
return new byte[][]{INodeDirectory.ROOT_NAME};
}
return parent.getPathComponents();
}
return constructRemainingPath(
inode.getPathComponents(), pathComponents, 4);
}
String path = "";
if (id != INodeId.ROOT_INODE_ID) {
path = inode.getFullPathName();
}
return constructRemainingPath(path, pathComponents, 4);
}
private static String constructRemainingPath(String pathPrefix,
byte[][] pathComponents, int startAt) {
StringBuilder path = new StringBuilder(pathPrefix);
for (int i = startAt; i < pathComponents.length; i++) {
path.append(Path.SEPARATOR).append(
DFSUtil.bytes2String(pathComponents[i]));
private static byte[][] constructRemainingPath(byte[][] components,
byte[][] extraComponents, int startAt) {
int remainder = extraComponents.length - startAt;
if (remainder > 0) {
// grow the array and copy in the remaining components
int pos = components.length;
components = Arrays.copyOf(components, pos + remainder);
System.arraycopy(extraComponents, startAt, components, pos, remainder);
}
if (NameNode.LOG.isDebugEnabled()) {
NameNode.LOG.debug("Resolved path is " + path);
NameNode.LOG.debug(
"Resolved path is " + DFSUtil.byteArray2PathString(components));
}
return path.toString();
return components;
}
INode getINode4DotSnapshot(String src) throws UnresolvedLinkException {

47
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -1778,8 +1778,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
logAuditEvent(true, "open", srcArg);
if (res.updateAccessTime()) {
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(
srcArg);
String src = srcArg;
writeLock();
final long now = now();
@ -1803,7 +1801,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* HDFS-7463. A better fix is to change the edit log of SetTime to
* use inode id instead of a path.
*/
src = dir.resolvePath(pc, srcArg, pathComponents);
src = dir.resolvePath(pc, srcArg);
final INodesInPath iip = dir.getINodesInPath(src, true);
INode inode = iip.getLastINode();
boolean updateAccessTime = inode != null &&
@ -1880,8 +1878,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean needBlockToken)
throws IOException {
String src = srcArg;
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = dir.resolvePath(pc, srcArg, pathComponents);
src = dir.resolvePath(pc, srcArg);
final INodesInPath iip = dir.getINodesInPath(src, true);
final INodeFile inode = INodeFile.valueOf(iip.getLastINode(), src);
if (isPermissionEnabled) {
@ -2075,13 +2072,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
boolean res;
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
writeLock();
BlocksMapUpdateInfo toRemoveBlocks = new BlocksMapUpdateInfo();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot truncate for " + src);
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
res = truncateInternal(src, newLength, clientName,
clientMachine, mtime, pc, toRemoveBlocks);
stat = dir.getAuditFileInfo(dir.getINodesInPath4Write(src, false));
@ -2438,7 +2434,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
" minimum value (" + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY
+ "): " + blockSize + " < " + minBlockSize);
}
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean create = flag.contains(CreateFlag.CREATE);
boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
boolean isLazyPersist = flag.contains(CreateFlag.LAZY_PERSIST);
@ -2465,7 +2460,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (provider != null) {
readLock();
try {
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
INodesInPath iip = dir.getINodesInPath4Write(src);
// Nothing to do if the path is not within an EZ
final EncryptionZone zone = dir.getEZForPath(iip);
@ -2503,7 +2498,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkNameNodeSafeMode("Cannot create file" + src);
dir.writeLock();
try {
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
final INodesInPath iip = dir.getINodesInPath4Write(src);
toRemoveBlocks = startFileInternal(
pc, iip, permissions, holder,
@ -2877,12 +2872,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean skipSync = false;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot recover the lease of " + src);
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
final INodesInPath iip = dir.getINodesInPath4Write(src);
final INodeFile inode = INodeFile.valueOf(iip.getLastINode(), src);
if (!inode.isUnderConstruction()) {
@ -3030,12 +3024,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
LocatedBlock lb = null;
HdfsFileStatus stat = null;
FSPermissionChecker pc = getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot append to file" + src);
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
final INodesInPath iip = dir.getINodesInPath4Write(src);
lb = appendFileInternal(pc, iip, holder, clientMachine, newBlock,
logRetryCache);
@ -3120,12 +3113,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
" for {}", src, fileId, clientName);
checkOperation(OperationCategory.READ);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
readLock();
try {
checkOperation(OperationCategory.READ);
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
FileState fileState = analyzeFileState(
src, fileId, clientName, previous, onRetryBlock);
if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length > 0) {
@ -3380,14 +3372,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final byte storagePolicyID;
final List<DatanodeStorageInfo> chosen;
checkOperation(OperationCategory.READ);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
readLock();
try {
checkOperation(OperationCategory.READ);
//check safe mode
checkNameNodeSafeMode("Cannot add datanode; src=" + src + ", blk=" + blk);
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
//check lease
final INode inode;
@ -3436,14 +3427,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
NameNode.stateChangeLog.debug(
"BLOCK* NameSystem.abandonBlock: {} of file {}", b, src);
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot abandon block " + b + " for file" + src);
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
final INode inode;
final INodesInPath iip;
@ -3534,14 +3524,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkBlock(last);
boolean success = false;
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot complete file " + src);
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
success = completeFileInternal(src, holder,
ExtendedBlock.getLocalBlock(last), fileId);
} finally {
@ -4032,7 +4021,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throws IOException {
NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName);
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
waitForLoadingFSImage();
@ -4040,7 +4028,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot fsync file " + src);
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
final INode inode;
if (fileId == INodeId.GRANDFATHER_INODE_ID) {
// Older clients may not have given us an inode ID to work with.
@ -8100,15 +8088,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
String src = srcArg;
HdfsFileStatus resultingStat = null;
checkSuperuserPrivilege();
final byte[][] pathComponents =
FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
writeLock();
try {
checkSuperuserPrivilege();
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot create encryption zone on " + src);
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
final CipherSuite suite = CipherSuite.convert(cipher);
// For now this is hardcoded, as we only support one method.
@ -8140,15 +8126,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throws AccessControlException, UnresolvedLinkException, IOException {
String src = srcArg;
HdfsFileStatus resultingStat = null;
final byte[][] pathComponents =
FSDirectory.getPathComponentsForReservedPath(src);
boolean success = false;
final FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.READ);
readLock();
try {
checkOperation(OperationCategory.READ);
src = dir.resolvePath(pc, src, pathComponents);
src = dir.resolvePath(pc, src);
final INodesInPath iip = dir.getINodesInPath(src, true);
if (isPermissionEnabled) {
dir.checkPathAccess(pc, iip, FsAction.READ);
@ -8250,11 +8234,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void checkAccess(String src, FsAction mode) throws IOException {
checkOperation(OperationCategory.READ);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
readLock();
try {
checkOperation(OperationCategory.READ);
src = FSDirectory.resolvePath(src, pathComponents, dir);
src = FSDirectory.resolvePath(src, dir);
final INodesInPath iip = dir.getINodesInPath(src, true);
INode inode = iip.getLastINode();
if (inode == null) {

33
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
View File

@ -844,7 +844,7 @@ public class TestINodeFile {
}
/**
* Tests for {@link FSDirectory#resolvePath(String, byte[][], FSDirectory)}
* Tests for {@link FSDirectory#resolvePath(String, FSDirectory)}
*/
@Test
public void testInodePath() throws IOException {
@ -855,53 +855,46 @@ public class TestINodeFile {
FSDirectory fsd = Mockito.mock(FSDirectory.class);
Mockito.doReturn(inode).when(fsd).getInode(Mockito.anyLong());
// Null components
assertEquals("/test", FSDirectory.resolvePath("/test", null, fsd));
// Tests for FSDirectory#resolvePath()
// Non inode regular path
byte[][] components = INode.getPathComponents(path);
String resolvedPath = FSDirectory.resolvePath(path, components, fsd);
String resolvedPath = FSDirectory.resolvePath(path, fsd);
assertEquals(path, resolvedPath);
// Inode path with no trailing separator
components = INode.getPathComponents("/.reserved/.inodes/1");
resolvedPath = FSDirectory.resolvePath(path, components, fsd);
String testPath = "/.reserved/.inodes/1";
resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals(path, resolvedPath);
// Inode path with trailing separator
components = INode.getPathComponents("/.reserved/.inodes/1/");
testPath = "/.reserved/.inodes/1/";
resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals(path, resolvedPath);
// Inode relative path
components = INode.getPathComponents("/.reserved/.inodes/1/d/e/f");
resolvedPath = FSDirectory.resolvePath(path, components, fsd);
testPath = "/.reserved/.inodes/1/d/e/f";
resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals("/a/b/c/d/e/f", resolvedPath);
// A path with just .inodes returns the path as is
String testPath = "/.reserved/.inodes";
components = INode.getPathComponents(testPath);
resolvedPath = FSDirectory.resolvePath(testPath, components, fsd);
testPath = "/.reserved/.inodes";
resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals(testPath, resolvedPath);
// Root inode path
testPath = "/.reserved/.inodes/" + INodeId.ROOT_INODE_ID;
components = INode.getPathComponents(testPath);
resolvedPath = FSDirectory.resolvePath(testPath, components, fsd);
resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals("/", resolvedPath);
// An invalid inode path should remain unresolved
testPath = "/.invalid/.inodes/1";
components = INode.getPathComponents(testPath);
resolvedPath = FSDirectory.resolvePath(testPath, components, fsd);
resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals(testPath, resolvedPath);
// Test path with nonexistent(deleted or wrong id) inode
Mockito.doReturn(null).when(fsd).getInode(Mockito.anyLong());
testPath = "/.reserved/.inodes/1234";
components = INode.getPathComponents(testPath);
try {
String realPath = FSDirectory.resolvePath(testPath, components, fsd);
String realPath = FSDirectory.resolvePath(testPath, fsd);
fail("Path should not be resolved:" + realPath);
} catch (IOException e) {
assertTrue(e instanceof FileNotFoundException);