diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index e0416b795cd..a82c779f16f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -245,12 +245,14 @@ class FSDirStatAndListingOp { .BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; if (!targetNode.isDirectory()) { + // return the file's status. note that the iip already includes the + // target INode INodeAttributes nodeAttrs = getINodeAttributes( fsd, src, HdfsFileStatus.EMPTY_NAME, targetNode, snapshot); return new DirectoryListing( new HdfsFileStatus[]{ createFileStatus( - fsd, HdfsFileStatus.EMPTY_NAME, targetNode, nodeAttrs, + fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs, needLocation, parentStoragePolicy, snapshot, isRawPath, iip) }, 0); } @@ -264,7 +266,7 @@ class FSDirStatAndListingOp { int locationBudget = fsd.getLsLimit(); int listingCnt = 0; HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing]; - for (int i=0; i0; i++) { + for (int i = 0; i < numOfListing && locationBudget > 0; i++) { INode cur = contents.get(startChild+i); byte curPolicy = isSuperUser && !cur.isSymlink()? cur.getLocalStoragePolicyID(): @@ -272,9 +274,11 @@ class FSDirStatAndListingOp { INodeAttributes nodeAttrs = getINodeAttributes( fsd, src, cur.getLocalNameBytes(), cur, snapshot); - listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), - cur, nodeAttrs, needLocation, getStoragePolicyID(curPolicy, - parentStoragePolicy), snapshot, isRawPath, iip); + final INodesInPath iipWithChild = INodesInPath.append(iip, cur, + cur.getLocalNameBytes()); + listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), nodeAttrs, + needLocation, getStoragePolicyID(curPolicy, parentStoragePolicy), + snapshot, isRawPath, iipWithChild); listingCnt++; if (needLocation) { // Once we hit lsLimit locations, stop. @@ -329,8 +333,7 @@ class FSDirStatAndListingOp { fsd, src, sRoot.getLocalNameBytes(), node, Snapshot.CURRENT_STATE_ID); listing[i] = createFileStatus( - fsd, sRoot.getLocalNameBytes(), - sRoot, nodeAttrs, + fsd, sRoot.getLocalNameBytes(), nodeAttrs, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID, false, INodesInPath.fromINode(sRoot)); @@ -350,31 +353,31 @@ class FSDirStatAndListingOp { /** Get the file info for a specific file. * @param fsd FSDirectory - * @param src The string representation of the path to the file + * @param iip The path to the file, the file is included * @param isRawPath true if a /.reserved/raw pathname was passed by the user * @param includeStoragePolicy whether to include storage policy * @return object containing information regarding the file * or null if file not found */ static HdfsFileStatus getFileInfo( - FSDirectory fsd, String path, INodesInPath src, boolean isRawPath, + FSDirectory fsd, String path, INodesInPath iip, boolean isRawPath, boolean includeStoragePolicy) throws IOException { fsd.readLock(); try { - final INode i = src.getLastINode(); - if (i == null) { + final INode node = iip.getLastINode(); + if (node == null) { return null; } - byte policyId = includeStoragePolicy && !i.isSymlink() ? - i.getStoragePolicyID() : + byte policyId = includeStoragePolicy && !node.isSymlink() ? + node.getStoragePolicyID() : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; INodeAttributes nodeAttrs = getINodeAttributes(fsd, path, HdfsFileStatus.EMPTY_NAME, - i, src.getPathSnapshotId()); - return createFileStatus(fsd, HdfsFileStatus.EMPTY_NAME, i, nodeAttrs, - policyId, src.getPathSnapshotId(), isRawPath, src); + node, iip.getPathSnapshotId()); + return createFileStatus(fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs, + policyId, iip.getPathSnapshotId(), isRawPath, iip); } finally { fsd.readUnlock(); } @@ -407,51 +410,54 @@ class FSDirStatAndListingOp { * * @param fsd FSDirectory * @param path the local name - * @param node inode * @param needLocation if block locations need to be included or not * @param isRawPath true if this is being called on behalf of a path in * /.reserved/raw + * @param iip the INodesInPath containing the target INode and its ancestors * @return a file status * @throws java.io.IOException if any error occurs */ private static HdfsFileStatus createFileStatus( - FSDirectory fsd, byte[] path, INode node, INodeAttributes nodeAttrs, + FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs, boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath, INodesInPath iip) throws IOException { if (needLocation) { - return createLocatedFileStatus(fsd, path, node, nodeAttrs, storagePolicy, + return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy, snapshot, isRawPath, iip); } else { - return createFileStatus(fsd, path, node, nodeAttrs, storagePolicy, + return createFileStatus(fsd, path, nodeAttrs, storagePolicy, snapshot, isRawPath, iip); } } /** - * Create FileStatus by file INode + * Create FileStatus for an given INodeFile. + * @param iip The INodesInPath containing the INodeFile and its ancestors */ static HdfsFileStatus createFileStatusForEditLog( - FSDirectory fsd, String fullPath, byte[] path, INode node, + FSDirectory fsd, String fullPath, byte[] path, byte storagePolicy, int snapshot, boolean isRawPath, INodesInPath iip) throws IOException { INodeAttributes nodeAttrs = getINodeAttributes( - fsd, fullPath, path, node, snapshot); - return createFileStatus(fsd, path, node, nodeAttrs, - storagePolicy, snapshot, isRawPath, iip); + fsd, fullPath, path, iip.getLastINode(), snapshot); + return createFileStatus(fsd, path, nodeAttrs, storagePolicy, + snapshot, isRawPath, iip); } /** - * Create FileStatus by file INode + * create file status for a given INode + * @param iip the INodesInPath containing the target INode and its ancestors */ static HdfsFileStatus createFileStatus( - FSDirectory fsd, byte[] path, INode node, + FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs, byte storagePolicy, int snapshot, boolean isRawPath, INodesInPath iip) throws IOException { long size = 0; // length is zero for directories short replication = 0; long blocksize = 0; final boolean isEncrypted; + final INode node = iip.getLastINode(); final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp .getFileEncryptionInfo(fsd, node, snapshot, iip); @@ -462,11 +468,9 @@ class FSDirStatAndListingOp { replication = fileNode.getFileReplication(snapshot); blocksize = fileNode.getPreferredBlockSize(); isEncrypted = (feInfo != null) - || (isRawPath && FSDirEncryptionZoneOp.isInAnEZ(fsd, - INodesInPath.fromINode(node))); + || (isRawPath && FSDirEncryptionZoneOp.isInAnEZ(fsd, iip)); } else { - isEncrypted = FSDirEncryptionZoneOp.isInAnEZ(fsd, - INodesInPath.fromINode(node)); + isEncrypted = FSDirEncryptionZoneOp.isInAnEZ(fsd, iip); } int childrenNum = node.isDirectory() ? @@ -497,9 +501,10 @@ class FSDirStatAndListingOp { /** * Create FileStatus with location info by file INode + * @param iip the INodesInPath containing the target INode and its ancestors */ private static HdfsLocatedFileStatus createLocatedFileStatus( - FSDirectory fsd, byte[] path, INode node, INodeAttributes nodeAttrs, + FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs, byte storagePolicy, int snapshot, boolean isRawPath, INodesInPath iip) throws IOException { assert fsd.hasReadLock(); @@ -508,6 +513,8 @@ class FSDirStatAndListingOp { long blocksize = 0; LocatedBlocks loc = null; final boolean isEncrypted; + final INode node = iip.getLastINode(); + final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp .getFileEncryptionInfo(fsd, node, snapshot, iip); if (node.isFile()) { @@ -528,11 +535,9 @@ class FSDirStatAndListingOp { loc = new LocatedBlocks(); } isEncrypted = (feInfo != null) - || (isRawPath && FSDirEncryptionZoneOp.isInAnEZ(fsd, - INodesInPath.fromINode(node))); + || (isRawPath && FSDirEncryptionZoneOp.isInAnEZ(fsd, iip)); } else { - isEncrypted = FSDirEncryptionZoneOp.isInAnEZ(fsd, - INodesInPath.fromINode(node)); + isEncrypted = FSDirEncryptionZoneOp.isInAnEZ(fsd, iip); } int childrenNum = node.isDirectory() ? node.asDirectory().getChildrenNum(snapshot) : 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index b339966b969..68d0c05d25f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -1674,8 +1674,8 @@ public class FSDirectory implements Closeable { INode node, int snapshot) { INodeAttributes nodeAttrs = node.getSnapshotINode(snapshot); if (attributeProvider != null) { - fullPath = fullPath + (fullPath.endsWith(Path.SEPARATOR) ? "" - : Path.SEPARATOR) + fullPath = fullPath + + (fullPath.endsWith(Path.SEPARATOR) ? "" : Path.SEPARATOR) + DFSUtil.bytes2String(path); nodeAttrs = attributeProvider.getAttributes(fullPath, nodeAttrs); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index 094bb9e1aae..e5431439d7e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -370,13 +370,14 @@ public class FSEditLogLoader { addCloseOp.atime, addCloseOp.blockSize, true, addCloseOp.clientName, addCloseOp.clientMachine, addCloseOp.storagePolicyId); + assert newFile != null; iip = INodesInPath.replace(iip, iip.length() - 1, newFile); fsNamesys.leaseManager.addLease(addCloseOp.clientName, newFile.getId()); // add the op into retry cache if necessary if (toAddRetryCache) { HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog( - fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, newFile, + fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID, false, iip); fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId, @@ -395,8 +396,7 @@ public class FSEditLogLoader { // add the op into retry cache if necessary if (toAddRetryCache) { HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog( - fsNamesys.dir, path, - HdfsFileStatus.EMPTY_NAME, newFile, + fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID, false, iip); fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId, @@ -470,7 +470,7 @@ public class FSEditLogLoader { // add the op into retry cache if necessary if (toAddRetryCache) { HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog( - fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, file, + fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID, false, iip); fsNamesys.addCacheEntryWithPayload(appendOp.rpcClientId, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java index 071bb35e877..86cab28ef00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java @@ -258,7 +258,7 @@ public class INodesInPath { */ public static INodesInPath append(INodesInPath iip, INode child, byte[] childName) { - Preconditions.checkArgument(!iip.isSnapshot && iip.length() > 0); + Preconditions.checkArgument(iip.length() > 0); Preconditions.checkArgument(iip.getLastINode() != null && iip .getLastINode().isDirectory()); INode[] inodes = new INode[iip.length() + 1]; @@ -267,7 +267,7 @@ public class INodesInPath { byte[][] path = new byte[iip.path.length + 1][]; System.arraycopy(iip.path, 0, path, 0, path.length - 1); path[path.length - 1] = childName; - return new INodesInPath(inodes, path, false, iip.snapshotId); + return new INodesInPath(inodes, path, iip.isSnapshot, iip.snapshotId); } private final byte[][] path;