Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-9621. Consolidate FSDirStatAndListingOp#createFileStatus to let its INodesInPath parameter always include the target INode. Contributed by Jing Zhao. 

(cherry picked from commit 313f03bfda)

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java

(cherry picked from commit c2708bc5fb)
This commit is contained in:
Kihwal Lee 2016-09-02 12:16:08 -05:00 committed by Zhe Zhang
parent 6f44510d87
commit a8b7817739
4 changed files with 56 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
View File

@ -171,12 +171,14 @@ class FSDirStatAndListingOp {
.ID_UNSPECIFIED; .ID_UNSPECIFIED;
if (!targetNode.isDirectory()) { if (!targetNode.isDirectory()) {
// return the file's status. note that the iip already includes the
// target INode
INodeAttributes nodeAttrs = getINodeAttributes( INodeAttributes nodeAttrs = getINodeAttributes(
fsd, src, HdfsFileStatus.EMPTY_NAME, targetNode, fsd, src, HdfsFileStatus.EMPTY_NAME, targetNode,
snapshot); snapshot);
return new DirectoryListing( return new DirectoryListing(
new HdfsFileStatus[]{ createFileStatus( new HdfsFileStatus[]{ createFileStatus(
fsd, HdfsFileStatus.EMPTY_NAME, targetNode, nodeAttrs, fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
needLocation, parentStoragePolicy, snapshot, isRawPath, iip) needLocation, parentStoragePolicy, snapshot, isRawPath, iip)
}, 0); }, 0);
} }
@ -190,7 +192,7 @@ class FSDirStatAndListingOp {
int locationBudget = fsd.getLsLimit(); int locationBudget = fsd.getLsLimit();
int listingCnt = 0; int listingCnt = 0;
HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing]; HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing];
for (int i=0; i<numOfListing && locationBudget>0; i++) { for (int i = 0; i < numOfListing && locationBudget > 0; i++) {
INode cur = contents.get(startChild+i); INode cur = contents.get(startChild+i);
byte curPolicy = isSuperUser && !cur.isSymlink()? byte curPolicy = isSuperUser && !cur.isSymlink()?
cur.getLocalStoragePolicyID(): cur.getLocalStoragePolicyID():
@ -198,9 +200,11 @@ class FSDirStatAndListingOp {
INodeAttributes nodeAttrs = getINodeAttributes( INodeAttributes nodeAttrs = getINodeAttributes(
fsd, src, cur.getLocalNameBytes(), cur, fsd, src, cur.getLocalNameBytes(), cur,
snapshot); snapshot);
listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), final INodesInPath iipWithChild = INodesInPath.append(iip, cur,
cur, nodeAttrs, needLocation, getStoragePolicyID(curPolicy, cur.getLocalNameBytes());
parentStoragePolicy), snapshot, isRawPath, iip); listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), nodeAttrs,
needLocation, getStoragePolicyID(curPolicy, parentStoragePolicy),
snapshot, isRawPath, iipWithChild);
listingCnt++; listingCnt++;
if (needLocation) { if (needLocation) {
// Once we hit lsLimit locations, stop. // Once we hit lsLimit locations, stop.
@ -255,9 +259,8 @@ class FSDirStatAndListingOp {
fsd, src, sRoot.getLocalNameBytes(), fsd, src, sRoot.getLocalNameBytes(),
node, Snapshot.CURRENT_STATE_ID); node, Snapshot.CURRENT_STATE_ID);
listing[i] = createFileStatus( listing[i] = createFileStatus(
fsd, sRoot.getLocalNameBytes(), fsd, sRoot.getLocalNameBytes(), nodeAttrs,
sRoot, nodeAttrs, BlockStoragePolicySuite.ID_UNSPECIFIED,
BlockStoragePolicySuite.ID_UNSPECIFIED,
Snapshot.CURRENT_STATE_ID, false, Snapshot.CURRENT_STATE_ID, false,
INodesInPath.fromINode(sRoot)); INodesInPath.fromINode(sRoot));
} }
@ -267,32 +270,31 @@ class FSDirStatAndListingOp {
/** Get the file info for a specific file. /** Get the file info for a specific file.
* @param fsd FSDirectory * @param fsd FSDirectory
* @param src The string representation of the path to the file * @param iip The path to the file, the file is included
* @param isRawPath true if a /.reserved/raw pathname was passed by the user * @param isRawPath true if a /.reserved/raw pathname was passed by the user
* @param includeStoragePolicy whether to include storage policy * @param includeStoragePolicy whether to include storage policy
* @return object containing information regarding the file * @return object containing information regarding the file
* or null if file not found * or null if file not found
*/ */
static HdfsFileStatus getFileInfo( static HdfsFileStatus getFileInfo(
FSDirectory fsd, String path, INodesInPath src, boolean isRawPath, FSDirectory fsd, String path, INodesInPath iip, boolean isRawPath,
boolean includeStoragePolicy) boolean includeStoragePolicy)
throws IOException { throws IOException {
fsd.readLock(); fsd.readLock();
try { try {
final INode i = src.getLastINode(); final INode node = iip.getLastINode();
if (i == null) { if (node == null) {
return null; return null;
} }
byte policyId = includeStoragePolicy && !i.isSymlink() ? byte policyId = includeStoragePolicy && !node.isSymlink() ?
i.getStoragePolicyID() : BlockStoragePolicySuite.ID_UNSPECIFIED; node.getStoragePolicyID() :
INodeAttributes nodeAttrs = getINodeAttributes( BlockStoragePolicySuite.ID_UNSPECIFIED;
fsd, path, HdfsFileStatus.EMPTY_NAME, i, src.getPathSnapshotId()); INodeAttributes nodeAttrs = getINodeAttributes(fsd, path,
return createFileStatus( HdfsFileStatus.EMPTY_NAME,
fsd, HdfsFileStatus.EMPTY_NAME, node, iip.getPathSnapshotId());
i, nodeAttrs, policyId, return createFileStatus(fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
src.getPathSnapshotId(), policyId, iip.getPathSnapshotId(), isRawPath, iip);
isRawPath, src);
} finally { } finally {
fsd.readUnlock(); fsd.readUnlock();
} }
@ -326,51 +328,54 @@ class FSDirStatAndListingOp {
* *
* @param fsd FSDirectory * @param fsd FSDirectory
* @param path the local name * @param path the local name
* @param node inode
* @param needLocation if block locations need to be included or not * @param needLocation if block locations need to be included or not
* @param isRawPath true if this is being called on behalf of a path in * @param isRawPath true if this is being called on behalf of a path in
* /.reserved/raw * /.reserved/raw
* @param iip the INodesInPath containing the target INode and its ancestors
* @return a file status * @return a file status
* @throws java.io.IOException if any error occurs * @throws java.io.IOException if any error occurs
*/ */
private static HdfsFileStatus createFileStatus( private static HdfsFileStatus createFileStatus(
FSDirectory fsd, byte[] path, INode node, INodeAttributes nodeAttrs, FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath, boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath,
INodesInPath iip) INodesInPath iip)
throws IOException { throws IOException {
if (needLocation) { if (needLocation) {
return createLocatedFileStatus(fsd, path, node, nodeAttrs, storagePolicy, return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy,
snapshot, isRawPath, iip); snapshot, isRawPath, iip);
} else { } else {
return createFileStatus(fsd, path, node, nodeAttrs, storagePolicy, return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
snapshot, isRawPath, iip); snapshot, isRawPath, iip);
} }
} }
/** /**
* Create FileStatus by file INode * Create FileStatus for an given INodeFile.
* @param iip The INodesInPath containing the INodeFile and its ancestors
*/ */
static HdfsFileStatus createFileStatusForEditLog( static HdfsFileStatus createFileStatusForEditLog(
FSDirectory fsd, String fullPath, byte[] path, INode node, FSDirectory fsd, String fullPath, byte[] path,
byte storagePolicy, int snapshot, boolean isRawPath, byte storagePolicy, int snapshot, boolean isRawPath,
INodesInPath iip) throws IOException { INodesInPath iip) throws IOException {
INodeAttributes nodeAttrs = getINodeAttributes( INodeAttributes nodeAttrs = getINodeAttributes(
fsd, fullPath, path, node, snapshot); fsd, fullPath, path, iip.getLastINode(), snapshot);
return createFileStatus(fsd, path, node, nodeAttrs, return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
storagePolicy, snapshot, isRawPath, iip); snapshot, isRawPath, iip);
} }
/** /**
* Create FileStatus by file INode * create file status for a given INode
* @param iip the INodesInPath containing the target INode and its ancestors
*/ */
static HdfsFileStatus createFileStatus( static HdfsFileStatus createFileStatus(
FSDirectory fsd, byte[] path, INode node, FSDirectory fsd, byte[] path,
INodeAttributes nodeAttrs, byte storagePolicy, int snapshot, INodeAttributes nodeAttrs, byte storagePolicy, int snapshot,
boolean isRawPath, INodesInPath iip) throws IOException { boolean isRawPath, INodesInPath iip) throws IOException {
long size = 0; // length is zero for directories long size = 0; // length is zero for directories
short replication = 0; short replication = 0;
long blocksize = 0; long blocksize = 0;
final boolean isEncrypted; final boolean isEncrypted;
final INode node = iip.getLastINode();
final FileEncryptionInfo feInfo = isRawPath ? null : final FileEncryptionInfo feInfo = isRawPath ? null :
fsd.getFileEncryptionInfo(node, snapshot, iip); fsd.getFileEncryptionInfo(node, snapshot, iip);
@ -381,9 +386,9 @@ class FSDirStatAndListingOp {
replication = fileNode.getFileReplication(snapshot); replication = fileNode.getFileReplication(snapshot);
blocksize = fileNode.getPreferredBlockSize(); blocksize = fileNode.getPreferredBlockSize();
isEncrypted = (feInfo != null) || isEncrypted = (feInfo != null) ||
(isRawPath && fsd.isInAnEZ(INodesInPath.fromINode(node))); (isRawPath && fsd.isInAnEZ(iip));
} else { } else {
isEncrypted = fsd.isInAnEZ(INodesInPath.fromINode(node)); isEncrypted = fsd.isInAnEZ(iip);
} }
int childrenNum = node.isDirectory() ? int childrenNum = node.isDirectory() ?
@ -414,9 +419,10 @@ class FSDirStatAndListingOp {
/** /**
* Create FileStatus with location info by file INode * Create FileStatus with location info by file INode
* @param iip the INodesInPath containing the target INode and its ancestors
*/ */
private static HdfsLocatedFileStatus createLocatedFileStatus( private static HdfsLocatedFileStatus createLocatedFileStatus(
FSDirectory fsd, byte[] path, INode node, INodeAttributes nodeAttrs, FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
byte storagePolicy, int snapshot, byte storagePolicy, int snapshot,
boolean isRawPath, INodesInPath iip) throws IOException { boolean isRawPath, INodesInPath iip) throws IOException {
assert fsd.hasReadLock(); assert fsd.hasReadLock();
@ -425,6 +431,8 @@ class FSDirStatAndListingOp {
long blocksize = 0; long blocksize = 0;
LocatedBlocks loc = null; LocatedBlocks loc = null;
final boolean isEncrypted; final boolean isEncrypted;
final INode node = iip.getLastINode();
final FileEncryptionInfo feInfo = isRawPath ? null : final FileEncryptionInfo feInfo = isRawPath ? null :
fsd.getFileEncryptionInfo(node, snapshot, iip); fsd.getFileEncryptionInfo(node, snapshot, iip);
if (node.isFile()) { if (node.isFile()) {
@ -445,9 +453,9 @@ class FSDirStatAndListingOp {
loc = new LocatedBlocks(); loc = new LocatedBlocks();
} }
isEncrypted = (feInfo != null) || isEncrypted = (feInfo != null) ||
(isRawPath && fsd.isInAnEZ(INodesInPath.fromINode(node))); (isRawPath && fsd.isInAnEZ(iip));
} else { } else {
isEncrypted = fsd.isInAnEZ(INodesInPath.fromINode(node)); isEncrypted = fsd.isInAnEZ(iip);
} }
int childrenNum = node.isDirectory() ? int childrenNum = node.isDirectory() ?
node.asDirectory().getChildrenNum(snapshot) : 0; node.asDirectory().getChildrenNum(snapshot) : 0;

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
View File

@ -1722,8 +1722,8 @@ public class FSDirectory implements Closeable {
INode node, int snapshot) { INode node, int snapshot) {
INodeAttributes nodeAttrs = node.getSnapshotINode(snapshot); INodeAttributes nodeAttrs = node.getSnapshotINode(snapshot);
if (attributeProvider != null) { if (attributeProvider != null) {
fullPath = fullPath + (fullPath.endsWith(Path.SEPARATOR) ? "" fullPath = fullPath
: Path.SEPARATOR) + (fullPath.endsWith(Path.SEPARATOR) ? "" : Path.SEPARATOR)
+ DFSUtil.bytes2String(path); + DFSUtil.bytes2String(path);
nodeAttrs = attributeProvider.getAttributes(fullPath, nodeAttrs); nodeAttrs = attributeProvider.getAttributes(fullPath, nodeAttrs);
} }

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
View File

@ -374,14 +374,15 @@ public class FSEditLogLoader {
addCloseOp.clientName, addCloseOp.clientName,
addCloseOp.clientMachine, addCloseOp.clientMachine,
addCloseOp.storagePolicyId); addCloseOp.storagePolicyId);
assert newFile != null;
iip = INodesInPath.replace(iip, iip.length() - 1, newFile); iip = INodesInPath.replace(iip, iip.length() - 1, newFile);
fsNamesys.leaseManager.addLease(addCloseOp.clientName, path); fsNamesys.leaseManager.addLease(addCloseOp.clientName, path);
// add the op into retry cache if necessary // add the op into retry cache if necessary
if (toAddRetryCache) { if (toAddRetryCache) {
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog( HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(
fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, newFile, fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME,
BlockStoragePolicySuite.ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID, BlockStoragePolicySuite.ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID,
false, iip); false, iip);
fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId, fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId,
addCloseOp.rpcCallId, stat); addCloseOp.rpcCallId, stat);
@ -399,9 +400,8 @@ public class FSEditLogLoader {
// add the op into retry cache if necessary // add the op into retry cache if necessary
if (toAddRetryCache) { if (toAddRetryCache) {
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog( HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(
fsNamesys.dir, path, fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME,
HdfsFileStatus.EMPTY_NAME, newFile, BlockStoragePolicySuite.ID_UNSPECIFIED,
BlockStoragePolicySuite.ID_UNSPECIFIED,
Snapshot.CURRENT_STATE_ID, false, iip); Snapshot.CURRENT_STATE_ID, false, iip);
fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId, fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId,
addCloseOp.rpcCallId, new LastBlockWithStatus(lb, stat)); addCloseOp.rpcCallId, new LastBlockWithStatus(lb, stat));
@ -473,8 +473,8 @@ public class FSEditLogLoader {
// add the op into retry cache if necessary // add the op into retry cache if necessary
if (toAddRetryCache) { if (toAddRetryCache) {
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog( HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(
fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, file, fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME,
BlockStoragePolicySuite.ID_UNSPECIFIED, BlockStoragePolicySuite.ID_UNSPECIFIED,
Snapshot.CURRENT_STATE_ID, false, iip); Snapshot.CURRENT_STATE_ID, false, iip);
fsNamesys.addCacheEntryWithPayload(appendOp.rpcClientId, fsNamesys.addCacheEntryWithPayload(appendOp.rpcClientId,
appendOp.rpcCallId, new LastBlockWithStatus(lb, stat)); appendOp.rpcCallId, new LastBlockWithStatus(lb, stat));

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
View File

@ -253,7 +253,7 @@ public class INodesInPath {
*/ */
public static INodesInPath append(INodesInPath iip, INode child, public static INodesInPath append(INodesInPath iip, INode child,
byte[] childName) { byte[] childName) {
Preconditions.checkArgument(!iip.isSnapshot && iip.length() > 0); Preconditions.checkArgument(iip.length() > 0);
Preconditions.checkArgument(iip.getLastINode() != null && iip Preconditions.checkArgument(iip.getLastINode() != null && iip
.getLastINode().isDirectory()); .getLastINode().isDirectory());
INode[] inodes = new INode[iip.length() + 1]; INode[] inodes = new INode[iip.length() + 1];
@ -262,7 +262,7 @@ public class INodesInPath {
byte[][] path = new byte[iip.path.length + 1][]; byte[][] path = new byte[iip.path.length + 1][];
System.arraycopy(iip.path, 0, path, 0, path.length - 1); System.arraycopy(iip.path, 0, path, 0, path.length - 1);
path[path.length - 1] = childName; path[path.length - 1] = childName;
return new INodesInPath(inodes, path, false, iip.snapshotId); return new INodesInPath(inodes, path, iip.isSnapshot, iip.snapshotId);
} }
private final byte[][] path; private final byte[][] path;