HDFS-8200. Refactor FSDirStatAndListingOp. Contributed by Haohui Mai.
This commit is contained in:
parent
c9ee316045
commit
01acf3a3af
|
@ -162,6 +162,8 @@ Release 2.8.0 - UNRELEASED
|
||||||
HDFS-5574. Remove buffer copy in BlockReader.skip.
|
HDFS-5574. Remove buffer copy in BlockReader.skip.
|
||||||
(Binglin Chang via aajisaka)
|
(Binglin Chang via aajisaka)
|
||||||
|
|
||||||
|
HDFS-8200. Refactor FSDirStatAndListingOp. (wheat9)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
|
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
|
||||||
|
|
|
@ -24,7 +24,6 @@ import org.apache.hadoop.fs.ContentSummary;
|
||||||
import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
|
import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
|
||||||
import org.apache.hadoop.fs.FileEncryptionInfo;
|
import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
import org.apache.hadoop.fs.InvalidPathException;
|
import org.apache.hadoop.fs.InvalidPathException;
|
||||||
import org.apache.hadoop.fs.UnresolvedLinkException;
|
|
||||||
import org.apache.hadoop.fs.permission.FsAction;
|
import org.apache.hadoop.fs.permission.FsAction;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
import org.apache.hadoop.hdfs.DFSUtil;
|
import org.apache.hadoop.hdfs.DFSUtil;
|
||||||
|
@ -180,10 +179,14 @@ class FSDirStatAndListingOp {
|
||||||
.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
|
.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
|
||||||
|
|
||||||
if (!targetNode.isDirectory()) {
|
if (!targetNode.isDirectory()) {
|
||||||
|
INodeAttributes nodeAttrs = getINodeAttributes(
|
||||||
|
fsd, src, HdfsFileStatus.EMPTY_NAME, targetNode,
|
||||||
|
snapshot);
|
||||||
return new DirectoryListing(
|
return new DirectoryListing(
|
||||||
new HdfsFileStatus[]{createFileStatus(fsd, src,
|
new HdfsFileStatus[]{ createFileStatus(
|
||||||
HdfsFileStatus.EMPTY_NAME, targetNode, needLocation,
|
fsd, HdfsFileStatus.EMPTY_NAME, targetNode, nodeAttrs,
|
||||||
parentStoragePolicy, snapshot, isRawPath, iip)}, 0);
|
needLocation, parentStoragePolicy, snapshot, isRawPath, iip)
|
||||||
|
}, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
final INodeDirectory dirInode = targetNode.asDirectory();
|
final INodeDirectory dirInode = targetNode.asDirectory();
|
||||||
|
@ -200,8 +203,11 @@ class FSDirStatAndListingOp {
|
||||||
byte curPolicy = isSuperUser && !cur.isSymlink()?
|
byte curPolicy = isSuperUser && !cur.isSymlink()?
|
||||||
cur.getLocalStoragePolicyID():
|
cur.getLocalStoragePolicyID():
|
||||||
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
|
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
|
||||||
listing[i] = createFileStatus(fsd, src, cur.getLocalNameBytes(), cur,
|
INodeAttributes nodeAttrs = getINodeAttributes(
|
||||||
needLocation, getStoragePolicyID(curPolicy,
|
fsd, src, cur.getLocalNameBytes(), cur,
|
||||||
|
snapshot);
|
||||||
|
listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(),
|
||||||
|
cur, nodeAttrs, needLocation, getStoragePolicyID(curPolicy,
|
||||||
parentStoragePolicy), snapshot, isRawPath, iip);
|
parentStoragePolicy), snapshot, isRawPath, iip);
|
||||||
listingCnt++;
|
listingCnt++;
|
||||||
if (needLocation) {
|
if (needLocation) {
|
||||||
|
@ -253,9 +259,15 @@ class FSDirStatAndListingOp {
|
||||||
final HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing];
|
final HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing];
|
||||||
for (int i = 0; i < numOfListing; i++) {
|
for (int i = 0; i < numOfListing; i++) {
|
||||||
Snapshot.Root sRoot = snapshots.get(i + skipSize).getRoot();
|
Snapshot.Root sRoot = snapshots.get(i + skipSize).getRoot();
|
||||||
listing[i] = createFileStatus(fsd, src, sRoot.getLocalNameBytes(), sRoot,
|
INodeAttributes nodeAttrs = getINodeAttributes(
|
||||||
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID,
|
fsd, src, sRoot.getLocalNameBytes(),
|
||||||
false, INodesInPath.fromINode(sRoot));
|
node, Snapshot.CURRENT_STATE_ID);
|
||||||
|
listing[i] = createFileStatus(
|
||||||
|
fsd, sRoot.getLocalNameBytes(),
|
||||||
|
sRoot, nodeAttrs,
|
||||||
|
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
|
||||||
|
Snapshot.CURRENT_STATE_ID, false,
|
||||||
|
INodesInPath.fromINode(sRoot));
|
||||||
}
|
}
|
||||||
return new DirectoryListing(
|
return new DirectoryListing(
|
||||||
listing, snapshots.size() - skipSize - numOfListing);
|
listing, snapshots.size() - skipSize - numOfListing);
|
||||||
|
@ -276,11 +288,20 @@ class FSDirStatAndListingOp {
|
||||||
fsd.readLock();
|
fsd.readLock();
|
||||||
try {
|
try {
|
||||||
final INode i = src.getLastINode();
|
final INode i = src.getLastINode();
|
||||||
byte policyId = includeStoragePolicy && i != null && !i.isSymlink() ?
|
if (i == null) {
|
||||||
i.getStoragePolicyID() : HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
|
return null;
|
||||||
return i == null ? null : createFileStatus(
|
}
|
||||||
fsd, path, HdfsFileStatus.EMPTY_NAME, i, policyId,
|
|
||||||
src.getPathSnapshotId(), isRawPath, src);
|
byte policyId = includeStoragePolicy && !i.isSymlink() ?
|
||||||
|
i.getStoragePolicyID() : HdfsConstantsClient
|
||||||
|
.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
|
||||||
|
INodeAttributes nodeAttrs = getINodeAttributes(
|
||||||
|
fsd, path, HdfsFileStatus.EMPTY_NAME, i, src.getPathSnapshotId());
|
||||||
|
return createFileStatus(
|
||||||
|
fsd, HdfsFileStatus.EMPTY_NAME,
|
||||||
|
i, nodeAttrs, policyId,
|
||||||
|
src.getPathSnapshotId(),
|
||||||
|
isRawPath, src);
|
||||||
} finally {
|
} finally {
|
||||||
fsd.readUnlock();
|
fsd.readUnlock();
|
||||||
}
|
}
|
||||||
|
@ -309,23 +330,6 @@ class FSDirStatAndListingOp {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Currently we only support "ls /xxx/.snapshot" which will return all the
|
|
||||||
* snapshots of a directory. The FSCommand Ls will first call getFileInfo to
|
|
||||||
* make sure the file/directory exists (before the real getListing call).
|
|
||||||
* Since we do not have a real INode for ".snapshot", we return an empty
|
|
||||||
* non-null HdfsFileStatus here.
|
|
||||||
*/
|
|
||||||
private static HdfsFileStatus getFileInfo4DotSnapshot(
|
|
||||||
FSDirectory fsd, String src)
|
|
||||||
throws UnresolvedLinkException {
|
|
||||||
if (fsd.getINode4DotSnapshot(src) != null) {
|
|
||||||
return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
|
|
||||||
HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
|
|
||||||
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* create an hdfs file status from an inode
|
* create an hdfs file status from an inode
|
||||||
|
@ -339,27 +343,40 @@ class FSDirStatAndListingOp {
|
||||||
* @return a file status
|
* @return a file status
|
||||||
* @throws java.io.IOException if any error occurs
|
* @throws java.io.IOException if any error occurs
|
||||||
*/
|
*/
|
||||||
static HdfsFileStatus createFileStatus(
|
private static HdfsFileStatus createFileStatus(
|
||||||
FSDirectory fsd, String fullPath, byte[] path, INode node,
|
FSDirectory fsd, byte[] path, INode node, INodeAttributes nodeAttrs,
|
||||||
boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath,
|
boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath,
|
||||||
INodesInPath iip)
|
INodesInPath iip)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (needLocation) {
|
if (needLocation) {
|
||||||
return createLocatedFileStatus(fsd, fullPath, path, node, storagePolicy,
|
return createLocatedFileStatus(fsd, path, node, nodeAttrs, storagePolicy,
|
||||||
snapshot, isRawPath, iip);
|
snapshot, isRawPath, iip);
|
||||||
} else {
|
} else {
|
||||||
return createFileStatus(fsd, fullPath, path, node, storagePolicy, snapshot,
|
return createFileStatus(fsd, path, node, nodeAttrs, storagePolicy,
|
||||||
isRawPath, iip);
|
snapshot, isRawPath, iip);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create FileStatus by file INode
|
* Create FileStatus by file INode
|
||||||
*/
|
*/
|
||||||
static HdfsFileStatus createFileStatus(
|
static HdfsFileStatus createFileStatusForEditLog(
|
||||||
FSDirectory fsd, String fullPath, byte[] path, INode node,
|
FSDirectory fsd, String fullPath, byte[] path, INode node,
|
||||||
byte storagePolicy, int snapshot, boolean isRawPath,
|
byte storagePolicy, int snapshot, boolean isRawPath,
|
||||||
INodesInPath iip) throws IOException {
|
INodesInPath iip) throws IOException {
|
||||||
|
INodeAttributes nodeAttrs = getINodeAttributes(
|
||||||
|
fsd, fullPath, path, node, snapshot);
|
||||||
|
return createFileStatus(fsd, path, node, nodeAttrs,
|
||||||
|
storagePolicy, snapshot, isRawPath, iip);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create FileStatus by file INode
|
||||||
|
*/
|
||||||
|
static HdfsFileStatus createFileStatus(
|
||||||
|
FSDirectory fsd, byte[] path, INode node,
|
||||||
|
INodeAttributes nodeAttrs, byte storagePolicy, int snapshot,
|
||||||
|
boolean isRawPath, INodesInPath iip) throws IOException {
|
||||||
long size = 0; // length is zero for directories
|
long size = 0; // length is zero for directories
|
||||||
short replication = 0;
|
short replication = 0;
|
||||||
long blocksize = 0;
|
long blocksize = 0;
|
||||||
|
@ -382,8 +399,6 @@ class FSDirStatAndListingOp {
|
||||||
int childrenNum = node.isDirectory() ?
|
int childrenNum = node.isDirectory() ?
|
||||||
node.asDirectory().getChildrenNum(snapshot) : 0;
|
node.asDirectory().getChildrenNum(snapshot) : 0;
|
||||||
|
|
||||||
INodeAttributes nodeAttrs =
|
|
||||||
fsd.getAttributes(fullPath, path, node, snapshot);
|
|
||||||
return new HdfsFileStatus(
|
return new HdfsFileStatus(
|
||||||
size,
|
size,
|
||||||
node.isDirectory(),
|
node.isDirectory(),
|
||||||
|
@ -402,13 +417,18 @@ class FSDirStatAndListingOp {
|
||||||
storagePolicy);
|
storagePolicy);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static INodeAttributes getINodeAttributes(
|
||||||
|
FSDirectory fsd, String fullPath, byte[] path, INode node, int snapshot) {
|
||||||
|
return fsd.getAttributes(fullPath, path, node, snapshot);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create FileStatus with location info by file INode
|
* Create FileStatus with location info by file INode
|
||||||
*/
|
*/
|
||||||
private static HdfsLocatedFileStatus createLocatedFileStatus(
|
private static HdfsLocatedFileStatus createLocatedFileStatus(
|
||||||
FSDirectory fsd, String fullPath, byte[] path, INode node,
|
FSDirectory fsd, byte[] path, INode node, INodeAttributes nodeAttrs,
|
||||||
byte storagePolicy, int snapshot, boolean isRawPath,
|
byte storagePolicy, int snapshot,
|
||||||
INodesInPath iip) throws IOException {
|
boolean isRawPath, INodesInPath iip) throws IOException {
|
||||||
assert fsd.hasReadLock();
|
assert fsd.hasReadLock();
|
||||||
long size = 0; // length is zero for directories
|
long size = 0; // length is zero for directories
|
||||||
short replication = 0;
|
short replication = 0;
|
||||||
|
@ -442,8 +462,6 @@ class FSDirStatAndListingOp {
|
||||||
int childrenNum = node.isDirectory() ?
|
int childrenNum = node.isDirectory() ?
|
||||||
node.asDirectory().getChildrenNum(snapshot) : 0;
|
node.asDirectory().getChildrenNum(snapshot) : 0;
|
||||||
|
|
||||||
INodeAttributes nodeAttrs =
|
|
||||||
fsd.getAttributes(fullPath, path, node, snapshot);
|
|
||||||
HdfsLocatedFileStatus status =
|
HdfsLocatedFileStatus status =
|
||||||
new HdfsLocatedFileStatus(size, node.isDirectory(), replication,
|
new HdfsLocatedFileStatus(size, node.isDirectory(), replication,
|
||||||
blocksize, node.getModificationTime(snapshot),
|
blocksize, node.getModificationTime(snapshot),
|
||||||
|
@ -468,7 +486,6 @@ class FSDirStatAndListingOp {
|
||||||
* return an FsPermissionExtension.
|
* return an FsPermissionExtension.
|
||||||
*
|
*
|
||||||
* @param node INode to check
|
* @param node INode to check
|
||||||
* @param snapshot int snapshot ID
|
|
||||||
* @param isEncrypted boolean true if the file/dir is encrypted
|
* @param isEncrypted boolean true if the file/dir is encrypted
|
||||||
* @return FsPermission from inode, with ACL bit on if the inode has an ACL
|
* @return FsPermission from inode, with ACL bit on if the inode has an ACL
|
||||||
* and encrypted bit on if it represents an encrypted file/dir.
|
* and encrypted bit on if it represents an encrypted file/dir.
|
||||||
|
|
|
@ -379,7 +379,7 @@ public class FSEditLogLoader {
|
||||||
|
|
||||||
// add the op into retry cache if necessary
|
// add the op into retry cache if necessary
|
||||||
if (toAddRetryCache) {
|
if (toAddRetryCache) {
|
||||||
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatus(
|
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(
|
||||||
fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, newFile,
|
fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, newFile,
|
||||||
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID,
|
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID,
|
||||||
false, iip);
|
false, iip);
|
||||||
|
@ -398,7 +398,7 @@ public class FSEditLogLoader {
|
||||||
false);
|
false);
|
||||||
// add the op into retry cache if necessary
|
// add the op into retry cache if necessary
|
||||||
if (toAddRetryCache) {
|
if (toAddRetryCache) {
|
||||||
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatus(
|
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(
|
||||||
fsNamesys.dir, path,
|
fsNamesys.dir, path,
|
||||||
HdfsFileStatus.EMPTY_NAME, newFile,
|
HdfsFileStatus.EMPTY_NAME, newFile,
|
||||||
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
|
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
|
||||||
|
@ -472,7 +472,7 @@ public class FSEditLogLoader {
|
||||||
false, false);
|
false, false);
|
||||||
// add the op into retry cache if necessary
|
// add the op into retry cache if necessary
|
||||||
if (toAddRetryCache) {
|
if (toAddRetryCache) {
|
||||||
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatus(
|
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(
|
||||||
fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, file,
|
fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, file,
|
||||||
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
|
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
|
||||||
Snapshot.CURRENT_STATE_ID, false, iip);
|
Snapshot.CURRENT_STATE_ID, false, iip);
|
||||||
|
|
Loading…
Reference in New Issue