Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-17060. Clarify listStatus and getFileStatus behaviors inconsistent in the case of ViewFs implementation for isDirectory. Contributed by Uma Maheswara Rao G.

This commit is contained in:
Uma Maheswara Rao G 2020-06-10 15:00:02 -07:00 committed by GitHub
parent b735a77717
commit 93b121a971
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 94 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -488,6 +488,14 @@ private static FileStatus wrapLocalFileStatus(FileStatus orig,
: new ViewFsFileStatus(orig, qualified);
}
/**
* {@inheritDoc}
*
* If the given path is a symlink(mount link), the path will be resolved to a
* target path and it will get the resolved path's FileStatus object. It will
* not be represented as a symlink and isDirectory API returns true if the
* resolved path is a directory, false otherwise.
*/
@Override
public FileStatus getFileStatus(final Path f) throws AccessControlException,
FileNotFoundException, IOException {
@ -505,6 +513,25 @@ public void access(Path path, FsAction mode) throws AccessControlException,
res.targetFileSystem.access(res.remainingPath, mode);
}
/**
* {@inheritDoc}
*
* Note: listStatus on root("/") considers listing from fallbackLink if
* available. If the same directory name is present in configured mount path
* as well as in fallback link, then only the configured mount path will be
* listed in the returned result.
*
* If any of the the immediate children of the given path f is a symlink(mount
* link), the returned FileStatus object of that children would be represented
* as a symlink. It will not be resolved to the target path and will not get
* the target path FileStatus object. The target path will be available via
* getSymlink on that children's FileStatus object. Since it represents as
* symlink, isDirectory on that children's FileStatus will return false.
*
* If you want to get the FileStatus of target path for that children, you may
* want to use GetFileStatus API with that children's symlink path. Please see
* {@link ViewFileSystem#getFileStatus(Path f)}
*/
@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
FileNotFoundException, IOException {
@ -1174,20 +1201,11 @@ public FileStatus getFileStatus(Path f) throws IOException {
checkPathIsSlash(f);
return new FileStatus(0, true, 0, 0, creationTime, creationTime,
PERMISSION_555, ugi.getShortUserName(), ugi.getPrimaryGroupName(),
new Path(theInternalDir.fullPath).makeQualified(
myUri, ROOT_PATH));
}
/**
* {@inheritDoc}
*
* Note: listStatus on root("/") considers listing from fallbackLink if
* available. If the same directory name is present in configured mount
* path as well as in fallback link, then only the configured mount path
* will be listed in the returned result.
*/
@Override
public FileStatus[] listStatus(Path f) throws AccessControlException,
FileNotFoundException, IOException {

24
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
View File

@ -351,6 +351,14 @@ public FileChecksum getFileChecksum(final Path f)
return res.targetFileSystem.getFileChecksum(res.remainingPath);
}
/**
* {@inheritDoc}
*
* If the given path is a symlink(mount link), the path will be resolved to a
* target path and it will get the resolved path's FileStatus object. It will
* not be represented as a symlink and isDirectory API returns true if the
* resolved path is a directory, false otherwise.
*/
@Override
public FileStatus getFileStatus(final Path f) throws AccessControlException,
FileNotFoundException, UnresolvedLinkException, IOException {
@ -436,6 +444,22 @@ public LocatedFileStatus getViewFsFileStatus(LocatedFileStatus stat,
};
}
/**
* {@inheritDoc}
*
* If any of the the immediate children of the given path f is a symlink(mount
* link), the returned FileStatus object of that children would be represented
* as a symlink. It will not be resolved to the target path and will not get
* the target path FileStatus object. The target path will be available via
* getSymlink on that children's FileStatus object. Since it represents as
* symlink, isDirectory on that children's FileStatus will return false.
*
* If you want to get the FileStatus of target path for that children, you may
* want to use GetFileStatus API with that children's symlink path. Please see
* {@link ViewFs#getFileStatus(Path f)}
*
* Note: In ViewFs, the mount links are represented as symlinks.
*/
@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
FileNotFoundException, UnresolvedLinkException, IOException {

22
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
View File

@ -135,6 +135,14 @@ public FileChecksum getFileChecksum(Path f)
return dfs.getFileChecksumWithCombineMode(getUriPath(f), Long.MAX_VALUE);
}
/**
* {@inheritDoc}
*
* If the given path is a symlink, the path will be resolved to a target path
* and it will get the resolved path's FileStatus object. It will not be
* represented as a symlink and isDirectory API returns true if the resolved
* path is a directory, false otherwise.
*/
@Override
public FileStatus getFileStatus(Path f)
throws IOException, UnresolvedLinkException {
@ -269,6 +277,20 @@ public HdfsFileStatus getNext() throws IOException {
}
}
/**
* {@inheritDoc}
*
* If any of the the immediate children of the given path f is a symlink, the
* returned FileStatus object of that children would be represented as a
* symlink. It will not be resolved to the target path and will not get the
* target path FileStatus object. The target path will be available via
* getSymlink on that children's FileStatus object. Since it represents as
* symlink, isDirectory on that children's FileStatus will return false.
*
* If you want to get the FileStatus of target path for that children, you may
* want to use GetFileStatus API with that children's symlink path. Please see
* {@link Hdfs#getFileStatus(Path f)}
*/
@Override
public FileStatus[] listStatus(Path f)
throws IOException, UnresolvedLinkException {

25
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -1143,10 +1143,21 @@ private FileStatus[] listStatusInternal(Path p) throws IOException {
/**
* List all the entries of a directory
*
* Note that this operation is not atomic for a large directory.
* The entries of a directory may be fetched from NameNode multiple times.
* It only guarantees that each name occurs once if a directory
* undergoes changes between the calls.
* Note that this operation is not atomic for a large directory. The entries
* of a directory may be fetched from NameNode multiple times. It only
* guarantees that each name occurs once if a directory undergoes changes
* between the calls.
*
* If any of the the immediate children of the given path f is a symlink, the
* returned FileStatus object of that children would be represented as a
* symlink. It will not be resolved to the target path and will not get the
* target path FileStatus object. The target path will be available via
* getSymlink on that children's FileStatus object. Since it represents as
* symlink, isDirectory on that children's FileStatus will return false.
*
* If you want to get the FileStatus of target path for that children, you may
* want to use GetFileStatus API with that children's symlink path. Please see
* {@link DistributedFileSystem#getFileStatus(Path f)}
*/
@Override
public FileStatus[] listStatus(Path p) throws IOException {
@ -1712,6 +1723,12 @@ public FsServerDefaults getServerDefaults() throws IOException {
/**
* Returns the stat information about the file.
*
* If the given path is a symlink, the path will be resolved to a target path
* and it will get the resolved path's FileStatus object. It will not be
* represented as a symlink and isDirectory API returns true if the resolved
* path is a directory, false otherwise.
*
* @throws FileNotFoundException if the file does not exist.
*/
@Override