Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-15532: listFiles on root/InternalDir will fail if fallback root has file. (#2298). Contributed by Uma Maheswara Rao G. 

(cherry picked from commit d2779de3f5)
This commit is contained in:
Uma Maheswara Rao G 2020-09-12 17:06:39 -07:00 committed by Uma Maheswara Rao G
parent 2d5ca83078
commit bfa145dd7c
4 changed files with 56 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -1277,6 +1277,23 @@ public class ViewFileSystem extends FileSystem {
public BlockLocation[] getFileBlockLocations(final FileStatus fs,
final long start, final long len) throws
FileNotFoundException, IOException {
// When application calls listFiles on internalDir, it would return
// RemoteIterator from InternalDirOfViewFs. If there is a fallBack, there
// is a chance of files exists under that internalDir in fallback.
// Iterator#next will call getFileBlockLocations with that files. So, we
// should return getFileBlockLocations on fallback. See HDFS-15532.
if (!InodeTree.SlashPath.equals(fs.getPath()) && this.fsState
.getRootFallbackLink() != null) {
FileSystem linkedFallbackFs =
this.fsState.getRootFallbackLink().getTargetFileSystem();
Path parent = Path.getPathWithoutSchemeAndAuthority(
new Path(theInternalDir.fullPath));
Path pathToFallbackFs = new Path(parent, fs.getPath().getName());
return linkedFallbackFs
.getFileBlockLocations(pathToFallbackFs, start, len);
}
checkPathIsSlash(fs.getPath());
throw new FileNotFoundException("Path points to dir not a file");
}

15
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
View File

@ -981,6 +981,21 @@ public class ViewFs extends AbstractFileSystem {
@Override
public BlockLocation[] getFileBlockLocations(final Path f, final long start,
final long len) throws FileNotFoundException, IOException {
// When application calls listFiles on internalDir, it would return
// RemoteIterator from InternalDirOfViewFs. If there is a fallBack, there
// is a chance of files exists under that internalDir in fallback.
// Iterator#next will call getFileBlockLocations with that files. So, we
// should return getFileBlockLocations on fallback. See HDFS-15532.
if (!InodeTree.SlashPath.equals(f) && this.fsState
.getRootFallbackLink() != null) {
AbstractFileSystem linkedFallbackFs =
this.fsState.getRootFallbackLink().getTargetFileSystem();
Path parent = Path.getPathWithoutSchemeAndAuthority(
new Path(theInternalDir.fullPath));
Path pathToFallbackFs = new Path(parent, f.getName());
return linkedFallbackFs
.getFileBlockLocations(pathToFallbackFs, start, len);
}
checkPathIsSlash(f);
throw new FileNotFoundException("Path points to dir not a file");
}

24
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsLinkFallback.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.fs.viewfs;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@ -30,11 +31,14 @@ import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@ -448,4 +452,24 @@ public class TestViewFsLinkFallback {
vfs.create(vfsTestDir, EnumSet.of(CREATE),
Options.CreateOpts.perms(FsPermission.getDefault())).close();
}
/**
* Tests that, when fallBack has files under matching internal dir, listFiles
* should work.
*/
@Test
public void testListFiles() throws Exception {
Configuration conf = new Configuration();
Path fallbackTarget = new Path(targetTestRoot, "fallbackDir");
fsTarget.mkdirs(fallbackTarget);
Path fileInFallBackRoot = new Path(fallbackTarget, "GetFileBlockLocations");
fsTarget.create(fileInFallBackRoot).close();
ConfigUtil.addLinkFallback(conf, fallbackTarget.toUri());
FileContext fc = FileContext.getFileContext(viewFsDefaultClusterUri, conf);
RemoteIterator<LocatedFileStatus> iterator =
fc.util().listFiles(new Path("/"), false);
assertTrue(iterator.hasNext());
assertEquals(fileInFallBackRoot.getName(),
iterator.next().getPath().getName());
}
}

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystemContract.java
View File

@ -29,7 +29,6 @@ import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
@ -86,9 +85,4 @@ public class TestViewDistributedFileSystemContract
super.testRenameRootDirForbidden();
});
}
@Ignore("Ignore this test until HDFS-15532")
@Override
public void testLSRootDir() throws Throwable {
}
}