Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added webhdfs test.

This commit is contained in:
Tsz-Wo Nicholas Sze 2023-03-24 16:28:03 +08:00
parent 0ec6f5c374
commit ad7ad57bd0
3 changed files with 52 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -3873,7 +3873,8 @@ public class DistributedFileSystem extends FileSystem
throws IOException {
// qualify the path to make sure that it refers to the current FS.
final Path p = makeQualified(path);
if (DfsPathCapabilities.hasPathCapability(p, capability)) {
if (DfsPathCapabilities.hasPathCapability(p, capability)
&& supportsSymlinks()) {
return true;
}
// this switch is for features which are in the DFS client but not

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
View File

@ -2206,7 +2206,8 @@ public class WebHdfsFileSystem extends FileSystem
throws IOException {
// qualify the path to make sure that it refers to the current FS.
final Path p = makeQualified(path);
if (DfsPathCapabilities.hasPathCapability(p, capability)) {
if (DfsPathCapabilities.hasPathCapability(p, capability)
&& supportsSymlinks()) {
return true;
}
return super.hasPathCapability(p, capability);

71
hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/TestDfsPathCapabilities.java
View File

@ -21,46 +21,71 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.fs.CommonPathCapabilities.FS_SYMLINKS;
public class TestDfsPathCapabilities {
public static final Logger LOG = LoggerFactory.getLogger(TestDfsPathCapabilities.class);
static FileSystem getFileSystem(String url) throws Exception {
final HdfsConfiguration conf = new HdfsConfiguration();
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, url);
final FileSystem fs = FileSystem.get(conf);
Assert.assertTrue(fs.supportsSymlinks());
return fs;
}
/**
* Test if {@link DistributedFileSystem#hasPathCapability(Path, String)}
* returns correct result for FS_SYMLINKS.
*/
@Test
public void testFS_SYMLINKS() throws Exception {
final HdfsConfiguration conf = new HdfsConfiguration();
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://localhost/");
final FileSystem fs = FileSystem.get(conf);
Assert.assertTrue(fs instanceof DistributedFileSystem);
final DistributedFileSystem dfs = Mockito.spy((DistributedFileSystem) fs);
public void testSymlinks() throws Exception {
final DistributedFileSystem dfs = Mockito.spy(
(DistributedFileSystem) getFileSystem("hdfs://localhost/"));
final WebHdfsFileSystem webhdfs = Mockito.spy(
(WebHdfsFileSystem) getFileSystem("webhdfs://localhost/"));
final FileSystem[] fileSystems = {dfs, webhdfs};
final Path path = new Path("/");
// Symlinks support disabled
Mockito.doReturn(false).when(dfs).supportsSymlinks();
Assert.assertFalse(FileSystem.areSymlinksEnabled());
Assert.assertFalse(dfs.hasPathCapability(path, FS_SYMLINKS));
for (FileSystem fs : fileSystems) {
LOG.info("fs is {}", fs.getClass().getSimpleName());
// Symlinks support disabled
Mockito.doReturn(false).when(fs).supportsSymlinks();
Assert.assertFalse(fs.supportsSymlinks());
Assert.assertFalse(FileSystem.areSymlinksEnabled());
Assert.assertFalse(fs.hasPathCapability(path, FS_SYMLINKS));
// Symlinks support enabled
Mockito.doReturn(true).when(dfs).supportsSymlinks();
Assert.assertFalse(FileSystem.areSymlinksEnabled());
Assert.assertFalse(dfs.hasPathCapability(path, FS_SYMLINKS));
// Symlinks support enabled
Mockito.doReturn(true).when(fs).supportsSymlinks();
Assert.assertTrue(fs.supportsSymlinks());
Assert.assertFalse(FileSystem.areSymlinksEnabled());
Assert.assertFalse(fs.hasPathCapability(path, FS_SYMLINKS));
}
// Symlinks enabled
// Once it is enabled, it cannot be disabled.
FileSystem.enableSymlinks();
Mockito.doReturn(true).when(dfs).supportsSymlinks();
Assert.assertTrue(FileSystem.areSymlinksEnabled());
Assert.assertTrue(dfs.hasPathCapability(path, FS_SYMLINKS));
// Symlinks enabled but symlink-support is disabled
FileSystem.enableSymlinks();
Mockito.doReturn(false).when(dfs).supportsSymlinks();
Assert.assertTrue(FileSystem.areSymlinksEnabled());
Assert.assertTrue(dfs.hasPathCapability(path, FS_SYMLINKS));
for (FileSystem fs : fileSystems) {
LOG.info("fs is {}", fs.getClass().getSimpleName());
// Symlinks enabled
Mockito.doReturn(true).when(fs).supportsSymlinks();
Assert.assertTrue(fs.supportsSymlinks());
Assert.assertTrue(FileSystem.areSymlinksEnabled());
Assert.assertTrue(fs.hasPathCapability(path, FS_SYMLINKS));
// Symlinks enabled but symlink-support is disabled
Mockito.doReturn(false).when(fs).supportsSymlinks();
Assert.assertFalse(fs.supportsSymlinks());
Assert.assertTrue(FileSystem.areSymlinksEnabled());
Assert.assertFalse(fs.hasPathCapability(path, FS_SYMLINKS));
}
}
}