Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added webhdfs test.

This commit is contained in:
Tsz-Wo Nicholas Sze 2023-03-24 16:28:03 +08:00
parent 0ec6f5c374
commit ad7ad57bd0
3 changed files with 52 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -3873,7 +3873,8 @@ public class DistributedFileSystem extends FileSystem
throws IOException { throws IOException {
// qualify the path to make sure that it refers to the current FS. // qualify the path to make sure that it refers to the current FS.
final Path p = makeQualified(path); final Path p = makeQualified(path);
if (DfsPathCapabilities.hasPathCapability(p, capability)) { if (DfsPathCapabilities.hasPathCapability(p, capability)
&& supportsSymlinks()) {
return true; return true;
} }
// this switch is for features which are in the DFS client but not // this switch is for features which are in the DFS client but not

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
View File

@ -2206,7 +2206,8 @@ public class WebHdfsFileSystem extends FileSystem
throws IOException { throws IOException {
// qualify the path to make sure that it refers to the current FS. // qualify the path to make sure that it refers to the current FS.
final Path p = makeQualified(path); final Path p = makeQualified(path);
if (DfsPathCapabilities.hasPathCapability(p, capability)) { if (DfsPathCapabilities.hasPathCapability(p, capability)
&& supportsSymlinks()) {
return true; return true;
} }
return super.hasPathCapability(p, capability); return super.hasPathCapability(p, capability);

71
hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/TestDfsPathCapabilities.java
View File

@ -21,46 +21,71 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.fs.CommonPathCapabilities.FS_SYMLINKS; import static org.apache.hadoop.fs.CommonPathCapabilities.FS_SYMLINKS;
public class TestDfsPathCapabilities { public class TestDfsPathCapabilities {
public static final Logger LOG = LoggerFactory.getLogger(TestDfsPathCapabilities.class);
static FileSystem getFileSystem(String url) throws Exception {
final HdfsConfiguration conf = new HdfsConfiguration();
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, url);
final FileSystem fs = FileSystem.get(conf);
Assert.assertTrue(fs.supportsSymlinks());
return fs;
}
/** /**
* Test if {@link DistributedFileSystem#hasPathCapability(Path, String)} * Test if {@link DistributedFileSystem#hasPathCapability(Path, String)}
* returns correct result for FS_SYMLINKS. * returns correct result for FS_SYMLINKS.
*/ */
@Test @Test
public void testFS_SYMLINKS() throws Exception { public void testSymlinks() throws Exception {
final HdfsConfiguration conf = new HdfsConfiguration(); final DistributedFileSystem dfs = Mockito.spy(
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://localhost/"); (DistributedFileSystem) getFileSystem("hdfs://localhost/"));
final FileSystem fs = FileSystem.get(conf); final WebHdfsFileSystem webhdfs = Mockito.spy(
Assert.assertTrue(fs instanceof DistributedFileSystem); (WebHdfsFileSystem) getFileSystem("webhdfs://localhost/"));
final DistributedFileSystem dfs = Mockito.spy((DistributedFileSystem) fs);
final FileSystem[] fileSystems = {dfs, webhdfs};
final Path path = new Path("/"); final Path path = new Path("/");
// Symlinks support disabled for (FileSystem fs : fileSystems) {
Mockito.doReturn(false).when(dfs).supportsSymlinks(); LOG.info("fs is {}", fs.getClass().getSimpleName());
Assert.assertFalse(FileSystem.areSymlinksEnabled()); // Symlinks support disabled
Assert.assertFalse(dfs.hasPathCapability(path, FS_SYMLINKS)); Mockito.doReturn(false).when(fs).supportsSymlinks();
Assert.assertFalse(fs.supportsSymlinks());
Assert.assertFalse(FileSystem.areSymlinksEnabled());
Assert.assertFalse(fs.hasPathCapability(path, FS_SYMLINKS));
// Symlinks support enabled // Symlinks support enabled
Mockito.doReturn(true).when(dfs).supportsSymlinks(); Mockito.doReturn(true).when(fs).supportsSymlinks();
Assert.assertFalse(FileSystem.areSymlinksEnabled()); Assert.assertTrue(fs.supportsSymlinks());
Assert.assertFalse(dfs.hasPathCapability(path, FS_SYMLINKS)); Assert.assertFalse(FileSystem.areSymlinksEnabled());
Assert.assertFalse(fs.hasPathCapability(path, FS_SYMLINKS));
}
// Symlinks enabled // Once it is enabled, it cannot be disabled.
FileSystem.enableSymlinks(); FileSystem.enableSymlinks();
Mockito.doReturn(true).when(dfs).supportsSymlinks();
Assert.assertTrue(FileSystem.areSymlinksEnabled());
Assert.assertTrue(dfs.hasPathCapability(path, FS_SYMLINKS));
// Symlinks enabled but symlink-support is disabled for (FileSystem fs : fileSystems) {
FileSystem.enableSymlinks(); LOG.info("fs is {}", fs.getClass().getSimpleName());
Mockito.doReturn(false).when(dfs).supportsSymlinks(); // Symlinks enabled
Assert.assertTrue(FileSystem.areSymlinksEnabled()); Mockito.doReturn(true).when(fs).supportsSymlinks();
Assert.assertTrue(dfs.hasPathCapability(path, FS_SYMLINKS)); Assert.assertTrue(fs.supportsSymlinks());
Assert.assertTrue(FileSystem.areSymlinksEnabled());
Assert.assertTrue(fs.hasPathCapability(path, FS_SYMLINKS));
// Symlinks enabled but symlink-support is disabled
Mockito.doReturn(false).when(fs).supportsSymlinks();
Assert.assertFalse(fs.supportsSymlinks());
Assert.assertTrue(FileSystem.areSymlinksEnabled());
Assert.assertFalse(fs.hasPathCapability(path, FS_SYMLINKS));
}
} }
} }