HDFS-15394. Add all available fs.viewfs.overload.scheme.target.<scheme>.impl classes in core-default.xml bydefault. Contributed by Uma Maheswara Rao G.
(cherry picked from commit 3ca15292c5
)
This commit is contained in:
parent
7b175739a9
commit
f85ce2570e
|
@ -952,6 +952,116 @@
|
|||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.hdfs.impl</name>
|
||||
<value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
|
||||
<description>The DistributedFileSystem for view file system overload scheme
|
||||
when child file system and ViewFSOverloadScheme's schemes are hdfs.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.s3a.impl</name>
|
||||
<value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
|
||||
<description>The S3AFileSystem for view file system overload scheme when
|
||||
child file system and ViewFSOverloadScheme's schemes are s3a.</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.o3fs.impl</name>
|
||||
<value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
|
||||
<description>The OzoneFileSystem for view file system overload scheme when
|
||||
child file system and ViewFSOverloadScheme's schemes are o3fs.</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.ftp.impl</name>
|
||||
<value>org.apache.hadoop.fs.ftp.FTPFileSystem</value>
|
||||
<description>The FTPFileSystem for view file system overload scheme when
|
||||
child file system and ViewFSOverloadScheme's schemes are ftp.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.webhdfs.impl</name>
|
||||
<value>org.apache.hadoop.hdfs.web.WebHdfsFileSystem</value>
|
||||
<description>The WebHdfsFileSystem for view file system overload scheme when
|
||||
child file system and ViewFSOverloadScheme's schemes are webhdfs.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.swebhdfs.impl</name>
|
||||
<value>org.apache.hadoop.hdfs.web.SWebHdfsFileSystem</value>
|
||||
<description>The SWebHdfsFileSystem for view file system overload scheme when
|
||||
child file system and ViewFSOverloadScheme's schemes are swebhdfs.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.file.impl</name>
|
||||
<value>org.apache.hadoop.fs.LocalFileSystem</value>
|
||||
<description>The LocalFileSystem for view file system overload scheme when
|
||||
child file system and ViewFSOverloadScheme's schemes are file.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.abfs.impl</name>
|
||||
<value>org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem</value>
|
||||
<description>The AzureBlobFileSystem for view file system overload scheme
|
||||
when child file system and ViewFSOverloadScheme's schemes are abfs.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.abfss.impl</name>
|
||||
<value>org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem</value>
|
||||
<description>The SecureAzureBlobFileSystem for view file system overload
|
||||
scheme when child file system and ViewFSOverloadScheme's schemes are abfss.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.wasb.impl</name>
|
||||
<value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>
|
||||
<description>The NativeAzureFileSystem for view file system overload scheme
|
||||
when child file system and ViewFSOverloadScheme's schemes are wasb.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.swift.impl</name>
|
||||
<value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
|
||||
<description>The SwiftNativeFileSystem for view file system overload scheme
|
||||
when child file system and ViewFSOverloadScheme's schemes are swift.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.oss.impl</name>
|
||||
<value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value>
|
||||
<description>The AliyunOSSFileSystem for view file system overload scheme
|
||||
when child file system and ViewFSOverloadScheme's schemes are oss.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.http.impl</name>
|
||||
<value>org.apache.hadoop.fs.http.HttpFileSystem</value>
|
||||
<description>The HttpFileSystem for view file system overload scheme
|
||||
when child file system and ViewFSOverloadScheme's schemes are http.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.viewfs.overload.scheme.target.https.impl</name>
|
||||
<value>org.apache.hadoop.fs.http.HttpsFileSystem</value>
|
||||
<description>The HttpsFileSystem for view file system overload scheme
|
||||
when child file system and ViewFSOverloadScheme's schemes are https.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.AbstractFileSystem.ftp.impl</name>
|
||||
<value>org.apache.hadoop.fs.ftp.FtpFs</value>
|
||||
|
|
|
@ -124,6 +124,24 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
|
|||
xmlPrefixToSkipCompare.add("fs.adl.");
|
||||
xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl");
|
||||
|
||||
// ViewfsOverloadScheme target fs impl property keys are dynamically
|
||||
// constructed and they are advanced props.
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfs.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfss.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.file.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.ftp.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.hdfs.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.http.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.https.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.o3fs.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.oss.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.s3a.impl");
|
||||
xmlPropsToSkipCompare.
|
||||
add("fs.viewfs.overload.scheme.target.swebhdfs.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.webhdfs.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.wasb.impl");
|
||||
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.swift.impl");
|
||||
|
||||
// Azure properties are in a different class
|
||||
// - org.apache.hadoop.fs.azure.AzureNativeFileSystemStore
|
||||
// - org.apache.hadoop.fs.azure.SASKeyGeneratorImpl
|
||||
|
|
|
@ -382,13 +382,17 @@ public class TestViewFileSystemOverloadSchemeWithHdfsScheme {
|
|||
* Note: Above links created because to make fs initialization success.
|
||||
* Otherwise will not proceed if no mount links.
|
||||
*
|
||||
* Don't set fs.viewfs.overload.scheme.target.hdfs.impl property.
|
||||
* Unset fs.viewfs.overload.scheme.target.hdfs.impl property.
|
||||
* So, OverloadScheme target fs initialization will fail.
|
||||
*/
|
||||
@Test(expected = UnsupportedFileSystemException.class, timeout = 30000)
|
||||
public void testInvalidOverloadSchemeTargetFS() throws Exception {
|
||||
final Path hdfsTargetPath = new Path(defaultFSURI + HDFS_USER_FOLDER);
|
||||
String mountTableIfSet = conf.get(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH);
|
||||
conf = new Configuration();
|
||||
if (mountTableIfSet != null) {
|
||||
conf.set(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH, mountTableIfSet);
|
||||
}
|
||||
addMountLinks(defaultFSURI.getAuthority(),
|
||||
new String[] {HDFS_USER_FOLDER, LOCAL_FOLDER,
|
||||
Constants.CONFIG_VIEWFS_LINK_FALLBACK },
|
||||
|
@ -400,6 +404,9 @@ public class TestViewFileSystemOverloadSchemeWithHdfsScheme {
|
|||
defaultFSURI.toString());
|
||||
conf.set(String.format(FS_IMPL_PATTERN_KEY, HDFS_SCHEME),
|
||||
ViewFileSystemOverloadScheme.class.getName());
|
||||
conf.unset(String.format(
|
||||
FsConstants.FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN,
|
||||
HDFS_SCHEME));
|
||||
|
||||
try (FileSystem fs = FileSystem.get(conf)) {
|
||||
fs.createNewFile(new Path("/onRootWhenFallBack"));
|
||||
|
|
Loading…
Reference in New Issue