Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-15635. ViewFileSystemOverloadScheme support specifying mount table loader imp through conf (#2389). Contributed by Junfan Zhang.

This commit is contained in:
zhang_jf 2020-11-20 12:21:16 +08:00 committed by GitHub
parent 07050339e0
commit 8fa699b53f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 37 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
View File

@ -125,4 +125,11 @@ public interface Constants {
"fs.viewfs.ignore.port.in.mount.table.name"; "fs.viewfs.ignore.port.in.mount.table.name";
boolean CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME_DEFAULT = false; boolean CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME_DEFAULT = false;
String CONFIG_VIEWFS_MOUNTTABLE_LOADER_IMPL =
CONFIG_VIEWFS_PREFIX + ".config.loader.impl";
Class<? extends MountTableConfigLoader>
DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL =
HCFSMountTableConfigLoader.class;
} }

33
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
View File

@ -17,6 +17,10 @@
*/ */
package org.apache.hadoop.fs.viewfs; package org.apache.hadoop.fs.viewfs;
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME;
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNTTABLE_LOADER_IMPL;
import static org.apache.hadoop.fs.viewfs.Constants.DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
@ -30,8 +34,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.util.ReflectionUtils;
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME;
/** /**
* <p> This class is extended from the ViewFileSystem for the overloaded * <p> This class is extended from the ViewFileSystem for the overloaded
@ -160,7 +163,7 @@ public class ViewFileSystemOverloadScheme extends ViewFileSystem {
conf.getBoolean(Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME, conf.getBoolean(Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME,
true)); true));
if (null != mountTableConfigPath) { if (null != mountTableConfigPath) {
MountTableConfigLoader loader = new HCFSMountTableConfigLoader(); MountTableConfigLoader loader = getMountTableConfigLoader(conf);
loader.load(mountTableConfigPath, conf); loader.load(mountTableConfigPath, conf);
} else { } else {
// TODO: Should we fail here.? // TODO: Should we fail here.?
@ -173,6 +176,30 @@ public class ViewFileSystemOverloadScheme extends ViewFileSystem {
super.initialize(theUri, conf); super.initialize(theUri, conf);
} }
private MountTableConfigLoader getMountTableConfigLoader(
final Configuration conf) {
Class<? extends MountTableConfigLoader> clazz =
conf.getClass(CONFIG_VIEWFS_MOUNTTABLE_LOADER_IMPL,
DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL,
MountTableConfigLoader.class);
if (clazz == null) {
throw new RuntimeException(
String.format("Errors on getting mount table loader class. "
+ "The fs.viewfs.mounttable.config.loader.impl conf is %s. ",
conf.get(CONFIG_VIEWFS_MOUNTTABLE_LOADER_IMPL,
DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL.getName())));
}
try {
MountTableConfigLoader mountTableConfigLoader =
ReflectionUtils.newInstance(clazz, conf);
return mountTableConfigLoader;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/** /**
* This method is overridden because in ViewFileSystemOverloadScheme if * This method is overridden because in ViewFileSystemOverloadScheme if
* overloaded scheme matches with mounted target fs scheme, file system * overloaded scheme matches with mounted target fs scheme, file system