Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9875. Improve fair scheduler configuration store on HDFS. (#3262)

Browse Source 
Contributed by Prabhu Joseph

(cherry picked from commit 155864da006346a500ff35c2f6b69281093195b1)

 Conflicts:
	hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java

Co-authored-by: Eric Yang <eyang@apache.org>
This commit is contained in:
Akira Ajisaka 2021-08-04 16:12:54 +09:00 committed by GitHub
parent 2f3c0b8dd1
commit d830e84217
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
View File

@ -67,7 +67,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore {
private Path configVersionFile; private Path configVersionFile;
@Override @Override
public void initialize(Configuration conf, Configuration vSchedConf, public void initialize(Configuration fsConf, Configuration vSchedConf,
RMContext rmContext) throws Exception { RMContext rmContext) throws Exception {
this.configFilePathFilter = new PathFilter() { this.configFilePathFilter = new PathFilter() {
@Override @Override
@ -81,6 +81,7 @@ public boolean accept(Path path) {
} }
}; };
Configuration conf = new Configuration(fsConf);
String schedulerConfPathStr = conf.get( String schedulerConfPathStr = conf.get(
YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH); YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH);
if (schedulerConfPathStr == null || schedulerConfPathStr.isEmpty()) { if (schedulerConfPathStr == null || schedulerConfPathStr.isEmpty()) {
@ -89,6 +90,15 @@ public boolean accept(Path path) {
+ " must be set"); + " must be set");
} }
this.schedulerConfDir = new Path(schedulerConfPathStr); this.schedulerConfDir = new Path(schedulerConfPathStr);
String scheme = schedulerConfDir.toUri().getScheme();
if (scheme == null) {
scheme = FileSystem.getDefaultUri(conf).getScheme();
}
if (scheme != null) {
String disableCacheName = String.format("fs.%s.impl.disable.cache",
scheme);
conf.setBoolean(disableCacheName, true);
}
this.fileSystem = this.schedulerConfDir.getFileSystem(conf); this.fileSystem = this.schedulerConfDir.getFileSystem(conf);
this.maxVersion = conf.getInt( this.maxVersion = conf.getInt(
YarnConfiguration.SCHEDULER_CONFIGURATION_FS_MAX_VERSION, YarnConfiguration.SCHEDULER_CONFIGURATION_FS_MAX_VERSION,