From d830e8421777181a8e28e0cfc2e33fd0a5cfea1c Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 4 Aug 2021 16:12:54 +0900 Subject: [PATCH] YARN-9875. Improve fair scheduler configuration store on HDFS. (#3262) Contributed by Prabhu Joseph (cherry picked from commit 155864da006346a500ff35c2f6b69281093195b1) Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java Co-authored-by: Eric Yang --- .../capacity/conf/FSSchedulerConfigurationStore.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java index c85506f5761..e32e725c0f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java @@ -67,7 +67,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { private Path configVersionFile; @Override - public void initialize(Configuration conf, Configuration vSchedConf, + public void initialize(Configuration fsConf, Configuration vSchedConf, RMContext rmContext) throws Exception { this.configFilePathFilter = new PathFilter() { @Override @@ -81,6 +81,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { } }; + Configuration conf = new Configuration(fsConf); String schedulerConfPathStr = conf.get( YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH); if (schedulerConfPathStr == null || schedulerConfPathStr.isEmpty()) { @@ -89,6 +90,15 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { + " must be set"); } this.schedulerConfDir = new Path(schedulerConfPathStr); + String scheme = schedulerConfDir.toUri().getScheme(); + if (scheme == null) { + scheme = FileSystem.getDefaultUri(conf).getScheme(); + } + if (scheme != null) { + String disableCacheName = String.format("fs.%s.impl.disable.cache", + scheme); + conf.setBoolean(disableCacheName, true); + } this.fileSystem = this.schedulerConfDir.getFileSystem(conf); this.maxVersion = conf.getInt( YarnConfiguration.SCHEDULER_CONFIGURATION_FS_MAX_VERSION,