Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9875. Improve fair scheduler configuration store on HDFS. 

Contributed by Prabhu Joseph
This commit is contained in:
Eric Yang 2019-10-18 20:30:11 -04:00
parent 72b1bed998
commit 155864da00
2 changed files with 62 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
View File

@ -66,7 +66,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore {
private Path configVersionFile;
@Override
public void initialize(Configuration conf, Configuration vSchedConf,
public void initialize(Configuration fsConf, Configuration vSchedConf,
RMContext rmContext) throws Exception {
this.configFilePathFilter = new PathFilter() {
@Override
@ -80,6 +80,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore {
}
};
Configuration conf = new Configuration(fsConf);
String schedulerConfPathStr = conf.get(
YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH);
if (schedulerConfPathStr == null || schedulerConfPathStr.isEmpty()) {
@ -88,6 +89,15 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore {
+ " must be set");
}
this.schedulerConfDir = new Path(schedulerConfPathStr);
String scheme = schedulerConfDir.toUri().getScheme();
if (scheme == null) {
scheme = FileSystem.getDefaultUri(conf).getScheme();
}
if (scheme != null) {
String disableCacheName = String.format("fs.%s.impl.disable.cache",
scheme);
conf.setBoolean(disableCacheName, true);
}
this.fileSystem = this.schedulerConfDir.getFileSystem(conf);
this.maxVersion = conf.getInt(
YarnConfiguration.SCHEDULER_CONFIGURATION_FS_MAX_VERSION,

51
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
View File

@ -28,6 +28,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
import org.junit.After;
@ -137,6 +139,55 @@ public class TestFSSchedulerConfigurationStore {
compareConfig(conf, storeConf);
}
@Test
public void testFileSystemClose() throws Exception {
MiniDFSCluster hdfsCluster = null;
FileSystem fs = null;
Path path = new Path("/tmp/confstore");
try {
HdfsConfiguration hdfsConfig = new HdfsConfiguration();
hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
.numDataNodes(1).build();
fs = hdfsCluster.getFileSystem();
if (!fs.exists(path)) {
fs.mkdirs(path);
}
FSSchedulerConfigurationStore configStore =
new FSSchedulerConfigurationStore();
hdfsConfig.set(YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH,
path.toString());
configStore.initialize(hdfsConfig, hdfsConfig, null);
// Close the FileSystem object and validate
fs.close();
try {
Map<String, String> updates = new HashMap<>();
updates.put("testkey", "testvalue");
LogMutation logMutation = new LogMutation(updates, "test");
configStore.logMutation(logMutation);
configStore.confirmMutation(true);
} catch (IOException e) {
if (e.getMessage().contains("Filesystem closed")) {
fail("FSSchedulerConfigurationStore failed to handle " +
"FileSystem close");
} else {
fail("Should not get any exceptions");
}
}
} finally {
fs = hdfsCluster.getFileSystem();
if (fs.exists(path)) {
fs.delete(path, true);
}
if (hdfsCluster != null) {
hdfsCluster.shutdown();
}
}
}
@Test
public void testFormatConfiguration() throws Exception {
Configuration schedulerConf = new Configuration();