Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8817. [Submarine] In cases when user doesn't ask HDFS path while submitting job but framework requires user to set HDFS related environments. Contributed by Wangda Tan.

This commit is contained in:
Sunil G 2018-09-25 11:23:34 +05:30
parent 9c3fbbc4f6
commit d0bc6a27fd
1 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
View File

@ -102,6 +102,13 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
} }
} }
private boolean needHdfs(String content) {
if (content != null && content.contains("hdfs://")) {
return true;
}
return false;
}
private void addHdfsClassPathIfNeeded(RunJobParameters parameters, private void addHdfsClassPathIfNeeded(RunJobParameters parameters,
PrintWriter fw, Component comp) throws IOException { PrintWriter fw, Component comp) throws IOException {
// Find envs to use HDFS // Find envs to use HDFS
@ -121,14 +128,11 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
boolean lackingEnvs = false; boolean lackingEnvs = false;
if ((parameters.getInputPath() != null && parameters.getInputPath() if (needHdfs(parameters.getInputPath()) || needHdfs(
.contains("hdfs://")) || (parameters.getCheckpointPath() != null parameters.getPSLaunchCmd()) || needHdfs(
&& parameters.getCheckpointPath().contains("hdfs://")) || ( parameters.getWorkerLaunchCmd()) || hadoopEnv) {
parameters.getSavedModelPath() != null && parameters.getSavedModelPath()
.contains("hdfs://")) || hadoopEnv) {
// HDFS is asked either in input or output, set LD_LIBRARY_PATH // HDFS is asked either in input or output, set LD_LIBRARY_PATH
// and classpath // and classpath
if (hdfsHome != null) { if (hdfsHome != null) {
// Unset HADOOP_HOME/HADOOP_YARN_HOME to make sure host machine's envs // Unset HADOOP_HOME/HADOOP_YARN_HOME to make sure host machine's envs
// won't pollute docker's env. // won't pollute docker's env.