YARN-8817. [Submarine] In cases when user doesn't ask HDFS path while submitting job but framework requires user to set HDFS related environments. Contributed by Wangda Tan.
This commit is contained in:
parent
9c3fbbc4f6
commit
d0bc6a27fd
|
@ -102,6 +102,13 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
|
|||
}
|
||||
}
|
||||
|
||||
private boolean needHdfs(String content) {
|
||||
if (content != null && content.contains("hdfs://")) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void addHdfsClassPathIfNeeded(RunJobParameters parameters,
|
||||
PrintWriter fw, Component comp) throws IOException {
|
||||
// Find envs to use HDFS
|
||||
|
@ -121,14 +128,11 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
|
|||
|
||||
boolean lackingEnvs = false;
|
||||
|
||||
if ((parameters.getInputPath() != null && parameters.getInputPath()
|
||||
.contains("hdfs://")) || (parameters.getCheckpointPath() != null
|
||||
&& parameters.getCheckpointPath().contains("hdfs://")) || (
|
||||
parameters.getSavedModelPath() != null && parameters.getSavedModelPath()
|
||||
.contains("hdfs://")) || hadoopEnv) {
|
||||
if (needHdfs(parameters.getInputPath()) || needHdfs(
|
||||
parameters.getPSLaunchCmd()) || needHdfs(
|
||||
parameters.getWorkerLaunchCmd()) || hadoopEnv) {
|
||||
// HDFS is asked either in input or output, set LD_LIBRARY_PATH
|
||||
// and classpath
|
||||
|
||||
if (hdfsHome != null) {
|
||||
// Unset HADOOP_HOME/HADOOP_YARN_HOME to make sure host machine's envs
|
||||
// won't pollute docker's env.
|
||||
|
|
Loading…
Reference in New Issue