YARN-8561. [Submarine] Initial implementation: Training job submission and job history retrieval. Contributed by Wangda Tan.
This commit is contained in:
Sunil G
parent
a8dae0047c
commit
cadbc8b57f
|
|
@ -0,0 +1,53 @@
|
|||
<!---
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
|
||||
# Overview
|
||||
|
||||
```$xslt
|
||||
_ _
|
||||
| | (_)
|
||||
___ _ _ | |__ _ __ ___ __ _ _ __ _ _ __ ___
|
||||
/ __|| | | || '_ \ | '_ ` _ \ / _` || '__|| || '_ \ / _ \
|
||||
\__ \| |_| || |_) || | | | | || (_| || | | || | | || __/
|
||||
|___/ \__,_||_.__/ |_| |_| |_| \__,_||_| |_||_| |_| \___|
|
||||
|
||||
?
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~|^"~~~~~~~~~~~~~~~~~~~~~~~~~o~~~~~~~~~~~
|
||||
o | o __o
|
||||
o | o |X__>
|
||||
___o | __o
|
||||
(X___>-- __|__ |X__> o
|
||||
| \ __o
|
||||
| \ |X__>
|
||||
_______________________|_______\________________
|
||||
< \____________ _
|
||||
\ \ (_)
|
||||
\ O O O >=)
|
||||
\__________________________________________________________/ (_)
|
||||
```
|
||||
|
||||
Submarine is a project which allows infra engineer / data scientist to run *unmodified* Tensorflow programs on YARN.
|
||||
|
||||
Goals of Submarine:
|
||||
- It allows jobs easy access data/models in HDFS and other storages.
|
||||
- Can launch services to serve Tensorflow/MXNet models.
|
||||
- Support run distributed Tensorflow jobs with simple configs.
|
||||
- Support run user-specified Docker images.
|
||||
- Support specify GPU and other resources.
|
||||
- Support launch tensorboard for training jobs if user specified.
|
||||
- Support customized DNS name for roles (like tensorboard.$user.$domain:6006)
|
||||
|
||||
Please jump to [QuickStart](src/site/QuickStart.md) guide to quickly understand how to use this framework.
|
||||
|
||||
If you're a developer, please find [Developer](src/site/DeveloperGuide.md) guide for more details.
|
||||
|
|
@ -0,0 +1,213 @@
|
|||
<?xml version="1.0"?>
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>hadoop-yarn-applications</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<version>3.2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>hadoop-yarn-submarine</artifactId>
|
||||
<version>3.2.0-SNAPSHOT</version>
|
||||
<name>Yet Another Learning Platform</name>
|
||||
|
||||
<properties>
|
||||
<!-- Needed for generating FindBugs warnings using parent pom -->
|
||||
<yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-cli</groupId>
|
||||
<artifactId>commons-cli</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-server-timelineservice</artifactId>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-annotations</artifactId>
|
||||
</dependency>
|
||||
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-api</artifactId>
|
||||
</dependency>
|
||||
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-common</artifactId>
|
||||
</dependency>
|
||||
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-client</artifactId>
|
||||
</dependency>
|
||||
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-server-tests</artifactId>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-all</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-common</artifactId>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdfs</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdfs-client</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdfs</artifactId>
|
||||
<scope>test</scope>
|
||||
<type>test-jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-fs2img</artifactId>
|
||||
<version>3.2.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-services-api</artifactId>
|
||||
<version>3.2.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-services-core</artifactId>
|
||||
<version>3.2.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>jar</goal>
|
||||
</goals>
|
||||
<!-- strictly speaking, the unit test is really a regression test. It
|
||||
needs the main jar to be available to be able to run. -->
|
||||
<phase>test-compile</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<archive>
|
||||
<manifest>
|
||||
<mainClass>org.apache.hadoop.yarn.submarine.client.cli.Cli</mainClass>
|
||||
</manifest>
|
||||
</archive>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<environmentVariables>
|
||||
<JAVA_HOME>${java.home}</JAVA_HOME>
|
||||
</environmentVariables>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli;
|
||||
|
||||
import org.apache.commons.cli.ParseException;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.util.Tool;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.submarine.common.exception.SubmarineException;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public abstract class AbstractCli implements Tool {
|
||||
protected ClientContext clientContext;
|
||||
|
||||
public AbstractCli(ClientContext cliContext) {
|
||||
this.clientContext = cliContext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public abstract int run(String[] args)
|
||||
throws ParseException, IOException, YarnException, InterruptedException,
|
||||
SubmarineException;
|
||||
|
||||
@Override
|
||||
public void setConf(Configuration conf) {
|
||||
clientContext.setSubmarineConfig(conf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Configuration getConf() {
|
||||
return clientContext.getSubmarineConfig();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.fs.DefaultRemoteDirectoryManager;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.RuntimeFactory;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
public class Cli {
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(Cli.class);
|
||||
|
||||
private static void printHelp() {
|
||||
StringBuilder helpMsg = new StringBuilder();
|
||||
helpMsg.append("\n\nUsage: <object> [<action>] [<args>]\n");
|
||||
helpMsg.append(" Below are all objects / actions:\n");
|
||||
helpMsg.append(" job \n");
|
||||
helpMsg.append(" run : run a job, please see 'job run --help' for usage \n");
|
||||
helpMsg.append(" show : get status of job, please see 'job show --help' for usage \n");
|
||||
|
||||
System.out.println(helpMsg.toString());
|
||||
}
|
||||
|
||||
private static ClientContext getClientContext() {
|
||||
Configuration conf = new YarnConfiguration();
|
||||
ClientContext clientContext = new ClientContext();
|
||||
clientContext.setConfiguration(conf);
|
||||
clientContext.setRemoteDirectoryManager(
|
||||
new DefaultRemoteDirectoryManager(clientContext));
|
||||
RuntimeFactory runtimeFactory = RuntimeFactory.getRuntimeFactory(
|
||||
clientContext);
|
||||
clientContext.setRuntimeFactory(runtimeFactory);
|
||||
return clientContext;
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
System.out.println(" _ _ \n"
|
||||
+ " | | (_) \n"
|
||||
+ " ___ _ _ | |__ _ __ ___ __ _ _ __ _ _ __ ___ \n"
|
||||
+ " / __|| | | || '_ \\ | '_ ` _ \\ / _` || '__|| || '_ \\ / _ \\\n"
|
||||
+ " \\__ \\| |_| || |_) || | | | | || (_| || | | || | | || __/\n"
|
||||
+ " |___/ \\__,_||_.__/ |_| |_| |_| \\__,_||_| |_||_| |_| \\___|\n"
|
||||
+ " \n"
|
||||
+ " ?\n"
|
||||
+ " ~~~~~~~~~~~~~~~~~~~~~~~~~~~|^\"~~~~~~~~~~~~~~~~~~~~~~~~~o~~~~~~~~~~~\n"
|
||||
+ " o | o __o\n"
|
||||
+ " o | o |X__>\n"
|
||||
+ " ___o | __o\n"
|
||||
+ " (X___>-- __|__ |X__> o\n"
|
||||
+ " | \\ __o\n"
|
||||
+ " | \\ |X__>\n"
|
||||
+ " _______________________|_______\\________________\n"
|
||||
+ " < \\____________ _\n"
|
||||
+ " \\ \\ (_)\n"
|
||||
+ " \\ O O O >=)\n"
|
||||
+ " \\__________________________________________________________/ (_)\n"
|
||||
+ "\n");
|
||||
|
||||
if (CliUtils.argsForHelp(args)) {
|
||||
printHelp();
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
if (args.length < 2) {
|
||||
LOG.error("Bad parameters specified.");
|
||||
printHelp();
|
||||
System.exit(-1);
|
||||
}
|
||||
|
||||
String[] moduleArgs = Arrays.copyOfRange(args, 2, args.length);
|
||||
ClientContext clientContext = getClientContext();
|
||||
|
||||
if (args[0].equals("job")) {
|
||||
String subCmd = args[1];
|
||||
if (subCmd.equals(CliConstants.RUN)) {
|
||||
new RunJobCli(clientContext).run(moduleArgs);
|
||||
} else if (subCmd.equals(CliConstants.SHOW)) {
|
||||
new ShowJobCli(clientContext).run(moduleArgs);
|
||||
} else {
|
||||
printHelp();
|
||||
throw new IllegalArgumentException("Unknown option for job");
|
||||
}
|
||||
} else {
|
||||
printHelp();
|
||||
throw new IllegalArgumentException("Bad parameters <TODO>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli;
|
||||
|
||||
/*
|
||||
* NOTE: use lowercase + "_" for the option name
|
||||
*/
|
||||
public class CliConstants {
|
||||
public static final String RUN = "run";
|
||||
public static final String SERVE = "serve";
|
||||
public static final String LIST = "list";
|
||||
public static final String SHOW = "show";
|
||||
public static final String NAME = "name";
|
||||
public static final String INPUT_PATH = "input_path";
|
||||
public static final String CHECKPOINT_PATH = "checkpoint_path";
|
||||
public static final String SAVED_MODEL_PATH = "saved_model_path";
|
||||
public static final String N_WORKERS = "num_workers";
|
||||
public static final String N_SERVING_TASKS = "num_serving_tasks";
|
||||
public static final String N_PS = "num_ps";
|
||||
public static final String WORKER_RES = "worker_resources";
|
||||
public static final String SERVING_RES = "serving_resources";
|
||||
public static final String PS_RES = "ps_resources";
|
||||
public static final String DOCKER_IMAGE = "docker_image";
|
||||
public static final String QUEUE = "queue";
|
||||
public static final String TENSORBOARD = "tensorboard";
|
||||
public static final String WORKER_LAUNCH_CMD = "worker_launch_cmd";
|
||||
public static final String SERVING_LAUNCH_CMD = "serving_launch_cmd";
|
||||
public static final String PS_LAUNCH_CMD = "ps_launch_cmd";
|
||||
public static final String ENV = "env";
|
||||
public static final String VERBOSE = "verbose";
|
||||
public static final String SERVING_FRAMEWORK = "serving_framework";
|
||||
public static final String STOP = "stop";
|
||||
public static final String WAIT_JOB_FINISH = "wait_job_finish";
|
||||
public static final String PS_DOCKER_IMAGE = "ps_docker_image";
|
||||
public static final String WORKER_DOCKER_IMAGE = "worker_docker_image";
|
||||
}
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
|
||||
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.param.RunJobParameters;
|
||||
import org.apache.hadoop.yarn.submarine.common.fs.RemoteDirectoryManager;
|
||||
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class CliUtils {
|
||||
private final static String RES_PATTERN = "^[^=]+=\\d+\\s?\\w*$";
|
||||
/**
|
||||
* Replace patterns inside cli
|
||||
*
|
||||
* @return launch command after pattern replace
|
||||
*/
|
||||
public static String replacePatternsInLaunchCommand(String specifiedCli,
|
||||
RunJobParameters jobRunParameters,
|
||||
RemoteDirectoryManager directoryManager) throws IOException {
|
||||
String jobDir = jobRunParameters.getCheckpointPath();
|
||||
if (null == jobDir) {
|
||||
jobDir = directoryManager.getJobCheckpointDir(jobRunParameters.getName(),
|
||||
true).toString();
|
||||
}
|
||||
|
||||
String input = jobRunParameters.getInputPath();
|
||||
String savedModelDir = jobRunParameters.getSavedModelPath();
|
||||
if (null == savedModelDir) {
|
||||
savedModelDir = jobDir;
|
||||
}
|
||||
|
||||
Map<String, String> replacePattern = new HashMap<>();
|
||||
if (jobDir != null) {
|
||||
replacePattern.put("%" + CliConstants.CHECKPOINT_PATH + "%", jobDir);
|
||||
}
|
||||
if (input != null) {
|
||||
replacePattern.put("%" + CliConstants.INPUT_PATH + "%", input);
|
||||
}
|
||||
if (savedModelDir != null) {
|
||||
replacePattern.put("%" + CliConstants.SAVED_MODEL_PATH + "%",
|
||||
savedModelDir);
|
||||
}
|
||||
|
||||
String newCli = specifiedCli;
|
||||
for (Map.Entry<String, String> replace : replacePattern.entrySet()) {
|
||||
newCli = newCli.replace(replace.getKey(), replace.getValue());
|
||||
}
|
||||
|
||||
return newCli;
|
||||
}
|
||||
|
||||
// TODO, this duplicated to Client of distributed shell, should cleanup
|
||||
private static Map<String, Long> parseResourcesString(String resourcesStr) {
|
||||
Map<String, Long> resources = new HashMap<>();
|
||||
|
||||
// Ignore the grouping "[]"
|
||||
if (resourcesStr.startsWith("[")) {
|
||||
resourcesStr = resourcesStr.substring(1);
|
||||
}
|
||||
if (resourcesStr.endsWith("]")) {
|
||||
resourcesStr = resourcesStr.substring(0, resourcesStr.length());
|
||||
}
|
||||
|
||||
for (String resource : resourcesStr.trim().split(",")) {
|
||||
resource = resource.trim();
|
||||
if (!resource.matches(RES_PATTERN)) {
|
||||
throw new IllegalArgumentException("\"" + resource + "\" is not a "
|
||||
+ "valid resource type/amount pair. "
|
||||
+ "Please provide key=amount pairs separated by commas.");
|
||||
}
|
||||
String[] splits = resource.split("=");
|
||||
String key = splits[0], value = splits[1];
|
||||
String units = ResourceUtils.getUnits(value);
|
||||
|
||||
String valueWithoutUnit = value.substring(0, value.length() - units.length()).trim();
|
||||
Long resourceValue = Long.valueOf(valueWithoutUnit);
|
||||
|
||||
// Convert commandline unit to standard YARN unit.
|
||||
if (units.equals("M") || units.equals("m")) {
|
||||
units = "Mi";
|
||||
} else if (units.equals("G") || units.equals("g")) {
|
||||
units = "Gi";
|
||||
} else if (units.isEmpty()) {
|
||||
// do nothing;
|
||||
} else{
|
||||
throw new IllegalArgumentException("Acceptable units are M/G or empty");
|
||||
}
|
||||
|
||||
// special handle memory-mb and memory
|
||||
if (key.equals(ResourceInformation.MEMORY_URI)) {
|
||||
if (!units.isEmpty()) {
|
||||
resourceValue = UnitsConversionUtil.convert(units, "Mi",
|
||||
resourceValue);
|
||||
}
|
||||
}
|
||||
|
||||
if (key.equals("memory")) {
|
||||
key = ResourceInformation.MEMORY_URI;
|
||||
resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue);
|
||||
}
|
||||
|
||||
// special handle gpu
|
||||
if (key.equals("gpu")) {
|
||||
key = ResourceInformation.GPU_URI;
|
||||
}
|
||||
|
||||
// special handle fpga
|
||||
if (key.equals("fpga")) {
|
||||
key = ResourceInformation.FPGA_URI;
|
||||
}
|
||||
|
||||
resources.put(key, resourceValue);
|
||||
}
|
||||
return resources;
|
||||
}
|
||||
|
||||
private static void validateResourceTypes(Iterable<String> resourceNames,
|
||||
List<ResourceTypeInfo> resourceTypes) throws IOException, YarnException {
|
||||
for (String resourceName : resourceNames) {
|
||||
if (!resourceTypes.stream().anyMatch(
|
||||
e -> e.getName().equals(resourceName))) {
|
||||
throw new ResourceNotFoundException(
|
||||
"Unknown resource: " + resourceName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static Resource createResourceFromString(String resourceStr,
|
||||
List<ResourceTypeInfo> resourceTypes) throws IOException, YarnException {
|
||||
Map<String, Long> typeToValue = parseResourcesString(resourceStr);
|
||||
validateResourceTypes(typeToValue.keySet(), resourceTypes);
|
||||
Resource resource = Resource.newInstance(0, 0);
|
||||
for (Map.Entry<String, Long> entry : typeToValue.entrySet()) {
|
||||
resource.setResourceValue(entry.getKey(), entry.getValue());
|
||||
}
|
||||
return resource;
|
||||
}
|
||||
|
||||
// Is it for help?
|
||||
public static boolean argsForHelp(String[] args) {
|
||||
if (args == null || args.length == 0)
|
||||
return true;
|
||||
|
||||
if (args.length == 1) {
|
||||
if (args[0].equals("-h") || args[0].equals("--help")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,204 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.GnuParser;
|
||||
import org.apache.commons.cli.HelpFormatter;
|
||||
import org.apache.commons.cli.Options;
|
||||
import org.apache.commons.cli.ParseException;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.param.RunJobParameters;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.exception.SubmarineException;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.JobMonitor;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.JobSubmitter;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.StorageKeyConstants;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class RunJobCli extends AbstractCli {
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(RunJobCli.class);
|
||||
|
||||
private Options options;
|
||||
private RunJobParameters parameters = new RunJobParameters();
|
||||
|
||||
private JobSubmitter jobSubmitter;
|
||||
private JobMonitor jobMonitor;
|
||||
|
||||
public RunJobCli(ClientContext cliContext) {
|
||||
this(cliContext, cliContext.getRuntimeFactory().getJobSubmitterInstance(),
|
||||
cliContext.getRuntimeFactory().getJobMonitorInstance());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public RunJobCli(ClientContext cliContext, JobSubmitter jobSubmitter,
|
||||
JobMonitor jobMonitor) {
|
||||
super(cliContext);
|
||||
options = generateOptions();
|
||||
this.jobSubmitter = jobSubmitter;
|
||||
this.jobMonitor = jobMonitor;
|
||||
}
|
||||
|
||||
public void printUsages() {
|
||||
new HelpFormatter().printHelp("job run", options);
|
||||
}
|
||||
|
||||
private Options generateOptions() {
|
||||
Options options = new Options();
|
||||
options.addOption(CliConstants.NAME, true, "Name of the job");
|
||||
options.addOption(CliConstants.INPUT_PATH, true,
|
||||
"Input of the job, could be local or other FS directory");
|
||||
options.addOption(CliConstants.CHECKPOINT_PATH, true,
|
||||
"Training output directory of the job, "
|
||||
+ "could be local or other FS directory. This typically includes "
|
||||
+ "checkpoint files and exported model ");
|
||||
options.addOption(CliConstants.SAVED_MODEL_PATH, true,
|
||||
"Model exported path (savedmodel) of the job, which is needed when "
|
||||
+ "exported model is not placed under ${checkpoint_path}"
|
||||
+ "could be local or other FS directory. This will be used to serve.");
|
||||
options.addOption(CliConstants.N_WORKERS, true,
|
||||
"Numnber of worker tasks of the job, by default it's 1");
|
||||
options.addOption(CliConstants.N_PS, true,
|
||||
"Number of PS tasks of the job, by default it's 0");
|
||||
options.addOption(CliConstants.WORKER_RES, true,
|
||||
"Resource of each worker, for example "
|
||||
+ "memory-mb=2048,vcores=2,yarn.io/gpu=2");
|
||||
options.addOption(CliConstants.PS_RES, true,
|
||||
"Resource of each PS, for example "
|
||||
+ "memory-mb=2048,vcores=2,yarn.io/gpu=2");
|
||||
options.addOption(CliConstants.DOCKER_IMAGE, true, "Docker image name/tag");
|
||||
options.addOption(CliConstants.QUEUE, true,
|
||||
"Name of queue to run the job, by default it uses default queue");
|
||||
options.addOption(CliConstants.TENSORBOARD, true,
|
||||
"Should we run TensorBoard" + " for this job? By default it's true");
|
||||
options.addOption(CliConstants.WORKER_LAUNCH_CMD, true,
|
||||
"Commandline of worker, arguments will be "
|
||||
+ "directly used to launch the worker");
|
||||
options.addOption(CliConstants.PS_LAUNCH_CMD, true,
|
||||
"Commandline of worker, arguments will be "
|
||||
+ "directly used to launch the PS");
|
||||
options.addOption(CliConstants.ENV, true,
|
||||
"Common environment variable of worker/ps");
|
||||
options.addOption(CliConstants.VERBOSE, false,
|
||||
"Print verbose log for troubleshooting");
|
||||
options.addOption(CliConstants.WAIT_JOB_FINISH, false,
|
||||
"Specified when user want to wait the job finish");
|
||||
options.addOption(CliConstants.PS_DOCKER_IMAGE, true,
|
||||
"Specify docker image for PS, when this is not specified, PS uses --"
|
||||
+ CliConstants.DOCKER_IMAGE + " as default.");
|
||||
options.addOption(CliConstants.WORKER_DOCKER_IMAGE, true,
|
||||
"Specify docker image for WORKER, when this is not specified, WORKER "
|
||||
+ "uses --" + CliConstants.DOCKER_IMAGE + " as default.");
|
||||
options.addOption("h", "help", false, "Print help");
|
||||
return options;
|
||||
}
|
||||
|
||||
private void replacePatternsInParameters() throws IOException {
|
||||
if (parameters.getPSLaunchCmd() != null && !parameters.getPSLaunchCmd()
|
||||
.isEmpty()) {
|
||||
String afterReplace = CliUtils.replacePatternsInLaunchCommand(
|
||||
parameters.getPSLaunchCmd(), parameters,
|
||||
clientContext.getRemoteDirectoryManager());
|
||||
parameters.setPSLaunchCmd(afterReplace);
|
||||
}
|
||||
|
||||
if (parameters.getWorkerLaunchCmd() != null && !parameters
|
||||
.getWorkerLaunchCmd().isEmpty()) {
|
||||
String afterReplace = CliUtils.replacePatternsInLaunchCommand(
|
||||
parameters.getWorkerLaunchCmd(), parameters,
|
||||
clientContext.getRemoteDirectoryManager());
|
||||
parameters.setWorkerLaunchCmd(afterReplace);
|
||||
}
|
||||
}
|
||||
|
||||
private void parseCommandLineAndGetRunJobParameters(String[] args)
|
||||
throws ParseException, IOException, YarnException {
|
||||
try {
|
||||
// Do parsing
|
||||
GnuParser parser = new GnuParser();
|
||||
CommandLine cli = parser.parse(options, args);
|
||||
parameters.updateParametersByParsedCommandline(cli, options, clientContext);
|
||||
} catch (ParseException e) {
|
||||
LOG.error("Exception in parse:", e.getMessage());
|
||||
printUsages();
|
||||
throw e;
|
||||
}
|
||||
|
||||
// replace patterns
|
||||
replacePatternsInParameters();
|
||||
}
|
||||
|
||||
private void storeJobInformation(String jobName, ApplicationId applicationId,
|
||||
String[] args) throws IOException {
|
||||
Map<String, String> jobInfo = new HashMap<>();
|
||||
jobInfo.put(StorageKeyConstants.JOB_NAME, jobName);
|
||||
jobInfo.put(StorageKeyConstants.APPLICATION_ID, applicationId.toString());
|
||||
|
||||
if (parameters.getCheckpointPath() != null) {
|
||||
jobInfo.put(StorageKeyConstants.CHECKPOINT_PATH,
|
||||
parameters.getCheckpointPath());
|
||||
}
|
||||
if (parameters.getInputPath() != null) {
|
||||
jobInfo.put(StorageKeyConstants.INPUT_PATH,
|
||||
parameters.getInputPath());
|
||||
}
|
||||
if (parameters.getSavedModelPath() != null) {
|
||||
jobInfo.put(StorageKeyConstants.SAVED_MODEL_PATH,
|
||||
parameters.getSavedModelPath());
|
||||
}
|
||||
|
||||
String joinedArgs = String.join(" ", args);
|
||||
jobInfo.put(StorageKeyConstants.JOB_RUN_ARGS, joinedArgs);
|
||||
clientContext.getRuntimeFactory().getSubmarineStorage().addNewJob(jobName,
|
||||
jobInfo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int run(String[] args)
|
||||
throws ParseException, IOException, YarnException, InterruptedException,
|
||||
SubmarineException {
|
||||
if (CliUtils.argsForHelp(args)) {
|
||||
printUsages();
|
||||
return 0;
|
||||
}
|
||||
|
||||
parseCommandLineAndGetRunJobParameters(args);
|
||||
ApplicationId applicationId = this.jobSubmitter.submitJob(parameters);
|
||||
storeJobInformation(parameters.getName(), applicationId, args);
|
||||
if (parameters.isWaitJobFinish()) {
|
||||
this.jobMonitor.waitTrainingFinal(parameters.getName());
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public JobSubmitter getJobSubmitter() {
|
||||
return jobSubmitter;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
RunJobParameters getRunJobParameters() {
|
||||
return parameters;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.GnuParser;
|
||||
import org.apache.commons.cli.HelpFormatter;
|
||||
import org.apache.commons.cli.Options;
|
||||
import org.apache.commons.cli.ParseException;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.param.ShowJobParameters;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.exception.SubmarineException;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.StorageKeyConstants;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.SubmarineStorage;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
public class ShowJobCli extends AbstractCli {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(ShowJobCli.class);
|
||||
|
||||
private Options options;
|
||||
private ShowJobParameters parameters = new ShowJobParameters();
|
||||
|
||||
public ShowJobCli(ClientContext cliContext) {
|
||||
super(cliContext);
|
||||
options = generateOptions();
|
||||
}
|
||||
|
||||
public void printUsages() {
|
||||
new HelpFormatter().printHelp("job show", options);
|
||||
}
|
||||
|
||||
private Options generateOptions() {
|
||||
Options options = new Options();
|
||||
options.addOption(CliConstants.NAME, true, "Name of the job");
|
||||
options.addOption("h", "help", false, "Print help");
|
||||
return options;
|
||||
}
|
||||
|
||||
private void parseCommandLineAndGetShowJobParameters(String[] args)
|
||||
throws IOException, YarnException {
|
||||
// Do parsing
|
||||
GnuParser parser = new GnuParser();
|
||||
CommandLine cli;
|
||||
try {
|
||||
cli = parser.parse(options, args);
|
||||
parameters.updateParametersByParsedCommandline(cli, options,
|
||||
clientContext);
|
||||
} catch (ParseException e) {
|
||||
printUsages();
|
||||
}
|
||||
}
|
||||
|
||||
private void printIfNotNull(String keyForPrint, String keyInStorage,
|
||||
Map<String, String> jobInfo) {
|
||||
if (jobInfo.containsKey(keyInStorage)) {
|
||||
System.out.println("\t" + keyForPrint + ": " + jobInfo.get(keyInStorage));
|
||||
}
|
||||
}
|
||||
|
||||
private void printJobInfo(Map<String, String> jobInfo) {
|
||||
System.out.println("Job Meta Info:");
|
||||
printIfNotNull("Application Id", StorageKeyConstants.APPLICATION_ID,
|
||||
jobInfo);
|
||||
printIfNotNull("Input Path", StorageKeyConstants.INPUT_PATH, jobInfo);
|
||||
printIfNotNull("Saved Model Path", StorageKeyConstants.SAVED_MODEL_PATH,
|
||||
jobInfo);
|
||||
printIfNotNull("Checkpoint Path", StorageKeyConstants.CHECKPOINT_PATH,
|
||||
jobInfo);
|
||||
printIfNotNull("Run Parameters", StorageKeyConstants.JOB_RUN_ARGS,
|
||||
jobInfo);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected void getAndPrintJobInfo() throws IOException {
|
||||
SubmarineStorage storage =
|
||||
clientContext.getRuntimeFactory().getSubmarineStorage();
|
||||
|
||||
Map<String, String> jobInfo = null;
|
||||
try {
|
||||
jobInfo = storage.getJobInfoByName(parameters.getName());
|
||||
} catch (IOException e) {
|
||||
LOG.error("Failed to retrieve job info", e);
|
||||
throw e;
|
||||
}
|
||||
|
||||
printJobInfo(jobInfo);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public ShowJobParameters getParameters() {
|
||||
return parameters;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int run(String[] args)
|
||||
throws ParseException, IOException, YarnException, InterruptedException,
|
||||
SubmarineException {
|
||||
if (CliUtils.argsForHelp(args)) {
|
||||
printUsages();
|
||||
return 0;
|
||||
}
|
||||
|
||||
parseCommandLineAndGetShowJobParameters(args);
|
||||
getAndPrintJobInfo();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli.param;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.Options;
|
||||
import org.apache.commons.cli.ParseException;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.CliConstants;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.conf.SubmarineLogs;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Base class of all parameters.
|
||||
*/
|
||||
public abstract class BaseParameters {
|
||||
private String name;
|
||||
|
||||
public void updateParametersByParsedCommandline(CommandLine parsedCommandLine,
|
||||
Options options, ClientContext clientContext)
|
||||
throws ParseException, IOException, YarnException {
|
||||
String name = parsedCommandLine.getOptionValue(CliConstants.NAME);
|
||||
if (name == null) {
|
||||
throw new ParseException("--name is absent");
|
||||
}
|
||||
|
||||
if (parsedCommandLine.hasOption(CliConstants.VERBOSE)) {
|
||||
SubmarineLogs.verboseOn();
|
||||
}
|
||||
|
||||
this.setName(name);
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public BaseParameters setName(String name) {
|
||||
this.name = name;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,222 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli.param;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.Options;
|
||||
import org.apache.commons.cli.ParseException;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.CliConstants;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.CliUtils;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Parameters used to run a job
|
||||
*/
|
||||
public class RunJobParameters extends RunParameters {
|
||||
private String input;
|
||||
private String checkpointPath;
|
||||
|
||||
private int numWorkers;
|
||||
private int numPS;
|
||||
private Resource workerResource;
|
||||
private Resource psResource;
|
||||
private boolean tensorboardEnabled;
|
||||
private String workerLaunchCmd;
|
||||
private String psLaunchCmd;
|
||||
|
||||
private String psDockerImage = null;
|
||||
private String workerDockerImage = null;
|
||||
|
||||
private boolean waitJobFinish = false;
|
||||
private boolean distributed = false;
|
||||
|
||||
@Override
|
||||
public void updateParametersByParsedCommandline(CommandLine parsedCommandLine,
|
||||
Options options, ClientContext clientContext)
|
||||
throws ParseException, IOException, YarnException {
|
||||
|
||||
String input = parsedCommandLine.getOptionValue(CliConstants.INPUT_PATH);
|
||||
String jobDir = parsedCommandLine.getOptionValue(CliConstants.CHECKPOINT_PATH);
|
||||
int nWorkers = 1;
|
||||
if (parsedCommandLine.getOptionValue(CliConstants.N_WORKERS) != null) {
|
||||
nWorkers = Integer.parseInt(
|
||||
parsedCommandLine.getOptionValue(CliConstants.N_WORKERS));
|
||||
}
|
||||
|
||||
int nPS = 0;
|
||||
if (parsedCommandLine.getOptionValue(CliConstants.N_PS) != null) {
|
||||
nPS = Integer.parseInt(
|
||||
parsedCommandLine.getOptionValue(CliConstants.N_PS));
|
||||
}
|
||||
|
||||
// Check #workers and #ps.
|
||||
// When distributed training is required
|
||||
if (nWorkers >= 2 && nPS > 0) {
|
||||
distributed = true;
|
||||
} else if (nWorkers == 1 && nPS > 0) {
|
||||
throw new ParseException("Only specified one worker but non-zero PS, "
|
||||
+ "please double check.");
|
||||
}
|
||||
|
||||
String workerResourceStr = parsedCommandLine.getOptionValue(
|
||||
CliConstants.WORKER_RES);
|
||||
if (workerResourceStr == null) {
|
||||
throw new ParseException("--" + CliConstants.WORKER_RES + " is absent.");
|
||||
}
|
||||
Resource workerResource = CliUtils.createResourceFromString(
|
||||
workerResourceStr,
|
||||
clientContext.getOrCreateYarnClient().getResourceTypeInfo());
|
||||
|
||||
Resource psResource = null;
|
||||
if (nPS > 0) {
|
||||
String psResourceStr = parsedCommandLine.getOptionValue(CliConstants.PS_RES);
|
||||
if (psResourceStr == null) {
|
||||
throw new ParseException("--" + CliConstants.PS_RES + " is absent.");
|
||||
}
|
||||
psResource = CliUtils.createResourceFromString(psResourceStr,
|
||||
clientContext.getOrCreateYarnClient().getResourceTypeInfo());
|
||||
}
|
||||
|
||||
boolean tensorboard = false;
|
||||
if (parsedCommandLine.getOptionValue(CliConstants.TENSORBOARD) != null) {
|
||||
tensorboard = Boolean.parseBoolean(
|
||||
parsedCommandLine.getOptionValue(CliConstants.TENSORBOARD));
|
||||
}
|
||||
|
||||
if (parsedCommandLine.hasOption(CliConstants.WAIT_JOB_FINISH)) {
|
||||
this.waitJobFinish = true;
|
||||
}
|
||||
|
||||
psDockerImage = parsedCommandLine.getOptionValue(
|
||||
CliConstants.PS_DOCKER_IMAGE);
|
||||
workerDockerImage = parsedCommandLine.getOptionValue(
|
||||
CliConstants.WORKER_DOCKER_IMAGE);
|
||||
|
||||
String workerLaunchCmd = parsedCommandLine.getOptionValue(
|
||||
CliConstants.WORKER_LAUNCH_CMD);
|
||||
String psLaunchCommand = parsedCommandLine.getOptionValue(
|
||||
CliConstants.PS_LAUNCH_CMD);
|
||||
|
||||
this.setInputPath(input).setCheckpointPath(jobDir).setNumPS(nPS).setNumWorkers(nWorkers)
|
||||
.setPSLaunchCmd(psLaunchCommand).setWorkerLaunchCmd(workerLaunchCmd)
|
||||
.setPsResource(psResource).setWorkerResource(workerResource)
|
||||
.setTensorboardEnabled(tensorboard);
|
||||
|
||||
super.updateParametersByParsedCommandline(parsedCommandLine,
|
||||
options, clientContext);
|
||||
}
|
||||
|
||||
public String getInputPath() {
|
||||
return input;
|
||||
}
|
||||
|
||||
public RunJobParameters setInputPath(String input) {
|
||||
this.input = input;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getCheckpointPath() {
|
||||
return checkpointPath;
|
||||
}
|
||||
|
||||
public RunJobParameters setCheckpointPath(String checkpointPath) {
|
||||
this.checkpointPath = checkpointPath;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getNumWorkers() {
|
||||
return numWorkers;
|
||||
}
|
||||
|
||||
public RunJobParameters setNumWorkers(int numWorkers) {
|
||||
this.numWorkers = numWorkers;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getNumPS() {
|
||||
return numPS;
|
||||
}
|
||||
|
||||
public RunJobParameters setNumPS(int numPS) {
|
||||
this.numPS = numPS;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Resource getWorkerResource() {
|
||||
return workerResource;
|
||||
}
|
||||
|
||||
public RunJobParameters setWorkerResource(Resource workerResource) {
|
||||
this.workerResource = workerResource;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Resource getPsResource() {
|
||||
return psResource;
|
||||
}
|
||||
|
||||
public RunJobParameters setPsResource(Resource psResource) {
|
||||
this.psResource = psResource;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean isTensorboardEnabled() {
|
||||
return tensorboardEnabled;
|
||||
}
|
||||
|
||||
public RunJobParameters setTensorboardEnabled(boolean tensorboardEnabled) {
|
||||
this.tensorboardEnabled = tensorboardEnabled;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getWorkerLaunchCmd() {
|
||||
return workerLaunchCmd;
|
||||
}
|
||||
|
||||
public RunJobParameters setWorkerLaunchCmd(String workerLaunchCmd) {
|
||||
this.workerLaunchCmd = workerLaunchCmd;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getPSLaunchCmd() {
|
||||
return psLaunchCmd;
|
||||
}
|
||||
|
||||
public RunJobParameters setPSLaunchCmd(String psLaunchCmd) {
|
||||
this.psLaunchCmd = psLaunchCmd;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean isWaitJobFinish() {
|
||||
return waitJobFinish;
|
||||
}
|
||||
|
||||
|
||||
public String getPsDockerImage() {
|
||||
return psDockerImage;
|
||||
}
|
||||
|
||||
public String getWorkerDockerImage() {
|
||||
return workerDockerImage;
|
||||
}
|
||||
|
||||
public boolean isDistributed() {
|
||||
return distributed;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli.param;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.Options;
|
||||
import org.apache.commons.cli.ParseException;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.CliConstants;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Parameters required to run anything on cluster. Such as run job / serve model
|
||||
*/
|
||||
public abstract class RunParameters extends BaseParameters {
|
||||
private String savedModelPath;
|
||||
private String dockerImageName;
|
||||
private List<String> envars = new ArrayList<>();
|
||||
private String queue;
|
||||
|
||||
@Override
|
||||
public void updateParametersByParsedCommandline(CommandLine parsedCommandLine,
|
||||
Options options, ClientContext clientContext) throws ParseException,
|
||||
IOException, YarnException {
|
||||
String savedModelPath = parsedCommandLine.getOptionValue(
|
||||
CliConstants.SAVED_MODEL_PATH);
|
||||
this.setSavedModelPath(savedModelPath);
|
||||
|
||||
// Envars
|
||||
List<String> envarsList = new ArrayList<>();
|
||||
String[] envars = parsedCommandLine.getOptionValues(CliConstants.ENV);
|
||||
if (envars != null) {
|
||||
for (String envar : envars) {
|
||||
envarsList.add(envar);
|
||||
}
|
||||
}
|
||||
this.setEnvars(envarsList);
|
||||
|
||||
String queue = parsedCommandLine.getOptionValue(
|
||||
CliConstants.QUEUE);
|
||||
this.setQueue(queue);
|
||||
|
||||
String dockerImage = parsedCommandLine.getOptionValue(
|
||||
CliConstants.DOCKER_IMAGE);
|
||||
this.setDockerImageName(dockerImage);
|
||||
|
||||
super.updateParametersByParsedCommandline(parsedCommandLine,
|
||||
options, clientContext);
|
||||
}
|
||||
|
||||
public String getQueue() {
|
||||
return queue;
|
||||
}
|
||||
|
||||
public RunParameters setQueue(String queue) {
|
||||
this.queue = queue;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getDockerImageName() {
|
||||
return dockerImageName;
|
||||
}
|
||||
|
||||
public RunParameters setDockerImageName(String dockerImageName) {
|
||||
this.dockerImageName = dockerImageName;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
public List<String> getEnvars() {
|
||||
return envars;
|
||||
}
|
||||
|
||||
public RunParameters setEnvars(List<String> envars) {
|
||||
this.envars = envars;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getSavedModelPath() {
|
||||
return savedModelPath;
|
||||
}
|
||||
|
||||
public RunParameters setSavedModelPath(String savedModelPath) {
|
||||
this.savedModelPath = savedModelPath;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli.param;
|
||||
|
||||
public class ShowJobParameters extends BaseParameters {
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.client.api.YarnClient;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.submarine.common.conf.SubmarineConfiguration;
|
||||
import org.apache.hadoop.yarn.submarine.common.fs.RemoteDirectoryManager;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.RuntimeFactory;
|
||||
|
||||
public class ClientContext {
|
||||
private Configuration yarnConf = new YarnConfiguration();
|
||||
|
||||
private RemoteDirectoryManager remoteDirectoryManager;
|
||||
private YarnClient yarnClient;
|
||||
private Configuration submarineConfig;
|
||||
private RuntimeFactory runtimeFactory;
|
||||
|
||||
public ClientContext() {
|
||||
submarineConfig = new SubmarineConfiguration();
|
||||
}
|
||||
|
||||
public synchronized YarnClient getOrCreateYarnClient() {
|
||||
if (yarnClient == null) {
|
||||
yarnClient = YarnClient.createYarnClient();
|
||||
yarnClient.init(yarnConf);
|
||||
yarnClient.start();
|
||||
}
|
||||
return yarnClient;
|
||||
}
|
||||
|
||||
public Configuration getYarnConfig() {
|
||||
return yarnConf;
|
||||
}
|
||||
|
||||
public void setConfiguration(Configuration conf) {
|
||||
this.yarnConf = conf;
|
||||
}
|
||||
|
||||
public RemoteDirectoryManager getRemoteDirectoryManager() {
|
||||
return remoteDirectoryManager;
|
||||
}
|
||||
|
||||
public void setRemoteDirectoryManager(
|
||||
RemoteDirectoryManager remoteDirectoryManager) {
|
||||
this.remoteDirectoryManager = remoteDirectoryManager;
|
||||
}
|
||||
|
||||
public Configuration getSubmarineConfig() {
|
||||
return submarineConfig;
|
||||
}
|
||||
|
||||
public void setSubmarineConfig(Configuration submarineConfig) {
|
||||
this.submarineConfig = submarineConfig;
|
||||
}
|
||||
|
||||
public RuntimeFactory getRuntimeFactory() {
|
||||
return runtimeFactory;
|
||||
}
|
||||
|
||||
public void setRuntimeFactory(RuntimeFactory runtimeFactory) {
|
||||
this.runtimeFactory = runtimeFactory;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common;
|
||||
|
||||
public class Envs {
|
||||
public static final String TASK_TYPE_ENV = "_TASK_TYPE";
|
||||
public static final String TASK_INDEX_ENV = "_TASK_INDEX";
|
||||
|
||||
/*
|
||||
* HDFS/HADOOP-related configs
|
||||
*/
|
||||
public static final String HADOOP_HDFS_HOME = "HADOOP_HDFS_HOME";
|
||||
public static final String JAVA_HOME = "JAVA_HOME";
|
||||
public static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR";
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.api;
|
||||
|
||||
import org.apache.hadoop.yarn.service.api.records.Component;
|
||||
import org.apache.hadoop.yarn.service.api.records.Container;
|
||||
import org.apache.hadoop.yarn.service.api.records.ContainerState;
|
||||
|
||||
/**
|
||||
* Status of component of training job
|
||||
*/
|
||||
public class JobComponentStatus {
|
||||
private String compName;
|
||||
private long numReadyContainers = 0;
|
||||
private long numRunningButUnreadyContainers = 0;
|
||||
private long totalAskedContainers;
|
||||
|
||||
public JobComponentStatus(String compName, long nReadyContainers,
|
||||
long nRunningButUnreadyContainers, long totalAskedContainers) {
|
||||
this.compName = compName;
|
||||
this.numReadyContainers = nReadyContainers;
|
||||
this.numRunningButUnreadyContainers = nRunningButUnreadyContainers;
|
||||
this.totalAskedContainers = totalAskedContainers;
|
||||
}
|
||||
|
||||
public String getCompName() {
|
||||
return compName;
|
||||
}
|
||||
|
||||
public void setCompName(String compName) {
|
||||
this.compName = compName;
|
||||
}
|
||||
|
||||
public long getNumReadyContainers() {
|
||||
return numReadyContainers;
|
||||
}
|
||||
|
||||
public void setNumReadyContainers(long numReadyContainers) {
|
||||
this.numReadyContainers = numReadyContainers;
|
||||
}
|
||||
|
||||
public long getNumRunningButUnreadyContainers() {
|
||||
return numRunningButUnreadyContainers;
|
||||
}
|
||||
|
||||
public void setNumRunningButUnreadyContainers(
|
||||
long numRunningButUnreadyContainers) {
|
||||
this.numRunningButUnreadyContainers = numRunningButUnreadyContainers;
|
||||
}
|
||||
|
||||
public long getTotalAskedContainers() {
|
||||
return totalAskedContainers;
|
||||
}
|
||||
|
||||
public void setTotalAskedContainers(long totalAskedContainers) {
|
||||
this.totalAskedContainers = totalAskedContainers;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.api;
|
||||
|
||||
/**
|
||||
* State of training job
|
||||
*/
|
||||
public enum JobState {
|
||||
/**
|
||||
* Job accepted by scheduler and start running
|
||||
*/
|
||||
RUNNING,
|
||||
|
||||
/**
|
||||
* Job killed by user
|
||||
*/
|
||||
KILLED,
|
||||
|
||||
/**
|
||||
* Job failed
|
||||
*/
|
||||
FAILED,
|
||||
|
||||
/**
|
||||
* Job succeeded
|
||||
*/
|
||||
SUCCEEDED,
|
||||
|
||||
/**
|
||||
* Job paused by user
|
||||
*/
|
||||
PAUSED;
|
||||
|
||||
public static boolean isFinal(JobState state) {
|
||||
return state == KILLED || state == SUCCEEDED || state == FAILED;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.api;
|
||||
|
||||
import java.io.PrintStream;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Status of training job.
|
||||
*/
|
||||
public class JobStatus {
|
||||
|
||||
protected String jobName;
|
||||
protected JobState state;
|
||||
protected String tensorboardLink = "N/A";
|
||||
protected List<JobComponentStatus> componentStatus;
|
||||
|
||||
public void nicePrint(PrintStream out) {
|
||||
out.println(
|
||||
"Job Name=" + this.jobName + ", status=" + state.name() + " time="
|
||||
+ Instant.now());
|
||||
if (JobState.isFinal(this.state)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (tensorboardLink.startsWith("http")) {
|
||||
out.println(" Tensorboard link: " + tensorboardLink);
|
||||
}
|
||||
|
||||
out.println(" Components:");
|
||||
for (JobComponentStatus comp : componentStatus) {
|
||||
out.println(" [" + comp.getCompName() + "] Ready=" + comp
|
||||
.getNumReadyContainers() + " + Running-But-Non-Ready=" + comp
|
||||
.getNumRunningButUnreadyContainers() + " | Asked=" + comp
|
||||
.getTotalAskedContainers());
|
||||
}
|
||||
out.println("------------------");
|
||||
}
|
||||
|
||||
public JobState getState() {
|
||||
return state;
|
||||
}
|
||||
|
||||
public String getTensorboardLink() {
|
||||
return tensorboardLink;
|
||||
}
|
||||
|
||||
public List<JobComponentStatus> getComponentStatus() {
|
||||
return componentStatus;
|
||||
}
|
||||
|
||||
public String getJobName() {
|
||||
return jobName;
|
||||
}
|
||||
|
||||
public void setJobName(String jobName) {
|
||||
this.jobName = jobName;
|
||||
}
|
||||
|
||||
public void setState(JobState state) {
|
||||
this.state = state;
|
||||
}
|
||||
|
||||
public void setTensorboardLink(String tensorboardLink) {
|
||||
this.tensorboardLink = tensorboardLink;
|
||||
}
|
||||
|
||||
public void setComponentStatus(List<JobComponentStatus> componentStatus) {
|
||||
this.componentStatus = componentStatus;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.api;
|
||||
|
||||
public enum TaskType {
|
||||
PRIMARY_WORKER("master"),
|
||||
WORKER("worker"),
|
||||
PS("ps"),
|
||||
TENSORBOARD("tensorboard");
|
||||
|
||||
private String compName;
|
||||
|
||||
TaskType(String compName) {
|
||||
this.compName = compName;
|
||||
}
|
||||
|
||||
public String getComponentName() {
|
||||
return compName;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.api.builder;
|
||||
|
||||
import org.apache.hadoop.yarn.service.api.records.Component;
|
||||
import org.apache.hadoop.yarn.service.api.records.Container;
|
||||
import org.apache.hadoop.yarn.service.api.records.ContainerState;
|
||||
import org.apache.hadoop.yarn.submarine.common.api.JobComponentStatus;
|
||||
|
||||
public class JobComponentStatusBuilder {
|
||||
public static JobComponentStatus fromServiceComponent(Component component) {
|
||||
long totalAskedContainers = component.getNumberOfContainers();
|
||||
int numReadyContainers = 0;
|
||||
int numRunningButUnreadyContainers = 0;
|
||||
String compName = component.getName();
|
||||
|
||||
for (Container c : component.getContainers()) {
|
||||
if (c.getState() == ContainerState.READY) {
|
||||
numReadyContainers++;
|
||||
} else if (c.getState() == ContainerState.RUNNING_BUT_UNREADY) {
|
||||
numRunningButUnreadyContainers++;
|
||||
}
|
||||
}
|
||||
|
||||
return new JobComponentStatus(compName, numReadyContainers,
|
||||
numRunningButUnreadyContainers, totalAskedContainers);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.api.builder;
|
||||
|
||||
import org.apache.hadoop.yarn.service.api.records.Component;
|
||||
import org.apache.hadoop.yarn.service.api.records.Service;
|
||||
import org.apache.hadoop.yarn.service.api.records.ServiceState;
|
||||
import org.apache.hadoop.yarn.submarine.common.api.JobComponentStatus;
|
||||
import org.apache.hadoop.yarn.submarine.common.api.JobState;
|
||||
import org.apache.hadoop.yarn.submarine.common.api.JobStatus;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class JobStatusBuilder {
|
||||
public static JobStatus fromServiceSpec(Service serviceSpec) {
|
||||
JobStatus status = new JobStatus();
|
||||
status.setState(fromServiceState(serviceSpec.getState()));
|
||||
|
||||
// If it is a final state, return.
|
||||
if (JobState.isFinal(status.getState())) {
|
||||
return status;
|
||||
}
|
||||
|
||||
List<JobComponentStatus> componentStatusList = new ArrayList<>();
|
||||
|
||||
for (Component component : serviceSpec.getComponents()) {
|
||||
componentStatusList.add(
|
||||
JobComponentStatusBuilder.fromServiceComponent(component));
|
||||
}
|
||||
status.setComponentStatus(componentStatusList);
|
||||
|
||||
// TODO, handle tensorboard differently.
|
||||
// status.setTensorboardLink(getTensorboardLink(serviceSpec, clientContext));
|
||||
|
||||
status.setJobName(serviceSpec.getName());
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
private static JobState fromServiceState(ServiceState serviceState) {
|
||||
switch (serviceState) {
|
||||
case STOPPED:
|
||||
// TODO, once YARN-8488 gets committed, we need to update this.
|
||||
return JobState.SUCCEEDED;
|
||||
case FAILED:
|
||||
return JobState.FAILED;
|
||||
}
|
||||
|
||||
return JobState.RUNNING;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.conf;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
|
||||
public class SubmarineConfiguration extends Configuration {
|
||||
private static final String SUBMARINE_CONFIGURATION_FILE = "submarine.xml";
|
||||
|
||||
public SubmarineConfiguration() {
|
||||
this(new Configuration(false), true);
|
||||
}
|
||||
|
||||
public SubmarineConfiguration(Configuration configuration) {
|
||||
this(configuration, false);
|
||||
}
|
||||
|
||||
public SubmarineConfiguration(Configuration configuration,
|
||||
boolean loadLocalConfig) {
|
||||
super(configuration);
|
||||
if (loadLocalConfig) {
|
||||
addResource(SUBMARINE_CONFIGURATION_FILE);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Runtime of submarine
|
||||
*/
|
||||
|
||||
private static final String PREFIX = "submarine.";
|
||||
|
||||
public static final String RUNTIME_CLASS = PREFIX + "runtime.class";
|
||||
public static final String DEFAULT_RUNTIME_CLASS =
|
||||
"org.apache.hadoop.yarn.submarine.runtimes.yarnservice.YarnServiceRuntimeFactory";
|
||||
|
||||
public void setSubmarineRuntimeClass(String runtimeClass) {
|
||||
set(RUNTIME_CLASS, runtimeClass);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.conf;
|
||||
|
||||
public class SubmarineLogs {
|
||||
private static volatile boolean verbose = false;
|
||||
|
||||
public static boolean isVerbose() {
|
||||
return SubmarineLogs.verbose;
|
||||
}
|
||||
|
||||
public static void verboseOn() {
|
||||
SubmarineLogs.verbose = true;
|
||||
}
|
||||
|
||||
public static void verboseOff() {
|
||||
SubmarineLogs.verbose = false;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.exception;
|
||||
|
||||
public class SubmarineException extends Exception {
|
||||
public SubmarineException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.exception;
|
||||
|
||||
public class SubmarineRuntimeException extends RuntimeException {
|
||||
public SubmarineRuntimeException(String s) {
|
||||
super(s);
|
||||
}
|
||||
|
||||
public SubmarineRuntimeException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.fs;
|
||||
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.CliConstants;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Manages remote directories for staging, log, etc.
|
||||
* TODO, need to properly handle permission / name validation, etc.
|
||||
*/
|
||||
public class DefaultRemoteDirectoryManager implements RemoteDirectoryManager {
|
||||
FileSystem fs;
|
||||
|
||||
public DefaultRemoteDirectoryManager(ClientContext context) {
|
||||
try {
|
||||
this.fs = FileSystem.get(context.getYarnConfig());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getJobStagingArea(String jobName, boolean create) throws IOException {
|
||||
Path staging = new Path(getJobRootFolder(jobName), "staging");
|
||||
if (create) {
|
||||
createFolderIfNotExist(staging);
|
||||
}
|
||||
return staging;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getJobCheckpointDir(String jobName, boolean create)
|
||||
throws IOException {
|
||||
Path jobDir = new Path(getJobStagingArea(jobName, create),
|
||||
CliConstants.CHECKPOINT_PATH);
|
||||
if (create) {
|
||||
createFolderIfNotExist(jobDir);
|
||||
}
|
||||
return jobDir;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getModelDir(String modelName, boolean create) throws IOException {
|
||||
Path modelDir = new Path(new Path("submarine", "models"), modelName);
|
||||
if (create) {
|
||||
createFolderIfNotExist(modelDir);
|
||||
}
|
||||
return modelDir;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileSystem getFileSystem() {
|
||||
return fs;
|
||||
}
|
||||
|
||||
private Path getJobRootFolder(String jobName) throws IOException {
|
||||
return new Path(new Path("submarine", "jobs"), jobName);
|
||||
}
|
||||
|
||||
private void createFolderIfNotExist(Path path) throws IOException {
|
||||
if (!fs.exists(path)) {
|
||||
if (!fs.mkdirs(path)) {
|
||||
throw new IOException("Failed to create folder=" + path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.fs;
|
||||
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public interface RemoteDirectoryManager {
|
||||
Path getJobStagingArea(String jobName, boolean create) throws IOException;
|
||||
|
||||
Path getJobCheckpointDir(String jobName, boolean create) throws IOException;
|
||||
|
||||
Path getModelDir(String modelName, boolean create) throws IOException;
|
||||
|
||||
FileSystem getFileSystem() throws IOException;
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.conf.SubmarineConfiguration;
|
||||
import org.apache.hadoop.yarn.submarine.common.exception.SubmarineRuntimeException;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.FSBasedSubmarineStorageImpl;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.JobMonitor;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.JobSubmitter;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.SubmarineStorage;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.yarnservice.YarnServiceJobMonitor;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.yarnservice.YarnServiceJobSubmitter;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
|
||||
public abstract class RuntimeFactory {
|
||||
protected ClientContext clientContext;
|
||||
private JobSubmitter jobSubmitter;
|
||||
private JobMonitor jobMonitor;
|
||||
private SubmarineStorage submarineStorage;
|
||||
|
||||
public RuntimeFactory(ClientContext clientContext) {
|
||||
this.clientContext = clientContext;
|
||||
}
|
||||
|
||||
public static RuntimeFactory getRuntimeFactory(
|
||||
ClientContext clientContext) {
|
||||
Configuration submarineConfiguration =
|
||||
clientContext.getSubmarineConfig();
|
||||
String runtimeClass = submarineConfiguration.get(
|
||||
SubmarineConfiguration.RUNTIME_CLASS,
|
||||
SubmarineConfiguration.DEFAULT_RUNTIME_CLASS);
|
||||
|
||||
try {
|
||||
Class<?> runtimeClazz = Class.forName(runtimeClass);
|
||||
if (RuntimeFactory.class.isAssignableFrom(runtimeClazz)) {
|
||||
return (RuntimeFactory) runtimeClazz.getConstructor(ClientContext.class).newInstance(clientContext);
|
||||
} else {
|
||||
throw new SubmarineRuntimeException("Class: " + runtimeClass
|
||||
+ " not instance of " + RuntimeFactory.class.getCanonicalName());
|
||||
}
|
||||
} catch (ClassNotFoundException | IllegalAccessException |
|
||||
InstantiationException | NoSuchMethodException |
|
||||
InvocationTargetException e) {
|
||||
throw new SubmarineRuntimeException(
|
||||
"Could not instantiate RuntimeFactory: " + runtimeClass, e);
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract JobSubmitter internalCreateJobSubmitter();
|
||||
|
||||
protected abstract JobMonitor internalCreateJobMonitor();
|
||||
|
||||
protected abstract SubmarineStorage internalCreateSubmarineStorage();
|
||||
|
||||
public synchronized JobSubmitter getJobSubmitterInstance() {
|
||||
if (jobSubmitter == null) {
|
||||
jobSubmitter = internalCreateJobSubmitter();
|
||||
}
|
||||
return jobSubmitter;
|
||||
}
|
||||
|
||||
public synchronized JobMonitor getJobMonitorInstance() {
|
||||
if (jobMonitor == null) {
|
||||
jobMonitor = internalCreateJobMonitor();
|
||||
}
|
||||
return jobMonitor;
|
||||
}
|
||||
|
||||
public synchronized SubmarineStorage getSubmarineStorage() {
|
||||
if (submarineStorage == null) {
|
||||
submarineStorage = internalCreateSubmarineStorage();
|
||||
}
|
||||
return submarineStorage;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public synchronized void setJobSubmitterInstance(JobSubmitter jobSubmitter) {
|
||||
this.jobSubmitter = jobSubmitter;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public synchronized void setJobMonitorInstance(JobMonitor jobMonitor) {
|
||||
this.jobMonitor = jobMonitor;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public synchronized void setSubmarineStorage(SubmarineStorage storage) {
|
||||
this.submarineStorage = storage;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.common;
|
||||
|
||||
import org.apache.hadoop.fs.FSDataInputStream;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.fs.RemoteDirectoryManager;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInput;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.ObjectOutput;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* A super naive FS-based storage.
|
||||
*/
|
||||
public class FSBasedSubmarineStorageImpl extends SubmarineStorage {
|
||||
ClientContext clientContext;
|
||||
RemoteDirectoryManager rdm;
|
||||
|
||||
public FSBasedSubmarineStorageImpl(ClientContext clientContext) {
|
||||
this.clientContext = clientContext;
|
||||
rdm = clientContext.getRemoteDirectoryManager();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addNewJob(String jobName, Map<String, String> jobInfo)
|
||||
throws IOException {
|
||||
Path jobInfoPath = getJobInfoPath(jobName, true);
|
||||
FSDataOutputStream fos = rdm.getFileSystem().create(jobInfoPath);
|
||||
serializeMap(fos, jobInfo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, String> getJobInfoByName(String jobName)
|
||||
throws IOException {
|
||||
Path jobInfoPath = getJobInfoPath(jobName, false);
|
||||
FSDataInputStream fis = rdm.getFileSystem().open(jobInfoPath);
|
||||
return deserializeMap(fis);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addNewModel(String modelName, String version,
|
||||
Map<String, String> modelInfo) throws IOException {
|
||||
Path modelInfoPath = getModelInfoPath(modelName, version, true);
|
||||
FSDataOutputStream fos = rdm.getFileSystem().create(modelInfoPath);
|
||||
serializeMap(fos, modelInfo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, String> getModelInfoByName(String modelName,
|
||||
String version) throws IOException {
|
||||
Path modelInfoPath = getModelInfoPath(modelName, version, false);
|
||||
FSDataInputStream fis = rdm.getFileSystem().open(modelInfoPath);
|
||||
return deserializeMap(fis);
|
||||
}
|
||||
|
||||
private Path getModelInfoPath(String modelName, String version, boolean create)
|
||||
throws IOException {
|
||||
Path modelDir = rdm.getModelDir(modelName, create);
|
||||
Path modelInfo = new Path(modelDir, version + ".info");
|
||||
return modelInfo;
|
||||
}
|
||||
|
||||
private void serializeMap(FSDataOutputStream fos, Map<String, String> map)
|
||||
throws IOException {
|
||||
ObjectOutput oo = new ObjectOutputStream(fos);
|
||||
oo.writeObject(map);
|
||||
oo.close();
|
||||
}
|
||||
|
||||
private Map<String, String> deserializeMap(FSDataInputStream fis)
|
||||
throws IOException {
|
||||
ObjectInput oi = new ObjectInputStream(fis);
|
||||
Map<String, String> newMap = null;
|
||||
try {
|
||||
newMap = (Map<String, String>) oi.readObject();
|
||||
} catch (ClassNotFoundException e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
return newMap;
|
||||
}
|
||||
|
||||
private Path getJobInfoPath(String jobName, boolean create) throws IOException {
|
||||
Path path = rdm.getJobStagingArea(jobName, create);
|
||||
Path fileName = new Path(path, "job.info");
|
||||
return fileName;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.common;
|
||||
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.api.JobState;
|
||||
import org.apache.hadoop.yarn.submarine.common.api.JobStatus;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.submarine.common.exception.SubmarineException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Monitor status of job(s)
|
||||
*/
|
||||
public abstract class JobMonitor {
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(JobMonitor.class);
|
||||
protected ClientContext clientContext;
|
||||
|
||||
public JobMonitor(ClientContext clientContext) {
|
||||
this.clientContext = clientContext;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns status of training job.
|
||||
*
|
||||
* @param jobName name of job
|
||||
* @return job status
|
||||
* @throws IOException anything else happens
|
||||
* @throws YarnException anything related to YARN happens
|
||||
*/
|
||||
public abstract JobStatus getTrainingJobStatus(String jobName)
|
||||
throws IOException, YarnException;
|
||||
|
||||
/**
|
||||
* Continue wait and print status if job goes to ready or final state.
|
||||
* @param jobName
|
||||
* @throws IOException
|
||||
* @throws YarnException
|
||||
* @throws SubmarineException
|
||||
*/
|
||||
public void waitTrainingFinal(String jobName)
|
||||
throws IOException, YarnException, SubmarineException {
|
||||
// Wait 5 sec between each fetch.
|
||||
int waitIntervalSec = 5;
|
||||
JobStatus js;
|
||||
while (true) {
|
||||
js = getTrainingJobStatus(jobName);
|
||||
JobState jobState = js.getState();
|
||||
js.nicePrint(System.err);
|
||||
|
||||
if (JobState.isFinal(jobState)) {
|
||||
if (jobState.equals(JobState.FAILED)) {
|
||||
throw new SubmarineException("Job failed");
|
||||
} else if (jobState.equals(JobState.KILLED)) {
|
||||
throw new SubmarineException("Job killed");
|
||||
}
|
||||
LOG.info("Job exited with state=" + jobState);
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
Thread.sleep(waitIntervalSec * 1000);
|
||||
} catch (InterruptedException e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.common;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.param.RunJobParameters;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Submit job to cluster master
|
||||
*/
|
||||
public interface JobSubmitter {
|
||||
/**
|
||||
* Submit job to cluster
|
||||
* @param parameters run job parameters
|
||||
* @return applicatioId when successfully submitted
|
||||
* @throws YarnException for issues while contacting YARN daemons
|
||||
* @throws IOException for other issues.
|
||||
*/
|
||||
ApplicationId submitJob(RunJobParameters parameters)
|
||||
throws IOException, YarnException;
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.common;
|
||||
|
||||
public class StorageKeyConstants {
|
||||
public static final String JOB_NAME = "JOB_NAME";
|
||||
public static final String JOB_RUN_ARGS = "JOB_RUN_ARGS";
|
||||
public static final String APPLICATION_ID = "APPLICATION_ID";
|
||||
public static final String CHECKPOINT_PATH = "CHECKPOINT_PATH";
|
||||
public static final String INPUT_PATH = "INPUT_PATH";
|
||||
public static final String SAVED_MODEL_PATH = "SAVED_MODEL_PATH";
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.common;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Persistent job/model, etc.
|
||||
*/
|
||||
public abstract class SubmarineStorage {
|
||||
/**
|
||||
* Add a new job by name
|
||||
* @param jobName name of job.
|
||||
* @param jobInfo info of the job.
|
||||
*/
|
||||
public abstract void addNewJob(String jobName, Map<String, String> jobInfo)
|
||||
throws IOException;
|
||||
|
||||
/**
|
||||
* Get job info by job name.
|
||||
* @param jobName name of job
|
||||
* @return info of the job.
|
||||
*/
|
||||
public abstract Map<String, String> getJobInfoByName(String jobName)
|
||||
throws IOException;
|
||||
|
||||
/**
|
||||
* Add a new model
|
||||
* @param modelName name of model
|
||||
* @param version version of the model, when null is specified, it will be
|
||||
* "default"
|
||||
* @param modelInfo info of the model.
|
||||
*/
|
||||
public abstract void addNewModel(String modelName, String version,
|
||||
Map<String, String> modelInfo) throws IOException;
|
||||
|
||||
/**
|
||||
* Get model info by name and version.
|
||||
* @param modelName name of model.
|
||||
* @param version version of the model, when null is specifed, it will be
|
||||
*/
|
||||
public abstract Map<String, String> getModelInfoByName(String modelName, String version)
|
||||
throws IOException;
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.service.api.records.Service;
|
||||
import org.apache.hadoop.yarn.service.client.ServiceClient;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.api.JobStatus;
|
||||
import org.apache.hadoop.yarn.submarine.common.api.builder.JobStatusBuilder;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.JobMonitor;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class YarnServiceJobMonitor extends JobMonitor {
|
||||
private ServiceClient serviceClient = null;
|
||||
|
||||
public YarnServiceJobMonitor(ClientContext clientContext) {
|
||||
super(clientContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized JobStatus getTrainingJobStatus(String jobName)
|
||||
throws IOException, YarnException {
|
||||
if (this.serviceClient == null) {
|
||||
this.serviceClient = YarnServiceUtils.createServiceClient(
|
||||
clientContext.getYarnConfig());
|
||||
}
|
||||
|
||||
Service serviceSpec = this.serviceClient.getStatus(jobName);
|
||||
JobStatus jobStatus = JobStatusBuilder.fromServiceSpec(serviceSpec);
|
||||
return jobStatus;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,458 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.service.api.ServiceApiConstants;
|
||||
import org.apache.hadoop.yarn.service.api.records.Artifact;
|
||||
import org.apache.hadoop.yarn.service.api.records.Component;
|
||||
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
|
||||
import org.apache.hadoop.yarn.service.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.service.api.records.ResourceInformation;
|
||||
import org.apache.hadoop.yarn.service.api.records.Service;
|
||||
import org.apache.hadoop.yarn.service.client.ServiceClient;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.param.RunJobParameters;
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.Envs;
|
||||
import org.apache.hadoop.yarn.submarine.common.api.TaskType;
|
||||
import org.apache.hadoop.yarn.submarine.common.conf.SubmarineLogs;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.JobSubmitter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.StringTokenizer;
|
||||
|
||||
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
|
||||
|
||||
/**
|
||||
* Submit a job to cluster
|
||||
*/
|
||||
public class YarnServiceJobSubmitter implements JobSubmitter {
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(YarnServiceJobSubmitter.class);
|
||||
ClientContext clientContext;
|
||||
Service serviceSpec;
|
||||
private Set<Path> uploadedFiles = new HashSet<>();
|
||||
|
||||
public YarnServiceJobSubmitter(ClientContext clientContext) {
|
||||
this.clientContext = clientContext;
|
||||
}
|
||||
|
||||
private Resource getServiceResourceFromYarnResource(
|
||||
org.apache.hadoop.yarn.api.records.Resource yarnResource) {
|
||||
Resource serviceResource = new Resource();
|
||||
serviceResource.setCpus(yarnResource.getVirtualCores());
|
||||
serviceResource.setMemory(String.valueOf(yarnResource.getMemorySize()));
|
||||
|
||||
Map<String, ResourceInformation> riMap = new HashMap<>();
|
||||
for (org.apache.hadoop.yarn.api.records.ResourceInformation ri : yarnResource
|
||||
.getAllResourcesListCopy()) {
|
||||
ResourceInformation serviceRi =
|
||||
new ResourceInformation();
|
||||
serviceRi.setValue(ri.getValue());
|
||||
serviceRi.setUnit(ri.getUnits());
|
||||
riMap.put(ri.getName(), serviceRi);
|
||||
}
|
||||
serviceResource.setResourceInformations(riMap);
|
||||
|
||||
return serviceResource;
|
||||
}
|
||||
|
||||
private String getValueOfEnvionment(String envar) {
|
||||
// extract value from "key=value" form
|
||||
if (envar == null || !envar.contains("=")) {
|
||||
return "";
|
||||
} else {
|
||||
return envar.substring(envar.indexOf("=") + 1);
|
||||
}
|
||||
}
|
||||
|
||||
private void addHdfsClassPathIfNeeded(RunJobParameters parameters,
|
||||
FileWriter fw, Component comp) throws IOException {
|
||||
// Find envs to use HDFS
|
||||
String hdfsHome = null;
|
||||
String javaHome = null;
|
||||
|
||||
boolean hadoopEnv = false;
|
||||
|
||||
for (String envar : parameters.getEnvars()) {
|
||||
if (envar.startsWith("DOCKER_HADOOP_HDFS_HOME=")) {
|
||||
hdfsHome = getValueOfEnvionment(envar);
|
||||
hadoopEnv = true;
|
||||
} else if (envar.startsWith("DOCKER_JAVA_HOME=")) {
|
||||
javaHome = getValueOfEnvionment(envar);
|
||||
}
|
||||
}
|
||||
|
||||
boolean lackingEnvs = false;
|
||||
|
||||
if ((parameters.getInputPath() != null && parameters.getInputPath()
|
||||
.contains("hdfs://")) || (parameters.getCheckpointPath() != null
|
||||
&& parameters.getCheckpointPath().contains("hdfs://")) || (
|
||||
parameters.getSavedModelPath() != null && parameters.getSavedModelPath()
|
||||
.contains("hdfs://")) || hadoopEnv) {
|
||||
// HDFS is asked either in input or output, set LD_LIBRARY_PATH
|
||||
// and classpath
|
||||
|
||||
if (hdfsHome != null) {
|
||||
// Unset HADOOP_HOME/HADOOP_YARN_HOME to make sure host machine's envs
|
||||
// won't pollute docker's env.
|
||||
fw.append("export HADOOP_HOME=\n");
|
||||
fw.append("export HADOOP_YARN_HOME=\n");
|
||||
fw.append("export HADOOP_HDFS_HOME=" + hdfsHome + "\n");
|
||||
} else{
|
||||
lackingEnvs = true;
|
||||
}
|
||||
|
||||
// hadoop confs will be uploaded to HDFS and localized to container's
|
||||
// local folder, so here set $HADOOP_CONF_DIR to $WORK_DIR.
|
||||
fw.append("export HADOOP_CONF_DIR=$WORK_DIR\n");
|
||||
if (javaHome != null) {
|
||||
fw.append("export JAVA_HOME=" + javaHome + "\n");
|
||||
fw.append("export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"
|
||||
+ "$JAVA_HOME/lib/amd64/server\n");
|
||||
} else {
|
||||
lackingEnvs = true;
|
||||
}
|
||||
fw.append("export CLASSPATH=`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`\n");
|
||||
}
|
||||
|
||||
if (lackingEnvs) {
|
||||
LOG.error("When hdfs is being used to read/write models/data. Following"
|
||||
+ "envs are required: 1) DOCKER_HADOOP_HDFS_HOME=<HDFS_HOME inside"
|
||||
+ "docker container> 2) DOCKER_JAVA_HOME=<JAVA_HOME inside docker"
|
||||
+ "container>. You can use --env to pass these envars.");
|
||||
throw new IOException("Failed to detect HDFS-related environments.");
|
||||
}
|
||||
|
||||
// Trying to upload core-site.xml and hdfs-site.xml
|
||||
Path stagingDir =
|
||||
clientContext.getRemoteDirectoryManager().getJobStagingArea(
|
||||
parameters.getName(), true);
|
||||
File coreSite = findFileOnClassPath("core-site.xml");
|
||||
File hdfsSite = findFileOnClassPath("hdfs-site.xml");
|
||||
if (coreSite == null || hdfsSite == null) {
|
||||
LOG.error("hdfs is being used, however we couldn't locate core-site.xml/"
|
||||
+ "hdfs-site.xml from classpath, please double check you classpath"
|
||||
+ "setting and make sure they're included.");
|
||||
throw new IOException(
|
||||
"Failed to locate core-site.xml / hdfs-site.xml from class path");
|
||||
}
|
||||
uploadToRemoteFileAndLocalizeToContainerWorkDir(stagingDir,
|
||||
coreSite.getAbsolutePath(), "core-site.xml", comp);
|
||||
uploadToRemoteFileAndLocalizeToContainerWorkDir(stagingDir,
|
||||
hdfsSite.getAbsolutePath(), "hdfs-site.xml", comp);
|
||||
|
||||
// DEBUG
|
||||
if (SubmarineLogs.isVerbose()) {
|
||||
fw.append("echo $CLASSPATH\n");
|
||||
fw.append("echo $JAVA_HOME\n");
|
||||
fw.append("echo $LD_LIBRARY_PATH\n");
|
||||
fw.append("echo $HADOOP_HDFS_HOME\n");
|
||||
}
|
||||
}
|
||||
|
||||
private void addCommonEnvironments(Component component, TaskType taskType) {
|
||||
Map<String, String> envs = component.getConfiguration().getEnv();
|
||||
envs.put(Envs.TASK_INDEX_ENV, ServiceApiConstants.COMPONENT_ID);
|
||||
envs.put(Envs.TASK_TYPE_ENV, taskType.name());
|
||||
}
|
||||
|
||||
/*
|
||||
* Generate a command launch script on local disk, returns patch to the script
|
||||
*/
|
||||
private String generateCommandLaunchScript(RunJobParameters parameters,
|
||||
TaskType taskType, Component comp) throws IOException {
|
||||
File file = File.createTempFile(taskType.name() + "-launch-script", ".sh");
|
||||
FileWriter fw = new FileWriter(file);
|
||||
|
||||
fw.append("#!/bin/bash\n");
|
||||
|
||||
addHdfsClassPathIfNeeded(parameters, fw, comp);
|
||||
|
||||
// For primary_worker
|
||||
if (taskType == TaskType.PRIMARY_WORKER) {
|
||||
// Do we need tensorboard?
|
||||
if (parameters.isTensorboardEnabled()) {
|
||||
int tensorboardPort = 6006;
|
||||
// Run tensorboard at the background
|
||||
fw.append(
|
||||
"tensorboard --port " + tensorboardPort + " --logdir " + parameters
|
||||
.getCheckpointPath() + " &\n");
|
||||
}
|
||||
}
|
||||
|
||||
// When distributed training is required
|
||||
if (parameters.isDistributed()) {
|
||||
// Generated TF_CONFIG
|
||||
String tfConfigEnv = YarnServiceUtils.getTFConfigEnv(
|
||||
taskType.getComponentName(), parameters.getNumWorkers(),
|
||||
parameters.getNumPS(), parameters.getName(),
|
||||
System.getProperty("user.name"),
|
||||
clientContext.getYarnConfig().get("hadoop.registry.dns.domain-name"));
|
||||
fw.append("export TF_CONFIG=\"" + tfConfigEnv + "\"\n");
|
||||
}
|
||||
|
||||
// Print launch command
|
||||
if (taskType.equals(TaskType.WORKER) || taskType.equals(
|
||||
TaskType.PRIMARY_WORKER)) {
|
||||
fw.append(parameters.getWorkerLaunchCmd() + '\n');
|
||||
|
||||
if (SubmarineLogs.isVerbose()) {
|
||||
LOG.info("Worker command =[" + parameters.getWorkerLaunchCmd() + "]");
|
||||
}
|
||||
} else if (taskType.equals(TaskType.PS)) {
|
||||
fw.append(parameters.getPSLaunchCmd() + '\n');
|
||||
|
||||
if (SubmarineLogs.isVerbose()) {
|
||||
LOG.info("PS command =[" + parameters.getPSLaunchCmd() + "]");
|
||||
}
|
||||
}
|
||||
|
||||
fw.close();
|
||||
return file.getAbsolutePath();
|
||||
}
|
||||
|
||||
private String getScriptFileName(TaskType taskType) {
|
||||
return "run-" + taskType.name() + ".sh";
|
||||
}
|
||||
|
||||
private File findFileOnClassPath(final String fileName) {
|
||||
final String classpath = System.getProperty("java.class.path");
|
||||
final String pathSeparator = System.getProperty("path.separator");
|
||||
final StringTokenizer tokenizer = new StringTokenizer(classpath,
|
||||
pathSeparator);
|
||||
|
||||
while (tokenizer.hasMoreTokens()) {
|
||||
final String pathElement = tokenizer.nextToken();
|
||||
final File directoryOrJar = new File(pathElement);
|
||||
final File absoluteDirectoryOrJar = directoryOrJar.getAbsoluteFile();
|
||||
if (absoluteDirectoryOrJar.isFile()) {
|
||||
final File target = new File(absoluteDirectoryOrJar.getParent(),
|
||||
fileName);
|
||||
if (target.exists()) {
|
||||
return target;
|
||||
}
|
||||
} else{
|
||||
final File target = new File(directoryOrJar, fileName);
|
||||
if (target.exists()) {
|
||||
return target;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private void uploadToRemoteFileAndLocalizeToContainerWorkDir(Path stagingDir,
|
||||
String fileToUpload, String destFilename, Component comp)
|
||||
throws IOException {
|
||||
FileSystem fs = FileSystem.get(clientContext.getYarnConfig());
|
||||
|
||||
// Upload to remote FS under staging area
|
||||
File localFile = new File(fileToUpload);
|
||||
if (!localFile.exists()) {
|
||||
throw new FileNotFoundException(
|
||||
"Trying to upload file=" + localFile.getAbsolutePath()
|
||||
+ " to remote, but couldn't find local file.");
|
||||
}
|
||||
String filename = new File(fileToUpload).getName();
|
||||
|
||||
Path uploadedFilePath = new Path(stagingDir, filename);
|
||||
if (!uploadedFiles.contains(uploadedFilePath)) {
|
||||
if (SubmarineLogs.isVerbose()) {
|
||||
LOG.info("Copying local file=" + fileToUpload + " to remote="
|
||||
+ uploadedFilePath);
|
||||
}
|
||||
fs.copyFromLocalFile(new Path(fileToUpload), uploadedFilePath);
|
||||
uploadedFiles.add(uploadedFilePath);
|
||||
}
|
||||
|
||||
FileStatus fileStatus = fs.getFileStatus(uploadedFilePath);
|
||||
LOG.info("Uploaded file path = " + fileStatus.getPath());
|
||||
|
||||
// Set it to component's files list
|
||||
comp.getConfiguration().getFiles().add(new ConfigFile().srcFile(
|
||||
fileStatus.getPath().toUri().toString()).destFile(destFilename)
|
||||
.type(ConfigFile.TypeEnum.STATIC));
|
||||
}
|
||||
|
||||
private void handleLaunchCommand(RunJobParameters parameters,
|
||||
TaskType taskType, Component component) throws IOException {
|
||||
// Get staging area directory
|
||||
Path stagingDir =
|
||||
clientContext.getRemoteDirectoryManager().getJobStagingArea(
|
||||
parameters.getName(), true);
|
||||
|
||||
// Generate script file in the local disk
|
||||
String localScriptFile = generateCommandLaunchScript(parameters, taskType,
|
||||
component);
|
||||
String destScriptFileName = getScriptFileName(taskType);
|
||||
uploadToRemoteFileAndLocalizeToContainerWorkDir(stagingDir, localScriptFile,
|
||||
destScriptFileName, component);
|
||||
|
||||
component.setLaunchCommand("./" + destScriptFileName);
|
||||
}
|
||||
|
||||
private void addWorkerComponent(Service service,
|
||||
RunJobParameters parameters, TaskType taskType) throws IOException {
|
||||
Component workerComponent = new Component();
|
||||
addCommonEnvironments(workerComponent, taskType);
|
||||
|
||||
workerComponent.setName(taskType.getComponentName());
|
||||
|
||||
if (taskType.equals(TaskType.PRIMARY_WORKER)) {
|
||||
workerComponent.setNumberOfContainers(1L);
|
||||
} else{
|
||||
workerComponent.setNumberOfContainers(
|
||||
(long) parameters.getNumWorkers() - 1);
|
||||
}
|
||||
|
||||
if (parameters.getWorkerDockerImage() != null) {
|
||||
workerComponent.setArtifact(
|
||||
getDockerArtifact(parameters.getWorkerDockerImage()));
|
||||
}
|
||||
|
||||
workerComponent.setResource(
|
||||
getServiceResourceFromYarnResource(parameters.getWorkerResource()));
|
||||
handleLaunchCommand(parameters, taskType, workerComponent);
|
||||
workerComponent.setRestartPolicy(Component.RestartPolicyEnum.NEVER);
|
||||
service.addComponent(workerComponent);
|
||||
}
|
||||
|
||||
// Handle worker and primary_worker.
|
||||
private void addWorkerComponents(Service service, RunJobParameters parameters)
|
||||
throws IOException {
|
||||
addWorkerComponent(service, parameters, TaskType.PRIMARY_WORKER);
|
||||
|
||||
if (parameters.getNumWorkers() > 1) {
|
||||
addWorkerComponent(service, parameters, TaskType.WORKER);
|
||||
}
|
||||
}
|
||||
|
||||
private void appendToEnv(Service service, String key, String value,
|
||||
String delim) {
|
||||
Map<String, String> env = service.getConfiguration().getEnv();
|
||||
if (!env.containsKey(key)) {
|
||||
env.put(key, value);
|
||||
} else {
|
||||
if (!value.isEmpty()) {
|
||||
String existingValue = env.get(key);
|
||||
if (!existingValue.endsWith(delim)) {
|
||||
env.put(key, existingValue + delim + value);
|
||||
} else {
|
||||
env.put(key, existingValue + value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void handleServiceEnvs(Service service, RunJobParameters parameters) {
|
||||
if (parameters.getEnvars() != null) {
|
||||
for (String envarPair : parameters.getEnvars()) {
|
||||
String key, value;
|
||||
if (envarPair.contains("=")) {
|
||||
int idx = envarPair.indexOf('=');
|
||||
key = envarPair.substring(0, idx);
|
||||
value = envarPair.substring(idx + 1);
|
||||
} else{
|
||||
// No "=" found so use the whole key
|
||||
key = envarPair;
|
||||
value = "";
|
||||
}
|
||||
appendToEnv(service, key, value, ":");
|
||||
}
|
||||
}
|
||||
|
||||
// Append other configs like /etc/passwd, /etc/krb5.conf
|
||||
appendToEnv(service, "YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS",
|
||||
"/etc/passwd:/etc/passwd:ro", ",");
|
||||
|
||||
String authenication = clientContext.getYarnConfig().get(
|
||||
HADOOP_SECURITY_AUTHENTICATION);
|
||||
if (authenication != null && authenication.equals("kerberos")) {
|
||||
appendToEnv(service, "YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS",
|
||||
"/etc/krb5.conf:/etc/krb5.conf:ro", ",");
|
||||
}
|
||||
}
|
||||
|
||||
private Artifact getDockerArtifact(String dockerImageName) {
|
||||
return new Artifact().type(Artifact.TypeEnum.DOCKER).id(dockerImageName);
|
||||
}
|
||||
|
||||
private Service createServiceByParameters(RunJobParameters parameters)
|
||||
throws IOException {
|
||||
Service service = new Service();
|
||||
service.setName(parameters.getName());
|
||||
service.setVersion(String.valueOf(System.currentTimeMillis()));
|
||||
service.setArtifact(getDockerArtifact(parameters.getDockerImageName()));
|
||||
|
||||
handleServiceEnvs(service, parameters);
|
||||
|
||||
addWorkerComponents(service, parameters);
|
||||
|
||||
if (parameters.getNumPS() > 0) {
|
||||
Component psComponent = new Component();
|
||||
psComponent.setName(TaskType.PS.getComponentName());
|
||||
addCommonEnvironments(psComponent, TaskType.PS);
|
||||
psComponent.setNumberOfContainers((long) parameters.getNumPS());
|
||||
psComponent.setRestartPolicy(Component.RestartPolicyEnum.NEVER);
|
||||
psComponent.setResource(
|
||||
getServiceResourceFromYarnResource(parameters.getPsResource()));
|
||||
|
||||
// Override global docker image if needed.
|
||||
if (parameters.getPsDockerImage() != null) {
|
||||
psComponent.setArtifact(
|
||||
getDockerArtifact(parameters.getPsDockerImage()));
|
||||
}
|
||||
handleLaunchCommand(parameters, TaskType.PS, psComponent);
|
||||
service.addComponent(psComponent);
|
||||
}
|
||||
return service;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public ApplicationId submitJob(RunJobParameters parameters)
|
||||
throws IOException, YarnException {
|
||||
Service service = createServiceByParameters(parameters);
|
||||
ServiceClient serviceClient = YarnServiceUtils.createServiceClient(
|
||||
clientContext.getYarnConfig());
|
||||
ApplicationId appid = serviceClient.actionCreate(service);
|
||||
serviceClient.stop();
|
||||
this.serviceSpec = service;
|
||||
return appid;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public Service getServiceSpec() {
|
||||
return serviceSpec;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
|
||||
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.RuntimeFactory;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.FSBasedSubmarineStorageImpl;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.JobMonitor;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.JobSubmitter;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.SubmarineStorage;
|
||||
|
||||
public class YarnServiceRuntimeFactory extends RuntimeFactory {
|
||||
|
||||
public YarnServiceRuntimeFactory(ClientContext clientContext) {
|
||||
super(clientContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected JobSubmitter internalCreateJobSubmitter() {
|
||||
return new YarnServiceJobSubmitter(super.clientContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected JobMonitor internalCreateJobMonitor() {
|
||||
return new YarnServiceJobMonitor(super.clientContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SubmarineStorage internalCreateSubmarineStorage() {
|
||||
return new FSBasedSubmarineStorageImpl(super.clientContext);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.service.client.ServiceClient;
|
||||
import org.apache.hadoop.yarn.submarine.common.Envs;
|
||||
|
||||
public class YarnServiceUtils {
|
||||
// This will be true only in UT.
|
||||
private static ServiceClient stubServiceClient = null;
|
||||
|
||||
public static ServiceClient createServiceClient(
|
||||
Configuration yarnConfiguration) {
|
||||
if (stubServiceClient != null) {
|
||||
return stubServiceClient;
|
||||
}
|
||||
|
||||
ServiceClient serviceClient = new ServiceClient();
|
||||
serviceClient.init(yarnConfiguration);
|
||||
serviceClient.start();
|
||||
return serviceClient;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static void setStubServiceClient(ServiceClient stubServiceClient) {
|
||||
YarnServiceUtils.stubServiceClient = stubServiceClient;
|
||||
}
|
||||
|
||||
public static String getTFConfigEnv(String curCommponentName, int nWorkers,
|
||||
int nPs, String serviceName, String userName, String domain) {
|
||||
String commonEndpointSuffix =
|
||||
"." + serviceName + "." + userName + "." + domain + ":8000";
|
||||
|
||||
String json = "{\\\"cluster\\\":{";
|
||||
|
||||
String master = getComponentArrayJson("master", 1, commonEndpointSuffix)
|
||||
+ ",";
|
||||
String worker = getComponentArrayJson("worker", nWorkers - 1,
|
||||
commonEndpointSuffix) + ",";
|
||||
String ps = getComponentArrayJson("ps", nPs, commonEndpointSuffix) + "},";
|
||||
|
||||
String task =
|
||||
"\\\"task\\\":{" + " \\\"type\\\":\\\"" + curCommponentName + "\\\","
|
||||
+ " \\\"index\\\":" + '$' + Envs.TASK_INDEX_ENV + "},";
|
||||
String environment = "\\\"environment\\\":\\\"cloud\\\"}";
|
||||
|
||||
return json + master + worker + ps + task + environment;
|
||||
}
|
||||
|
||||
private static String getComponentArrayJson(String componentName, int count,
|
||||
String endpointSuffix) {
|
||||
String component = "\\\"" + componentName + "\\\":";
|
||||
String array = "[";
|
||||
for (int i = 0; i < count; i++) {
|
||||
array = array + "\\\"" + componentName + "-" + i
|
||||
+ endpointSuffix + "\\\"";
|
||||
if (i != count - 1) {
|
||||
array = array + ",";
|
||||
}
|
||||
}
|
||||
array = array + "]";
|
||||
return component + array;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
<!---
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
|
||||
# Developper Guide
|
||||
|
||||
(Need add more details)
|
||||
|
||||
By default, submarine uses YARN service framework as runtime. If you want to add your own implementation. You can add a new `RuntimeFactory` implementation and configure following option to `submarine.xml` (which should be placed under same `$HADOOP_CONF_DIR`)
|
||||
|
||||
```
|
||||
<property>
|
||||
<name>submarine.runtime.class</name>
|
||||
<value>... full qualified class name for your runtime factory ... </value>
|
||||
</property>
|
||||
```
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
<!---
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
|
||||
# Quick Start Guide
|
||||
|
||||
## Prerequisite
|
||||
|
||||
Must:
|
||||
- Apache Hadoop 3.1.0, YARN service enabled.
|
||||
|
||||
Optional:
|
||||
- Enable YARN DNS. (When distributed training required.)
|
||||
- Enable GPU on YARN support. (When GPU-based training required.)
|
||||
|
||||
## Run jobs
|
||||
|
||||
### Commandline options
|
||||
|
||||
```$xslt
|
||||
usage: job run
|
||||
-checkpoint_path <arg> Training output directory of the job, could
|
||||
be local or other FS directory. This
|
||||
typically includes checkpoint files and
|
||||
exported model
|
||||
-docker_image <arg> Docker image name/tag
|
||||
-env <arg> Common environment variable of worker/ps
|
||||
-input_path <arg> Input of the job, could be local or other FS
|
||||
directory
|
||||
-name <arg> Name of the job
|
||||
-num_ps <arg> Number of PS tasks of the job, by default
|
||||
it's 0
|
||||
-num_workers <arg> Numnber of worker tasks of the job, by
|
||||
default it's 1
|
||||
-ps_docker_image <arg> Specify docker image for PS, when this is
|
||||
not specified, PS uses --docker_image as
|
||||
default.
|
||||
-ps_launch_cmd <arg> Commandline of worker, arguments will be
|
||||
directly used to launch the PS
|
||||
-ps_resources <arg> Resource of each PS, for example
|
||||
memory-mb=2048,vcores=2,yarn.io/gpu=2
|
||||
-queue <arg> Name of queue to run the job, by default it
|
||||
uses default queue
|
||||
-saved_model_path <arg> Model exported path (savedmodel) of the job,
|
||||
which is needed when exported model is not
|
||||
placed under ${checkpoint_path}could be
|
||||
local or other FS directory. This will be
|
||||
used to serve.
|
||||
-tensorboard <arg> Should we run TensorBoard for this job? By
|
||||
default it's true
|
||||
-verbose Print verbose log for troubleshooting
|
||||
-wait_job_finish Specified when user want to wait the job
|
||||
finish
|
||||
-worker_docker_image <arg> Specify docker image for WORKER, when this
|
||||
is not specified, WORKER uses --docker_image
|
||||
as default.
|
||||
-worker_launch_cmd <arg> Commandline of worker, arguments will be
|
||||
directly used to launch the worker
|
||||
-worker_resources <arg> Resource of each worker, for example
|
||||
memory-mb=2048,vcores=2,yarn.io/gpu=2
|
||||
```
|
||||
|
||||
### Launch Standalone Tensorflow Application:
|
||||
|
||||
#### Commandline
|
||||
```
|
||||
yarn jar path-to/hadoop-yarn-applications-submarine-3.2.0-SNAPSHOT.jar job run \
|
||||
--env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/ \
|
||||
--env DOCKER_HADOOP_HDFS_HOME=/hadoop-3.1.0 --name tf-job-001 \
|
||||
--docker_image <your-docker-image> \
|
||||
--input_path hdfs://default/dataset/cifar-10-data \
|
||||
--checkpoint_path hdfs://default/tmp/cifar-10-jobdir \
|
||||
--worker_resources memory=4G,vcores=2,gpu=2 \
|
||||
--worker_launch_cmd "python ... (Your training application cmd)"
|
||||
```
|
||||
|
||||
#### Notes:
|
||||
|
||||
1) `DOCKER_JAVA_HOME` points to JAVA_HOME inside Docker image.
|
||||
2) `DOCKER_HADOOP_HDFS_HOME` points to HADOOP_HDFS_HOME inside Docker image.
|
||||
3) `--worker_resources` can include gpu when you need GPU to train your task.
|
||||
|
||||
### Launch Distributed Tensorflow Application:
|
||||
|
||||
#### Commandline
|
||||
|
||||
```
|
||||
yarn jar hadoop-yarn-applications-submarine-<version>.jar job run \
|
||||
--name tf-job-001 --docker_image <your docker image> \
|
||||
--input_path hdfs://default/dataset/cifar-10-data \
|
||||
--checkpoint_path hdfs://default/tmp/cifar-10-jobdir \
|
||||
--env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/ \
|
||||
--env DOCKER_HADOOP_HDFS_HOME=/hadoop-3.1.0 \
|
||||
--num_workers 2 \
|
||||
--worker_resources memory=8G,vcores=2,gpu=1 --worker_launch_cmd "cmd for worker ..." \
|
||||
--num_ps 2 \
|
||||
--ps_resources memory=4G,vcores=2,gpu=0 --ps_launch_cmd "cmd for ps" \
|
||||
```
|
||||
|
||||
#### Notes:
|
||||
|
||||
1) Very similar to standalone TF application, but you need to specify #worker/#ps
|
||||
2) Different resources can be specified for worker and PS.
|
||||
3) `TF_CONFIG` environment will be auto generated and set before executing user's launch command.
|
||||
|
||||
## Run jobs
|
||||
|
||||
### Get Job Status
|
||||
|
||||
```
|
||||
yarn jar hadoop-yarn-applications-submarine-3.2.0-SNAPSHOT.jar job show --name tf-job-001
|
||||
```
|
||||
|
||||
Output looks like:
|
||||
```
|
||||
Job Meta Info:
|
||||
Application Id: application_1532131617202_0005
|
||||
Input Path: hdfs://default/dataset/cifar-10-data
|
||||
Checkpoint Path: hdfs://default/tmp/cifar-10-jobdir
|
||||
Run Parameters: --name tf-job-001 --docker_image wtan/tf-1.8.0-gpu:0.0.3
|
||||
(... all your commandline before run the job)
|
||||
```
|
||||
|
||||
After that, you can run ```tensorboard --logdir=<checkpoint-path>``` to view Tensorboard of the job.
|
||||
|
|
@ -0,0 +1,229 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.param.RunJobParameters;
|
||||
import org.apache.hadoop.yarn.submarine.common.MockClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.conf.SubmarineLogs;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.RuntimeFactory;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.JobMonitor;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.JobSubmitter;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.SubmarineStorage;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class TestRunJobCliParsing {
|
||||
@Before
|
||||
public void before() {
|
||||
SubmarineLogs.verboseOff();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPrintHelp() {
|
||||
MockClientContext mockClientContext = new MockClientContext();
|
||||
JobSubmitter mockJobSubmitter = mock(JobSubmitter.class);
|
||||
JobMonitor mockJobMonitor = mock(JobMonitor.class);
|
||||
RunJobCli runJobCli = new RunJobCli(mockClientContext, mockJobSubmitter,
|
||||
mockJobMonitor);
|
||||
runJobCli.printUsages();
|
||||
}
|
||||
|
||||
private MockClientContext getMockClientContext()
|
||||
throws IOException, YarnException {
|
||||
MockClientContext mockClientContext = new MockClientContext();
|
||||
JobSubmitter mockJobSubmitter = mock(JobSubmitter.class);
|
||||
when(mockJobSubmitter.submitJob(any(RunJobParameters.class))).thenReturn(
|
||||
ApplicationId.newInstance(1234L, 1));
|
||||
JobMonitor mockJobMonitor = mock(JobMonitor.class);
|
||||
SubmarineStorage storage = mock(SubmarineStorage.class);
|
||||
RuntimeFactory rtFactory = mock(RuntimeFactory.class);
|
||||
|
||||
when(rtFactory.getJobSubmitterInstance()).thenReturn(mockJobSubmitter);
|
||||
when(rtFactory.getJobMonitorInstance()).thenReturn(mockJobMonitor);
|
||||
when(rtFactory.getSubmarineStorage()).thenReturn(storage);
|
||||
|
||||
mockClientContext.setRuntimeFactory(rtFactory);
|
||||
return mockClientContext;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBasicRunJobForDistributedTraining() throws Exception {
|
||||
RunJobCli runJobCli = new RunJobCli(getMockClientContext());
|
||||
|
||||
Assert.assertFalse(SubmarineLogs.isVerbose());
|
||||
|
||||
runJobCli.run(
|
||||
new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
|
||||
"--input_path", "hdfs://input", "--checkpoint_path", "hdfs://output",
|
||||
"--num_workers", "3", "--num_ps", "2", "--worker_launch_cmd",
|
||||
"python run-job.py", "--worker_resources", "memory=2048M,vcores=2",
|
||||
"--ps_resources", "memory=4G,vcores=4", "--tensorboard", "true",
|
||||
"--ps_launch_cmd", "python run-ps.py", "--verbose" });
|
||||
|
||||
RunJobParameters jobRunParameters = runJobCli.getRunJobParameters();
|
||||
|
||||
Assert.assertEquals(jobRunParameters.getInputPath(), "hdfs://input");
|
||||
Assert.assertEquals(jobRunParameters.getCheckpointPath(), "hdfs://output");
|
||||
Assert.assertEquals(jobRunParameters.getNumPS(), 2);
|
||||
Assert.assertEquals(jobRunParameters.getPSLaunchCmd(), "python run-ps.py");
|
||||
Assert.assertEquals(Resources.createResource(4096, 4),
|
||||
jobRunParameters.getPsResource());
|
||||
Assert.assertEquals(jobRunParameters.getWorkerLaunchCmd(),
|
||||
"python run-job.py");
|
||||
Assert.assertEquals(Resources.createResource(2048, 2),
|
||||
jobRunParameters.getWorkerResource());
|
||||
Assert.assertEquals(jobRunParameters.getDockerImageName(),
|
||||
"tf-docker:1.1.0");
|
||||
Assert.assertTrue(SubmarineLogs.isVerbose());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBasicRunJobForSingleNodeTraining() throws Exception {
|
||||
RunJobCli runJobCli = new RunJobCli(getMockClientContext());
|
||||
Assert.assertFalse(SubmarineLogs.isVerbose());
|
||||
|
||||
runJobCli.run(
|
||||
new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
|
||||
"--input_path", "hdfs://input", "--checkpoint_path", "hdfs://output",
|
||||
"--num_workers", "1", "--worker_launch_cmd", "python run-job.py",
|
||||
"--worker_resources", "memory=4g,vcores=2", "--tensorboard",
|
||||
"true", "--verbose", "--wait_job_finish" });
|
||||
|
||||
RunJobParameters jobRunParameters = runJobCli.getRunJobParameters();
|
||||
|
||||
Assert.assertEquals(jobRunParameters.getInputPath(), "hdfs://input");
|
||||
Assert.assertEquals(jobRunParameters.getCheckpointPath(), "hdfs://output");
|
||||
Assert.assertEquals(jobRunParameters.getNumWorkers(), 1);
|
||||
Assert.assertEquals(jobRunParameters.getWorkerLaunchCmd(),
|
||||
"python run-job.py");
|
||||
Assert.assertEquals(Resources.createResource(4096, 2),
|
||||
jobRunParameters.getWorkerResource());
|
||||
Assert.assertTrue(SubmarineLogs.isVerbose());
|
||||
Assert.assertTrue(jobRunParameters.isWaitJobFinish());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLaunchCommandPatternReplace() throws Exception {
|
||||
RunJobCli runJobCli = new RunJobCli(getMockClientContext());
|
||||
Assert.assertFalse(SubmarineLogs.isVerbose());
|
||||
|
||||
runJobCli.run(
|
||||
new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
|
||||
"--input_path", "hdfs://input", "--checkpoint_path", "hdfs://output",
|
||||
"--num_workers", "3", "--num_ps", "2", "--worker_launch_cmd",
|
||||
"python run-job.py --input=%input_path% --model_dir=%checkpoint_path% --export_dir=%saved_model_path%/savedmodel",
|
||||
"--worker_resources", "memory=2048,vcores=2", "--ps_resources",
|
||||
"memory=4096,vcores=4", "--tensorboard", "true", "--ps_launch_cmd",
|
||||
"python run-ps.py --input=%input_path% --model_dir=%checkpoint_path%/model",
|
||||
"--verbose" });
|
||||
|
||||
Assert.assertEquals(
|
||||
"python run-job.py --input=hdfs://input --model_dir=hdfs://output "
|
||||
+ "--export_dir=hdfs://output/savedmodel",
|
||||
runJobCli.getRunJobParameters().getWorkerLaunchCmd());
|
||||
Assert.assertEquals(
|
||||
"python run-ps.py --input=hdfs://input --model_dir=hdfs://output/model",
|
||||
runJobCli.getRunJobParameters().getPSLaunchCmd());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testResourceUnitParsing() throws Exception {
|
||||
Resource res = CliUtils.createResourceFromString("memory=20g,vcores=3",
|
||||
ResourceUtils.getResourcesTypeInfo());
|
||||
Assert.assertEquals(Resources.createResource(20 * 1024, 3), res);
|
||||
|
||||
res = CliUtils.createResourceFromString("memory=20G,vcores=3",
|
||||
ResourceUtils.getResourcesTypeInfo());
|
||||
Assert.assertEquals(Resources.createResource(20 * 1024, 3), res);
|
||||
|
||||
res = CliUtils.createResourceFromString("memory=20M,vcores=3",
|
||||
ResourceUtils.getResourcesTypeInfo());
|
||||
Assert.assertEquals(Resources.createResource(20, 3), res);
|
||||
|
||||
res = CliUtils.createResourceFromString("memory=20m,vcores=3",
|
||||
ResourceUtils.getResourcesTypeInfo());
|
||||
Assert.assertEquals(Resources.createResource(20, 3), res);
|
||||
|
||||
res = CliUtils.createResourceFromString("memory-mb=20,vcores=3",
|
||||
ResourceUtils.getResourcesTypeInfo());
|
||||
Assert.assertEquals(Resources.createResource(20, 3), res);
|
||||
|
||||
res = CliUtils.createResourceFromString("memory-mb=20m,vcores=3",
|
||||
ResourceUtils.getResourcesTypeInfo());
|
||||
Assert.assertEquals(Resources.createResource(20, 3), res);
|
||||
|
||||
res = CliUtils.createResourceFromString("memory-mb=20G,vcores=3",
|
||||
ResourceUtils.getResourcesTypeInfo());
|
||||
Assert.assertEquals(Resources.createResource(20 * 1024, 3), res);
|
||||
|
||||
// W/o unit for memory means bits, and 20 bits will be rounded to 0
|
||||
res = CliUtils.createResourceFromString("memory=20,vcores=3",
|
||||
ResourceUtils.getResourcesTypeInfo());
|
||||
Assert.assertEquals(Resources.createResource(0, 3), res);
|
||||
|
||||
// Test multiple resources
|
||||
List<ResourceTypeInfo> resTypes = new ArrayList<>(
|
||||
ResourceUtils.getResourcesTypeInfo());
|
||||
resTypes.add(ResourceTypeInfo.newInstance(ResourceInformation.GPU_URI, ""));
|
||||
ResourceUtils.reinitializeResources(resTypes);
|
||||
res = CliUtils.createResourceFromString("memory=2G,vcores=3,gpu=0",
|
||||
resTypes);
|
||||
Assert.assertEquals(2 * 1024, res.getMemorySize());
|
||||
Assert.assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI));
|
||||
|
||||
res = CliUtils.createResourceFromString("memory=2G,vcores=3,gpu=3",
|
||||
resTypes);
|
||||
Assert.assertEquals(2 * 1024, res.getMemorySize());
|
||||
Assert.assertEquals(3, res.getResourceValue(ResourceInformation.GPU_URI));
|
||||
|
||||
res = CliUtils.createResourceFromString("memory=2G,vcores=3",
|
||||
resTypes);
|
||||
Assert.assertEquals(2 * 1024, res.getMemorySize());
|
||||
Assert.assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI));
|
||||
|
||||
res = CliUtils.createResourceFromString("memory=2G,vcores=3,yarn.io/gpu=0",
|
||||
resTypes);
|
||||
Assert.assertEquals(2 * 1024, res.getMemorySize());
|
||||
Assert.assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI));
|
||||
|
||||
res = CliUtils.createResourceFromString("memory=2G,vcores=3,yarn.io/gpu=3",
|
||||
resTypes);
|
||||
Assert.assertEquals(2 * 1024, res.getMemorySize());
|
||||
Assert.assertEquals(3, res.getResourceValue(ResourceInformation.GPU_URI));
|
||||
|
||||
// TODO, add more negative tests.
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli;
|
||||
|
||||
import org.apache.commons.cli.ParseException;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.param.ShowJobParameters;
|
||||
import org.apache.hadoop.yarn.submarine.common.MockClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.conf.SubmarineLogs;
|
||||
import org.apache.hadoop.yarn.submarine.common.exception.SubmarineException;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.RuntimeFactory;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.MemorySubmarineStorage;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.StorageKeyConstants;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.SubmarineStorage;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class TestShowJobCliParsing {
|
||||
@Before
|
||||
public void before() {
|
||||
SubmarineLogs.verboseOff();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPrintHelp() {
|
||||
MockClientContext mockClientContext = new MockClientContext();
|
||||
ShowJobCli showJobCli = new ShowJobCli(mockClientContext);
|
||||
showJobCli.printUsages();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShowJob()
|
||||
throws InterruptedException, SubmarineException, YarnException,
|
||||
ParseException, IOException {
|
||||
MockClientContext mockClientContext = new MockClientContext();
|
||||
ShowJobCli showJobCli = new ShowJobCli(mockClientContext) {
|
||||
@Override
|
||||
protected void getAndPrintJobInfo() {
|
||||
// do nothing
|
||||
}
|
||||
};
|
||||
showJobCli.run(new String[] { "--name", "my-job" });
|
||||
ShowJobParameters parameters = showJobCli.getParameters();
|
||||
Assert.assertEquals(parameters.getName(), "my-job");
|
||||
}
|
||||
|
||||
private Map<String, String> getMockJobInfo(String jobName) {
|
||||
Map<String, String> map = new HashMap<>();
|
||||
map.put(StorageKeyConstants.APPLICATION_ID,
|
||||
ApplicationId.newInstance(1234L, 1).toString());
|
||||
map.put(StorageKeyConstants.JOB_RUN_ARGS, "job run 123456");
|
||||
map.put(StorageKeyConstants.INPUT_PATH, "hdfs://" + jobName);
|
||||
return map;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSimpleShowJob()
|
||||
throws InterruptedException, SubmarineException, YarnException,
|
||||
ParseException, IOException {
|
||||
SubmarineStorage storage = new MemorySubmarineStorage();
|
||||
MockClientContext mockClientContext = new MockClientContext();
|
||||
RuntimeFactory runtimeFactory = mock(RuntimeFactory.class);
|
||||
when(runtimeFactory.getSubmarineStorage()).thenReturn(storage);
|
||||
mockClientContext.setRuntimeFactory(runtimeFactory);
|
||||
|
||||
ShowJobCli showJobCli = new ShowJobCli(mockClientContext);
|
||||
|
||||
try {
|
||||
showJobCli.run(new String[] { "--name", "my-job" });
|
||||
} catch (IOException e) {
|
||||
// expected
|
||||
}
|
||||
|
||||
|
||||
storage.addNewJob("my-job", getMockJobInfo("my-job"));
|
||||
showJobCli.run(new String[] { "--name", "my-job" });
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,167 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli.yarnservice;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.service.api.records.Component;
|
||||
import org.apache.hadoop.yarn.service.api.records.Service;
|
||||
import org.apache.hadoop.yarn.service.client.ServiceClient;
|
||||
import org.apache.hadoop.yarn.submarine.client.cli.RunJobCli;
|
||||
import org.apache.hadoop.yarn.submarine.common.MockClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.api.TaskType;
|
||||
import org.apache.hadoop.yarn.submarine.common.conf.SubmarineLogs;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.JobSubmitter;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.StorageKeyConstants;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.SubmarineStorage;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.yarnservice.YarnServiceJobSubmitter;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.yarnservice.YarnServiceUtils;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class TestYarnServiceRunJobCli {
|
||||
@Before
|
||||
public void before() throws IOException, YarnException {
|
||||
SubmarineLogs.verboseOff();
|
||||
ServiceClient serviceClient = mock(ServiceClient.class);
|
||||
when(serviceClient.actionCreate(any(Service.class))).thenReturn(
|
||||
ApplicationId.newInstance(1234L, 1));
|
||||
YarnServiceUtils.setStubServiceClient(serviceClient);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPrintHelp() {
|
||||
MockClientContext mockClientContext =
|
||||
YarnServiceCliTestUtils.getMockClientContext();
|
||||
RunJobCli runJobCli = new RunJobCli(mockClientContext);
|
||||
runJobCli.printUsages();
|
||||
}
|
||||
|
||||
private Service getServiceSpecFromJobSubmitter(JobSubmitter jobSubmitter) {
|
||||
return ((YarnServiceJobSubmitter) jobSubmitter).getServiceSpec();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBasicRunJobForDistributedTraining() throws Exception {
|
||||
MockClientContext mockClientContext =
|
||||
YarnServiceCliTestUtils.getMockClientContext();
|
||||
RunJobCli runJobCli = new RunJobCli(mockClientContext);
|
||||
Assert.assertFalse(SubmarineLogs.isVerbose());
|
||||
|
||||
runJobCli.run(
|
||||
new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
|
||||
"--input_path", "s3://input", "--checkpoint_path",
|
||||
"s3://output", "--num_workers", "3", "--num_ps", "2",
|
||||
"--worker_launch_cmd", "python run-job.py", "--worker_resources",
|
||||
"memory=2048M,vcores=2", "--ps_resources", "memory=4096M,vcores=4",
|
||||
"--tensorboard", "true", "--ps_docker_image", "ps.image",
|
||||
"--worker_docker_image", "worker.image",
|
||||
"--ps_launch_cmd", "python run-ps.py", "--verbose" });
|
||||
Service serviceSpec = getServiceSpecFromJobSubmitter(
|
||||
runJobCli.getJobSubmitter());
|
||||
Assert.assertEquals(3, serviceSpec.getComponents().size());
|
||||
Assert.assertTrue(
|
||||
serviceSpec.getComponent(TaskType.WORKER.getComponentName()) != null);
|
||||
Assert.assertTrue(
|
||||
serviceSpec.getComponent(TaskType.PRIMARY_WORKER.getComponentName())
|
||||
!= null);
|
||||
Assert.assertTrue(
|
||||
serviceSpec.getComponent(TaskType.PS.getComponentName()) != null);
|
||||
Component primaryWorkerComp = serviceSpec.getComponent(
|
||||
TaskType.PRIMARY_WORKER.getComponentName());
|
||||
Assert.assertEquals(2048, primaryWorkerComp.getResource().calcMemoryMB());
|
||||
Assert.assertEquals(2,
|
||||
primaryWorkerComp.getResource().getCpus().intValue());
|
||||
|
||||
Component workerComp = serviceSpec.getComponent(
|
||||
TaskType.WORKER.getComponentName());
|
||||
Assert.assertEquals(2048, workerComp.getResource().calcMemoryMB());
|
||||
Assert.assertEquals(2, workerComp.getResource().getCpus().intValue());
|
||||
|
||||
Component psComp = serviceSpec.getComponent(TaskType.PS.getComponentName());
|
||||
Assert.assertEquals(4096, psComp.getResource().calcMemoryMB());
|
||||
Assert.assertEquals(4, psComp.getResource().getCpus().intValue());
|
||||
|
||||
Assert.assertEquals("worker.image", workerComp.getArtifact().getId());
|
||||
Assert.assertEquals("ps.image", psComp.getArtifact().getId());
|
||||
|
||||
Assert.assertTrue(SubmarineLogs.isVerbose());
|
||||
|
||||
// TODO, ADD TEST TO USE SERVICE CLIENT TO VALIDATE THE JSON SPEC
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBasicRunJobForSingleNodeTraining() throws Exception {
|
||||
MockClientContext mockClientContext =
|
||||
YarnServiceCliTestUtils.getMockClientContext();
|
||||
RunJobCli runJobCli = new RunJobCli(mockClientContext);
|
||||
Assert.assertFalse(SubmarineLogs.isVerbose());
|
||||
|
||||
runJobCli.run(
|
||||
new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
|
||||
"--input_path", "s3://input", "--checkpoint_path",
|
||||
"s3://output", "--num_workers", "1", "--worker_launch_cmd",
|
||||
"python run-job.py", "--worker_resources", "memory=2G,vcores=2",
|
||||
"--tensorboard", "true", "--verbose" });
|
||||
Service serviceSpec = getServiceSpecFromJobSubmitter(
|
||||
runJobCli.getJobSubmitter());
|
||||
Assert.assertEquals(1, serviceSpec.getComponents().size());
|
||||
Assert.assertTrue(
|
||||
serviceSpec.getComponent(TaskType.PRIMARY_WORKER.getComponentName())
|
||||
!= null);
|
||||
Component primaryWorkerComp = serviceSpec.getComponent(
|
||||
TaskType.PRIMARY_WORKER.getComponentName());
|
||||
Assert.assertEquals(2048, primaryWorkerComp.getResource().calcMemoryMB());
|
||||
Assert.assertEquals(2,
|
||||
primaryWorkerComp.getResource().getCpus().intValue());
|
||||
|
||||
Assert.assertTrue(SubmarineLogs.isVerbose());
|
||||
|
||||
// TODO, ADD TEST TO USE SERVICE CLIENT TO VALIDATE THE JSON SPEC
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testParameterStorageForTrainingJob() throws Exception {
|
||||
MockClientContext mockClientContext =
|
||||
YarnServiceCliTestUtils.getMockClientContext();
|
||||
RunJobCli runJobCli = new RunJobCli(mockClientContext);
|
||||
Assert.assertFalse(SubmarineLogs.isVerbose());
|
||||
|
||||
runJobCli.run(
|
||||
new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
|
||||
"--input_path", "s3://input", "--checkpoint_path",
|
||||
"s3://output", "--num_workers", "1", "--worker_launch_cmd",
|
||||
"python run-job.py", "--worker_resources", "memory=2G,vcores=2",
|
||||
"--tensorboard", "true", "--verbose" });
|
||||
SubmarineStorage storage =
|
||||
mockClientContext.getRuntimeFactory().getSubmarineStorage();
|
||||
Map<String, String> jobInfo = storage.getJobInfoByName("my-job");
|
||||
Assert.assertTrue(jobInfo.size() > 0);
|
||||
Assert.assertEquals(jobInfo.get(StorageKeyConstants.INPUT_PATH),
|
||||
"s3://input");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.client.cli.yarnservice;
|
||||
|
||||
import org.apache.hadoop.yarn.submarine.common.MockClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.RuntimeFactory;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.common.MemorySubmarineStorage;
|
||||
import org.apache.hadoop.yarn.submarine.runtimes.yarnservice.YarnServiceRuntimeFactory;
|
||||
|
||||
public class YarnServiceCliTestUtils {
|
||||
public static MockClientContext getMockClientContext() {
|
||||
MockClientContext mockClientContext = new MockClientContext();
|
||||
RuntimeFactory runtimeFactory = new YarnServiceRuntimeFactory(
|
||||
mockClientContext);
|
||||
mockClientContext.setRuntimeFactory(runtimeFactory);
|
||||
runtimeFactory.setSubmarineStorage(new MemorySubmarineStorage());
|
||||
return mockClientContext;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common;
|
||||
|
||||
import org.apache.hadoop.yarn.submarine.common.fs.MockRemoteDirectoryManager;
|
||||
import org.apache.hadoop.yarn.submarine.common.fs.RemoteDirectoryManager;
|
||||
import org.apache.hadoop.yarn.client.api.YarnClient;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.service.client.ServiceClient;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class MockClientContext extends ClientContext {
|
||||
private MockRemoteDirectoryManager remoteDirectoryMgr =
|
||||
new MockRemoteDirectoryManager();
|
||||
|
||||
@Override
|
||||
public RemoteDirectoryManager getRemoteDirectoryManager() {
|
||||
return remoteDirectoryMgr;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized YarnClient getOrCreateYarnClient() {
|
||||
YarnClient client = mock(YarnClient.class);
|
||||
try {
|
||||
when(client.getResourceTypeInfo()).thenReturn(
|
||||
ResourceUtils.getResourcesTypeInfo());
|
||||
} catch (YarnException e) {
|
||||
fail(e.getMessage());
|
||||
} catch (IOException e) {
|
||||
fail(e.getMessage());
|
||||
}
|
||||
return client;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.common.fs;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
public class MockRemoteDirectoryManager implements RemoteDirectoryManager {
|
||||
private File jobsParentDir = null;
|
||||
private File modelParentDir = null;
|
||||
|
||||
@Override
|
||||
public Path getJobStagingArea(String jobName, boolean create)
|
||||
throws IOException {
|
||||
if (jobsParentDir == null && create) {
|
||||
jobsParentDir = new File(
|
||||
"target/_staging_area_" + System.currentTimeMillis());
|
||||
if (!jobsParentDir.mkdirs()) {
|
||||
throw new IOException(
|
||||
"Failed to mkdirs for" + jobsParentDir.getAbsolutePath());
|
||||
}
|
||||
}
|
||||
|
||||
File jobDir = new File(jobsParentDir.getAbsolutePath(), jobName);
|
||||
if (create && !jobDir.exists()) {
|
||||
if (!jobDir.mkdirs()) {
|
||||
throw new IOException("Failed to mkdirs for " + jobDir.getAbsolutePath());
|
||||
}
|
||||
}
|
||||
return new Path(jobDir.getAbsolutePath());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getJobCheckpointDir(String jobName, boolean create)
|
||||
throws IOException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getModelDir(String modelName, boolean create) throws IOException {
|
||||
if (modelParentDir == null && create) {
|
||||
modelParentDir = new File(
|
||||
"target/_models_" + System.currentTimeMillis());
|
||||
if (!modelParentDir.mkdirs()) {
|
||||
throw new IOException(
|
||||
"Failed to mkdirs for " + modelParentDir.getAbsolutePath());
|
||||
}
|
||||
}
|
||||
|
||||
File modelDir = new File(modelParentDir.getAbsolutePath(), modelName);
|
||||
if (create) {
|
||||
if (!modelDir.exists() && !modelDir.mkdirs()) {
|
||||
throw new IOException("Failed to mkdirs for " + modelDir.getAbsolutePath());
|
||||
}
|
||||
}
|
||||
return new Path(modelDir.getAbsolutePath());
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileSystem getFileSystem() throws IOException {
|
||||
return FileSystem.getLocal(new Configuration());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.common;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class MemorySubmarineStorage extends SubmarineStorage {
|
||||
private Map<String, Map<String, String>> jobsInfo = new HashMap<>();
|
||||
private Map<String, Map<String, Map<String, String>>> modelsInfo =
|
||||
new HashMap<>();
|
||||
|
||||
@Override
|
||||
public synchronized void addNewJob(String jobName, Map<String, String> jobInfo)
|
||||
throws IOException {
|
||||
jobsInfo.put(jobName, jobInfo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized Map<String, String> getJobInfoByName(String jobName)
|
||||
throws IOException {
|
||||
Map<String, String> info = jobsInfo.get(jobName);
|
||||
if (info == null) {
|
||||
throw new IOException("Failed to find job=" + jobName);
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void addNewModel(String modelName, String version,
|
||||
Map<String, String> modelInfo) throws IOException {
|
||||
if (!modelsInfo.containsKey(modelName)) {
|
||||
modelsInfo.put(modelName, new HashMap<>());
|
||||
}
|
||||
modelsInfo.get(modelName).put(version, modelInfo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized Map<String, String> getModelInfoByName(String modelName,
|
||||
String version) throws IOException {
|
||||
|
||||
boolean notFound = false;
|
||||
Map<String, String> info = null;
|
||||
try {
|
||||
info = modelsInfo.get(modelName).get(version);
|
||||
} catch (NullPointerException e) {
|
||||
notFound = true;
|
||||
}
|
||||
|
||||
if (notFound || info == null) {
|
||||
throw new IOException(
|
||||
"Failed to find, model=" + modelName + " version=" + version);
|
||||
}
|
||||
|
||||
return info;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.common;
|
||||
|
||||
import org.apache.hadoop.yarn.submarine.common.ClientContext;
|
||||
import org.apache.hadoop.yarn.submarine.common.fs.MockRemoteDirectoryManager;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class TestFSBasedSubmarineStorage {
|
||||
private Map<String, String> getMap(String prefix) {
|
||||
Map<String, String> map = new HashMap<>();
|
||||
map.put(prefix + "1", "1");
|
||||
map.put(prefix + "2", "2");
|
||||
map.put(prefix + "3", "3");
|
||||
map.put(prefix + "4", "4");
|
||||
return map;
|
||||
}
|
||||
|
||||
private void compareMap(Map<String, String> map1, Map<String, String> map2) {
|
||||
Assert.assertEquals(map1.size(), map2.size());
|
||||
for (String k : map1.keySet()) {
|
||||
Assert.assertEquals(map1.get(k), map2.get(k));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStorageOps() throws IOException {
|
||||
MockRemoteDirectoryManager remoteDirectoryManager = new MockRemoteDirectoryManager();
|
||||
ClientContext clientContext = mock(ClientContext.class);
|
||||
when(clientContext.getRemoteDirectoryManager()).thenReturn(remoteDirectoryManager);
|
||||
FSBasedSubmarineStorageImpl storage = new FSBasedSubmarineStorageImpl(
|
||||
clientContext);
|
||||
storage.addNewJob("job1", getMap("job1"));
|
||||
storage.addNewJob("job2", getMap("job2"));
|
||||
storage.addNewJob("job3", getMap("job3"));
|
||||
storage.addNewJob("job4", new HashMap<>());
|
||||
storage.addNewModel("model1", "1.0", getMap("model1_1.0"));
|
||||
storage.addNewModel("model1", "2.0.0", getMap("model1_2.0.0"));
|
||||
storage.addNewModel("model2", null, getMap("model1_default"));
|
||||
storage.addNewModel("model2", "1.0", getMap("model2_1.0"));
|
||||
|
||||
// create a new storage and read it back.
|
||||
storage = new FSBasedSubmarineStorageImpl(
|
||||
clientContext);
|
||||
compareMap(getMap("job1"), storage.getJobInfoByName("job1"));
|
||||
compareMap(getMap("job2"), storage.getJobInfoByName("job2"));
|
||||
compareMap(getMap("job3"), storage.getJobInfoByName("job3"));
|
||||
compareMap(new HashMap<>(), storage.getJobInfoByName("job4"));
|
||||
compareMap(getMap("model1_1.0"), storage.getModelInfoByName("model1", "1.0"));
|
||||
compareMap(getMap("model1_2.0.0"), storage.getModelInfoByName("model1", "2.0.0"));
|
||||
compareMap(getMap("model2_1.0"), storage.getModelInfoByName("model2", "1.0"));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/**
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. See accompanying LICENSE file.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
|
||||
|
||||
import org.codehaus.jettison.json.JSONException;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestTFConfigGenerator {
|
||||
@Test
|
||||
public void testSimpleDistributedTFConfigGenerator() throws JSONException {
|
||||
String json = YarnServiceUtils.getTFConfigEnv("worker", 5, 3, "wtan",
|
||||
"tf-job-001", "example.com");
|
||||
String expected =
|
||||
"{\\\"cluster\\\":{\\\"master\\\":[\\\"master-0.wtan.tf-job-001.example.com:8000\\\"],\\\"worker\\\":[\\\"worker-0.wtan.tf-job-001.example.com:8000\\\",\\\"worker-1.wtan.tf-job-001.example.com:8000\\\",\\\"worker-2.wtan.tf-job-001.example.com:8000\\\",\\\"worker-3.wtan.tf-job-001.example.com:8000\\\"],\\\"ps\\\":[\\\"ps-0.wtan.tf-job-001.example.com:8000\\\",\\\"ps-1.wtan.tf-job-001.example.com:8000\\\",\\\"ps-2.wtan.tf-job-001.example.com:8000\\\"]},\\\"task\\\":{ \\\"type\\\":\\\"worker\\\", \\\"index\\\":$_TASK_INDEX},\\\"environment\\\":\\\"cloud\\\"}";
|
||||
Assert.assertEquals(expected, json);
|
||||
|
||||
json = YarnServiceUtils.getTFConfigEnv("ps", 5, 3, "wtan", "tf-job-001",
|
||||
"example.com");
|
||||
expected =
|
||||
"{\\\"cluster\\\":{\\\"master\\\":[\\\"master-0.wtan.tf-job-001.example.com:8000\\\"],\\\"worker\\\":[\\\"worker-0.wtan.tf-job-001.example.com:8000\\\",\\\"worker-1.wtan.tf-job-001.example.com:8000\\\",\\\"worker-2.wtan.tf-job-001.example.com:8000\\\",\\\"worker-3.wtan.tf-job-001.example.com:8000\\\"],\\\"ps\\\":[\\\"ps-0.wtan.tf-job-001.example.com:8000\\\",\\\"ps-1.wtan.tf-job-001.example.com:8000\\\",\\\"ps-2.wtan.tf-job-001.example.com:8000\\\"]},\\\"task\\\":{ \\\"type\\\":\\\"ps\\\", \\\"index\\\":$_TASK_INDEX},\\\"environment\\\":\\\"cloud\\\"}";
|
||||
Assert.assertEquals(expected, json);
|
||||
|
||||
json = YarnServiceUtils.getTFConfigEnv("master", 2, 1, "wtan", "tf-job-001",
|
||||
"example.com");
|
||||
expected =
|
||||
"{\\\"cluster\\\":{\\\"master\\\":[\\\"master-0.wtan.tf-job-001.example.com:8000\\\"],\\\"worker\\\":[\\\"worker-0.wtan.tf-job-001.example.com:8000\\\"],\\\"ps\\\":[\\\"ps-0.wtan.tf-job-001.example.com:8000\\\"]},\\\"task\\\":{ \\\"type\\\":\\\"master\\\", \\\"index\\\":$_TASK_INDEX},\\\"environment\\\":\\\"cloud\\\"}";
|
||||
Assert.assertEquals(expected, json);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
|
||||
<!-- Put site-specific property overrides in this file. -->
|
||||
|
||||
<configuration>
|
||||
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
|
||||
<!-- Put site-specific property overrides in this file. -->
|
||||
|
||||
<configuration>
|
||||
|
||||
</configuration>
|
||||
|
|
@ -37,6 +37,7 @@
|
|||
<module>hadoop-yarn-applications-distributedshell</module>
|
||||
<module>hadoop-yarn-applications-unmanaged-am-launcher</module>
|
||||
<module>hadoop-yarn-services</module>
|
||||
<module>hadoop-yarn-submarine</module>
|
||||
</modules>
|
||||
|
||||
<profiles>
|
||||
|
|
|
|||
Loading…
Reference in New Issue