YARN-6626. Embed REST API service into RM. Contributed by Eric Yang

This commit is contained in:
Jian He 2017-09-28 16:29:22 -07:00
parent 8851209788
commit 9e677fa05c
9 changed files with 89 additions and 11 deletions

View File

@ -336,6 +336,8 @@ private static void addDeprecatedKeys() {
public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn." public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
+ "webapp.ui2.war-file-path"; + "webapp.ui2.war-file-path";
public static final String YARN_API_SERVICES_ENABLE = "yarn."
+ "webapp.api-service.enable";
public static final String RM_RESOURCE_TRACKER_ADDRESS = public static final String RM_RESOURCE_TRACKER_ADDRESS =
RM_PREFIX + "resource-tracker.address"; RM_PREFIX + "resource-tracker.address";

View File

@ -17,6 +17,7 @@
package org.apache.hadoop.yarn.service.webapp; package org.apache.hadoop.yarn.service.webapp;
import com.google.inject.Inject;
import com.google.inject.Singleton; import com.google.inject.Singleton;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.util.VersionInfo;
@ -57,6 +58,12 @@
@Singleton @Singleton
@Path(CONTEXT_ROOT) @Path(CONTEXT_ROOT)
public class ApiServer { public class ApiServer {
@Inject
public ApiServer(Configuration conf) {
super();
}
private static final Logger LOG = private static final Logger LOG =
LoggerFactory.getLogger(ApiServer.class); LoggerFactory.getLogger(ApiServer.class);
private static Configuration YARN_CONFIG = new YarnConfiguration(); private static Configuration YARN_CONFIG = new YarnConfiguration();

View File

@ -244,12 +244,6 @@
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.apache.curator</groupId> <groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId> <artifactId>curator-test</artifactId>

View File

@ -23,6 +23,9 @@
import java.io.Serializable; import java.io.Serializable;
import java.util.Objects; import java.util.Objects;
import javax.xml.bind.annotation.XmlEnum;
import javax.xml.bind.annotation.XmlType;
import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue; import com.fasterxml.jackson.annotation.JsonValue;
@ -42,6 +45,11 @@ public class Artifact implements Serializable {
private String id = null; private String id = null;
/**
* Artifact Type. DOCKER, TARBALL or SERVICE
**/
@XmlType(name = "artifact_type")
@XmlEnum
public enum TypeEnum { public enum TypeEnum {
DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE"); DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE");

View File

@ -26,7 +26,10 @@
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlEnum;
import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
import java.io.Serializable; import java.io.Serializable;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -45,6 +48,11 @@
public class ConfigFile implements Serializable { public class ConfigFile implements Serializable {
private static final long serialVersionUID = -7009402089417704612L; private static final long serialVersionUID = -7009402089417704612L;
/**
* Config Type. XML, JSON, YAML, TEMPLATE, ENV and HADOOP_XML are supported.
**/
@XmlType(name = "config_type")
@XmlEnum
public enum TypeEnum { public enum TypeEnum {
XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE( XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
"TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),; "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;

View File

@ -25,6 +25,9 @@
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import javax.xml.bind.annotation.XmlEnum;
import javax.xml.bind.annotation.XmlType;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue; import com.fasterxml.jackson.annotation.JsonValue;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
@ -43,6 +46,11 @@
public class ReadinessCheck implements Serializable { public class ReadinessCheck implements Serializable {
private static final long serialVersionUID = -3836839816887186801L; private static final long serialVersionUID = -3836839816887186801L;
/**
* Type. HTTP and PORT
**/
@XmlType(name = "type")
@XmlEnum
public enum TypeEnum { public enum TypeEnum {
HTTP("HTTP"), HTTP("HTTP"),
PORT("PORT"); PORT("PORT");

View File

@ -211,6 +211,14 @@
<value></value> <value></value>
</property> </property>
<property>
<description>
Enable services rest api on ResourceManager.
</description>
<name>yarn.webapp.api-service.enable</name>
<value>false</value>
</property>
<property> <property>
<name>yarn.resourcemanager.resource-tracker.address</name> <name>yarn.resourcemanager.resource-tracker.address</name>
<value>${yarn.resourcemanager.hostname}:8031</value> <value>${yarn.resourcemanager.hostname}:8031</value>

View File

@ -22,6 +22,8 @@
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.RMHAUtils; import org.apache.hadoop.yarn.util.RMHAUtils;
@ -38,8 +40,12 @@
*/ */
public class RMWebApp extends WebApp implements YarnWebParams { public class RMWebApp extends WebApp implements YarnWebParams {
private static final Log LOG =
LogFactory.getLog(RMWebApp.class.getName());
private final ResourceManager rm; private final ResourceManager rm;
private boolean standby = false; private boolean standby = false;
private final static String APISERVER =
"org.apache.hadoop.yarn.service.webapp.ApiServer";
public RMWebApp(ResourceManager rm) { public RMWebApp(ResourceManager rm) {
this.rm = rm; this.rm = rm;
@ -53,6 +59,19 @@ public void setup() {
bind(RMWebApp.class).toInstance(this); bind(RMWebApp.class).toInstance(this);
if (rm != null) { if (rm != null) {
boolean enableServiceApi = rm.getConfig()
.getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, false);
if (enableServiceApi) {
try {
// Use reflection here to load ApiServer class,
// this is done to avoid creating cyclic dependency
// between maven projects.
Class<?> apiServer = Class.forName(APISERVER);
bind(apiServer);
} catch (ClassNotFoundException e) {
LOG.warn("ApiServer REST API is not activated.");
}
}
bind(ResourceManager.class).toInstance(rm); bind(ResourceManager.class).toInstance(rm);
} }
route("/", RmController.class); route("/", RmController.class);

View File

@ -101,17 +101,41 @@ yarn service destroy ${SERVICE_NAME}
``` ```
## Manage services on YARN via REST API ## Manage services on YARN via REST API
Below steps walk you through deploying services on YARN via REST API.
Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn. YARN API Server REST API can be activated in two modes: embedded or standalone.
### Start API-Server for deploying services on YARN
API server is the service that sits in front of YARN ResourceManager and lets users submit their API specs via HTTP. ### Start Embedded API-Server as part of ResourceManager
For running inside ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager.
```
<property>
<description>
Enable services rest api on ResourceManager.
</description>
<name>yarn.webapp.api-service.enable</name>
<value>true</value>
</property>
```
Services can be deployed on YARN through the ResourceManager web endpoint.
### Start Standalone API-Server for deploying services on YARN
API server is the service that sits in front of YARN ResourceManager and lets users submit their service specs via HTTP.
``` ```
yarn --daemon start apiserver yarn --daemon start apiserver
``` ```
The above command starts the API Server on the localhost at port 9191 by default. The above command starts the API Server on the localhost at port 9191 by default.
Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
### Deploy a service ### Deploy a service
POST the aforementioned example service definition to the api-server endpoint:
POST the aforementioned example service definition to the ResourceManager api-server endpoint:
```
POST http://localhost:8088/ws/v1/services
```
Or standalone API server:
``` ```
POST http://localhost:9191/ws/v1/services POST http://localhost:9191/ws/v1/services
``` ```