Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

This commit is contained in:
Billie Rinaldi 2017-08-23 12:31:47 -07:00 committed by Jian He
parent 79bf5c084d
commit 2191f5b533
496 changed files with 1205 additions and 54575 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml → hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
View File

8
hadoop-yarn-project/hadoop-yarn/bin/yarn
View File

@ -49,7 +49,7 @@ function hadoop_usage
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
hadoop_add_subcommand "servicesapi" "run yarn-service rest server"
hadoop_add_subcommand "servicesapi" "run yarn-native-service rest server"
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "service" "run a service"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
@ -155,7 +155,7 @@ function yarncmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp'
HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApplicationApiWebApp'
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
@ -164,7 +164,7 @@ ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
hadoop_translate_cygwin_path sld
hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
;;
sharedcachemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@ -181,7 +181,7 @@ ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
hadoop_translate_cygwin_path sld
hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
;;
timelinereader)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"

83
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java
View File

@ -15,23 +15,21 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.services.api.impl;
package org.apache.hadoop.yarn.service.webapp;
import com.google.inject.Singleton;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.client.ServiceClient;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.ApplicationState;
import org.apache.slider.api.resource.ApplicationStatus;
import org.apache.slider.api.resource.Component;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.api.records.ApplicationState;
import org.apache.hadoop.yarn.service.api.records.ApplicationStatus;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -51,12 +49,10 @@ import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import static org.apache.slider.util.RestApiConstants.*;
import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
@Singleton
@Path(APPLICATIONS_API_RESOURCE_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
@Path(CONTEXT_ROOT)
public class ApplicationApiService {
private static final Logger LOG =
LoggerFactory.getLogger(ApplicationApiService.class);
@ -75,15 +71,18 @@ public class ApplicationApiService {
}
@GET
@Path("/versions/yarn-service-version")
@Path(VERSION)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON }) public Response getSliderVersion() {
@Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
public Response getVersion() {
String version = VersionInfo.getBuildVersion();
LOG.info(version);
return Response.ok(version).build();
}
@POST @Consumes({ MediaType.APPLICATION_JSON })
@POST
@Path(APP_ROOT_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
public Response createApplication(Application application) {
LOG.info("POST: createApplication = {}", application);
@ -94,7 +93,7 @@ public class ApplicationApiService {
+ " applicationId = " + applicationId);
applicationStatus.setState(ApplicationState.ACCEPTED);
applicationStatus.setUri(
CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH + "/" + application
CONTEXT_ROOT + APP_ROOT_PATH + "/" + application
.getName());
return Response.status(Status.CREATED).entity(applicationStatus).build();
} catch (IllegalArgumentException e) {
@ -110,10 +109,11 @@ public class ApplicationApiService {
}
}
@GET @Path("/{app_name}")
@GET
@Path(APP_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
public Response getApplication(@PathParam("app_name") String appName) {
public Response getApplication(@PathParam(APP_NAME) String appName) {
LOG.info("GET: getApplication for appName = {}", appName);
ApplicationStatus applicationStatus = new ApplicationStatus();
@ -127,22 +127,7 @@ public class ApplicationApiService {
try {
Application app = SERVICE_CLIENT.getStatus(appName);
ApplicationReport report = SERVICE_CLIENT.getYarnClient()
.getApplicationReport(ApplicationId.fromString(app.getId()));
if (report != null) {
app.setLifetime(
report.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME)
.getRemainingTime());
LOG.info("Application = {}", app);
return Response.ok(app).build();
} else {
String message = "Application " + appName + " does not exist.";
LOG.info(message);
applicationStatus.setCode(ERROR_CODE_APP_DOES_NOT_EXIST);
applicationStatus.setDiagnostics(message);
return Response.status(Status.NOT_FOUND).entity(applicationStatus)
.build();
}
return Response.ok(app).build();
} catch (Exception e) {
LOG.error("Get application failed", e);
applicationStatus
@ -153,17 +138,17 @@ public class ApplicationApiService {
}
@DELETE
@Path("/{app_name}")
@Path(APP_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
public Response deleteApplication(@PathParam("app_name") String appName) {
public Response deleteApplication(@PathParam(APP_NAME) String appName) {
LOG.info("DELETE: deleteApplication for appName = {}", appName);
return stopApplication(appName, true);
}
private Response stopApplication(String appName, boolean destroy) {
try {
SERVICE_CLIENT.actionStop(appName);
SERVICE_CLIENT.actionStop(appName, destroy);
if (destroy) {
SERVICE_CLIENT.actionDestroy(appName);
LOG.info("Successfully deleted application {}", appName);
@ -185,11 +170,12 @@ public class ApplicationApiService {
}
}
@PUT @Path("/{app_name}/components/{component_name}")
@PUT
@Path(COMPONENT_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
public Response updateComponent(@PathParam("app_name") String appName,
@PathParam("component_name") String componentName, Component component) {
@Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
public Response updateComponent(@PathParam(APP_NAME) String appName,
@PathParam(COMPONENT_NAME) String componentName, Component component) {
if (component.getNumberOfContainers() < 0) {
return Response.status(Status.BAD_REQUEST).entity(
@ -201,9 +187,10 @@ public class ApplicationApiService {
Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName,
Collections.singletonMap(component.getName(),
component.getNumberOfContainers()));
return Response.ok().entity("Updating " + componentName + " size from "
+ original.get(componentName) + " to "
+ component.getNumberOfContainers()).build();
return Response.ok().entity(
"Updating component " + componentName + " size from " + original
.get(componentName) + " to " + component.getNumberOfContainers())
.build();
} catch (YarnException | IOException e) {
ApplicationStatus status = new ApplicationStatus();
status.setDiagnostics(e.getMessage());
@ -212,10 +199,11 @@ public class ApplicationApiService {
}
}
@PUT @Path("/{app_name}")
@PUT
@Path(APP_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
public Response updateApplication(@PathParam("app_name") String appName,
public Response updateApplication(@PathParam(APP_NAME) String appName,
Application updateAppData) {
LOG.info("PUT: updateApplication for app = {} with data = {}", appName,
updateAppData);
@ -246,8 +234,7 @@ public class ApplicationApiService {
// flex a single component app
if (updateAppData.getNumberOfContainers() != null && !ServiceApiUtil
.hasComponent(
updateAppData)) {
.hasComponent(updateAppData)) {
Component defaultComp = ServiceApiUtil.createDefaultComponent(updateAppData);
return updateComponent(updateAppData.getName(), defaultComp.getName(),
defaultComp);

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java
View File

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.services.webapp;
package org.apache.hadoop.yarn.service.webapp;
import static org.apache.slider.util.RestApiConstants.*;
import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
import java.io.IOException;
import java.net.InetAddress;
@ -27,7 +27,6 @@ import java.util.Arrays;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.services.api.impl.ApplicationApiService;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.eclipse.jetty.webapp.Configuration;
@ -100,8 +99,7 @@ public class ApplicationApiWebApp extends AbstractService {
ApplicationApiService.class.getPackage().getName() + SEP
+ GenericExceptionHandler.class.getPackage().getName() + SEP
+ YarnJacksonJaxbJsonProvider.class.getPackage().getName();
applicationApiServer.addJerseyResourcePackage(apiPackages, CONTEXT_ROOT
+ "/*");
applicationApiServer.addJerseyResourcePackage(apiPackages, "/*");
try {
logger.info("Application starting up. Logging start...");

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
View File

@ -1,36 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.services.api;
import javax.ws.rs.core.Response;
import org.apache.slider.api.resource.Application;
/**
* Apache Hadoop YARN Services REST API interface.
*
*/
public interface ApplicationApi {
Response createApplication(Application application);
Response getApplications(String state);
Response deleteApplication(String appName);
Response updateApplication(String appName, Application updateAppData);
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
View File

@ -277,9 +277,6 @@ definitions:
type: integer
format: int64
description: Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.
unique_component_support:
type: boolean
description: Certain applications need to define multiple components using the same artifact and resource profile, differing only in configurations. In such cases, this field helps app owners to avoid creating multiple component definitions with repeated information. The number_of_containers field dictates the initial number of components created. Component names typically differ with a trailing id, but assumptions should not be made on that, as the algorithm can change at any time. Configurations section will be able to use placeholders like ${USER}, ${CLUSTER_NAME} and ${COMPONENT_NAME} to be replaced at runtime with user the app is submitted as, application name and application component name respectively. Launch command can use placeholders like ${APP_COMPONENT_NAME} and ${APP_NAME} to get its component name and app name respectively at runtime. The best part of this feature is that when the component is flexed up, entirely new components (with new trailing ids) are created.
run_privileged_container:
type: boolean
description: Run all containers of this component in privileged mode (YARN-4262).

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
View File

@ -21,7 +21,7 @@
<servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
<init-param>
<param-name>com.sun.jersey.config.property.packages</param-name>
<param-value>org.apache.hadoop.yarn.services.webapp,org.apache.hadoop.yarn.services.api,org.apache.slider.api.resource,org.apache.hadoop.yarn.services.api.impl</param-value>
<param-value>org.apache.hadoop.yarn.service.webapp,org.apache.hadoop.yarn.service.api,org.apache.hadoop.yarn.service.api.records</param-value>
</init-param>
<init-param>
<param-name>com.sun.jersey.api.json.POJOMappingFeature</param-name>

93
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
View File

@ -23,99 +23,26 @@
<class name="org.apache.hadoop.yarn.service.utils.ServiceApiUtil" />
<Bug pattern="MS_CANNOT_BE_FINAL" />
</Match>
<Match>
<Package name="org.apache.slider.api.proto" />
</Match>
<Match>
<class name="~org\.apache\.slider\.*" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
<Match>
<Class name="org.apache.slider.core.zk.BlockingZKWatcher" />
<Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
</Match>
<Match>
<Class name="org.apache.slider.server.appmaster.state.ProviderAppState" />
<Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
</Match>
<Match>
<Class name="org.apache.slider.providers.ProviderUtils" />
<Bug pattern="SWL_SLEEP_WITH_LOCK_HELD" />
</Match>
<Match>
<Class name="org.apache.slider.server.appmaster.SliderAppMaster" />
<Bug pattern="WA_AWAIT_NOT_IN_LOOP" />
</Match>
<Match>
<Class name="org.apache.slider.core.zk.ZKIntegration" />
<Bug pattern="SBSC_USE_STRINGBUFFER_CONCATENATION" />
</Match>
<Match>
<Bug pattern="NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE" />
</Match>
<Match>
<Class name="org.apache.slider.server.servicemonitor.ProbeWorker" />
<Bug pattern="SF_SWITCH_FALLTHROUGH" />
</Match>
<Match>
<Class name="org.apache.slider.core.persist.JsonSerDeser" />
<Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" />
</Match>
<Match>
<Class name="org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider" />
<Bug pattern="EI_EXPOSE_REP" />
</Match>
<Match>
<Class name="org.apache.slider.server.appmaster.state.OutstandingRequest" />
<Bug pattern="EQ_DOESNT_OVERRIDE_EQUALS" />
</Match>
<Match>
<Class name="org.apache.slider.server.appmaster.web.SliderAMWebApp" />
<Bug pattern="LG_LOST_LOGGER_DUE_TO_WEAK_REFERENCE" />
</Match>
<Match>
<Class name="org.apache.slider.server.servicemonitor.Probe" />
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
<Match>
<Class name="org.apache.slider.server.appmaster.web.rest.registry.PathEntryResource" />
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
<Match>
<Class name="org.apache.slider.server.appmaster.web.rest.application.actions.StopResponse" />
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
<Match>
<Class name="org.apache.slider.server.appmaster.state.ContainerAllocationResults" />
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
<Match>
<Class name="org.apache.slider.core.persist.JsonSerDeser" />
<Method name="save" />
<Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
<Bug pattern="OBL_UNSATISFIED_OBLIGATION" />
</Match>
<Match>
<Class name="org.apache.slider.server.services.workflow.ForkedProcessService" />
<Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
<Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
<Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" />
</Match>
<Match>
<Class name="org.apache.slider.server.appmaster.state.RoleInstance"/>
<Package name="org.apache.hadoop.yarn.service.client.params"/>
<Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/>
</Match>
<Match>
<Class name="org.apache.slider.client.SliderClient" />
<Method name="actionRegistryListConfigsYarn" />
<Bug pattern="OS_OPEN_STREAM" />
<Package name="org.apache.hadoop.yarn.service.client.params"/>
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
</Match>
<Match>
<Class name="org.apache.slider.client.SliderClient" />
<Method name="actionRegistryListExports" />
<Bug pattern="OS_OPEN_STREAM" />
</Match>
<Match>
<Class name="org.apache.slider.common.tools.SliderUtils" />
<Method name="getApplicationResourceInputStream" />
<Bug pattern="OS_OPEN_STREAM" />
<Class name="org.apache.hadoop.yarn.service.client.ServiceClient"/>
<Field name="registryClient" />
<Bug pattern="IS2_INCONSISTENT_SYNC"/>
</Match>
</FindBugsFilter>

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
View File

@ -17,14 +17,14 @@
# This is the log4j configuration for Slider Application Master
# Log rotation based on size (100KB) with a max of 10 backup files
# Log rotation based on size (256MB) with a max of 20 backup files
log4j.rootLogger=INFO, amlog
log4j.threshhold=ALL
log4j.appender.amlog=org.apache.log4j.RollingFileAppender
log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
log4j.appender.amlog.File=${LOG_DIR}/slider.log
log4j.appender.amlog.MaxFileSize=1MB
log4j.appender.amlog.MaxBackupIndex=10
log4j.appender.amlog.File=${LOG_DIR}/serviceam.log
log4j.appender.amlog.MaxFileSize=256MB
log4j.appender.amlog.MaxBackupIndex=20
# log layout skips stack-trace creation operations by avoiding line numbers and method
log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
@ -43,13 +43,8 @@ log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%
log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
# for debugging Slider
#log4j.logger.org.apache.slider=DEBUG
# uncomment to debug service lifecycle issues
#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
# for debugging yarn-service framework
#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
# uncomment for YARN operations
@ -61,7 +56,6 @@ log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
#crank back on some noise
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
log4j.logger.org.apache.hadoop.hdfs=WARN
log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.apache.curator.framework.state=ERROR

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
View File

@ -59,8 +59,6 @@
<source>
<directory>${basedir}/src/main/proto</directory>
<includes>
<include>SliderClusterMessages.proto</include>
<include>SliderClusterProtocol.proto</include>
<include>ClientAMProtocol.proto</include>
</includes>
</source>
@ -91,33 +89,12 @@
</configuration>
</plugin>
<plugin>
<groupId>org.apache.avro</groupId>
<artifactId>avro-maven-plugin</artifactId>
<version>${avro.version}</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>schema</goal>
</goals>
<configuration>
<sourceDirectory>${project.basedir}/src/main/avro/
</sourceDirectory>
<outputDirectory>${project.build.directory}/generated-sources/java
</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/*.json</exclude>
<exclude>src/main/resources/webapps/slideram/.keep</exclude>
</excludes>
</configuration>
</plugin>
@ -217,11 +194,6 @@
<artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
@ -422,7 +394,7 @@
<attach>false</attach>
<finalName>${project.artifactId}-${project.version}</finalName>
<descriptorRefs>
<descriptorRef>hadoop-yarn-slider-dist</descriptorRef>
<descriptorRef>hadoop-yarn-services-dist</descriptorRef>
</descriptorRefs>
</configuration>
</execution>

114
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
View File

@ -1,114 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
[
{
"type": "record",
"name": "NodeEntryRecord",
"namespace": "org.apache.slider.server.avro",
"fields": [
{
"name": "host",
"type": "string"
},
{
"name": "role",
"type": "int"
},
{
"name": "active",
"type": "boolean"
},
{
"name": "last_used",
"type": "long"
}
]
},
{
"type": "record",
"name": "RoleHistoryHeader",
"namespace": "org.apache.slider.server.avro",
"fields": [
{
"name": "version",
"type": "int"
},
{
"name": "saved",
"type": "long"
},
{
"name": "savedx",
"type": "string"
},
{
"name": "savedate",
"type": "string",
"default": ""
},
{
"name": "roles",
"type": "int"
}
]
},
{
"type": "record",
"name": "RoleHistoryMapping",
"namespace": "org.apache.slider.server.avro",
"fields": [
{
"name": "rolemap",
"type": {
"type": "map",
"values": "int"
}
}
]
},
{
"type": "record",
"name": "RoleHistoryFooter",
"namespace": "org.apache.slider.server.avro",
"fields": [
{
"name": "count",
"type": "long"
}
]
},
{
"type": "record",
"name": "RoleHistoryRecord",
"namespace": "org.apache.slider.server.avro",
"fields": [
{
"name": "entry",
"type": [
"org.apache.slider.server.avro.NodeEntryRecord",
"org.apache.slider.server.avro.RoleHistoryHeader",
"org.apache.slider.server.avro.RoleHistoryFooter",
"org.apache.slider.server.avro.RoleHistoryMapping"
]
}
]
}
]

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
View File

@ -29,7 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.slider.api.ResourceKeys.NODE_FAILURE_THRESHOLD;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.NODE_BLACKLIST_THRESHOLD;
/**
* This tracks the container failures per node. If the failure counter exceeds
@ -52,7 +52,7 @@ public class ContainerFailureTracker {
this.context = context;
this.component = component;
maxFailurePerNode = component.getComponentSpec().getConfiguration()
.getPropertyInt(NODE_FAILURE_THRESHOLD, 3);
.getPropertyInt(NODE_BLACKLIST_THRESHOLD, 3);
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
View File

@ -21,9 +21,9 @@ package org.apache.hadoop.yarn.service;
import com.google.common.cache.LoadingCache;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
public class ServiceContext {
public Application application = null;

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
View File

@ -35,10 +35,12 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.servicemonitor.ServiceMonitor;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -68,6 +70,7 @@ public class ServiceMaster extends CompositeService {
context = new ServiceContext();
Path appDir = getAppDir();
context.serviceHdfsDir = appDir.toString();
SliderFileSystem fs = new SliderFileSystem(conf);
context.fs = fs;
fs.setAppDir(appDir);

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
View File

@ -54,26 +54,25 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants;
import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType;
import org.apache.hadoop.yarn.service.component.Component;
import org.apache.hadoop.yarn.service.component.ComponentEvent;
import org.apache.hadoop.yarn.service.component.ComponentEventType;
import org.apache.hadoop.yarn.service.conf.SliderKeys;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
import org.apache.hadoop.yarn.util.BoundedAppender;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.ServiceApiConstants;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.core.zk.ZKIntegration;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -93,8 +92,8 @@ import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
import static org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*;
import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
import static org.apache.slider.api.ServiceApiConstants.*;
/**
*
@ -203,7 +202,7 @@ public class ServiceScheduler extends CompositeService {
protected YarnRegistryViewForProviders createYarnRegistryOperations(
ServiceContext context, RegistryOperations registryClient) {
return new YarnRegistryViewForProviders(registryClient,
RegistryUtils.currentUser(), SliderKeys.APP_TYPE, app.getName(),
RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, app.getName(),
context.attemptId);
}
@ -236,7 +235,8 @@ public class ServiceScheduler extends CompositeService {
DefaultMetricsSystem.shutdown();
if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
serviceTimelinePublisher.serviceAttemptUnregistered(context);
serviceTimelinePublisher
.serviceAttemptUnregistered(context, diagnostics.toString());
}
// Cleanup each component instance. no need to release containers as
// they will be automatically released by RM
@ -285,7 +285,7 @@ public class ServiceScheduler extends CompositeService {
LOG.error("Failed to get user.", e);
}
globalTokens
.put(SERVICE_ZK_PATH, ZKIntegration.mkClusterPath(user, app.getName()));
.put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkClusterPath(user, app.getName()));
globalTokens.put(ServiceApiConstants.USER, user);
String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN);
@ -336,7 +336,7 @@ public class ServiceScheduler extends CompositeService {
context.configCache = configFileCache;
}
protected void registerServiceInstance(ApplicationAttemptId attemptId,
private void registerServiceInstance(ApplicationAttemptId attemptId,
Application application) throws IOException {
LOG.info("Registering " + attemptId + ", " + application.getName()
+ " into registry");
@ -345,11 +345,11 @@ public class ServiceScheduler extends CompositeService {
attemptId.getApplicationId().toString());
serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
PersistencePolicies.APPLICATION);
serviceRecord.description = "Slider Application Master";
serviceRecord.description = "Yarn Service Master";
serviceRecord.addExternalEndpoint(RegistryTypeUtils
.ipcEndpoint(CustomRegistryConstants.AM_IPC_PROTOCOL,
new InetSocketAddress(5000))); // FIXME
.ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
context.clientAMService.getBindAddress()));
// set any provided attributes
setUserProvidedServiceRecordAttributes(application.getConfiguration(),
@ -376,13 +376,13 @@ public class ServiceScheduler extends CompositeService {
}
});
if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
serviceTimelinePublisher.serviceAttemptRegistered(app);
serviceTimelinePublisher.serviceAttemptRegistered(app, getConfig());
}
}
private void setUserProvidedServiceRecordAttributes(
org.apache.slider.api.resource.Configuration conf, ServiceRecord record) {
String prefix = RoleKeys.SERVICE_RECORD_ATTRIBUTE_PREFIX;
org.apache.hadoop.yarn.service.api.records.Configuration conf, ServiceRecord record) {
String prefix = "service.record.attribute";
for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
if (entry.getKey().startsWith(prefix)) {
String key = entry.getKey().substring(prefix.length() + 1);
@ -395,10 +395,10 @@ public class ServiceScheduler extends CompositeService {
long allocateId = 0;
// sort components by dependencies
Collection<org.apache.slider.api.resource.Component> sortedComponents =
Collection<org.apache.hadoop.yarn.service.api.records.Component> sortedComponents =
ServiceApiUtil.sortByDependencies(app.getComponents());
for (org.apache.slider.api.resource.Component compSpec : sortedComponents) {
for (org.apache.hadoop.yarn.service.api.records.Component compSpec : sortedComponents) {
Component component = new Component(compSpec, allocateId, context);
componentsById.put(allocateId, component);
componentsByName.put(component.getName(), component);
@ -517,7 +517,7 @@ public class ServiceScheduler extends CompositeService {
@Override public float getProgress() {
// get running containers over desired containers
long total = 0;
for (org.apache.slider.api.resource.Component component : app
for (org.apache.hadoop.yarn.service.api.records.Component component : app
.getComponents()) {
total += component.getNumberOfContainers();
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.api;
package org.apache.hadoop.yarn.service.api.constants;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java
View File

@ -16,13 +16,15 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
@ -36,7 +38,8 @@ import java.util.Objects;
/**
* An Application resource has the following attributes.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "An Application resource has the following attributes.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ApplicationState.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java
View File

@ -15,14 +15,17 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import io.swagger.annotations.ApiModel;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* The current state of an application.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "The current state of an application.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public enum ApplicationState {

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ApplicationStatus.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@ -26,12 +26,15 @@ import javax.xml.bind.annotation.XmlRootElement;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* The current status of a submitted application, returned as a response to the
* GET API.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "The current status of a submitted application, returned as a response to the GET API.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Artifact.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@ -26,11 +26,14 @@ import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* Artifact of an application component.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "Artifact of an application component")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@JsonInclude(JsonInclude.Include.NON_NULL)

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/BaseResource.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
View File

@ -15,10 +15,14 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import java.io.Serializable;
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class BaseResource implements Serializable {
private static final long serialVersionUID = 1492603053176889431L;

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@ -31,6 +31,8 @@ import javax.xml.bind.annotation.XmlRootElement;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* One or more components of the application. If the application is HBase say,
@ -39,7 +41,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
* applications say Kafka or Storm. Thereby it opens up the support for complex
* and nested applications.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@ -54,7 +57,6 @@ public class Component implements Serializable {
private String launchCommand = null;
private Resource resource = null;
private Long numberOfContainers = null;
private Boolean uniqueComponentSupport = false;
private Boolean runPrivilegedContainer = false;
private PlacementPolicy placementPolicy = null;
private Configuration configuration = new Configuration();
@ -225,38 +227,6 @@ public class Component implements Serializable {
return null;
}
/**
* Certain applications need to define multiple components using the same
* artifact and resource profile, differing only in configurations. In such
* cases, this field helps app owners to avoid creating multiple component
* definitions with repeated information. The number_of_containers field
* dictates the initial number of components created. Component names
* typically differ with a trailing id, but assumptions should not be made on
* that, as the algorithm can change at any time. Configurations section will
* be able to use placeholders like ${USER}, ${CLUSTER_NAME} and
* ${COMPONENT_NAME} to be replaced at runtime with user the app is submitted
* as, application name and application component name respectively. Launch
* command can use placeholders like ${APP_COMPONENT_NAME} and ${APP_NAME} to
* get its component name and app name respectively at runtime. The best part
* of this feature is that when the component is flexed up, entirely new
* components (with new trailing ids) are created.
**/
public Component uniqueComponentSupport(Boolean uniqueComponentSupport) {
this.uniqueComponentSupport = uniqueComponentSupport;
return this;
}
@ApiModelProperty(example = "null", value = "Certain applications need to define multiple components using the same artifact and resource profile, differing only in configurations. In such cases, this field helps app owners to avoid creating multiple component definitions with repeated information. The number_of_containers field dictates the initial number of components created. Component names typically differ with a trailing id, but assumptions should not be made on that, as the algorithm can change at any time. Configurations section will be able to use placeholders like ${USER}, ${CLUSTER_NAME} and ${COMPONENT_NAME} to be replaced at runtime with user the app is submitted as, application name and application component name respectively. Launch command can use placeholders like ${APP_COMPONENT_NAME} and ${APP_NAME} to get its component name and app name respectively at runtime. The best part of this feature is that when the component is flexed up, entirely new components (with new trailing ids) are created.")
@JsonProperty("unique_component_support")
public Boolean getUniqueComponentSupport() {
return uniqueComponentSupport;
}
@XmlElement(name = "unique_component_support")
public void setUniqueComponentSupport(Boolean uniqueComponentSupport) {
this.uniqueComponentSupport = uniqueComponentSupport;
}
/**
* Run all containers of this component in privileged mode (YARN-4262).
**/
@ -351,8 +321,6 @@ public class Component implements Serializable {
&& Objects.equals(this.launchCommand, component.launchCommand)
&& Objects.equals(this.resource, component.resource)
&& Objects.equals(this.numberOfContainers, component.numberOfContainers)
&& Objects.equals(this.uniqueComponentSupport,
component.uniqueComponentSupport)
&& Objects.equals(this.runPrivilegedContainer,
component.runPrivilegedContainer)
&& Objects.equals(this.placementPolicy, component.placementPolicy)
@ -363,7 +331,7 @@ public class Component implements Serializable {
@Override
public int hashCode() {
return Objects.hash(name, dependencies, readinessCheck, artifact,
launchCommand, resource, numberOfContainers, uniqueComponentSupport,
launchCommand, resource, numberOfContainers,
runPrivilegedContainer, placementPolicy, configuration, quicklinks);
}
@ -385,8 +353,6 @@ public class Component implements Serializable {
.append(toIndentedString(numberOfContainers)).append("\n");
sb.append(" containers: ").append(toIndentedString(containers))
.append("\n");
sb.append(" uniqueComponentSupport: ")
.append(toIndentedString(uniqueComponentSupport)).append("\n");
sb.append(" runPrivilegedContainer: ")
.append(toIndentedString(runPrivilegedContainer)).append("\n");
sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
@ -433,9 +399,6 @@ public class Component implements Serializable {
if (this.getRunPrivilegedContainer() == null) {
this.setRunPrivilegedContainer(that.getRunPrivilegedContainer());
}
if (this.getUniqueComponentSupport() == null) {
this.setUniqueComponentSupport(that.getUniqueComponentSupport());
}
if (this.getDependencies() == null) {
this.setDependencies(that.getDependencies());
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
View File

@ -15,13 +15,15 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
@ -34,7 +36,8 @@ import java.util.Objects;
* A config file that needs to be created and made available as a volume in an
* application component container.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "A config file that needs to be created and made available as a volume in an application component container.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
View File

@ -16,10 +16,15 @@
* limitations under the License.
*/
package org.apache.slider.core.registry.docstore;
package org.apache.hadoop.yarn.service.api.records;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import java.util.Locale;
@InterfaceAudience.Public
@InterfaceStability.Unstable
public enum ConfigFormat {
JSON("json"),

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
View File

@ -15,14 +15,16 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import org.apache.commons.lang.StringUtils;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import java.io.Serializable;
import java.util.ArrayList;
@ -37,7 +39,8 @@ import java.util.Objects;
* Files of several standard formats like xml, properties, json, yaml and
* templates will be supported.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@JsonInclude(JsonInclude.Include.NON_NULL)

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Container.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@ -28,11 +28,14 @@ import javax.xml.bind.annotation.XmlRootElement;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* An instance of a running application container.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "An instance of a running application container")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ContainerState.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
View File

@ -15,11 +15,16 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* The current state of the container of an application.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
public enum ContainerState {
RUNNING_BUT_UNREADY, READY, STOPPED
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Error.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
View File

@ -15,14 +15,18 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import io.swagger.annotations.ApiModelProperty;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Unstable
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public class Error {

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/PlacementPolicy.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@ -24,12 +24,15 @@ import java.io.Serializable;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* Placement policy of an instance of an application. This feature is in the
* works in YARN-4902.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "Placement policy of an instance of an application. This feature is in the works in YARN-4902.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public class PlacementPolicy implements Serializable {

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ReadinessCheck.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@ -27,6 +27,8 @@ import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* A custom command or a pluggable helper container to determine the readiness
@ -34,7 +36,8 @@ import com.fasterxml.jackson.annotation.JsonValue;
* Hence the need for a simple interface, with scope to support advanced
* usecases.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public class ReadinessCheck implements Serializable {

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.slider.api.resource;
package org.apache.hadoop.yarn.service.api.records;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@ -23,6 +23,8 @@ import io.swagger.annotations.ApiModelProperty;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* Resource determines the amount of resources (vcores, memory, network, etc.)
@ -32,7 +34,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
* of profile OR cpu &amp; memory are exepected. It raises a validation
* exception otherwise.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public class Resource extends BaseResource implements Cloneable {

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
View File

@ -20,8 +20,9 @@ package org.apache.hadoop.yarn.service.client;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.api.resource.Application;
import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -32,7 +33,7 @@ public class ServiceCLI {
LoggerFactory.getLogger(ServiceClient.class);
protected ServiceClient client;
public int exec(ClientArgs args) throws Throwable {
int exec(ClientArgs args) throws Throwable {
if (StringUtils.isEmpty(args.getAction())) {
System.out.println(args.usage());
return -1;
@ -55,7 +56,7 @@ public class ServiceCLI {
client.actionFlexByCLI(args);
break;
case ACTION_STOP:
client.actionStop(args.getClusterName());
client.actionStop(args.getClusterName(), false);
break;
case ACTION_DESTROY: // Destroy can happen only if app is already stopped
client.actionDestroy(args.getClusterName());
@ -90,7 +91,12 @@ public class ServiceCLI {
public static void main(String[] args) throws Throwable {
ClientArgs clientArgs = new ClientArgs(args);
clientArgs.parse();
try {
clientArgs.parse();
} catch (BadCommandArgumentsException e) {
System.err.println(e.getMessage());
System.exit(-1);
}
ServiceCLI cli = new ServiceCLI();
int res = cli.exec(clientArgs);
System.exit(res);

259
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
View File

@ -22,6 +22,8 @@ import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryNTimes;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@ -40,6 +42,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsReque
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
@ -58,34 +61,32 @@ import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
import org.apache.hadoop.yarn.service.ClientAMProtocol;
import org.apache.hadoop.yarn.service.ServiceMaster;
import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
import org.apache.hadoop.yarn.service.client.params.Arguments;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.client.params.CommonArgs;
import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
import org.apache.hadoop.yarn.service.conf.SliderKeys;
import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.Times;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Component;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.UsageException;
import org.apache.slider.core.launch.ClasspathConstructor;
import org.apache.slider.core.launch.JavaCommandLineBuilder;
import org.apache.slider.core.registry.SliderRegistryUtils;
import org.apache.slider.core.zk.ZKIntegration;
import org.apache.slider.core.zk.ZookeeperUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -105,17 +106,21 @@ import java.util.concurrent.ConcurrentHashMap;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX;
import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
import static org.apache.slider.common.tools.SliderUtils.*;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class ServiceClient extends CompositeService
implements SliderExitCodes, SliderKeys {
implements SliderExitCodes, YarnServiceConstants {
private static final Logger LOG =
LoggerFactory.getLogger(ServiceClient.class);
private SliderFileSystem fs;
private YarnClient yarnClient;
// Avoid looking up applicationId from fs all the time.
private Map<String, ApplicationId> cachedAppIds = new ConcurrentHashMap<>();
private Map<String, ClientAMProtocol> cachedAMProxies = new ConcurrentHashMap<>();
private RegistryOperations registryClient;
private CuratorFramework curatorClient;
private YarnRPC rpc;
@ -293,7 +298,12 @@ public class ServiceClient extends CompositeService
ServiceApiUtil.jsonSerDeser
.save(fs.getFileSystem(), ServiceApiUtil.getAppJsonPath(fs, appName),
persistedApp, true);
ClientAMProtocol proxy = connectToAM(appName);
ClientAMProtocol proxy = getAMProxy(appName);
if (proxy == null) {
String message = appName + " is not running";
LOG.error(message);
throw new YarnException(message);
}
proxy.flexComponents(requestBuilder.build());
for (Map.Entry<String, Long> entry : original.entrySet()) {
LOG.info("[COMPONENT {}]: number of containers changed from {} to {}",
@ -303,9 +313,10 @@ public class ServiceClient extends CompositeService
return original;
}
public int actionStop(String appName) throws YarnException, IOException {
public int actionStop(String appName, boolean waitForAppStopped)
throws YarnException, IOException {
validateClusterName(appName);
getAppIdFromPersistedApp(appName);
getAppId(appName);
ApplicationId currentAppId = cachedAppIds.get(appName);
ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
if (terminatedStates.contains(report.getYarnApplicationState())) {
@ -315,17 +326,29 @@ public class ServiceClient extends CompositeService
}
LOG.info("Stopping application {}, with appId = {}", appName, currentAppId);
try {
// try to stop the app gracefully.
ClientAMProtocol proxy = connectToAM(appName);
StopRequestProto request = StopRequestProto.newBuilder().build();
proxy.stop(request);
LOG.info("Application " + appName + " is being gracefully stopped...");
ClientAMProtocol proxy = getAMProxy(appName, report);
cachedAppIds.remove(appName);
cachedAMProxies.remove(appName);
if (proxy != null) {
// try to stop the app gracefully.
StopRequestProto request = StopRequestProto.newBuilder().build();
proxy.stop(request);
LOG.info("Application " + appName + " is being gracefully stopped...");
} else {
yarnClient.killApplication(currentAppId,
appName + " is forcefully killed by user!");
LOG.info("Forcefully kill the application: " + appName);
return EXIT_SUCCESS;
}
if (!waitForAppStopped) {
return EXIT_SUCCESS;
}
// Wait until the app is killed.
long startTime = System.currentTimeMillis();
int pollCount = 0;
while (true) {
Thread.sleep(1000);
Thread.sleep(2000);
report = yarnClient.getApplicationReport(currentAppId);
if (terminatedStates.contains(report.getYarnApplicationState())) {
LOG.info("Application " + appName + " is stopped.");
@ -358,6 +381,7 @@ public class ServiceClient extends CompositeService
FileSystem fileSystem = fs.getFileSystem();
// remove from the appId cache
cachedAppIds.remove(appName);
cachedAMProxies.remove(appName);
if (fileSystem.exists(appDir)) {
if (fileSystem.delete(appDir, true)) {
LOG.info("Successfully deleted application dir for " + appName + ": "
@ -370,7 +394,7 @@ public class ServiceClient extends CompositeService
}
}
deleteZKNode(appName);
String registryPath = SliderRegistryUtils.registryPathForInstance(appName);
String registryPath = ServiceRegistryUtils.registryPathForInstance(appName);
try {
getRegistryClient().delete(registryPath, true);
} catch (IOException e) {
@ -395,7 +419,7 @@ public class ServiceClient extends CompositeService
private void deleteZKNode(String clusterName) throws Exception {
CuratorFramework curatorFramework = getCuratorClient();
String user = RegistryUtils.currentUser();
String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
String zkPath = ServiceRegistryUtils.mkClusterPath(user, clusterName);
if (curatorFramework.checkExists().forPath(zkPath) != null) {
curatorFramework.delete().deletingChildrenIfNeeded().forPath(zkPath);
LOG.info("Deleted zookeeper path: " + zkPath);
@ -418,7 +442,7 @@ public class ServiceClient extends CompositeService
if (curatorClient == null) {
curatorClient =
CuratorFrameworkFactory.builder().connectString(registryQuorum)
.sessionTimeoutMs(10000).retryPolicy(new RetryNTimes(10, 2000))
.sessionTimeoutMs(10000).retryPolicy(new RetryNTimes(5, 2000))
.build();
curatorClient.start();
}
@ -433,7 +457,7 @@ public class ServiceClient extends CompositeService
private void verifyNoLiveAppInRM(String appname, String action)
throws IOException, YarnException {
Set<String> types = new HashSet<>(1);
types.add(SliderKeys.APP_TYPE);
types.add(YarnServiceConstants.APP_TYPE);
Set<String> tags = null;
if (appname != null) {
tags = Collections.singleton(SliderUtils.createNameTag(appname));
@ -469,12 +493,13 @@ public class ServiceClient extends CompositeService
appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime());
submissionContext.setApplicationTimeouts(appTimeout);
}
submissionContext.setMaxAppAttempts(conf.getInt(KEY_AM_RESTART_LIMIT, 2));
submissionContext.setMaxAppAttempts(conf.getInt(
YarnServiceConf.AM_RESTART_MAX, 2));
Map<String, LocalResource> localResources = new HashMap<>();
// copy local slideram-log4j.properties to hdfs and add to localResources
boolean hasSliderAMLog4j =
boolean hasAMLog4j =
addAMLog4jResource(appName, conf, localResources);
// copy jars to hdfs and add to localResources
addJarResource(appName, localResources);
@ -487,17 +512,17 @@ public class ServiceClient extends CompositeService
// create AM CLI
String cmdStr =
buildCommandLine(appName, conf, appRootDir, hasSliderAMLog4j);
buildCommandLine(appName, conf, appRootDir, hasAMLog4j);
submissionContext.setResource(Resource.newInstance(YarnServiceConf
.getLong(KEY_AM_RESOURCE_MEM, DEFAULT_KEY_AM_RESOURCE_MEM,
.getLong(YarnServiceConf.AM_RESOURCE_MEM, YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM,
app.getConfiguration(), conf), 1));
String queue = app.getQueue();
if (StringUtils.isEmpty(queue)) {
queue = conf.get(KEY_YARN_QUEUE, "default");
queue = conf.get(YARN_QUEUE, "default");
}
submissionContext.setQueue(queue);
submissionContext.setApplicationName(appName);
submissionContext.setApplicationType(SliderKeys.APP_TYPE);
submissionContext.setApplicationType(YarnServiceConstants.APP_TYPE);
Set<String> appTags =
AbstractClientProvider.createApplicationTags(appName, null, null);
if (!appTags.isEmpty()) {
@ -531,7 +556,7 @@ public class ServiceClient extends CompositeService
//TODO CLI.setJVMHeap
//TODO CLI.addJVMOPTS
if (hasSliderAMLog4j) {
CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME);
CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, YARN_SERVICE_LOG4J_FILENAME);
CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
}
CLI.add(ServiceMaster.class.getCanonicalName());
@ -553,15 +578,15 @@ public class ServiceClient extends CompositeService
private Map<String, String> addAMEnv(Configuration conf) throws IOException {
Map<String, String> env = new HashMap<>();
ClasspathConstructor classpath =
buildClasspath(SliderKeys.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
env.put("CLASSPATH", classpath.buildClasspath());
env.put("LANG", "en_US.UTF-8");
env.put("LC_ALL", "en_US.UTF-8");
env.put("LANGUAGE", "en_US.UTF-8");
String jaas = System.getenv(HADOOP_JAAS_DEBUG);
String jaas = System.getenv("HADOOP_JAAS_DEBUG");
if (jaas != null) {
env.put(HADOOP_JAAS_DEBUG, jaas);
env.put("HADOOP_JAAS_DEBUG", jaas);
}
if (!UserGroupInformation.isSecurityEnabled()) {
String userName = UserGroupInformation.getCurrentUser().getUserName();
@ -579,11 +604,11 @@ public class ServiceClient extends CompositeService
throws IOException, SliderException {
Path libPath = fs.buildClusterDirPath(appName);
ProviderUtils
.addProviderJar(localResources, ServiceMaster.class, SLIDER_JAR, fs,
.addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs,
libPath, "lib", false);
Path dependencyLibTarGzip = fs.getDependencyTarGzip();
if (fs.isFile(dependencyLibTarGzip)) {
LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ": "
LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
+ dependencyLibTarGzip);
SliderUtils.putAmTarGzipAndUpdate(localResources, fs);
} else {
@ -599,27 +624,29 @@ public class ServiceClient extends CompositeService
private boolean addAMLog4jResource(String appName, Configuration conf,
Map<String, LocalResource> localResources)
throws IOException, BadClusterStateException {
boolean hasSliderAMLog4j = false;
boolean hasAMLog4j = false;
String hadoopConfDir =
System.getenv(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
if (hadoopConfDir != null) {
File localFile =
new File(hadoopConfDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
new File(hadoopConfDir, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
if (localFile.exists()) {
Path localFilePath = createLocalPath(localFile);
Path appDirPath = fs.buildClusterDirPath(appName);
Path remoteConfPath =
new Path(appDirPath, SliderKeys.SUBMITTED_CONF_DIR);
new Path(appDirPath, YarnServiceConstants.SUBMITTED_CONF_DIR);
Path remoteFilePath =
new Path(remoteConfPath, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
new Path(remoteConfPath, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
copy(conf, localFilePath, remoteFilePath);
LocalResource localResource =
fs.createAmResource(remoteConfPath, LocalResourceType.FILE);
localResources.put(localFilePath.getName(), localResource);
hasSliderAMLog4j = true;
hasAMLog4j = true;
} else {
LOG.warn("AM log4j property file doesn't exist: " + localFile);
}
}
return hasSliderAMLog4j;
return hasAMLog4j;
}
public int actionStart(String appName) throws YarnException, IOException {
@ -674,22 +701,22 @@ public class ServiceClient extends CompositeService
return;
}
String keytabPreInstalledOnHost =
conf.get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
conf.get(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
if (StringUtils.isEmpty(keytabPreInstalledOnHost)) {
String amKeytabName =
conf.get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
String keytabDir = conf.get(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
conf.get(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
String keytabDir = conf.get(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
Path keytabPath =
fileSystem.buildKeytabPath(keytabDir, amKeytabName, appName);
if (fileSystem.getFileSystem().exists(keytabPath)) {
LocalResource keytabRes =
fileSystem.createAmResource(keytabPath, LocalResourceType.FILE);
localResource
.put(SliderKeys.KEYTAB_DIR + "/" + amKeytabName, keytabRes);
.put(YarnServiceConstants.KEYTAB_DIR + "/" + amKeytabName, keytabRes);
LOG.info("Adding AM keytab on hdfs: " + keytabPath);
} else {
LOG.warn("No keytab file was found at {}.", keytabPath);
if (conf.getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
if (conf.getBoolean(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
throw new BadConfigException("No keytab file was found at %s.",
keytabPath);
} else {
@ -704,7 +731,7 @@ public class ServiceClient extends CompositeService
public String updateLifetime(String appName, long lifetime)
throws YarnException, IOException {
getAppIdFromPersistedApp(appName);
getAppId(appName);
ApplicationId currentAppId = cachedAppIds.get(appName);
ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
if (report == null) {
@ -729,11 +756,25 @@ public class ServiceClient extends CompositeService
public Application getStatus(String appName)
throws IOException, YarnException {
ClientAMProtocol proxy = connectToAM(appName);
GetStatusResponseProto response =
proxy.getStatus(GetStatusRequestProto.newBuilder().build());
return ServiceApiUtil.jsonSerDeser.fromJson(response.getStatus());
validateClusterName(appName);
ApplicationId currentAppId = getAppId(appName);
ApplicationReport appReport = yarnClient.getApplicationReport(currentAppId);
ClientAMProtocol amProxy = getAMProxy(appName, appReport);
Application appSpec;
if (amProxy != null) {
GetStatusResponseProto response =
amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
appSpec = ServiceApiUtil.jsonSerDeser.fromJson(response.getStatus());
} else {
appSpec = new Application();
appSpec.setName(appName);
}
ApplicationTimeout lifetime =
appReport.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME);
if (lifetime != null) {
appSpec.setLifetime(lifetime.getRemainingTime());
}
return appSpec;
}
public YarnClient getYarnClient() {
@ -760,71 +801,61 @@ public class ServiceClient extends CompositeService
String[] libDirs = SliderUtils.getLibDirs();
if (libDirs.length > 0) {
File tempLibTarGzipFile = File.createTempFile(
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_",
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
// copy all jars
tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
LOG.info("Uploading dependency for AM (version {}) from {} to {}",
VersionInfo.getBuildVersion(), tempLibTarGzipFile.toURI(),
dependencyLibTarGzip.toUri());
LOG.info("Version Info: " + VersionInfo.getBuildVersion());
fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
new FsPermission(SliderKeys.SLIDER_DEPENDENCY_DIR_PERMISSIONS));
new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
return EXIT_SUCCESS;
} else {
return EXIT_FALSE;
}
}
protected ClientAMProtocol connectToAM(String appName)
throws IOException, YarnException {
ApplicationId currentAppId = getAppIdFromPersistedApp(appName);
// Wait until app becomes running.
long startTime = System.currentTimeMillis();
int pollCount = 0;
ApplicationReport appReport = null;
while (true) {
appReport = yarnClient.getApplicationReport(currentAppId);
YarnApplicationState state = appReport.getYarnApplicationState();
if (state == RUNNING) {
break;
}
if (terminatedStates.contains(state)) {
throw new YarnException(
"Failed to getStatus " + currentAppId + ": " + appReport
.getDiagnostics());
}
long elapsedMillis = System.currentTimeMillis() - startTime;
// if over 5 min, quit
if (elapsedMillis >= 300000) {
throw new YarnException(
"Timed out while waiting for application " + currentAppId
+ " to be running");
}
if (++pollCount % 10 == 0) {
LOG.info(
"Waiting for application {} to be running, current state is {}",
currentAppId, state);
}
try {
Thread.sleep(3000);
} catch (InterruptedException ie) {
String msg =
"Interrupted while waiting for application " + currentAppId
+ " to be running.";
throw new YarnException(msg, ie);
}
// Get AMProxy with the appReport provided
protected ClientAMProtocol getAMProxy(String appName, ApplicationReport report)
throws IOException {
if (!cachedAMProxies.containsKey(appName) && !StringUtils
.isEmpty(report.getHost())) {
insertAMProxy(appName, report.getHost(), report.getRpcPort());
}
// Make the connection
InetSocketAddress address = NetUtils
.createSocketAddrForHost(appReport.getHost(), appReport.getRpcPort());
return ClientAMProxy.createProxy(getConfig(), ClientAMProtocol.class,
UserGroupInformation.getCurrentUser(), rpc, address);
return cachedAMProxies.get(appName);
}
private synchronized ApplicationId getAppIdFromPersistedApp(String appName)
// Get AMProxy without appReport provided - it'll getAppReport from RM
protected ClientAMProtocol getAMProxy(String appName)
throws IOException, YarnException {
ApplicationId currentAppId = getAppId(appName);
if (cachedAMProxies.containsKey(appName)) {
return cachedAMProxies.get(appName);
} else {
ApplicationReport appReport =
yarnClient.getApplicationReport(currentAppId);
String host = appReport.getHost();
int port = appReport.getRpcPort();
if (!StringUtils.isEmpty(host)) {
return insertAMProxy(appName, host, port);
}
return null;
}
}
private ClientAMProtocol insertAMProxy(String appName, String host, int port)
throws IOException {
InetSocketAddress address =
NetUtils.createSocketAddrForHost(host, port);
ClientAMProtocol amProxy =
ClientAMProxy.createProxy(getConfig(), ClientAMProtocol.class,
UserGroupInformation.getCurrentUser(), rpc, address);
cachedAMProxies.put(appName, amProxy);
return amProxy;
}
private synchronized ApplicationId getAppId(String appName)
throws IOException, YarnException {
if (cachedAppIds.containsKey(appName)) {
return cachedAppIds.get(appName);

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
View File

@ -20,10 +20,9 @@ package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.params.PathArgumentConverter;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.UsageException;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -49,7 +48,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
public String filesystemBinding;
@Parameter(names = {ARG_BASE_PATH},
description = "Slider base path on the filesystem",
description = "Service base path on the filesystem",
converter = PathArgumentConverter.class)
public Path basePath;
@ -129,7 +128,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
if (minArgs > actionArgSize) {
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
" Expected minimum " + minArgs + " but got " + actionArgSize);
", Expected minimum " + minArgs + " but got " + actionArgSize);
}
int maxArgs = getMaxParams();
if (maxArgs == -1) {
@ -146,7 +145,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
StringBuilder buf = new StringBuilder(message);
for (String actionArg : parameters) {
log.error("[{}] \"{}\"", index++, actionArg);
buf.append(" \"" + actionArg + "\" ");
buf.append(" \"").append(actionArg).append("\" ");
}
throw new BadCommandArgumentsException(buf.toString());
}
@ -156,25 +155,4 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
public String toString() {
return super.toString() + ": " + getActionName();
}
/**
* Override point:
* Flag to indicate that core hadoop API services are needed (HDFS, YARN, etc)
* —and that validation of the client state should take place.
*
* @return a flag to indicate that the core hadoop services will be needed.
*/
public boolean getHadoopServicesRequired() {
return true;
}
/**
* Flag to disable secure login.
* This MUST only be set if the action is bypassing security or setting
* it itself
* @return true if login at slider client init time is to be skipped
*/
public boolean disableSecureLogin() {
return false;
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.yarn.service.client.params.ArgOps;
import org.apache.hadoop.yarn.service.client.params.Arguments;

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
View File

@ -16,14 +16,12 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParametersDelegate;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.ComponentArgsDelegate;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import java.io.File;
import java.util.List;
@ -57,35 +55,4 @@ public abstract class AbstractClusterBuildingActionArgs
@ParametersDelegate
public OptionArgsDelegate optionsDelegate =
new OptionArgsDelegate();
public Map<String, String> getOptionsMap() throws
BadCommandArgumentsException {
return optionsDelegate.getOptionsMap();
}
/**
* Get the role heap mapping (may be empty, but never null).
* @return role heap mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, Map<String, String>> getCompOptionMap() throws
BadCommandArgumentsException {
return optionsDelegate.getCompOptionMap();
}
@VisibleForTesting
public List<String> getComponentTuples() {
return componentDelegate.getComponentTuples();
}
/**
* Get the role mapping (may be empty, but never null).
* @return role mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, String> getComponentMap() throws
BadCommandArgumentsException {
return componentDelegate.getComponentMap();
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
View File

@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
@Parameters(commandNames = { SliderActions.ACTION_BUILD},
commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
View File

@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
@Parameters(commandNames = { SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
View File

@ -17,8 +17,8 @@
*/
package org.apache.hadoop.yarn.service.client.params;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
View File

@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import java.util.List;
import java.util.Map;
@ -47,9 +47,4 @@ public class ActionFlexArgs extends AbstractActionArgs {
BadCommandArgumentsException {
return componentDelegate.getComponentMap();
}
public List<String> getComponentTuples() {
return componentDelegate.getComponentTuples();
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
View File

@ -16,13 +16,11 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
@Parameters(commandNames = { SliderActions.ACTION_STOP },
commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
@ -41,13 +41,4 @@ public class ActionHelpArgs extends AbstractActionArgs {
public int getMinParams() {
return 0;
}
/**
* This action does not need hadoop services
* @return false
*/
@Override
public boolean getHadoopServicesRequired() {
return false;
}
}

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKDiagArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
View File

@ -16,15 +16,13 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import java.io.File;
import java.util.ArrayList;
@ -65,16 +63,6 @@ public class ActionKDiagArgs extends AbstractActionArgs {
return 0;
}
@Override
public boolean getHadoopServicesRequired() {
return false;
}
@Override
public boolean disableSecureLogin() {
return true;
}
@Override
public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import java.util.HashSet;
import java.util.Set;

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
View File

@ -16,17 +16,14 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.Arguments;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.hadoop.yarn.service.conf.SliderKeys;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
import org.apache.slider.core.registry.docstore.ConfigFormat;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
@ -128,7 +125,7 @@ public class ActionRegistryArgs extends AbstractActionArgs {
@Parameter(names = {ARG_SERVICETYPE},
description = "optional service type")
public String serviceType = SliderKeys.APP_TYPE;
public String serviceType = YarnServiceConstants.APP_TYPE;
@Parameter(names = {ARG_VERBOSE},
description = "verbose output")

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
View File

@ -16,15 +16,13 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import java.io.File;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
View File

@ -16,13 +16,11 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import java.io.File;

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
View File

@ -16,14 +16,12 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import java.io.File;

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpdateArgs.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
View File

@ -16,10 +16,9 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
View File

@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
View File

@ -26,59 +26,38 @@ package org.apache.hadoop.yarn.service.client.params;
* so it is easier to see what arguments are there
*/
public interface Arguments {
String ARG_ADDON = "--addon";
String ARG_ALL = "--all";
String ARG_ALIAS = "--alias";
String ARG_APPLICATION = "--application";
String ARG_APPDEF = "--appdef";
String ARG_APP_HOME = "--apphome";
String ARG_BASE_PATH = "--basepath";
String ARG_CLIENT = "--client";
String ARG_CONFDIR = "--appconf";
String ARG_COMPONENT = "--component";
String ARG_COUNT = "--count";
String ARG_COMPONENT_SHORT = "--comp";
String ARG_COMPONENTS = "--components";
String ARG_COMP_OPT= "--compopt";
String ARG_COMP_OPT_SHORT = "--co";
String ARG_CONFIG = "--config";
String ARG_CONTAINERS = "--containers";
String ARG_CREDENTIALS = "--credentials";
String ARG_DEBUG = "--debug";
String ARG_DEFINE = "-D";
String ARG_DELETE = "--delete";
String ARG_DEST = "--dest";
String ARG_DESTDIR = "--destdir";
String ARG_DESTFILE = "--destfile";
String ARG_EXITCODE = "--exitcode";
String ARG_FAIL = "--fail";
/**
filesystem-uri: {@value}
*/
String ARG_FILESYSTEM = "--fs";
String ARG_FILESYSTEM_LONG = "--filesystem";
String ARG_FOLDER = "--folder";
String ARG_FORCE = "--force";
String ARG_FORMAT = "--format";
String ARG_GETCERTSTORE = "--getcertstore";
String ARG_GETCONF = "--getconf";
String ARG_GETEXP = "--getexp";
String ARG_GETFILES = "--getfiles";
String ARG_HEALTHY= "--healthy";
String ARG_HELP = "--help";
String ARG_HOSTNAME = "--hostname";
String ARG_ID = "--id";
String ARG_IMAGE = "--image";
String ARG_INSTALL = "--install";
String ARG_INTERNAL = "--internal";
String ARG_KEYLEN = "--keylen";
String ARG_KEYTAB = "--keytab";
String ARG_KEYSTORE = "--keystore";
String ARG_KEYTABINSTALL = ARG_INSTALL;
String ARG_KEYTABDELETE = ARG_DELETE;
String ARG_KEYTABLIST = "--list";
String ARG_LABEL = "--label";
String ARG_LEVEL = "--level";
String ARG_LIST = "--list";
String ARG_LISTCONF = "--listconf";
String ARG_LISTEXP = "--listexp";
@ -87,8 +66,6 @@ public interface Arguments {
String ARG_MANAGER = "--manager";
String ARG_MANAGER_SHORT = "--m";
String ARG_MESSAGE = "--message";
String ARG_METAINFO = "--metainfo";
String ARG_METAINFO_JSON = "--metainfojson";
String ARG_NAME = "--name";
String ARG_OPTION = "--option";
String ARG_OPTION_SHORT = "-O";
@ -96,35 +73,23 @@ public interface Arguments {
String ARG_OUTPUT_SHORT = "-o";
String ARG_OVERWRITE = "--overwrite";
String ARG_PACKAGE = "--package";
String ARG_PASSWORD = "--password";
String ARG_PATH = "--path";
String ARG_PKGDELETE = ARG_DELETE;
String ARG_PKGINSTANCES = "--instances";
String ARG_PKGLIST = ARG_LIST;
String ARG_PRINCIPAL = "--principal";
String ARG_PROVIDER = "--provider";
String ARG_QUEUE = "--queue";
String ARG_LIFETIME = "--lifetime";
String ARG_REPLACE_PKG = "--replacepkg";
String ARG_RESOURCE = "--resource";
String ARG_RESOURCE_MANAGER = "--rm";
String ARG_SECURE = "--secure";
String ARG_SERVICETYPE = "--servicetype";
String ARG_SERVICES = "--services";
String ARG_SLIDER = "--slider";
String ARG_SOURCE = "--source";
String ARG_STATE = "--state";
String ARG_SYSPROP = "-S";
String ARG_TRUSTSTORE = "--truststore";
String ARG_USER = "--user";
String ARG_UPLOAD = "--upload";
String ARG_VERBOSE = "--verbose";
String ARG_VERSION = "--version";
String ARG_WAIT = "--wait";
String ARG_YARN = "--yarn";
String ARG_ZKHOSTS = "--zkhosts";
String ARG_ZKPATH = "--zkpath";
String ARG_ZKPORT = "--zkport";
/*
STOP: DO NOT ADD YOUR ARGUMENTS HERE. GO BACK AND INSERT THEM IN THE
RIGHT PLACE IN THE LIST

147
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
View File

@ -20,78 +20,36 @@ package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionClientArgs;
import org.apache.slider.common.params.ActionDiagnosticArgs;
import org.apache.slider.common.params.ActionExistsArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionHelpArgs;
import org.apache.slider.common.params.ActionKDiagArgs;
import org.apache.slider.common.params.ActionKeytabArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionLookupArgs;
import org.apache.slider.common.params.ActionNodesArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionResourceArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.ActionTokensArgs;
import org.apache.slider.common.params.ActionUpdateArgs;
import org.apache.slider.common.params.ActionUpgradeArgs;
import org.apache.slider.common.params.ActionVersionArgs;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import java.util.Collection;
/**
* Slider Client CLI Args
* Client CLI Args
*/
public class ClientArgs extends CommonArgs {
/*
All the arguments for specific actions
*/
/**
* This is not bonded to jcommander, it is set up
* after the construction to point to the relevant
* entry
*
* KEEP IN ALPHABETICAL ORDER
*/
private AbstractClusterBuildingActionArgs buildingActionArgs;
// =========================================================
// Keep all of these in alphabetical order. Thanks.
// =========================================================
private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs();
private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
private final ActionClientArgs actionClientArgs = new ActionClientArgs();
private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
private final ActionDiagnosticArgs actionDiagnosticArgs = new ActionDiagnosticArgs();
private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
private final ActionKDiagArgs actionKDiagArgs = new ActionKDiagArgs();
private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
private final ActionKillContainerArgs actionKillContainerArgs =
new ActionKillContainerArgs();
private final ActionListArgs actionListArgs = new ActionListArgs();
private final ActionLookupArgs actionLookupArgs = new ActionLookupArgs();
private final ActionNodesArgs actionNodesArgs = new ActionNodesArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
@ -99,8 +57,6 @@ public class ClientArgs extends CommonArgs {
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
private final ActionUpgradeArgs actionUpgradeArgs = new ActionUpgradeArgs();
private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
public ClientArgs(String[] args) {
super(args);
@ -114,32 +70,15 @@ public class ClientArgs extends CommonArgs {
protected void addActionArguments() {
addActions(
actionAMSuicideArgs,
actionBuildArgs,
actionClientArgs,
actionCreateArgs,
actionDependencyArgs,
actionDestroyArgs,
actionDiagnosticArgs,
actionExistsArgs,
actionFlexArgs,
actionFreezeArgs,
actionHelpArgs,
actionKDiagArgs,
actionKeytabArgs,
actionKillContainerArgs,
actionListArgs,
actionLookupArgs,
actionNodesArgs,
actionRegistryArgs,
actionResolveArgs,
actionResourceArgs,
actionStatusArgs,
actionThawArgs,
actionTokenArgs,
actionUpdateArgs,
actionUpgradeArgs,
actionVersionArgs
actionThawArgs
);
}
@ -154,43 +93,20 @@ public class ClientArgs extends CommonArgs {
}
if (getBasePath() != null) {
log.debug("Setting basePath to {}", getBasePath());
conf.set(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH,
conf.set(YarnServiceConf.YARN_SERVICE_BASE_PATH,
getBasePath().toString());
}
}
public ActionDiagnosticArgs getActionDiagnosticArgs() {
return actionDiagnosticArgs;
}
public AbstractClusterBuildingActionArgs getBuildingActionArgs() {
return buildingActionArgs;
}
public ActionAMSuicideArgs getActionAMSuicideArgs() {
return actionAMSuicideArgs;
}
public ActionBuildArgs getActionBuildArgs() {
return actionBuildArgs;
}
public ActionClientArgs getActionClientArgs() { return actionClientArgs; }
public ActionKDiagArgs getActionKDiagArgs() {
return actionKDiagArgs;
}
public ActionKeytabArgs getActionKeytabArgs() { return actionKeytabArgs; }
public ActionUpdateArgs getActionUpdateArgs() {
return actionUpdateArgs;
}
public ActionUpgradeArgs getActionUpgradeArgs() {
return actionUpgradeArgs;
}
public ActionCreateArgs getActionCreateArgs() {
return actionCreateArgs;
}
@ -215,21 +131,10 @@ public class ClientArgs extends CommonArgs {
return actionFreezeArgs;
}
public ActionKillContainerArgs getActionKillContainerArgs() {
return actionKillContainerArgs;
}
public ActionListArgs getActionListArgs() {
return actionListArgs;
}
public ActionNodesArgs getActionNodesArgs() {
return actionNodesArgs;
}
public ActionLookupArgs getActionLookupArgs() {
return actionLookupArgs;
}
public ActionRegistryArgs getActionRegistryArgs() {
return actionRegistryArgs;
@ -268,14 +173,10 @@ public class ClientArgs extends CommonArgs {
switch (action) {
case ACTION_BUILD:
bindCoreAction(actionBuildArgs);
//its a builder, so set those actions too
buildingActionArgs = actionBuildArgs;
break;
case ACTION_CREATE:
bindCoreAction(actionCreateArgs);
//its a builder, so set those actions too
buildingActionArgs = actionCreateArgs;
break;
case ACTION_STOP:
@ -286,14 +187,6 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionThawArgs);
break;
case ACTION_AM_SUICIDE:
bindCoreAction(actionAMSuicideArgs);
break;
case ACTION_CLIENT:
bindCoreAction(actionClientArgs);
break;
case ACTION_DEPENDENCY:
bindCoreAction(actionDependencyArgs);
break;
@ -302,10 +195,6 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionDestroyArgs);
break;
case ACTION_DIAGNOSTICS:
bindCoreAction(actionDiagnosticArgs);
break;
case ACTION_EXISTS:
bindCoreAction(actionExistsArgs);
break;
@ -326,22 +215,10 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionKeytabArgs);
break;
case ACTION_KILL_CONTAINER:
bindCoreAction(actionKillContainerArgs);
break;
case ACTION_LIST:
bindCoreAction(actionListArgs);
break;
case ACTION_LOOKUP:
bindCoreAction(actionLookupArgs);
break;
case ACTION_NODES:
bindCoreAction(actionNodesArgs);
break;
case ACTION_REGISTRY:
bindCoreAction(actionRegistryArgs);
break;
@ -366,14 +243,6 @@ public class ClientArgs extends CommonArgs {
bindCoreAction(actionUpdateArgs);
break;
case ACTION_UPGRADE:
bindCoreAction(actionUpgradeArgs);
break;
case ACTION_VERSION:
bindCoreAction(actionVersionArgs);
break;
default:
throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
+ " " + action);

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
View File

@ -25,11 +25,11 @@ import com.beust.jcommander.ParameterException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.UsageException;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -112,7 +112,7 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
if (commandOfInterest == null) {
// JCommander.usage is too verbose for a command with many options like
// slider no short version of that is found Instead, we compose our msg by
helperMessage.append("\nUsage: slider COMMAND [options]\n");
helperMessage.append("\nUsage: service COMMAND [options]\n");
helperMessage.append("where COMMAND is one of\n");
for (String jcommand : serviceArgs.commander.getCommands().keySet()) {
helperMessage.append(String.format("\t%-"
@ -123,7 +123,7 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
.append("Most commands print help when invoked without parameters or with --help");
result = helperMessage.toString();
} else {
helperMessage.append("\nUsage: slider ").append(commandOfInterest);
helperMessage.append("\nUsage: service ").append(commandOfInterest);
helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <application>" : "");
helperMessage.append("\n");
for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
@ -223,14 +223,6 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
splitPairs(coreAction.sysprops, syspropsMap);
}
/**
* Get the core action -type depends on the action
* @return the action class
*/
public AbstractActionArgs getCoreAction() {
return coreAction;
}
/**
* Validate the arguments against the action requested
*/
@ -244,7 +236,8 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
coreAction.validate();
} catch (BadCommandArgumentsException e) {
String badArgMsgBuilder =
e.toString() + "\n" + usage(this, coreAction.getActionName());
e.getMessage() + System.lineSeparator() + usage(this,
coreAction.getActionName());
throw new BadCommandArgumentsException(badArgMsgBuilder);
}
}
@ -286,9 +279,4 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
public String getAction() {
return commander.getParsedCommand();
}
public List<String> getActionArgs() {
return coreAction.parameters;
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
View File

@ -19,9 +19,7 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import org.apache.slider.common.params.AbstractArgsDelegate;
import org.apache.slider.common.params.DontSplitArguments;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import java.util.ArrayList;
import java.util.List;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.converters.IParameterSplitter;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import java.io.File;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/OptionArgsDelegate.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
View File

@ -16,10 +16,10 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import java.util.ArrayList;
import java.util.List;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.converters.BaseConverter;
import org.apache.hadoop.fs.Path;

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
View File

@ -21,10 +21,6 @@ package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.slider.common.params.LaunchArgsAccessor;
import org.apache.slider.common.params.LaunchArgsDelegate;
import java.io.File;

48
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
View File

@ -24,16 +24,13 @@ package org.apache.hadoop.yarn.service.client.params;
* are listed here to ensure the names are consistent
*/
public interface SliderActions {
String ACTION_AM_SUICIDE = "am-suicide";
String ACTION_BUILD = "build";
String ACTION_CLIENT = "client";
String ACTION_CREATE = "create";
String ACTION_DIAGNOSTICS = "diagnostics";
String ACTION_DEPENDENCY = "dependency";
String ACTION_UPDATE = "update";
String ACTION_UPGRADE = "upgrade";
String ACTION_DESTROY = "destroy";
String ACTION_ECHO = "echo";
String ACTION_EXISTS = "exists";
String ACTION_FLEX = "flex";
String ACTION_STOP = "stop";
@ -41,12 +38,8 @@ public interface SliderActions {
String ACTION_INSTALL_KEYTAB = "install-keytab";
String ACTION_KDIAG = "kdiag";
String ACTION_KEYTAB = "keytab";
String ACTION_KILL_CONTAINER = "kill-container";
String ACTION_LIST = "list";
String ACTION_LOOKUP = "lookup";
String ACTION_NODES = "nodes";
String ACTION_PREFLIGHT = "preflight";
String ACTION_RECONFIGURE = "reconfigure";
String ACTION_REGISTRY = "registry";
String ACTION_RESOLVE = "resolve";
String ACTION_RESOURCE = "resource";
@ -54,52 +47,35 @@ public interface SliderActions {
String ACTION_START = "start";
String ACTION_TOKENS = "tokens";
String ACTION_VERSION = "version";
String DESCRIBE_ACTION_AM_SUICIDE =
"Tell the Slider Application Master to simulate a process failure by terminating itself";
String DESCRIBE_ACTION_BUILD =
"Build a Slider cluster specification, but do not start it";
"Build a service specification, but do not start it";
String DESCRIBE_ACTION_CREATE =
"Create a live Slider application";
"Build and start a service, it's equivalent to first invoke build and then start";
String DESCRIBE_ACTION_DEPENDENCY =
"Slider AM and agent dependency (libraries) management";
"Yarn service framework dependency (libraries) management";
String DESCRIBE_ACTION_UPDATE =
"Update template for a Slider application";
"Update template for service";
String DESCRIBE_ACTION_UPGRADE =
"Rolling upgrade/downgrade the component/containerto a newer/previous version";
String DESCRIBE_ACTION_DESTROY =
"Destroy a stopped Slider application";
"Destroy a stopped service, service must be stopped first before destroying.";
String DESCRIBE_ACTION_EXISTS =
"Probe for an application running";
String DESCRIBE_ACTION_FLEX = "Flex a Slider application";
String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
String DESCRIBE_ACTION_FREEZE =
"Stop a running application";
String DESCRIBE_ACTION_GETCONF =
"Get the configuration of an application";
"Stop a running service";
String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
String DESCRIBE_ACTION_KILL_CONTAINER =
"Kill a container in the application";
String DESCRIBE_ACTION_HELP = "Print help information";
String DESCRIBE_ACTION_LIST =
"List running Slider applications";
String DESCRIBE_ACTION_LOOKUP =
"look up a YARN application";
String DESCRIBE_ACTION_NODES = "List the node information for the YARN cluster or a running application";
String DESCRIBE_ACTION_MONITOR =
"Monitor a running application";
"List running services";
String DESCRIBE_ACTION_REGISTRY =
"Query the registry of a YARN application";
String DESCRIBE_ACTION_RESOLVE =
"Resolve or list records in the YARN registry";
"Query the registry of a service";
String DESCRIBE_ACTION_STATUS =
"Get the status of an application";
"Get the status of a service";
String DESCRIBE_ACTION_THAW =
"Start a stopped application";
String DESCRIBE_ACTION_VERSION =
"Print the Slider version information";
"Start a service with pre-built specification or a previously stopped service";
String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore";
String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.common.params;
package org.apache.hadoop.yarn.service.client.params;
public interface WaitTimeAccessor {
int getWaittime();

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
View File

@ -36,17 +36,17 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.service.ServiceScheduler;
import org.apache.hadoop.yarn.service.api.records.ContainerState;
import org.apache.hadoop.yarn.service.component.Component;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.BoundedAppender;
import org.apache.slider.api.resource.ContainerState;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
import org.apache.slider.server.servicemonitor.ProbeStatus;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus;
import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -88,7 +88,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
private volatile ContainerStatus status;
private long containerStartedTime = 0;
// This container object is used for rest API query
private org.apache.slider.api.resource.Container containerSpec;
private org.apache.hadoop.yarn.service.api.records.Container containerSpec;
private static final StateMachineFactory<ComponentInstance,
ComponentInstanceState, ComponentInstanceEventType, ComponentInstanceEvent>
@ -142,11 +142,11 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
compInstance.getContainerId(), compInstance), 0, 1,
TimeUnit.SECONDS);
org.apache.slider.api.resource.Container container =
new org.apache.slider.api.resource.Container();
org.apache.hadoop.yarn.service.api.records.Container container =
new org.apache.hadoop.yarn.service.api.records.Container();
container.setId(compInstance.getContainerId().toString());
container.setLaunchTime(new Date());
container.setState(org.apache.slider.api.resource.ContainerState.RUNNING_BUT_UNREADY);
container.setState(ContainerState.RUNNING_BUT_UNREADY);
container.setBareHost(compInstance.container.getNodeId().getHost());
container.setComponentName(compInstance.getCompInstanceName());
if (compInstance.containerSpec != null) {
@ -290,7 +290,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
public void updateContainerStatus(ContainerStatus status) {
this.status = status;
org.apache.slider.api.resource.Container container =
org.apache.hadoop.yarn.service.api.records.Container container =
getCompSpec().getContainer(getContainerId().toString());
if (container != null) {
container.setIp(StringUtils.join(",", status.getIPs()));
@ -330,7 +330,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
return this.container.getNodeId();
}
public org.apache.slider.api.resource.Component getCompSpec() {
public org.apache.hadoop.yarn.service.api.records.Component getCompSpec() {
return component.getComponentSpec();
}

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
View File

@ -39,9 +39,9 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.server.servicemonitor.MonitorUtils;
import org.apache.slider.server.servicemonitor.Probe;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.servicemonitor.probe.MonitorUtils;
import org.apache.hadoop.yarn.service.servicemonitor.probe.Probe;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -61,12 +61,12 @@ import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STARTED;
import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STOP;
import static org.apache.hadoop.yarn.service.component.ComponentState.*;
import static org.apache.slider.api.ResourceKeys.CONTAINER_FAILURE_THRESHOLD;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD;
public class Component implements EventHandler<ComponentEvent> {
private static final Logger LOG = LoggerFactory.getLogger(Component.class);
private org.apache.slider.api.resource.Component componentSpec;
private org.apache.hadoop.yarn.service.api.records.Component componentSpec;
private long allocateId;
private Priority priority;
private ServiceMetrics componentMetrics;
@ -124,7 +124,8 @@ public class Component implements EventHandler<ComponentEvent> {
FLEX, new FlexComponentTransition())
.installTopology();
public Component(org.apache.slider.api.resource.Component component,
public Component(
org.apache.hadoop.yarn.service.api.records.Component component,
long allocateId, ServiceContext context) {
this.allocateId = allocateId;
this.priority = Priority.newInstance((int) allocateId);
@ -418,7 +419,7 @@ public class Component implements EventHandler<ComponentEvent> {
return compInstances;
}
public org.apache.slider.api.resource.Component getComponentSpec() {
public org.apache.hadoop.yarn.service.api.records.Component getComponentSpec() {
return this.componentSpec;
}

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.conf;
public interface RestApiConstants {
// Rest endpoints
String CONTEXT_ROOT = "/services/v1";
String VERSION = "/version";
String APP_ROOT_PATH = "/applications";
String APP_PATH = "/applications/{app_name}";
String COMPONENT_PATH = "/applications/{app_name}/components/{component_name}";
// Query param
String APP_NAME = "app_name";
String COMPONENT_NAME = "component_name";
String DEFAULT_COMPONENT_NAME = "default";
String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
Long DEFAULT_UNLIMITED_LIFETIME = -1l;
Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
Integer ERROR_CODE_APP_NAME_INVALID = 404004;
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
View File

@ -18,7 +18,7 @@
package org.apache.hadoop.yarn.service.conf;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
public interface SliderExitCodes extends LauncherExitCodes {

195
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java
View File

@ -1,195 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.conf;
/**
* Keys and various constants for Slider
*/
public interface SliderKeys extends SliderXmlConfKeys {
/**
* This is the name of the slider appmaster in configurations :{@value}
*/
String COMPONENT_AM = "slider-appmaster";
/**
* Slider role is "special":{@value}
*/
int ROLE_AM_PRIORITY_INDEX = 0;
/**
* The path under which cluster and temp data are stored
* {@value}
*/
String SLIDER_BASE_DIRECTORY = ".slider";
/**
* The paths under which Slider AM dependency libraries are stored
*/
String SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK = "slider_dep";
String SLIDER_DEPENDENCY_HDP_PARENT_DIR = "/hdp";
String SLIDER_DEPENDENCY_DIR = "/apps/%s/slider";
String SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME = "slider-dep";
String SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz";
String SLIDER_DEPENDENCY_DIR_PERMISSIONS = "755";
/**
*
*/
String HDP_VERSION_PROP_NAME = "HDP_VERSION";
/**
* name of the relative path to expaned an image into: {@value}.
* The title of this path is to help people understand it when
* they see it in their error messages
*/
String LOCAL_TARBALL_INSTALL_SUBDIR = "expandedarchive";
/**
* Application type for YARN {@value}
*/
String APP_TYPE = "org-apache-slider";
/**
* A component type for an external app that has been predefined using the
* slider build command
*/
String COMPONENT_SEPARATOR = "-";
/**
* A component type for a client component
*/
String COMPONENT_TYPE_CLIENT = "client";
/**
* Key for application version.
*/
String APP_VERSION_UNKNOWN = "awaiting heartbeat...";
/**
* Keys for application container specific properties, like release timeout
*/
String APP_CONTAINER_RELEASE_TIMEOUT = "site.global.app_container.release_timeout_secs";
/**
* Subdirectories of HDFS cluster dir.
*/
String DATA_DIR_NAME = "data";
String HISTORY_DIR_NAME = "history";
String HISTORY_FILENAME_SUFFIX = "json";
String HISTORY_FILENAME_PREFIX = "rolehistory-";
String KEYTAB_DIR = "keytabs";
String RESOURCE_DIR = "resources";
/**
* Filename pattern is required to save in strict temporal order.
* Important: older files must sort less-than newer files when using
* case-sensitive name sort.
*/
String HISTORY_FILENAME_CREATION_PATTERN = HISTORY_FILENAME_PREFIX +"%016x."+
HISTORY_FILENAME_SUFFIX;
/**
* The posix regexp used to locate this
*/
String HISTORY_FILENAME_MATCH_PATTERN = HISTORY_FILENAME_PREFIX +"[0-9a-f]+\\."+
HISTORY_FILENAME_SUFFIX;
/**
* The posix regexp used to locate this
*/
String HISTORY_FILENAME_GLOB_PATTERN = HISTORY_FILENAME_PREFIX +"*."+
HISTORY_FILENAME_SUFFIX;
String CLUSTER_DIRECTORY = "cluster";
/**
* JVM property to define the slider lib directory;
* this is set by the slider script: {@value}
*/
String PROPERTY_LIB_DIR = "slider.libdir";
/**
* name of generated dir for this conf: {@value}
*/
String SUBMITTED_CONF_DIR = "conf";
/**
* Slider AM log4j file name : {@value}
*/
String LOG4J_SERVER_PROP_FILENAME = "slideram-log4j.properties";
/**
* Log4j sysprop to name the resource :{@value}
*/
String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
/**
* sysprop for Slider AM log4j directory :{@value}
*/
String SYSPROP_LOG_DIR = "LOG_DIR";
/**
* name of the Slider client resource
* loaded when the service is loaded.
*/
String SLIDER_CLIENT_XML = "slider-client.xml";
/**
* The name of the resource to put on the classpath
*/
String SLIDER_SERVER_XML = "slider-server.xml";
String TMP_DIR_PREFIX = "tmp";
/**
* Store the default app definition, e.g. metainfo file or content of a folder
*/
String APP_DEF_DIR = "appdef";
/**
* Store additional app defs - co-processors
*/
String ADDONS_DIR = "addons";
String SLIDER_JAR = "slider-core.jar";
String STDOUT_AM = "slider-out.txt";
String STDERR_AM = "slider-err.txt";
String HADOOP_USER_NAME = "HADOOP_USER_NAME";
/**
* Name of the AM filter to use: {@value}
*/
String AM_FILTER_NAME =
"org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer";
String YARN_CONTAINER_PATH = "/node/container/";
String APP_CONF_DIR = "conf";
String APP_LIB_DIR = "lib";
String OUT_FILE = "stdout.txt";
String ERR_FILE = "stderr.txt";
String QUICK_LINKS = "quicklinks";
String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
}

191
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java
View File

@ -1,191 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.conf;
/**
* These are the keys that can be added to <code>conf/slider-client.xml</code>.
*/
public interface SliderXmlConfKeys {
String PREFIX_PROVIDER = "slider.provider";
/**
* pattern to identify a provider
* {@value}
*/
String KEY_PROVIDER = PREFIX_PROVIDER + ".%s";
/**
* conf option set to point to where the config came from
* {@value}
*/
String KEY_TEMPLATE_ORIGIN = "slider.template.origin";
/**
* Original name for the default FS. This is still
* expected by applications deployed
*/
String FS_DEFAULT_NAME_CLASSIC = "fs.default.name";
/**
* Slider principal
*/
String KEY_KERBEROS_PRINCIPAL = "slider.kerberos.principal";
/**
* Name of the property for ACLs for Slider AM.
* {@value}
*/
String KEY_PROTOCOL_ACL = "slider.security.protocol.acl";
/**
* Limit on restarts for the AM
* {@value}
*/
String KEY_AM_RESTART_LIMIT = "slider.yarn.restart.limit";
/**
* queue name, by default let YARN pick the queue
*/
String KEY_YARN_QUEUE = "slider.yarn.queue";
String DEFAULT_YARN_QUEUE = null;
/**
* default priority
*/
String KEY_YARN_QUEUE_PRIORITY = "slider.yarn.queue.priority";
int DEFAULT_YARN_QUEUE_PRIORITY = 1;
String KEY_AM_RESOURCE_MEM = "slider.am.resource.memory";
long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
/**
* The slider base path: {@value}
* Defaults to HomeDir/.slider
*/
String KEY_SLIDER_BASE_PATH = "slider.base.path";
/**
* Option for the permissions for the cluster directory itself: {@value}
*/
String CLUSTER_DIRECTORY_PERMISSIONS =
"slider.cluster.directory.permissions";
/**
* Default value for the permissions :{@value}
*/
String DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS = "750";
/**
*
* Option for the permissions for the data directory itself: {@value}
*/
String DATA_DIRECTORY_PERMISSIONS = "slider.data.directory.permissions";
/**
* Default value for the data directory permissions: {@value}
*/
String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750";
String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH =
"ipc.client.fallback-to-simple-auth-allowed";
String HADOOP_HTTP_FILTER_INITIALIZERS =
"hadoop.http.filter.initializers";
String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location";
String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
/** Declare that a keytab must be provided */
String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
String KEY_SECURITY_ENABLED = "site.global.security_enabled";
/**
* Set to disable server-side checks for python, openssl &c.
* This should only be set for testing
*/
String KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED =
"slider.am.dependency.checks.disabled";
/**
* The path to the python executable utilized to launch the agent.
*/
String PYTHON_EXECUTABLE_PATH = "agent.python.exec.path";
/**
* Flag to enable the insecure AM filter: {@value}
*/
String X_DEV_INSECURE_WS = "slider.feature.ws.insecure";
/**
* Flag to indicate the insecure AM filter is enabled by default: {@value}.
*/
boolean X_DEV_INSECURE_DEFAULT = false;
/**
* Flag to indicate the insecure AM filter is required for
* complex REST Verbs: {@value}.
* When Slider switches to being Hadoop 2.7+ only, this flag
* can be set to false
*/
boolean X_DEV_INSECURE_REQUIRED = true;
/**
*
*/
String KEY_IPC_CLIENT_RETRY_POLICY_ENABLED =
"slider.ipc.client.retry.enabled";
boolean IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true;
String KEY_IPC_CLIENT_RETRY_POLICY_SPEC =
"slider.ipc.client.retry.policy.spec";
String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
"10000,6,60000,10"; //t1,n1,t2,n2,...
String KEY_AM_LAUNCH_ENV = "slider.am.launch.env";
/**
* From {@code DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY}
*/
String DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY = "dfs.namenode.kerberos.principal";
String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal";
//Delegation token related keys
String DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY
= "dfs.namenode.delegation.key.update-interval";
long DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60 *
1000; // 1 day
String DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY
= "dfs.namenode.delegation.token.renew-interval";
long DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 *
1000; // 1 day
String DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY
= "dfs.namenode.delegation.token.max-lifetime";
long DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 *
1000; // 7 days
String DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY
= "dfs.namenode.delegation.token.always-use"; // for tests
boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false;
String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file";
String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved";
String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary";
}

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
View File

@ -18,7 +18,7 @@
package org.apache.hadoop.yarn.service.conf;
import org.apache.slider.api.resource.Configuration;
import org.apache.hadoop.yarn.service.api.records.Configuration;
public class YarnServiceConf {
@ -30,8 +30,54 @@ public class YarnServiceConf {
public static final String CONTAINER_RETRY_MAX = "yarn.service.container-failure.retry.max";
public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval";
public static final String AM_RESTART_MAX = "yarn.service.am-restart.max-attempts";
public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory";
public static final long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
public static final String YARN_QUEUE = "yarn.service.queue";
/**
* Get long value for the property
* The yarn service base path:
* Defaults to HomeDir/.yarn/
*/
public static final String YARN_SERVICE_BASE_PATH = "yarn.service.base.path";
//TODO rename
/** Declare that a keytab must be provided */
public static final String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
public static final String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
public static final String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
public static final String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
/**
* maximum number of failed containers (in a single component)
* before the app exits
*/
public static final String CONTAINER_FAILURE_THRESHOLD =
"yarn.service.container-failure-per-component.threshold";
/**
* Maximum number of container failures on a node before the node is blacklisted
*/
public static final String NODE_BLACKLIST_THRESHOLD =
"yarn.service.node-blacklist.threshold";
/**
* The failure count for CONTAINER_FAILURE_THRESHOLD and NODE_BLACKLIST_THRESHOLD
* gets reset periodically, the unit is seconds.
*/
public static final String CONTAINER_FAILURE_WINDOW =
"yarn.service.failure-count-reset.window";
/**
* interval between readiness checks.
*/
public static final String READINESS_CHECK_INTERVAL = "yarn.service.readiness-check-interval.seconds";
public static final int DEFAULT_READINESS_CHECK_INTERVAL = 30; // seconds
/**
* Get long value for the property. First get from the userConf, if not
* present, get from systemConf.
*
* @param name name of the property
* @param defaultValue default value of the property, if it is not defined in
* userConf and systemConf.

90
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java Normal file
View File

@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.conf;
public interface YarnServiceConstants {
/**
* The path under which cluster and temp data are stored
*/
String SERVICE_BASE_DIRECTORY = ".yarn";
/**
* The paths under which Service AM dependency libraries are stored
*/
String DEPENDENCY_LOCALIZED_DIR_LINK = "service_dep";
String DEPENDENCY_DIR = "/yarn-services/%s/";
String DEPENDENCY_TAR_GZ_FILE_NAME = "service-dep";
String DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz";
String DEPENDENCY_DIR_PERMISSIONS = "755";
/**
* Application type for YARN service
*/
String APP_TYPE = "yarn-service";
String KEYTAB_DIR = "keytabs";
String RESOURCE_DIR = "resources";
String SERVICES_DIRECTORY = "services";
/**
* JVM property to define the service lib directory;
* this is set by the yarn.sh script
*/
String PROPERTY_LIB_DIR = "service.libdir";
/**
* name of generated dir for this conf
*/
String SUBMITTED_CONF_DIR = "conf";
/**
* Service AM log4j file name
*/
String YARN_SERVICE_LOG4J_FILENAME = "yarnservice-log4j.properties";
/**
* Log4j sysprop to name the resource
*/
String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
/**
* sysprop for Service AM log4j directory
*/
String SYSPROP_LOG_DIR = "LOG_DIR";
String TMP_DIR_PREFIX = "tmp";
String SERVICE_CORE_JAR = "yarn-service-core.jar";
String STDOUT_AM = "serviceam-out.txt";
String STDERR_AM = "serviceam-err.txt";
String HADOOP_USER_NAME = "HADOOP_USER_NAME";
String APP_CONF_DIR = "conf";
String APP_LIB_DIR = "lib";
String OUT_FILE = "stdout.txt";
String ERR_FILE = "stderr.txt";
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.core.launch;
package org.apache.hadoop.yarn.service.containerlaunch;
import com.google.common.base.Preconditions;
import org.apache.hadoop.security.Credentials;
@ -26,9 +26,9 @@ import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.service.conf.SliderKeys;
import org.apache.slider.common.tools.CoreFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -220,7 +220,7 @@ public class AbstractLauncher {
protected void propagateUsernameInInsecureCluster() throws IOException {
//insecure cluster: propagate user name via env variable
String userName = UserGroupInformation.getCurrentUser().getUserName();
envVars.put(SliderKeys.HADOOP_USER_NAME, userName);
envVars.put(YarnServiceConstants.HADOOP_USER_NAME, userName);
}
/**

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
View File

@ -16,13 +16,13 @@
* limitations under the License.
*/
package org.apache.slider.core.launch;
package org.apache.hadoop.yarn.service.containerlaunch;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import java.util.ArrayList;
import java.util.Arrays;

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
View File

@ -16,11 +16,11 @@
* limitations under the License.
*/
package org.apache.slider.core.launch;
package org.apache.hadoop.yarn.service.containerlaunch;
import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import java.util.ArrayList;
import java.util.List;
@ -42,10 +42,7 @@ public class CommandLineBuilder {
}
}
/**
* Get the number of arguments
* @return an integer >= 0
*/
// Get the number of arguments
public int size() {
return argumentList.size();
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
View File

@ -16,17 +16,17 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.service;
package org.apache.hadoop.yarn.service.containerlaunch;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.provider.ProviderService;
import org.apache.hadoop.yarn.service.provider.ProviderFactory;
import org.apache.slider.api.resource.Application;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.launch.AbstractLauncher;
import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -81,7 +81,7 @@ public class ContainerLaunchService extends AbstractService{
}
@Override public void run() {
org.apache.slider.api.resource.Component compSpec = instance.getCompSpec();
Component compSpec = instance.getCompSpec();
ProviderService provider = ProviderFactory.getProviderService(
compSpec.getArtifact());
AbstractLauncher launcher = new AbstractLauncher(fs, null);

62
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.core.launch;
package org.apache.hadoop.yarn.service.containerlaunch;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
@ -37,13 +37,11 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.Serializable;
@ -55,9 +53,7 @@ import java.util.Comparator;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import static org.apache.hadoop.security.UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
/**
@ -67,9 +63,6 @@ import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
*/
public final class CredentialUtils {
public static final String JOB_CREDENTIALS_BINARY
= SliderXmlConfKeys.MAPREDUCE_JOB_CREDENTIALS_BINARY;
private CredentialUtils() {
}
@ -97,59 +90,6 @@ public final class CredentialUtils {
return buffer;
}
public static File locateEnvCredentials(Map<String, String> env,
Configuration conf,
StringBuffer sourceTextOut) throws FileNotFoundException {
String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION);
String source = "environment variable " + HADOOP_TOKEN_FILE_LOCATION;
if (tokenFilename == null) {
tokenFilename = conf.get(JOB_CREDENTIALS_BINARY);
source = "configuration option " + JOB_CREDENTIALS_BINARY;
}
if (tokenFilename != null) {
// use delegation tokens, i.e. from Oozie
File file = new File(tokenFilename.trim());
String details = String.format(
"Token File %s from %s",
file,
source);
if (!file.exists()) {
throw new FileNotFoundException("No " + details);
}
if (!file.isFile() && !file.canRead()) {
throw new FileNotFoundException("Cannot read " + details);
}
sourceTextOut.append(details);
return file;
} else {
return null;
}
}
/**
* Load the credentials from the environment. This looks at
* the value of {@link UserGroupInformation#HADOOP_TOKEN_FILE_LOCATION}
* and attempts to read in the value
* @param env environment to resolve the variable from
* @param conf configuration use when reading the tokens
* @return a set of credentials, or null if the environment did not
* specify any
* @throws IOException if a location for credentials was defined, but
* the credentials could not be loaded.
*/
public static Credentials loadTokensFromEnvironment(Map<String, String> env,
Configuration conf)
throws IOException {
StringBuffer origin = new StringBuffer();
File file = locateEnvCredentials(env, conf, origin);
if (file != null) {
LOG.debug("Using {}", origin);
return Credentials.readTokenStorageFile(file, conf);
} else {
return null;
}
}
/**
* Save credentials to a file
* @param file file to save to (will be overwritten)

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
View File

@ -16,16 +16,15 @@
* limitations under the License.
*/
package org.apache.slider.core.launch;
package org.apache.hadoop.yarn.service.containerlaunch;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
import java.util.Iterator;
import java.util.Map;
/**

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
View File

@ -16,8 +16,9 @@
* limitations under the License.
*/
package org.apache.slider.core.exceptions;
package org.apache.hadoop.yarn.service.exceptions;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
/**
* The system is in a bad state

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadCommandArgumentsException.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.core.exceptions;
package org.apache.hadoop.yarn.service.exceptions;
public class BadCommandArgumentsException extends SliderException {
public BadCommandArgumentsException(String s, Object... args) {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/BadConfigException.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.core.exceptions;
package org.apache.hadoop.yarn.service.exceptions;
/**
* An exception to raise on a bad configuration

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.core.exceptions;
package org.apache.hadoop.yarn.service.exceptions;
public interface ErrorStrings {
String E_UNSTABLE_CLUSTER = "Unstable Application Instance :";

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ExitCodeProvider.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.core.main;
package org.apache.hadoop.yarn.service.exceptions;
/**
* Get the exit code of an exception. Making it an interface allows

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.core.main;
package org.apache.hadoop.yarn.service.exceptions;
/*
* Common Exit codes

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.slider.util;
package org.apache.hadoop.yarn.service.exceptions;
public interface RestApiErrorMessages {
String ERROR_APPLICATION_NAME_INVALID =

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.core.main;
package org.apache.hadoop.yarn.service.exceptions;
import org.apache.hadoop.yarn.exceptions.YarnException;

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
View File

@ -16,10 +16,9 @@
* limitations under the License.
*/
package org.apache.slider.core.exceptions;
package org.apache.hadoop.yarn.service.exceptions;
import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
import org.apache.slider.core.main.ServiceLaunchException;
public class SliderException extends ServiceLaunchException implements
SliderExitCodes {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.slider.core.exceptions;
package org.apache.hadoop.yarn.service.exceptions;
/**
* Used to raise a usage exception ... this has the exit code

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
View File

@ -19,18 +19,12 @@
package org.apache.hadoop.yarn.service.provider;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.SliderException;
import org.codehaus.jettison.json.JSONObject;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.HashSet;

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
View File

@ -20,14 +20,14 @@ package org.apache.hadoop.yarn.service.provider;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Component;
import org.apache.hadoop.yarn.service.conf.SliderKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.AbstractLauncher;
import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.ServiceContext;
import org.slf4j.Logger;
@ -42,7 +42,7 @@ import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETR
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
public abstract class AbstractProviderService implements ProviderService,
SliderKeys {
YarnServiceConstants {
protected static final Logger log =
LoggerFactory.getLogger(AbstractProviderService.class);

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
View File

@ -19,8 +19,7 @@
package org.apache.hadoop.yarn.service.provider;
import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
import org.slf4j.Logger;

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
View File

@ -19,10 +19,10 @@
package org.apache.hadoop.yarn.service.provider;
import org.apache.hadoop.conf.Configuration;
import org.apache.slider.api.resource.Application;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.AbstractLauncher;
import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import java.io.IOException;

111
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
View File

@ -24,30 +24,24 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Component;
import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.resource.Configuration;
import org.apache.hadoop.yarn.service.conf.SliderKeys;
import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.AbstractLauncher;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.registry.docstore.ConfigFormat;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.ServiceContext;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
import org.apache.hadoop.yarn.service.api.records.Configuration;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -55,23 +49,18 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.regex.Pattern;
import static org.apache.slider.api.ServiceApiConstants.*;
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
import static org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*;
/**
* This is a factoring out of methods handy for providers. It's bonded to a log
* at construction time.
*/
public class ProviderUtils implements RoleKeys, SliderKeys {
public class ProviderUtils implements YarnServiceConstants {
protected static final Logger log =
LoggerFactory.getLogger(ProviderUtils.class);
@ -173,47 +162,23 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
}
/**
* Get resource requirements from a String value. If value isn't specified,
* use the default value. If value is greater than max, use the max value.
* @param val string value
* @param defVal default value
* @param maxVal maximum value
* @return int resource requirement
*/
public int getRoleResourceRequirement(String val,
int defVal,
int maxVal) {
if (val==null) {
val = Integer.toString(defVal);
}
Integer intVal;
if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
intVal = maxVal;
} else {
intVal = Integer.decode(val);
}
return intVal;
}
/**
* Localize the service keytabs for the application.
* @param launcher container launcher
* @param fileSystem file system
* @throws IOException trouble uploading to HDFS
*/
public void localizeServiceKeytabs(ContainerLauncher launcher,
public void localizeServiceKeytabs(AbstractLauncher launcher,
SliderFileSystem fileSystem, Application application) throws IOException {
Configuration conf = application.getConfiguration();
String keytabPathOnHost =
conf.getProperty(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
if (SliderUtils.isUnset(keytabPathOnHost)) {
String amKeytabName =
conf.getProperty(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
String keytabDir =
conf.getProperty(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
conf.getProperty(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
// we need to localize the keytab files in the directory
Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
application.getName());
@ -434,38 +399,4 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
String.valueOf(instance.getCompInstanceId().getId()));
return tokens;
}
/**
* Add ROLE_HOST tokens for substitution into config values.
* @param tokens existing tokens
* @param amState access to AM state
*/
public static void addComponentHostTokens(Map<String, String> tokens,
StateAccessForProviders amState) {
if (amState == null) {
return;
}
for (Map.Entry<String, Map<String, ClusterNode>> entry :
amState.getRoleClusterNodeMapping().entrySet()) {
String tokenName = entry.getKey().toUpperCase(Locale.ENGLISH) + "_HOST";
String hosts = StringUtils .join(",",
getHostsList(entry.getValue().values(), true));
tokens.put($(tokenName), hosts);
}
}
/**
* Return a list of hosts based on current ClusterNodes.
* @param values cluster nodes
* @param hostOnly whether host or host/server name will be added to list
* @return list of hosts
*/
public static Iterable<String> getHostsList(Collection<ClusterNode> values,
boolean hostOnly) {
List<String> hosts = new ArrayList<>();
for (ClusterNode cn : values) {
hosts.add(hostOnly ? cn.host : cn.host + "/" + cn.name);
}
return hosts;
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
View File

@ -19,8 +19,8 @@ package org.apache.hadoop.yarn.service.provider.defaultImpl;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.ConfigFile;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import java.io.IOException;
import java.nio.file.Paths;

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
View File

@ -19,9 +19,9 @@ package org.apache.hadoop.yarn.service.provider.defaultImpl;
import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
import org.apache.slider.api.resource.Application;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.launch.AbstractLauncher;
import org.apache.hadoop.yarn.service.api.records.Application;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import java.io.IOException;

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
View File

@ -19,16 +19,16 @@ package org.apache.hadoop.yarn.service.provider.docker;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.ConfigFile;
import org.apache.hadoop.yarn.service.conf.SliderKeys;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
import org.apache.slider.util.RestApiErrorMessages;
import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
import java.io.IOException;
public class DockerClientProvider extends AbstractClientProvider
implements SliderKeys {
implements YarnServiceConstants {
public DockerClientProvider() {
super();

Some files were not shown because too many files have changed in this diff Show More